修复跨 Docker 版本灰度镜像校验
This commit is contained in:
@@ -24,7 +24,9 @@ done < "$release/release.env"
|
||||
[[ "$CHAT_IMAGE" =~ ^baiyue-logistics-chat-gray:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2
|
||||
[[ "$HOST_PORT" =~ ^[0-9]+$ ]] && ((HOST_PORT >= 1024 && HOST_PORT <= 65535)) || exit 2
|
||||
[[ -d "$CHAT_PROXY_DIR" && -r "$CHAT_TLS_CERT" && -r "$CHAT_TLS_KEY" ]] || exit 2
|
||||
[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_IMAGE")" == "$CHAT_IMAGE_ID" ]] || exit 2
|
||||
loaded_id=$(awk -F= '$1 == "CHAT_IMAGE_ID" {print $2}' "$release/loaded-image-ids.env")
|
||||
[[ "$loaded_id" =~ ^sha256:[a-f0-9]{64}$ ]] || exit 2
|
||||
[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_IMAGE")" == "$loaded_id" ]] || { echo 'Loaded application image changed.' >&2; exit 2; }
|
||||
[[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CHAT_IMAGE")" == "$SOURCE_COMMIT" ]] || exit 2
|
||||
openssl x509 -in "$CHAT_TLS_CERT" -checkhost "$CHAT_DOMAIN" -noout >/dev/null
|
||||
|
||||
|
||||
@@ -18,7 +18,9 @@ done < "$release/release.env"
|
||||
[[ "$CHAT_TLS_CERT" == "$root/tls/config/live/$CHAT_DOMAIN/fullchain.pem" ]] || exit 2
|
||||
[[ "$CHAT_TLS_KEY" == "$root/tls/config/live/$CHAT_DOMAIN/privkey.pem" ]] || exit 2
|
||||
[[ "$CHAT_CERTBOT_IMAGE" =~ ^baiyue-logistics-chat-certbot:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2
|
||||
[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_CERTBOT_IMAGE")" == "$CHAT_CERTBOT_IMAGE_ID" ]] || exit 2
|
||||
loaded_id=$(awk -F= '$1 == "CERTBOT_IMAGE_ID" {print $2}' "$release/loaded-image-ids.env")
|
||||
[[ "$loaded_id" =~ ^sha256:[a-f0-9]{64}$ ]] || exit 2
|
||||
[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_CERTBOT_IMAGE")" == "$loaded_id" ]] || { echo 'Loaded certificate image changed.' >&2; exit 2; }
|
||||
[[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CHAT_CERTBOT_IMAGE")" == "$SOURCE_COMMIT" ]] || exit 2
|
||||
|
||||
site="$CHAT_PROXY_DIR/baiyue-logistics-chat-gray.conf"
|
||||
|
||||
+33
-3
@@ -40,7 +40,7 @@ release=$2
|
||||
[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2
|
||||
[[ "$release" == "$root"/releases/* ]] || exit 2
|
||||
[[ "$(id -u)" == 0 ]] || { echo 'The managed gray deployment account must have the required host permissions.' >&2; exit 2; }
|
||||
for program in docker nginx openssl curl ss systemctl; do command -v "$program" >/dev/null; done
|
||||
for program in docker nginx openssl curl ss systemctl sha256sum; do command -v "$program" >/dev/null; done
|
||||
if [[ -e "$root" && ! -f "$root/.project" ]] && [[ -n "$(find "$root" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then
|
||||
echo 'Refusing to reuse an unowned deployment directory.' >&2
|
||||
exit 2
|
||||
@@ -55,8 +55,38 @@ chown 10001:10001 "$root/data" "$root/logs"
|
||||
install -d -m 700 "$release"
|
||||
PREPARE
|
||||
|
||||
# Transport only tested images built without runtime credentials.
|
||||
docker save "$CHAT_IMAGE" "$CERTBOT_IMAGE" | gzip | run_ssh "bash -o pipefail -c 'gzip -dc | docker load'"
|
||||
# Verify the exact exported archive before loading. Classic and containerd image
|
||||
# stores can report different IDs for the same archive, so retain target IDs too.
|
||||
docker save "$CHAT_IMAGE" "$CERTBOT_IMAGE" | gzip > "$temporary/images.tar.gz"
|
||||
archive_sha=$(sha256sum "$temporary/images.tar.gz" | awk '{print $1}')
|
||||
scp "${ssh_options[@]}" "$temporary/images.tar.gz" "$target:$release/images.tar.gz"
|
||||
run_ssh bash -s -- "$root" "$release" "$archive_sha" "$CHAT_IMAGE" "$CERTBOT_IMAGE" "$SOURCE_COMMIT" <<'LOAD'
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
root=$1
|
||||
release=$2
|
||||
checksum=$3
|
||||
app_image=$4
|
||||
certbot_image=$5
|
||||
source_commit=$6
|
||||
[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2
|
||||
[[ "$release" == "$root"/releases/* && "$release" != *..* ]] || exit 2
|
||||
[[ "$(cat "$root/.project")" == baiyue-logistics-chat-gray ]] || exit 2
|
||||
[[ "$checksum" =~ ^[a-f0-9]{64}$ && "$source_commit" =~ ^[a-f0-9]{40}$ ]] || exit 2
|
||||
[[ "$app_image" =~ ^baiyue-logistics-chat-gray:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2
|
||||
[[ "$certbot_image" =~ ^baiyue-logistics-chat-certbot:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2
|
||||
printf '%s %s\n' "$checksum" "$release/images.tar.gz" | sha256sum --check --status
|
||||
gzip -dc "$release/images.tar.gz" | docker load
|
||||
for image in "$app_image" "$certbot_image"; do
|
||||
[[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")" == "$source_commit" ]] || exit 2
|
||||
done
|
||||
app_id=$(docker image inspect --format '{{.Id}}' "$app_image")
|
||||
certbot_id=$(docker image inspect --format '{{.Id}}' "$certbot_image")
|
||||
[[ "$app_id" =~ ^sha256:[a-f0-9]{64}$ && "$certbot_id" =~ ^sha256:[a-f0-9]{64}$ ]] || exit 2
|
||||
printf 'ARCHIVE_SHA256=%s\nCHAT_IMAGE_ID=%s\nCERTBOT_IMAGE_ID=%s\n' "$checksum" "$app_id" "$certbot_id" > "$release/loaded-image-ids.env"
|
||||
rm -f -- "$release/images.tar.gz"
|
||||
echo 'Image archive checksum and source labels verified; target image IDs recorded.'
|
||||
LOAD
|
||||
scp "${ssh_options[@]}" "$temporary/release/"* "$target:$release/"
|
||||
run_ssh bash "$release/prepare-tls.sh" "$root" "$release"
|
||||
run_ssh bash "$release/activate.sh" "$root" "$release"
|
||||
|
||||
+2
-2
@@ -37,11 +37,11 @@
|
||||
1. 完成检查并提交代码，创建 `gray-chat-YYYYMMDD-<commit12>` 一类不可变发布标签。不要重写已有标签或强推覆盖远端分支；分支有其他更新时先明确发布基线。
|
||||
2. 推送标签，在任务 `SOURCE_REF` 选择该标签。Jenkins 按此引用加载流水线并检出代码，镜像记录完整提交 SHA。
|
||||
3. 流水线检查 Bash 语法，运行 Node 回归，再构建 `test` 目标执行 Python 回归；通过后构建 `production` 应用镜像及 `certbot` 运维镜像。
|
||||
4. 构建完成后才绑定运行凭据。发布脚本校验路径、域名、源提交和镜像 ID，生成私有配置并发送镜像及发布文件。
|
||||
4. 构建完成后才绑定运行凭据。发布脚本校验路径、域名和源提交，生成私有配置并发送镜像及发布文件。服务器验证镜像归档的 SHA-256 后加载，记录目标主机的镜像 ID。
|
||||
5. 首次签发域名证书，等待应用健康，再安装本服务的 Nginx 配置。已有业务的容器和站点配置不参与本次发布。
|
||||
6. Jenkins 使用验收账号验证公网 HTTPS `/healthz`；之后在浏览器验证 AI 流式回复、上传、转人工和工作台。
|
||||
|
||||
应用镜像必须选择 `production` 构建目标；`certbot` 仅用于证书运维。每个构建生成独立镜像标签，接收主机再次核验镜像 ID，不依赖 `latest`。
|
||||
应用镜像必须选择 `production` 构建目标；`certbot` 仅用于证书运维。每个构建生成独立镜像标签，不依赖 `latest`。Docker 的 classic 与 containerd 镜像存储可能返回不同 ID；跨主机以归档校验和与源提交标签核验传输，激活前再与 `loaded-image-ids.env` 中的目标 ID 比较，拒绝加载后被替换的镜像。归档加载后删除，构建机 ID 保留在 `release.env` 供追溯。
|
||||
|
||||
## 数据与失败回退
|
||||
|
||||
|
||||
@@ -66,9 +66,12 @@ class GrayActivationTests(unittest.TestCase):
|
||||
unrelated.write_text('# unrelated site\n')
|
||||
config = GrayConfig('chat.example.com', 29013, str(root), str(root / 'releases/new'),
|
||||
str(proxy), str(certificate), str(private_key), 'a'*40,
|
||||
'baiyue-logistics-chat-gray:' + 'a'*12 + '-b7', 'sha256:' + 'b'*64, 'sha256:' + 'c'*64)
|
||||
'baiyue-logistics-chat-gray:' + 'a'*12 + '-b7', 'sha256:' + 'd'*64, 'sha256:' + 'c'*64)
|
||||
release = Path(config.release)
|
||||
write_release(release, config, 'fixture-model-key')
|
||||
# Docker 26 and 29 may assign different IDs after an archive is loaded.
|
||||
loaded_id = ('e' if scenario == 'image-mismatch' else 'b') * 64
|
||||
(release / 'loaded-image-ids.env').write_text('CHAT_IMAGE_ID=sha256:' + loaded_id + '\n')
|
||||
(release / 'site.htpasswd').write_text('new-auth-hash\n')
|
||||
commands = directory / 'commands'
|
||||
commands.mkdir()
|
||||
@@ -105,6 +108,13 @@ class GrayActivationTests(unittest.TestCase):
|
||||
def test_failed_application_restores_previous_release(self):
|
||||
self.assert_rolled_back('app-fail')
|
||||
|
||||
def test_image_retagged_after_verified_load_is_rejected_before_activation(self):
|
||||
result, root, proxy, previous, release, old_site, calls = self.run_activation('image-mismatch')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual((root / 'current-release').read_text().strip(), str(previous))
|
||||
self.assertEqual((proxy / 'baiyue-logistics-chat-gray.conf').read_text(), old_site)
|
||||
self.assertFalse(any(name == 'docker' and 'compose' in args for name, args in calls))
|
||||
|
||||
def test_invalid_nginx_configuration_restores_previous_site_and_application(self):
|
||||
self.assert_rolled_back('proxy-fail')
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Verify the real remote image-load guard without contacting Docker or a host."""
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
@unittest.skipUnless(os.name == 'posix' and shutil.which('bash'), 'Image transfer requires Linux Bash')
|
||||
class GrayImageTransferTests(unittest.TestCase):
|
||||
def run_load(self, scenario):
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
directory = Path(temporary.name)
|
||||
root = directory / 'gray-logistics-chat'
|
||||
release = root / 'releases/build'
|
||||
release.mkdir(parents=True)
|
||||
(root / '.project').write_text('baiyue-logistics-chat-gray\n')
|
||||
archive = release / 'images.tar.gz'
|
||||
archive.write_bytes(gzip.compress(b'fixture-image-archive'))
|
||||
checksum = hashlib.sha256(archive.read_bytes()).hexdigest()
|
||||
if scenario == 'corrupt':
|
||||
archive.write_bytes(archive.read_bytes() + b'changed-in-transit')
|
||||
commands = directory / 'commands'
|
||||
commands.mkdir()
|
||||
docker = commands / 'docker'
|
||||
docker.write_text('#!' + sys.executable + '\n' + '''import os, pathlib, sys
|
||||
args = sys.argv[1:]
|
||||
with pathlib.Path(os.environ['FAKE_LOG']).open('a') as output:
|
||||
output.write(' '.join(args) + '\\n')
|
||||
if args == ['load']:
|
||||
assert sys.stdin.buffer.read() == b'fixture-image-archive'
|
||||
elif args[:2] == ['image', 'inspect']:
|
||||
if args[3] == '{{.Id}}':
|
||||
print('sha256:' + ('b' if '-gray:' in args[-1] else 'c') * 64)
|
||||
else:
|
||||
print(('e' if os.environ['FAKE_SCENARIO'] == 'wrong-source' else 'a') * 40)
|
||||
else:
|
||||
sys.exit(3)
|
||||
''')
|
||||
docker.chmod(0o755)
|
||||
publish = (Path(__file__).resolve().parents[1] / 'deploy/publish-gray.sh').read_text()
|
||||
remote = publish.split("<<'LOAD'\n", 1)[1].split('\nLOAD\n', 1)[0]
|
||||
log = directory / 'commands.log'
|
||||
environment = dict(os.environ, PATH=str(commands) + os.pathsep + os.environ['PATH'],
|
||||
FAKE_LOG=str(log), FAKE_SCENARIO=scenario)
|
||||
result = subprocess.run(['bash', '-s', '--', str(root), str(release), checksum,
|
||||
'baiyue-logistics-chat-gray:' + 'a' * 12 + '-b7',
|
||||
'baiyue-logistics-chat-certbot:' + 'a' * 12 + '-b7', 'a' * 40],
|
||||
input=remote, env=environment, capture_output=True, text=True, timeout=15)
|
||||
return result, release, log
|
||||
|
||||
def test_verified_archive_records_target_ids_and_removes_archive(self):
|
||||
result, release, log = self.run_load('success')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
receipt = (release / 'loaded-image-ids.env').read_text()
|
||||
self.assertIn('CHAT_IMAGE_ID=sha256:' + 'b' * 64, receipt)
|
||||
self.assertIn('CERTBOT_IMAGE_ID=sha256:' + 'c' * 64, receipt)
|
||||
self.assertFalse((release / 'images.tar.gz').exists())
|
||||
|
||||
def test_modified_archive_is_rejected_before_docker_load(self):
|
||||
result, release, log = self.run_load('corrupt')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(log.exists())
|
||||
self.assertFalse((release / 'loaded-image-ids.env').exists())
|
||||
|
||||
def test_wrong_source_cannot_create_a_verified_receipt(self):
|
||||
result, release, log = self.run_load('wrong-source')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse((release / 'loaded-image-ids.env').exists())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user