diff --git a/deploy/activate-gray.sh b/deploy/activate-gray.sh index 851d134..1490d41 100644 --- a/deploy/activate-gray.sh +++ b/deploy/activate-gray.sh @@ -24,7 +24,9 @@ done < "$release/release.env" [[ "$CHAT_IMAGE" =~ ^baiyue-logistics-chat-gray:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2 [[ "$HOST_PORT" =~ ^[0-9]+$ ]] && ((HOST_PORT >= 1024 && HOST_PORT <= 65535)) || exit 2 [[ -d "$CHAT_PROXY_DIR" && -r "$CHAT_TLS_CERT" && -r "$CHAT_TLS_KEY" ]] || exit 2 -[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_IMAGE")" == "$CHAT_IMAGE_ID" ]] || exit 2 +loaded_id=$(awk -F= '$1 == "CHAT_IMAGE_ID" {print $2}' "$release/loaded-image-ids.env") +[[ "$loaded_id" =~ ^sha256:[a-f0-9]{64}$ ]] || exit 2 +[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_IMAGE")" == "$loaded_id" ]] || { echo 'Loaded application image changed.' >&2; exit 2; } [[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CHAT_IMAGE")" == "$SOURCE_COMMIT" ]] || exit 2 openssl x509 -in "$CHAT_TLS_CERT" -checkhost "$CHAT_DOMAIN" -noout >/dev/null diff --git a/deploy/prepare-gray-tls.sh b/deploy/prepare-gray-tls.sh index b52eeb1..3216412 100644 --- a/deploy/prepare-gray-tls.sh +++ b/deploy/prepare-gray-tls.sh @@ -18,7 +18,9 @@ done < "$release/release.env" [[ "$CHAT_TLS_CERT" == "$root/tls/config/live/$CHAT_DOMAIN/fullchain.pem" ]] || exit 2 [[ "$CHAT_TLS_KEY" == "$root/tls/config/live/$CHAT_DOMAIN/privkey.pem" ]] || exit 2 [[ "$CHAT_CERTBOT_IMAGE" =~ ^baiyue-logistics-chat-certbot:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2 -[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_CERTBOT_IMAGE")" == "$CHAT_CERTBOT_IMAGE_ID" ]] || exit 2 +loaded_id=$(awk -F= '$1 == "CERTBOT_IMAGE_ID" {print $2}' "$release/loaded-image-ids.env") +[[ "$loaded_id" =~ ^sha256:[a-f0-9]{64}$ ]] || exit 2 +[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_CERTBOT_IMAGE")" == "$loaded_id" ]] || { echo 'Loaded certificate image changed.' >&2; exit 2; } [[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CHAT_CERTBOT_IMAGE")" == "$SOURCE_COMMIT" ]] || exit 2 site="$CHAT_PROXY_DIR/baiyue-logistics-chat-gray.conf" diff --git a/deploy/publish-gray.sh b/deploy/publish-gray.sh index d5b2c33..b8216b8 100644 --- a/deploy/publish-gray.sh +++ b/deploy/publish-gray.sh @@ -40,7 +40,7 @@ release=$2 [[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2 [[ "$release" == "$root"/releases/* ]] || exit 2 [[ "$(id -u)" == 0 ]] || { echo 'The managed gray deployment account must have the required host permissions.' >&2; exit 2; } -for program in docker nginx openssl curl ss systemctl; do command -v "$program" >/dev/null; done +for program in docker nginx openssl curl ss systemctl sha256sum; do command -v "$program" >/dev/null; done if [[ -e "$root" && ! -f "$root/.project" ]] && [[ -n "$(find "$root" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then echo 'Refusing to reuse an unowned deployment directory.' >&2 exit 2 @@ -55,8 +55,38 @@ chown 10001:10001 "$root/data" "$root/logs" install -d -m 700 "$release" PREPARE -# Transport only tested images built without runtime credentials. -docker save "$CHAT_IMAGE" "$CERTBOT_IMAGE" | gzip | run_ssh "bash -o pipefail -c 'gzip -dc | docker load'" +# Verify the exact exported archive before loading. Classic and containerd image +# stores can report different IDs for the same archive, so retain target IDs too. +docker save "$CHAT_IMAGE" "$CERTBOT_IMAGE" | gzip > "$temporary/images.tar.gz" +archive_sha=$(sha256sum "$temporary/images.tar.gz" | awk '{print $1}') +scp "${ssh_options[@]}" "$temporary/images.tar.gz" "$target:$release/images.tar.gz" +run_ssh bash -s -- "$root" "$release" "$archive_sha" "$CHAT_IMAGE" "$CERTBOT_IMAGE" "$SOURCE_COMMIT" <<'LOAD' +set -Eeuo pipefail +umask 077 +root=$1 +release=$2 +checksum=$3 +app_image=$4 +certbot_image=$5 +source_commit=$6 +[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2 +[[ "$release" == "$root"/releases/* && "$release" != *..* ]] || exit 2 +[[ "$(cat "$root/.project")" == baiyue-logistics-chat-gray ]] || exit 2 +[[ "$checksum" =~ ^[a-f0-9]{64}$ && "$source_commit" =~ ^[a-f0-9]{40}$ ]] || exit 2 +[[ "$app_image" =~ ^baiyue-logistics-chat-gray:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2 +[[ "$certbot_image" =~ ^baiyue-logistics-chat-certbot:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2 +printf '%s %s\n' "$checksum" "$release/images.tar.gz" | sha256sum --check --status +gzip -dc "$release/images.tar.gz" | docker load +for image in "$app_image" "$certbot_image"; do + [[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")" == "$source_commit" ]] || exit 2 +done +app_id=$(docker image inspect --format '{{.Id}}' "$app_image") +certbot_id=$(docker image inspect --format '{{.Id}}' "$certbot_image") +[[ "$app_id" =~ ^sha256:[a-f0-9]{64}$ && "$certbot_id" =~ ^sha256:[a-f0-9]{64}$ ]] || exit 2 +printf 'ARCHIVE_SHA256=%s\nCHAT_IMAGE_ID=%s\nCERTBOT_IMAGE_ID=%s\n' "$checksum" "$app_id" "$certbot_id" > "$release/loaded-image-ids.env" +rm -f -- "$release/images.tar.gz" +echo 'Image archive checksum and source labels verified; target image IDs recorded.' +LOAD scp "${ssh_options[@]}" "$temporary/release/"* "$target:$release/" run_ssh bash "$release/prepare-tls.sh" "$root" "$release" run_ssh bash "$release/activate.sh" "$root" "$release" diff --git a/docs/deployment.md b/docs/deployment.md index dff3644..5d4e17f 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -37,11 +37,11 @@ 1. 完成检查并提交代码,创建 `gray-chat-YYYYMMDD-` 一类不可变发布标签。不要重写已有标签或强推覆盖远端分支;分支有其他更新时先明确发布基线。 2. 推送标签,在任务 `SOURCE_REF` 选择该标签。Jenkins 按此引用加载流水线并检出代码,镜像记录完整提交 SHA。 3. 流水线检查 Bash 语法,运行 Node 回归,再构建 `test` 目标执行 Python 回归;通过后构建 `production` 应用镜像及 `certbot` 运维镜像。 -4. 构建完成后才绑定运行凭据。发布脚本校验路径、域名、源提交和镜像 ID,生成私有配置并发送镜像及发布文件。 +4. 构建完成后才绑定运行凭据。发布脚本校验路径、域名和源提交,生成私有配置并发送镜像及发布文件。服务器验证镜像归档的 SHA-256 后加载,记录目标主机的镜像 ID。 5. 首次签发域名证书,等待应用健康,再安装本服务的 Nginx 配置。已有业务的容器和站点配置不参与本次发布。 6. Jenkins 使用验收账号验证公网 HTTPS `/healthz`;之后在浏览器验证 AI 流式回复、上传、转人工和工作台。 -应用镜像必须选择 `production` 构建目标;`certbot` 仅用于证书运维。每个构建生成独立镜像标签,接收主机再次核验镜像 ID,不依赖 `latest`。 +应用镜像必须选择 `production` 构建目标;`certbot` 仅用于证书运维。每个构建生成独立镜像标签,不依赖 `latest`。Docker 的 classic 与 containerd 镜像存储可能返回不同 ID;跨主机以归档校验和与源提交标签核验传输,激活前再与 `loaded-image-ids.env` 中的目标 ID 比较,拒绝加载后被替换的镜像。归档加载后删除,构建机 ID 保留在 `release.env` 供追溯。 ## 数据与失败回退 diff --git a/tests/test_gray_activation.py b/tests/test_gray_activation.py index e9f4cde..9c9053c 100644 --- a/tests/test_gray_activation.py +++ b/tests/test_gray_activation.py @@ -66,9 +66,12 @@ class GrayActivationTests(unittest.TestCase): unrelated.write_text('# unrelated site\n') config = GrayConfig('chat.example.com', 29013, str(root), str(root / 'releases/new'), str(proxy), str(certificate), str(private_key), 'a'*40, - 'baiyue-logistics-chat-gray:' + 'a'*12 + '-b7', 'sha256:' + 'b'*64, 'sha256:' + 'c'*64) + 'baiyue-logistics-chat-gray:' + 'a'*12 + '-b7', 'sha256:' + 'd'*64, 'sha256:' + 'c'*64) release = Path(config.release) write_release(release, config, 'fixture-model-key') + # Docker 26 and 29 may assign different IDs after an archive is loaded. + loaded_id = ('e' if scenario == 'image-mismatch' else 'b') * 64 + (release / 'loaded-image-ids.env').write_text('CHAT_IMAGE_ID=sha256:' + loaded_id + '\n') (release / 'site.htpasswd').write_text('new-auth-hash\n') commands = directory / 'commands' commands.mkdir() @@ -105,6 +108,13 @@ class GrayActivationTests(unittest.TestCase): def test_failed_application_restores_previous_release(self): self.assert_rolled_back('app-fail') + def test_image_retagged_after_verified_load_is_rejected_before_activation(self): + result, root, proxy, previous, release, old_site, calls = self.run_activation('image-mismatch') + self.assertNotEqual(result.returncode, 0) + self.assertEqual((root / 'current-release').read_text().strip(), str(previous)) + self.assertEqual((proxy / 'baiyue-logistics-chat-gray.conf').read_text(), old_site) + self.assertFalse(any(name == 'docker' and 'compose' in args for name, args in calls)) + def test_invalid_nginx_configuration_restores_previous_site_and_application(self): self.assert_rolled_back('proxy-fail') diff --git a/tests/test_gray_image_transfer.py b/tests/test_gray_image_transfer.py new file mode 100644 index 0000000..c994178 --- /dev/null +++ b/tests/test_gray_image_transfer.py @@ -0,0 +1,79 @@ +"""Verify the real remote image-load guard without contacting Docker or a host.""" + +import gzip +import hashlib +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + + +@unittest.skipUnless(os.name == 'posix' and shutil.which('bash'), 'Image transfer requires Linux Bash') +class GrayImageTransferTests(unittest.TestCase): + def run_load(self, scenario): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + directory = Path(temporary.name) + root = directory / 'gray-logistics-chat' + release = root / 'releases/build' + release.mkdir(parents=True) + (root / '.project').write_text('baiyue-logistics-chat-gray\n') + archive = release / 'images.tar.gz' + archive.write_bytes(gzip.compress(b'fixture-image-archive')) + checksum = hashlib.sha256(archive.read_bytes()).hexdigest() + if scenario == 'corrupt': + archive.write_bytes(archive.read_bytes() + b'changed-in-transit') + commands = directory / 'commands' + commands.mkdir() + docker = commands / 'docker' + docker.write_text('#!' + sys.executable + '\n' + '''import os, pathlib, sys +args = sys.argv[1:] +with pathlib.Path(os.environ['FAKE_LOG']).open('a') as output: + output.write(' '.join(args) + '\\n') +if args == ['load']: + assert sys.stdin.buffer.read() == b'fixture-image-archive' +elif args[:2] == ['image', 'inspect']: + if args[3] == '{{.Id}}': + print('sha256:' + ('b' if '-gray:' in args[-1] else 'c') * 64) + else: + print(('e' if os.environ['FAKE_SCENARIO'] == 'wrong-source' else 'a') * 40) +else: + sys.exit(3) +''') + docker.chmod(0o755) + publish = (Path(__file__).resolve().parents[1] / 'deploy/publish-gray.sh').read_text() + remote = publish.split("<<'LOAD'\n", 1)[1].split('\nLOAD\n', 1)[0] + log = directory / 'commands.log' + environment = dict(os.environ, PATH=str(commands) + os.pathsep + os.environ['PATH'], + FAKE_LOG=str(log), FAKE_SCENARIO=scenario) + result = subprocess.run(['bash', '-s', '--', str(root), str(release), checksum, + 'baiyue-logistics-chat-gray:' + 'a' * 12 + '-b7', + 'baiyue-logistics-chat-certbot:' + 'a' * 12 + '-b7', 'a' * 40], + input=remote, env=environment, capture_output=True, text=True, timeout=15) + return result, release, log + + def test_verified_archive_records_target_ids_and_removes_archive(self): + result, release, log = self.run_load('success') + self.assertEqual(result.returncode, 0, result.stderr) + receipt = (release / 'loaded-image-ids.env').read_text() + self.assertIn('CHAT_IMAGE_ID=sha256:' + 'b' * 64, receipt) + self.assertIn('CERTBOT_IMAGE_ID=sha256:' + 'c' * 64, receipt) + self.assertFalse((release / 'images.tar.gz').exists()) + + def test_modified_archive_is_rejected_before_docker_load(self): + result, release, log = self.run_load('corrupt') + self.assertNotEqual(result.returncode, 0) + self.assertFalse(log.exists()) + self.assertFalse((release / 'loaded-image-ids.env').exists()) + + def test_wrong_source_cannot_create_a_verified_receipt(self): + result, release, log = self.run_load('wrong-source') + self.assertNotEqual(result.returncode, 0) + self.assertFalse((release / 'loaded-image-ids.env').exists()) + + +if __name__ == '__main__': + unittest.main()