feat(ci): rollback unhealthy deployments

2026-08-22 18:26:47 +08:00
parent 1df87e5d17
commit ae9238b3cf
3 changed files with 388 additions and 23 deletions
@@ -0,0 +1,148 @@
+import assert from 'node:assert/strict'
+import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { delimiter, join, resolve } from 'node:path'
+import { spawnSync } from 'node:child_process'
+import test from 'node:test'
+
+const repositoryRoot = resolve(import.meta.dirname, '..')
+const deployScript = resolve(repositoryRoot, 'deploy/scripts/deploy-built-service.sh')
+const shell = process.platform === 'win32'
+  ? 'C:/Program Files/Git/bin/bash.exe'
+  : '/bin/sh'
+
+const mockDocker = `#!/usr/bin/env sh
+set -eu
+printf '%s\\n' "$*" >> "$MOCK_LOG"
+command="$1"
+shift
+case "$command" in
+  login|logout|pull)
+    exit 0
+    ;;
+  compose)
+    action=''
+    for argument in "$@"; do
+      case "$argument" in
+        ps|up) action="$argument" ;;
+      esac
+    done
+    case "$action" in
+      ps)
+        echo mock-container
+        ;;
+      up)
+        service=''
+        for argument in "$@"; do service="$argument"; done
+        if [ "$IMAGE_TAG" = "\${MOCK_FAIL_UP_TAG:-never}" ]; then
+          exit 3
+        fi
+        printf '%s\\n' "$IMAGE_REPOSITORY/$service:$IMAGE_TAG" > "$MOCK_CURRENT_IMAGE"
+        if [ "$IMAGE_TAG" = "\${MOCK_FAIL_TAG:-never}" ]; then
+          printf '%s\\n' unhealthy > "$MOCK_HEALTH_STATE"
+        else
+          printf '%s\\n' healthy > "$MOCK_HEALTH_STATE"
+        fi
+        ;;
+    esac
+    ;;
+  inspect)
+    case "$2" in
+      *Config.Image*) cat "$MOCK_CURRENT_IMAGE" ;;
+      *State.Health*) cat "$MOCK_HEALTH_STATE" ;;
+      *) exit 2 ;;
+    esac
+    ;;
+  *)
+    echo "unexpected docker command: $command $*" >&2
+    exit 2
+    ;;
+esac
+`
+
+async function runDeployment({ imageTag, failingTag, failingStartTag }) {
+  const temporaryDirectory = await mkdtemp(join(tmpdir(), 'firstleg-rollback-'))
+  const mockBin = join(temporaryDirectory, 'bin')
+  const dockerPath = join(mockBin, 'docker')
+  const envFile = join(temporaryDirectory, 'deployment.env')
+  const logFile = join(temporaryDirectory, 'docker.log')
+  const currentImageFile = join(temporaryDirectory, 'current-image')
+  const healthStateFile = join(temporaryDirectory, 'health-state')
+
+  await mkdir(mockBin)
+  await writeFile(dockerPath, mockDocker, 'utf8')
+  await chmod(dockerPath, 0o755)
+  await writeFile(envFile, 'PLACEHOLDER=true\n', 'utf8')
+  await writeFile(logFile, '', 'utf8')
+  await writeFile(currentImageFile, 'registry.local/group/firstleg/audit-service:oldgood\n', 'utf8')
+  await writeFile(healthStateFile, 'healthy\n', 'utf8')
+
+  const result = spawnSync(shell, [
+    deployScript.replaceAll('\\', '/'),
+    'development',
+    'registry.local/group/firstleg',
+    imageTag,
+    'audit-service',
+    envFile.replaceAll('\\', '/')
+  ], {
+    cwd: repositoryRoot,
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      PATH: `${mockBin}${delimiter}${process.env.PATH ?? ''}`,
+      TARGET_IS_LOCAL: 'true',
+      REGISTRY_USER: 'ci-user',
+      REGISTRY_PASSWORD: 'ci-password',
+      DEPLOY_HEALTH_TIMEOUT_SECONDS: '2',
+      DEPLOY_HEALTH_POLL_SECONDS: '1',
+      MOCK_LOG: logFile,
+      MOCK_CURRENT_IMAGE: currentImageFile,
+      MOCK_HEALTH_STATE: healthStateFile,
+      MOCK_FAIL_TAG: failingTag ?? 'never',
+      MOCK_FAIL_UP_TAG: failingStartTag ?? 'never'
+    }
+  })
+
+  const dockerLog = await readFile(logFile, 'utf8')
+  const currentImage = (await readFile(currentImageFile, 'utf8')).trim()
+  await rm(temporaryDirectory, { recursive: true, force: true })
+  return { ...result, dockerLog, currentImage }
+}
+
+test('successful deployment keeps the new healthy image', async (context) => {
+  if (!existsSync(shell)) context.skip(`shell unavailable: ${shell}`)
+
+  const result = await runDeployment({ imageTag: 'newgood' })
+
+  assert.equal(result.status, 0, result.stderr)
+  assert.match(result.stdout, /部署前运行镜像: registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.match(result.stdout, /新镜像健康检查通过: registry\.local\/group\/firstleg\/audit-service:newgood/)
+  assert.equal(result.currentImage, 'registry.local/group/firstleg/audit-service:newgood')
+})
+
+test('unhealthy deployment restores the previous healthy image and still fails the job', async (context) => {
+  if (!existsSync(shell)) context.skip(`shell unavailable: ${shell}`)
+
+  const result = await runDeployment({ imageTag: 'newbad', failingTag: 'newbad' })
+  const output = `${result.stdout}\n${result.stderr}`
+
+  assert.equal(result.status, 1, output)
+  assert.match(output, /新镜像健康检查失败: registry\.local\/group\/firstleg\/audit-service:newbad/)
+  assert.match(output, /开始自动回滚到: registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.match(output, /自动回滚成功: registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.match(result.dockerLog, /pull registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.equal(result.currentImage, 'registry.local/group/firstleg/audit-service:oldgood')
+})
+
+test('failed compose start also restores the previous healthy image', async (context) => {
+  if (!existsSync(shell)) context.skip(`shell unavailable: ${shell}`)
+
+  const result = await runDeployment({ imageTag: 'newbroken', failingStartTag: 'newbroken' })
+  const output = `${result.stdout}\n${result.stderr}`
+
+  assert.equal(result.status, 1, output)
+  assert.match(output, /新镜像启动命令失败: registry\.local\/group\/firstleg\/audit-service:newbroken/)
+  assert.match(output, /自动回滚成功: registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.equal(result.currentImage, 'registry.local/group/firstleg/audit-service:oldgood')
+})
@@ -101,3 +101,13 @@ test('source workspaces are preserved for incremental Git fetches', () => {
  assert.equal((pipeline.match(/cleanWs\(/g) ?? []).length, 2)
  assert.doesNotMatch(coordinatorPipeline, /cleanWs\(/)
 })
+
+test('direct deployments wait for health and automatically restore the previous image', () => {
+  assert.match(directDeployScript, /HEALTH_TIMEOUT_SECONDS="\$\{DEPLOY_HEALTH_TIMEOUT_SECONDS:-240\}"/)
+  assert.match(directDeployScript, /previous_image="\$\(local_container_image \|\| true\)"/)
+  assert.match(directDeployScript, /if wait_for_local_health; then/)
+  assert.match(directDeployScript, /开始自动回滚到/)
+  assert.match(directDeployScript, /自动回滚成功/)
+  assert.match(directDeployScript, /自动回滚失败/)
+  assert.match(directDeployScript, /return 1/)
+})