feat(ci): rollback unhealthy deployments

deploy/scripts/deploy-built-service.sh

@@ -15,6 +15,18 @@ COMPOSE_FILE="deploy/docker-compose.yml"
 IMAGE="$IMAGE_REPOSITORY/$SERVICE:$IMAGE_TAG"
 REGISTRY_HOST="${IMAGE_REPOSITORY%%/*}"
 PROJECT_NAME="firstleg-$DEPLOY_ENV"
+HEALTH_TIMEOUT_SECONDS="${DEPLOY_HEALTH_TIMEOUT_SECONDS:-240}"
+HEALTH_POLL_SECONDS="${DEPLOY_HEALTH_POLL_SECONDS:-5}"
+
+for value_name in HEALTH_TIMEOUT_SECONDS HEALTH_POLL_SECONDS; do
+  eval "value=\${$value_name}"
+  case "$value" in
+    ''|*[!0-9]*|0)
+      echo "$value_name 必须为正整数，当前: $value" >&2
+      exit 2
+      ;;
+  esac
+done
 
 case "$DEPLOY_ENV" in
   development|test) ;;
@@ -165,23 +177,121 @@ local_registry_logout() {
   docker logout "$REGISTRY_HOST" >/dev/null 2>&1 || true
 }
 
+local_compose() {
+  docker compose \
+    --project-name "$PROJECT_NAME" \
+    --env-file "$ENV_FILE" \
+    -f "$COMPOSE_FILE" \
+    "$@"
+}
+
+local_container_id() {
+  local_compose ps --all --quiet "$SERVICE" | head -n 1
+}
+
+local_container_image() {
+  container_id="$(local_container_id)"
+  if [ -n "$container_id" ]; then
+    docker inspect --format '{{.Config.Image}}' "$container_id"
+  fi
+}
+
+local_health_state() {
+  container_id="$(local_container_id)"
+  if [ -z "$container_id" ]; then
+    echo missing
+    return
+  fi
+  docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-health:{{.State.Status}}{{end}}' "$container_id" 2>/dev/null || echo missing
+}
+
+wait_for_local_health() {
+  elapsed=0
+  while [ "$elapsed" -lt "$HEALTH_TIMEOUT_SECONDS" ]; do
+    health_state="$(local_health_state)"
+    echo "容器健康状态: $health_state（已等待 ${elapsed}s）"
+    case "$health_state" in
+      healthy)
+        return 0
+        ;;
+      no-health:running)
+        echo "容器未配置 HEALTHCHECK，以运行状态作为成功依据" >&2
+        return 0
+        ;;
+      unhealthy|no-health:exited|no-health:dead)
+        return 1
+        ;;
+    esac
+    sleep "$HEALTH_POLL_SECONDS"
+    elapsed=$((elapsed + HEALTH_POLL_SECONDS))
+  done
+  echo "等待容器健康超时: ${HEALTH_TIMEOUT_SECONDS}s" >&2
+  return 1
+}
+
 deploy_local() {
   printf '%s' "$REGISTRY_PASSWORD" | docker login -u "$REGISTRY_USER" --password-stdin "$REGISTRY_HOST"
   trap local_registry_logout EXIT HUP INT TERM
+  previous_image="$(local_container_image || true)"
+  if [ -n "$previous_image" ]; then
+    echo "部署前运行镜像: $previous_image"
+  else
+    echo "首次部署，未发现可回滚的旧镜像"
+  fi
   docker pull "$IMAGE"
   export DEPLOY_ENV IMAGE_REPOSITORY IMAGE_TAG
-  docker compose \
-    --project-name "$PROJECT_NAME" \
-    --env-file "$ENV_FILE" \
-    -f "$COMPOSE_FILE" \
-    up -d --remove-orphans --pull never "$SERVICE"
-  docker compose \
-    --project-name "$PROJECT_NAME" \
-    --env-file "$ENV_FILE" \
-    -f "$COMPOSE_FILE" \
-    ps "$SERVICE"
-  local_registry_logout
-  trap - EXIT HUP INT TERM
+  new_started=true
+  if ! local_compose up -d --remove-orphans --pull never "$SERVICE"; then
+    echo "新镜像启动命令失败: $IMAGE" >&2
+    new_started=false
+  fi
+  if [ "$new_started" = "true" ] && wait_for_local_health; then
+    echo "新镜像健康检查通过: $IMAGE"
+    local_compose ps "$SERVICE"
+    local_registry_logout
+    trap - EXIT HUP INT TERM
+    return 0
+  fi
+
+  echo "新镜像健康检查失败: $IMAGE" >&2
+  expected_prefix="$IMAGE_REPOSITORY/$SERVICE:"
+  case "$previous_image" in
+    "$expected_prefix"*)
+      previous_tag="${previous_image#"$expected_prefix"}"
+      ;;
+    '')
+      echo "首次部署没有可回滚镜像，部署失败" >&2
+      local_compose stop "$SERVICE" >/dev/null 2>&1 || true
+      return 1
+      ;;
+    *)
+      echo "旧镜像不属于当前服务，拒绝自动回滚: $previous_image" >&2
+      return 1
+      ;;
+  esac
+  if [ "$previous_image" = "$IMAGE" ]; then
+    echo "旧镜像与新镜像相同，无法自动回滚: $IMAGE" >&2
+    return 1
+  fi
+
+  echo "开始自动回滚到: $previous_image" >&2
+  if ! docker pull "$previous_image"; then
+    echo "自动回滚失败，无法拉取旧镜像: $previous_image" >&2
+    return 1
+  fi
+  IMAGE_TAG="$previous_tag"
+  export IMAGE_TAG
+  if ! local_compose up -d --remove-orphans --pull never "$SERVICE"; then
+    echo "自动回滚失败，旧镜像启动命令失败: $previous_image" >&2
+    return 1
+  fi
+  if wait_for_local_health; then
+    echo "自动回滚成功: $previous_image" >&2
+    local_compose ps "$SERVICE" || true
+  else
+    echo "自动回滚失败，旧镜像未恢复健康: $previous_image" >&2
+  fi
+  return 1
 }
 
 if [ "${TARGET_IS_LOCAL:-false}" = "true" ]; then
@@ -252,7 +362,7 @@ printf '%s' "$REGISTRY_PASSWORD" | run_ssh docker login -u "$REGISTRY_USER" --pa
 trap remote_registry_logout EXIT HUP INT TERM
 run_ssh docker pull "$IMAGE"
 
-run_ssh sh -s -- "$REMOTE_DIR" "$DEPLOY_ENV" "$IMAGE_REPOSITORY" "$IMAGE_TAG" "$SERVICE" "$PROJECT_NAME" "$REMOTE_ENV_NAME" "$REMOTE_COMPOSE_NAME" <<'REMOTE_DEPLOY'
+run_ssh sh -s -- "$REMOTE_DIR" "$DEPLOY_ENV" "$IMAGE_REPOSITORY" "$IMAGE_TAG" "$SERVICE" "$PROJECT_NAME" "$REMOTE_ENV_NAME" "$REMOTE_COMPOSE_NAME" "$HEALTH_TIMEOUT_SECONDS" "$HEALTH_POLL_SECONDS" <<'REMOTE_DEPLOY'
 set -eu
 remote_dir="$1"
 deploy_env="$2"
@@ -262,20 +372,117 @@ service="$5"
 project_name="$6"
 env_file="$7"
 compose_file="$8"
+health_timeout_seconds="$9"
+health_poll_seconds="${10}"
 cd "$remote_dir"
 export DEPLOY_ENV="$deploy_env"
 export IMAGE_REPOSITORY="$image_repository"
 export IMAGE_TAG="$image_tag"
-docker compose \
-  --project-name "$project_name" \
-  --env-file "$env_file" \
-  -f "$compose_file" \
-  up -d --remove-orphans --pull never "$service"
-docker compose \
-  --project-name "$project_name" \
-  --env-file "$env_file" \
-  -f "$compose_file" \
-  ps "$service"
+
+compose() {
+  docker compose \
+    --project-name "$project_name" \
+    --env-file "$env_file" \
+    -f "$compose_file" \
+    "$@"
+}
+
+container_id() {
+  compose ps --all --quiet "$service" | head -n 1
+}
+
+container_image() {
+  id="$(container_id)"
+  if [ -n "$id" ]; then
+    docker inspect --format '{{.Config.Image}}' "$id"
+  fi
+}
+
+health_state() {
+  id="$(container_id)"
+  if [ -z "$id" ]; then
+    echo missing
+    return
+  fi
+  docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-health:{{.State.Status}}{{end}}' "$id" 2>/dev/null || echo missing
+}
+
+wait_for_health() {
+  elapsed=0
+  while [ "$elapsed" -lt "$health_timeout_seconds" ]; do
+    state="$(health_state)"
+    echo "容器健康状态: $state（已等待 ${elapsed}s）"
+    case "$state" in
+      healthy) return 0 ;;
+      no-health:running)
+        echo "容器未配置 HEALTHCHECK，以运行状态作为成功依据" >&2
+        return 0
+        ;;
+      unhealthy|no-health:exited|no-health:dead) return 1 ;;
+    esac
+    sleep "$health_poll_seconds"
+    elapsed=$((elapsed + health_poll_seconds))
+  done
+  echo "等待容器健康超时: ${health_timeout_seconds}s" >&2
+  return 1
+}
+
+new_image="$image_repository/$service:$image_tag"
+previous_image="$(container_image || true)"
+if [ -n "$previous_image" ]; then
+  echo "部署前运行镜像: $previous_image"
+else
+  echo "首次部署，未发现可回滚的旧镜像"
+fi
+
+new_started=true
+if ! compose up -d --remove-orphans --pull never "$service"; then
+  echo "新镜像启动命令失败: $new_image" >&2
+  new_started=false
+fi
+if [ "$new_started" = "true" ] && wait_for_health; then
+  echo "新镜像健康检查通过: $new_image"
+  compose ps "$service"
+  exit 0
+fi
+
+echo "新镜像健康检查失败: $new_image" >&2
+expected_prefix="$image_repository/$service:"
+case "$previous_image" in
+  "$expected_prefix"*) previous_tag="${previous_image#"$expected_prefix"}" ;;
+  '')
+    echo "首次部署没有可回滚镜像，部署失败" >&2
+    compose stop "$service" >/dev/null 2>&1 || true
+    exit 1
+    ;;
+  *)
+    echo "旧镜像不属于当前服务，拒绝自动回滚: $previous_image" >&2
+    exit 1
+    ;;
+esac
+if [ "$previous_image" = "$new_image" ]; then
+  echo "旧镜像与新镜像相同，无法自动回滚: $new_image" >&2
+  exit 1
+fi
+
+echo "开始自动回滚到: $previous_image" >&2
+if ! docker pull "$previous_image"; then
+  echo "自动回滚失败，无法拉取旧镜像: $previous_image" >&2
+  exit 1
+fi
+IMAGE_TAG="$previous_tag"
+export IMAGE_TAG
+if ! compose up -d --remove-orphans --pull never "$service"; then
+  echo "自动回滚失败，旧镜像启动命令失败: $previous_image" >&2
+  exit 1
+fi
+if wait_for_health; then
+  echo "自动回滚成功: $previous_image" >&2
+  compose ps "$service" || true
+else
+  echo "自动回滚失败，旧镜像未恢复健康: $previous_image" >&2
+fi
+exit 1
 REMOTE_DEPLOY
 
 remote_registry_logout

scripts/deploy-built-service-rollback.test.mjs

@@ -0,0 +1,148 @@
+import assert from 'node:assert/strict'
+import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { delimiter, join, resolve } from 'node:path'
+import { spawnSync } from 'node:child_process'
+import test from 'node:test'
+
+const repositoryRoot = resolve(import.meta.dirname, '..')
+const deployScript = resolve(repositoryRoot, 'deploy/scripts/deploy-built-service.sh')
+const shell = process.platform === 'win32'
+  ? 'C:/Program Files/Git/bin/bash.exe'
+  : '/bin/sh'
+
+const mockDocker = `#!/usr/bin/env sh
+set -eu
+printf '%s\\n' "$*" >> "$MOCK_LOG"
+command="$1"
+shift
+case "$command" in
+  login|logout|pull)
+    exit 0
+    ;;
+  compose)
+    action=''
+    for argument in "$@"; do
+      case "$argument" in
+        ps|up) action="$argument" ;;
+      esac
+    done
+    case "$action" in
+      ps)
+        echo mock-container
+        ;;
+      up)
+        service=''
+        for argument in "$@"; do service="$argument"; done
+        if [ "$IMAGE_TAG" = "\${MOCK_FAIL_UP_TAG:-never}" ]; then
+          exit 3
+        fi
+        printf '%s\\n' "$IMAGE_REPOSITORY/$service:$IMAGE_TAG" > "$MOCK_CURRENT_IMAGE"
+        if [ "$IMAGE_TAG" = "\${MOCK_FAIL_TAG:-never}" ]; then
+          printf '%s\\n' unhealthy > "$MOCK_HEALTH_STATE"
+        else
+          printf '%s\\n' healthy > "$MOCK_HEALTH_STATE"
+        fi
+        ;;
+    esac
+    ;;
+  inspect)
+    case "$2" in
+      *Config.Image*) cat "$MOCK_CURRENT_IMAGE" ;;
+      *State.Health*) cat "$MOCK_HEALTH_STATE" ;;
+      *) exit 2 ;;
+    esac
+    ;;
+  *)
+    echo "unexpected docker command: $command $*" >&2
+    exit 2
+    ;;
+esac
+`
+
+async function runDeployment({ imageTag, failingTag, failingStartTag }) {
+  const temporaryDirectory = await mkdtemp(join(tmpdir(), 'firstleg-rollback-'))
+  const mockBin = join(temporaryDirectory, 'bin')
+  const dockerPath = join(mockBin, 'docker')
+  const envFile = join(temporaryDirectory, 'deployment.env')
+  const logFile = join(temporaryDirectory, 'docker.log')
+  const currentImageFile = join(temporaryDirectory, 'current-image')
+  const healthStateFile = join(temporaryDirectory, 'health-state')
+
+  await mkdir(mockBin)
+  await writeFile(dockerPath, mockDocker, 'utf8')
+  await chmod(dockerPath, 0o755)
+  await writeFile(envFile, 'PLACEHOLDER=true\n', 'utf8')
+  await writeFile(logFile, '', 'utf8')
+  await writeFile(currentImageFile, 'registry.local/group/firstleg/audit-service:oldgood\n', 'utf8')
+  await writeFile(healthStateFile, 'healthy\n', 'utf8')
+
+  const result = spawnSync(shell, [
+    deployScript.replaceAll('\\', '/'),
+    'development',
+    'registry.local/group/firstleg',
+    imageTag,
+    'audit-service',
+    envFile.replaceAll('\\', '/')
+  ], {
+    cwd: repositoryRoot,
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      PATH: `${mockBin}${delimiter}${process.env.PATH ?? ''}`,
+      TARGET_IS_LOCAL: 'true',
+      REGISTRY_USER: 'ci-user',
+      REGISTRY_PASSWORD: 'ci-password',
+      DEPLOY_HEALTH_TIMEOUT_SECONDS: '2',
+      DEPLOY_HEALTH_POLL_SECONDS: '1',
+      MOCK_LOG: logFile,
+      MOCK_CURRENT_IMAGE: currentImageFile,
+      MOCK_HEALTH_STATE: healthStateFile,
+      MOCK_FAIL_TAG: failingTag ?? 'never',
+      MOCK_FAIL_UP_TAG: failingStartTag ?? 'never'
+    }
+  })
+
+  const dockerLog = await readFile(logFile, 'utf8')
+  const currentImage = (await readFile(currentImageFile, 'utf8')).trim()
+  await rm(temporaryDirectory, { recursive: true, force: true })
+  return { ...result, dockerLog, currentImage }
+}
+
+test('successful deployment keeps the new healthy image', async (context) => {
+  if (!existsSync(shell)) context.skip(`shell unavailable: ${shell}`)
+
+  const result = await runDeployment({ imageTag: 'newgood' })
+
+  assert.equal(result.status, 0, result.stderr)
+  assert.match(result.stdout, /部署前运行镜像: registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.match(result.stdout, /新镜像健康检查通过: registry\.local\/group\/firstleg\/audit-service:newgood/)
+  assert.equal(result.currentImage, 'registry.local/group/firstleg/audit-service:newgood')
+})
+
+test('unhealthy deployment restores the previous healthy image and still fails the job', async (context) => {
+  if (!existsSync(shell)) context.skip(`shell unavailable: ${shell}`)
+
+  const result = await runDeployment({ imageTag: 'newbad', failingTag: 'newbad' })
+  const output = `${result.stdout}\n${result.stderr}`
+
+  assert.equal(result.status, 1, output)
+  assert.match(output, /新镜像健康检查失败: registry\.local\/group\/firstleg\/audit-service:newbad/)
+  assert.match(output, /开始自动回滚到: registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.match(output, /自动回滚成功: registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.match(result.dockerLog, /pull registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.equal(result.currentImage, 'registry.local/group/firstleg/audit-service:oldgood')
+})
+
+test('failed compose start also restores the previous healthy image', async (context) => {
+  if (!existsSync(shell)) context.skip(`shell unavailable: ${shell}`)
+
+  const result = await runDeployment({ imageTag: 'newbroken', failingStartTag: 'newbroken' })
+  const output = `${result.stdout}\n${result.stderr}`
+
+  assert.equal(result.status, 1, output)
+  assert.match(output, /新镜像启动命令失败: registry\.local\/group\/firstleg\/audit-service:newbroken/)
+  assert.match(output, /自动回滚成功: registry\.local\/group\/firstleg\/audit-service:oldgood/)
+  assert.equal(result.currentImage, 'registry.local/group/firstleg/audit-service:oldgood')
+})

scripts/test-environment-pipeline-contract.test.mjs

@@ -101,3 +101,13 @@ test('source workspaces are preserved for incremental Git fetches', () => {
   assert.equal((pipeline.match(/cleanWs\(/g) ?? []).length, 2)
   assert.doesNotMatch(coordinatorPipeline, /cleanWs\(/)
 })
+
+test('direct deployments wait for health and automatically restore the previous image', () => {
+  assert.match(directDeployScript, /HEALTH_TIMEOUT_SECONDS="\$\{DEPLOY_HEALTH_TIMEOUT_SECONDS:-240\}"/)
+  assert.match(directDeployScript, /previous_image="\$\(local_container_image \|\| true\)"/)
+  assert.match(directDeployScript, /if wait_for_local_health; then/)
+  assert.match(directDeployScript, /开始自动回滚到/)
+  assert.match(directDeployScript, /自动回滚成功/)
+  assert.match(directDeployScript, /自动回滚失败/)
+  assert.match(directDeployScript, /return 1/)
+})