等待 Nginx 新进程完成灰度入口切换

deploy/activate-gray.sh

@@ -70,6 +70,8 @@ proxy_changed=0
 rollback() {
   result=$?
   trap - ERR
+  # ERR is inherited by command substitutions; only the parent may roll back.
+  if ((BASH_SUBSHELL)); then exit "$result"; fi
   set +e
   if ((proxy_changed)); then
     if [[ -f "$release/previous-site.conf" ]]; then cp -p "$release/previous-site.conf" "$site"; else rm -f -- "$site"; fi
@@ -102,7 +104,14 @@ install -m 644 "$release/site.conf" "$site"
 nginx -t
 nginx -s reload
 for path in / /api/export_chat /api/get_history; do
-  code=$(curl --silent --show-error --resolve "$CHAT_DOMAIN:443:127.0.0.1" -o /dev/null -w '%{http_code}' "https://$CHAT_DOMAIN$path")
+  code=''
+  # A successful reload signal precedes the new workers accepting connections.
+  for attempt in {1..10}; do
+    if code=$(curl --silent --connect-timeout 3 --max-time 8 --resolve "$CHAT_DOMAIN:443:127.0.0.1" -o /dev/null -w '%{http_code}' "https://$CHAT_DOMAIN$path"); then
+      [[ "$code" != 401 ]] || break
+    fi
+    sleep 1
+  done
   [[ "$code" == 401 ]] || { echo 'Gray access protection verification failed.' >&2; false; }
 done
 printf '%s\n' "$release" > "$root/.current-release.new"

tests/test_gray_activation.py

@@ -35,7 +35,15 @@ elif name == 'nginx' and '-t' in args and scenario == 'proxy-fail':
     if sum(item[0] == 'nginx' and '-t' in item[1] for item in calls) == 1:
         sys.exit(19)
 elif name == 'curl' and '-w' in args:
+    calls = [json.loads(line) for line in log.read_text().splitlines()]
+    if scenario == 'reload-pending' and sum(item[0] == 'curl' and '-w' in item[1] for item in calls) == 1:
+        sys.exit(60)
+    if scenario == 'proxy-unsafe':
+        print('200', end='')
+        sys.exit(0)
     print('401', end='')
+elif name == 'sleep':
+    pass
 '''
 
 
@@ -75,7 +83,7 @@ class GrayActivationTests(unittest.TestCase):
         (release / 'site.htpasswd').write_text('new-auth-hash\n')
         commands = directory / 'commands'
         commands.mkdir()
-        for name in ('docker', 'nginx', 'openssl', 'ss', 'curl'):
+        for name in ('docker', 'nginx', 'openssl', 'ss', 'curl', 'sleep'):
             command = commands / name
             command.write_text('#!' + sys.executable + '\n' + FAKE_COMMAND)
             command.chmod(0o755)
@@ -108,6 +116,15 @@ class GrayActivationTests(unittest.TestCase):
     def test_failed_application_restores_previous_release(self):
         self.assert_rolled_back('app-fail')
 
+    def test_async_nginx_reload_waits_for_the_new_tls_workers(self):
+        result, root, proxy, previous, release, old_site, calls = self.run_activation('reload-pending')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual((root / 'current-release').read_text().strip(), str(release))
+        self.assertTrue(any(name == 'sleep' for name, args in calls))
+
+    def test_unprotected_proxy_never_publishes_the_release_pointer(self):
+        self.assert_rolled_back('proxy-unsafe')
+
     def test_image_retagged_after_verified_load_is_rejected_before_activation(self):
         result, root, proxy, previous, release, old_site, calls = self.run_activation('image-mismatch')
         self.assertNotEqual(result.returncode, 0)