diff --git a/Dockerfile b/Dockerfile index 871e413..f8a131b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ COPY logistics_chat/ ./logistics_chat/ COPY templates/ ./templates/ COPY static/ ./static/ COPY deploy/__init__.py deploy/wsgi.py ./deploy/ +COPY scripts/import_order_test_data.py ./scripts/ USER 10001:10001 EXPOSE 5000 HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=6 \ diff --git a/Jenkinsfile b/Jenkinsfile index 0e58301..719fda9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,6 +20,9 @@ pipeline { if (!(params.PYTHON_BASE_IMAGE ==~ /[A-Za-z0-9][A-Za-z0-9._\/:@-]+/)) { error('Invalid Python base image reference') } + if (!((params.ASR_PROVIDER ?: '') in ['', 'baidu'])) { + error('Gray speech provider must be baidu or disabled') + } } sh '''set -eu for file in deploy/*.sh; do bash -n "$file"; done @@ -50,16 +53,29 @@ pipeline { withEnv([ "PUBLIC_ORIGIN=${params.PUBLIC_ORIGIN}", "HOST_PORT=${params.HOST_PORT}", "GRAY_DEPLOY_DIR=${params.GRAY_DEPLOY_DIR}", "GRAY_PROXY_CONF_DIR=${params.GRAY_PROXY_CONF_DIR}", - "GRAY_TLS_CERT_FILE=${params.GRAY_TLS_CERT_FILE}", "GRAY_TLS_KEY_FILE=${params.GRAY_TLS_KEY_FILE}" + "GRAY_TLS_CERT_FILE=${params.GRAY_TLS_CERT_FILE}", "GRAY_TLS_KEY_FILE=${params.GRAY_TLS_KEY_FILE}", + "ASR_PROVIDER=${params.ASR_PROVIDER ?: ''}" ]) { withCredentials([ sshUserPrivateKey(credentialsId: params.SSH_CREDENTIAL_ID, keyFileVariable: 'TARGET_SSH_KEY', usernameVariable: 'TARGET_SSH_USER'), string(credentialsId: params.SSH_HOST_CREDENTIAL_ID, variable: 'TARGET_SSH_HOST'), file(credentialsId: params.SSH_KNOWN_HOSTS_CREDENTIAL_ID, variable: 'TARGET_SSH_KNOWN_HOSTS'), string(credentialsId: params.DEEPSEEK_CREDENTIAL_ID, variable: 'DEEPSEEK_API_KEY'), + string(credentialsId: params.BAIDU_ASR_API_KEY_CREDENTIAL_ID, variable: 'BAIDU_ASR_API_KEY'), + string(credentialsId: params.BAIDU_ASR_SECRET_KEY_CREDENTIAL_ID, variable: 'BAIDU_ASR_SECRET_KEY'), usernamePassword(credentialsId: params.PREVIEW_AUTH_CREDENTIAL_ID, usernameVariable: 'PREVIEW_USERNAME', passwordVariable: 'PREVIEW_PASSWORD') ]) { sh 'bash deploy/publish-gray.sh' + script { + if (params.IMPORT_ORDER_SAMPLES) { + withCredentials([ + file(credentialsId: params.ORDER_SNAPSHOT_CREDENTIAL_ID, variable: 'ORDER_SNAPSHOT_FILE'), + file(credentialsId: params.TRACKING_SNAPSHOT_CREDENTIAL_ID, variable: 'TRACKING_SNAPSHOT_FILE') + ]) { + sh 'bash deploy/import-gray-order-samples.sh' + } + } + } } } } diff --git a/deploy/gray_config.py b/deploy/gray_config.py index 0b2d8a3..6f82667 100644 --- a/deploy/gray_config.py +++ b/deploy/gray_config.py @@ -114,14 +114,30 @@ server {{ ''' -def write_release(directory, config, key): - if not key.strip() or any(character in key for character in ('\r', '\n', '\x00')): - raise ValueError('The model credential must be a nonempty single line') +def runtime_environment(key, speech=None): + fields = {'DEEPSEEK_API_KEY': key} + speech = speech or {} + provider = (speech.get('ASR_PROVIDER') or '').strip() + if provider: + if provider != 'baidu': + raise ValueError('Gray speech provider must be baidu or disabled') + fields.update(ASR_PROVIDER=provider, + BAIDU_ASR_API_KEY=speech.get('BAIDU_ASR_API_KEY', ''), + BAIDU_ASR_SECRET_KEY=speech.get('BAIDU_ASR_SECRET_KEY', '')) + for value in fields.values(): + if not isinstance(value, str) or not value.strip() or any(c in value for c in ('\r', '\n', '\x00')): + raise ValueError('Runtime credentials must be nonempty single lines') + def literal(value): + return "'" + value.strip().replace('\\', '\\\\').replace("'", "\\'") + "'" + return ''.join(name + '=' + literal(value) + '\n' for name, value in fields.items()) + + +def write_release(directory, config, key, speech=None): + runtime = runtime_environment(key, speech) directory.mkdir(parents=True, exist_ok=False) (directory / 'release.env').write_text(config.release_environment(), encoding='utf-8') (directory / 'site.conf').write_text(config.nginx_config(), encoding='utf-8') - escaped = key.strip().replace('\\', '\\\\').replace("'", "\\'") - (directory / 'runtime.env').write_text("DEEPSEEK_API_KEY='" + escaped + "'\n", encoding='utf-8') + (directory / 'runtime.env').write_text(runtime, encoding='utf-8') (directory / 'runtime.env').chmod(0o600) source = Path(__file__).resolve().parent shutil.copyfile(source / 'compose.gray.yml', directory / 'compose.yml') @@ -136,7 +152,7 @@ def main(): args = parser.parse_args() try: config = GrayConfig.from_environment(os.environ) - write_release(args.directory, config, os.environ['DEEPSEEK_API_KEY']) + write_release(args.directory, config, os.environ['DEEPSEEK_API_KEY'], os.environ) except (KeyError, ValueError) as error: parser.exit(2, 'Gray release configuration is incomplete or invalid.\n') diff --git a/deploy/import-gray-order-samples.sh b/deploy/import-gray-order-samples.sh new file mode 100644 index 0000000..44be2d9 --- /dev/null +++ b/deploy/import-gray-order-samples.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set +x +set -Eeuo pipefail +umask 077 + +: "${ORDER_SNAPSHOT_FILE:?}" "${TRACKING_SNAPSHOT_FILE:?}" +: "${TARGET_SSH_KEY:?}" "${TARGET_SSH_USER:?}" "${TARGET_SSH_HOST:?}" "${TARGET_SSH_KNOWN_HOSTS:?}" +: "${GRAY_DEPLOY_DIR:?}" "${SOURCE_COMMIT:?}" "${BUILD_NUMBER:?}" +[[ "$TARGET_SSH_USER" =~ ^[A-Za-z0-9_.-]+$ && "$TARGET_SSH_HOST" =~ ^[A-Za-z0-9.-]+$ ]] || exit 2 +[[ "$SOURCE_COMMIT" =~ ^[a-f0-9]{40}$ && "$BUILD_NUMBER" =~ ^[1-9][0-9]*$ ]] || exit 2 +root=$GRAY_DEPLOY_DIR +[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2 +release_id="${SOURCE_COMMIT:0:12}-b$BUILD_NUMBER" +release="$root/releases/$release_id" +order_sha=$(sha256sum "$ORDER_SNAPSHOT_FILE" | awk '{print $1}') +tracking_sha=$(sha256sum "$TRACKING_SNAPSHOT_FILE" | awk '{print $1}') +ssh_options=(-i "$TARGET_SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=$TARGET_SSH_KNOWN_HOSTS" -o ConnectTimeout=12) +target="$TARGET_SSH_USER@$TARGET_SSH_HOST" + +# Names are fixed; credentials and snapshot contents never enter the shell log. +scp "${ssh_options[@]}" "$ORDER_SNAPSHOT_FILE" "$target:$release/order-input.json" +scp "${ssh_options[@]}" "$TRACKING_SNAPSHOT_FILE" "$target:$release/tracking-input.json" +ssh "${ssh_options[@]}" "$target" bash -s -- "$root" "$release_id" "$SOURCE_COMMIT" "$order_sha" "$tracking_sha" <<'IMPORT' +set -Eeuo pipefail +umask 077 +root=$1 +release_id=$2 +source_commit=$3 +order_sha=$4 +tracking_sha=$5 +[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2 +[[ "$release_id" =~ ^[a-f0-9]{12}-b[0-9]+$ && "$source_commit" =~ ^[a-f0-9]{40}$ ]] || exit 2 +[[ "$order_sha" =~ ^[a-f0-9]{64}$ && "$tracking_sha" =~ ^[a-f0-9]{64}$ ]] || exit 2 +release="$root/releases/$release_id" +[[ "$(cat "$root/.project")" == baiyue-logistics-chat-gray && "$(cat "$root/current-release")" == "$release" ]] || exit 2 +container=baiyue-logistics-chat-gray +[[ "$(docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' "$container")" == baiyue-logistics-chat-gray ]] || exit 2 +[[ "$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container")" == "$source_commit" ]] || exit 2 +printf '%s %s\n' "$order_sha" "$release/order-input.json" "$tracking_sha" "$release/tracking-input.json" | sha256sum --check --status +input="$root/data/imports/$release_id" +[[ ! -e "$input" ]] || { echo 'Existing import input requires inspection.' >&2; exit 2; } +cleanup() { + rm -f -- "$input/orders.json" "$input/tracking.json" "$release/order-input.json" "$release/tracking-input.json" + rmdir -- "$input" 2>/dev/null || true +} +trap cleanup EXIT +install -d -m 700 -o 10001 -g 10001 "$root/data/imports" "$input" +install -m 600 -o 10001 -g 10001 "$release/order-input.json" "$input/orders.json" +install -m 600 -o 10001 -g 10001 "$release/tracking-input.json" "$input/tracking.json" +docker exec "$container" python -c 'import pathlib,sqlite3,sys; p=pathlib.Path("/var/lib/baiyue-logistics-chat/backups")/("pre-order-import-"+sys.argv[1]+".db"); assert not p.exists(); source=sqlite3.connect("/var/lib/baiyue-logistics-chat/history.db"); target=sqlite3.connect(str(p)); source.backup(target); target.close(); source.close(); print("Chat SQLite backup completed before sample import.")' "$release_id" +docker exec "$container" python scripts/import_order_test_data.py \ + --db /var/lib/baiyue-logistics-chat/history.db --snapshot "/var/lib/baiyue-logistics-chat/imports/$release_id/orders.json" +docker exec "$container" python scripts/import_order_test_data.py \ + --db /var/lib/baiyue-logistics-chat/history.db --snapshot "/var/lib/baiyue-logistics-chat/imports/$release_id/tracking.json" --tracking-only +echo 'Private snapshots imported with existing order fields and nonempty tracking preserved.' +IMPORT diff --git a/docs/deployment.md b/docs/deployment.md index d051fe3..c6234be 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -29,9 +29,17 @@ | `SSH_KNOWN_HOSTS_CREDENTIAL_ID` | 已核实的 known_hosts 文件引用 | | `DEEPSEEK_CREDENTIAL_ID` | 本服务模型密钥的 Secret text 引用 | | `PREVIEW_AUTH_CREDENTIAL_ID` | 专用验收用户名和密码引用 | +| `ASR_PROVIDER` | `baidu` 启用语音；空值关闭 | +| `BAIDU_ASR_API_KEY_CREDENTIAL_ID` / `BAIDU_ASR_SECRET_KEY_CREDENTIAL_ID` | 本服务百度语音专用 Secret text 引用 | +| `IMPORT_ORDER_SAMPLES` | 默认关闭；明确需要补样本时才启用 | +| `ORDER_SNAPSHOT_CREDENTIAL_ID` / `TRACKING_SNAPSHOT_CREDENTIAL_ID` | 本次导入的临时 Secret file 引用 | 运行密钥和验收账号保存在私有灰度配置与 Jenkins 凭据库，供后续发布持续复用；轮换时同步更新。不要将密钥放在构建参数、Git URL、镜像 ARG、日志或示例文件中。 +语音运行配置只注入 `ASR_PROVIDER`、`BAIDU_ASR_API_KEY`、`BAIDU_ASR_SECRET_KEY`，与 DS 密钥一起写入权限 600 的 `runtime.env`。容器使用镜像内的 FFmpeg，不传本机 `FFMPEG_BIN`，也不使用 OCR 凭据。缺少任一百度凭据时拒绝生成发布配置。 + +私有订单样本不进入 Git、构建上下文或镜像。可在明确授权后临时上传为 Jenkins Secret file 并启用 `IMPORT_ORDER_SAMPLES`；发布脚本核验快照校验和与实际运行提交，在线备份聊天 SQLite，再调用 `scripts/import_order_test_data.py` 默认导入订单、以 `--tracking-only` 补充轨迹。两步均不使用 `--replace`，保留手工订单字段和已有非空轨迹。临时文件在导入后清除；验收后移除临时 Jenkins 文件凭据并关闭导入开关。此操作只访问聊天服务自己的 SQLite。 + ## 发布步骤 1. 完成检查并提交代码，创建 `gray-chat-YYYYMMDD-` 一类不可变发布标签。不要重写已有标签或强推覆盖远端分支；分支有其他更新时先明确发布基线。 @@ -62,7 +70,7 @@ 更新前执行 SQLite 在线备份。应用启动或 Nginx 检查失败时，仅恢复本服务之前的 Compose 和站点配置，保留数据；不自动用旧库覆盖新记录。失败发布目录保留供定位，数据库降级及删除数据需另行处理。 -新环境自动创建八张业务表，不创建默认客服账号或演示数据。验收用全站密码直接进入工作台；Nginx 验收账号不是应用内客服账号。 +新环境自动创建业务表，不创建默认客服账号或演示数据。验收用全站密码直接进入工作台；Nginx 验收账号不是应用内客服账号。 ## HTTPS 及续期 diff --git a/tests/test_gray_config.py b/tests/test_gray_config.py index c16a63e..6179c5d 100644 --- a/tests/test_gray_config.py +++ b/tests/test_gray_config.py @@ -65,6 +65,32 @@ class GrayConfigTests(unittest.TestCase): write_release(path, config, 'fixture-key\nANOTHER_SETTING=bad') self.assertFalse(path.exists()) + def test_baidu_credentials_only_enter_private_runtime_file(self): + config = GrayConfig.from_environment(self.settings()) + speech = {'ASR_PROVIDER': 'baidu', 'BAIDU_ASR_API_KEY': 'fixture-${asr-key}', + 'BAIDU_ASR_SECRET_KEY': "fixture-asr-secret's", 'FFMPEG_BIN': r'C:\local\ffmpeg.exe'} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'release' + write_release(path, config, 'fixture-model-key', speech) + runtime = dotenv_values(path / 'runtime.env', interpolate=False) + self.assertEqual(runtime['ASR_PROVIDER'], 'baidu') + for key in ('BAIDU_ASR_API_KEY', 'BAIDU_ASR_SECRET_KEY'): + self.assertEqual(runtime[key], speech[key]) + for file in path.iterdir(): + if file.name != 'runtime.env': + self.assertNotIn(speech[key], file.read_text(encoding='utf-8')) + self.assertNotIn('FFMPEG_BIN', runtime) + + def test_incomplete_or_multiline_speech_credentials_are_rejected_before_writing(self): + config = GrayConfig.from_environment(self.settings()) + for speech in ({'ASR_PROVIDER': 'baidu'}, {'ASR_PROVIDER': 'unknown'}, + {'ASR_PROVIDER': 'baidu', 'BAIDU_ASR_API_KEY': 'key', 'BAIDU_ASR_SECRET_KEY': 'secret\nOTHER=bad'}): + with self.subTest(provider=speech['ASR_PROVIDER']), tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'release' + with self.assertRaises(ValueError): + write_release(path, config, 'fixture-key', speech) + self.assertFalse(path.exists()) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_gray_order_import.py b/tests/test_gray_order_import.py new file mode 100644 index 0000000..cf88180 --- /dev/null +++ b/tests/test_gray_order_import.py @@ -0,0 +1,102 @@ +"""Run the real private-import guard against a disposable database and fake Docker.""" + +import hashlib +import json +import os +from pathlib import Path +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import unittest + +from logistics_chat.order_data import import_records + + +@unittest.skipUnless(os.name == 'posix' and shutil.which('bash'), 'Private import requires Linux Bash') +class GrayOrderImportTests(unittest.TestCase): + def run_import(self, corrupt=False): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + directory = Path(temporary.name) + root = directory / 'gray-logistics-chat' + release_id = 'a' * 12 + '-b7' + release = root / 'releases' / release_id + release.mkdir(parents=True) + (root / '.project').write_text('baiyue-logistics-chat-gray\n') + (root / 'current-release').write_text(str(release) + '\n') + data = root / 'data' + (data / 'backups').mkdir(parents=True) + database = data / 'history.db' + with sqlite3.connect(database) as connection: + connection.execute('CREATE TABLE messages (content TEXT)') + connection.execute("INSERT INTO messages VALUES ('keep chat history')") + import_records(connection, [{'order_no': 'EXISTING1', 'source_order_id': '1', + 'tenant_id': 'test', 'gross_weight_kg': 'manual weight', + 'tracking_json': [{'manual': True}]}]) + orders = [{'order_no': 'EXISTING1', 'source_order_id': '1', 'tenant_id': 'test', 'gross_weight_kg': 'source weight'}, + {'order_no': 'NEW00002', 'source_order_id': '2', 'tenant_id': 'test'}] + tracking = [{'order_no': row['order_no'], 'source_order_id': row['source_order_id'], + 'tenant_id': 'test', 'tracking_json': [{'code': 'IN_TRANSIT', 'actual_at': '2026-10-30'}]} + for row in orders] + hashes = [] + for name, records in [('order-input.json', orders), ('tracking-input.json', tracking)]: + path = release / name + path.write_text(json.dumps({'orders': records})) + hashes.append(hashlib.sha256(path.read_bytes()).hexdigest()) + if corrupt: + (release / 'order-input.json').write_text('changed in transit') + commands = directory / 'commands' + commands.mkdir() + docker = commands / 'docker' + docker.write_text('#!' + sys.executable + '\n' + '''import os, pathlib, subprocess, sys +args = sys.argv[1:] +if args[0] == 'inspect': + print('a'*40 if 'revision' in args[2] else 'baiyue-logistics-chat-gray') +elif args[0] == 'exec': + command = [sys.executable] + args[3:] + command = [value.replace('/var/lib/baiyue-logistics-chat', os.environ['FAKE_DATA']) for value in command] + if len(command) > 1 and command[1] != '-c': + command[1] = str(pathlib.Path(os.environ['FAKE_SOURCE']) / command[1]) + sys.exit(subprocess.run(command).returncode) +else: + sys.exit(3) +''') + docker.chmod(0o755) + source = Path(__file__).resolve().parents[1] + script = (source / 'deploy/import-gray-order-samples.sh').read_text() + remote = script.split("<<'IMPORT'\n", 1)[1].split('\nIMPORT\n', 1)[0] + environment = dict(os.environ, PATH=str(commands) + os.pathsep + os.environ['PATH'], + FAKE_DATA=str(data), FAKE_SOURCE=str(source)) + result = subprocess.run(['bash', '-s', '--', str(root), release_id, 'a' * 40, *hashes], + input=remote, env=environment, capture_output=True, text=True, timeout=30) + return result, data, release, release_id + + def test_import_backs_up_and_preserves_manual_fields_tracking_and_chat_history(self): + result, data, release, release_id = self.run_import() + self.assertEqual(result.returncode, 0, result.stderr) + with sqlite3.connect(data / 'history.db') as connection: + self.assertEqual(connection.execute('SELECT content FROM messages').fetchall(), [('keep chat history',)]) + row = connection.execute('SELECT gross_weight_kg,tracking_json FROM order_test_data WHERE order_no="EXISTING1"').fetchone() + self.assertEqual(row[0], 'manual weight') + self.assertEqual(json.loads(row[1]), [{'manual': True}]) + self.assertEqual(connection.execute('SELECT COUNT(*) FROM order_test_data').fetchone()[0], 2) + row = connection.execute('SELECT tracking_json FROM order_test_data WHERE order_no="NEW00002"').fetchone() + self.assertEqual(json.loads(row[0])[0]['code'], 'IN_TRANSIT') + with sqlite3.connect(data / 'backups' / ('pre-order-import-' + release_id + '.db')) as backup: + self.assertEqual(backup.execute('SELECT COUNT(*) FROM order_test_data').fetchone()[0], 1) + self.assertFalse((data / 'imports' / release_id).exists()) + self.assertFalse((release / 'order-input.json').exists()) + self.assertFalse((release / 'tracking-input.json').exists()) + + def test_corrupt_snapshot_is_rejected_before_database_changes(self): + result, data, release, release_id = self.run_import(corrupt=True) + self.assertNotEqual(result.returncode, 0) + with sqlite3.connect(data / 'history.db') as connection: + self.assertEqual(connection.execute('SELECT COUNT(*) FROM order_test_data').fetchone()[0], 1) + self.assertFalse(list((data / 'backups').iterdir())) + + +if __name__ == '__main__': + unittest.main()