songxinnuo/baiyue-logistics-chat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

补齐灰度语音配置与订单样本私有导入

Browse Source
This commit is contained in:
ouyangxq
2026-10-30 11:28:07 +08:00
parent 3f1cee8006
commit aacdca6930
7 changed files with 234 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+1
View File
@@ -18,6 +18,7 @@ COPY logistics_chat/ ./logistics_chat/
COPY templates/ ./templates/
COPY static/ ./static/
COPY deploy/__init__.py deploy/wsgi.py ./deploy/
COPY scripts/import_order_test_data.py ./scripts/
USER 10001:10001
EXPOSE 5000
HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=6 \
Jenkinsfile
Vendored
+17 -1
View File
@@ -20,6 +20,9 @@ pipeline {
if (!(params.PYTHON_BASE_IMAGE ==~ /[A-Za-z0-9][A-Za-z0-9._\/:@-]+/)) {
error('Invalid Python base image reference')
}
if (!((params.ASR_PROVIDER ?: '') in ['', 'baidu'])) {
error('Gray speech provider must be baidu or disabled')
}
}
sh '''set -eu
for file in deploy/*.sh; do bash -n "$file"; done
@@ -50,16 +53,29 @@ pipeline {
withEnv([
"PUBLIC_ORIGIN=${params.PUBLIC_ORIGIN}", "HOST_PORT=${params.HOST_PORT}",
"GRAY_DEPLOY_DIR=${params.GRAY_DEPLOY_DIR}", "GRAY_PROXY_CONF_DIR=${params.GRAY_PROXY_CONF_DIR}",
"GRAY_TLS_CERT_FILE=${params.GRAY_TLS_CERT_FILE}", "GRAY_TLS_KEY_FILE=${params.GRAY_TLS_KEY_FILE}"
"GRAY_TLS_CERT_FILE=${params.GRAY_TLS_CERT_FILE}", "GRAY_TLS_KEY_FILE=${params.GRAY_TLS_KEY_FILE}",
"ASR_PROVIDER=${params.ASR_PROVIDER ?: ''}"
]) {
withCredentials([
sshUserPrivateKey(credentialsId: params.SSH_CREDENTIAL_ID, keyFileVariable: 'TARGET_SSH_KEY', usernameVariable: 'TARGET_SSH_USER'),
string(credentialsId: params.SSH_HOST_CREDENTIAL_ID, variable: 'TARGET_SSH_HOST'),
file(credentialsId: params.SSH_KNOWN_HOSTS_CREDENTIAL_ID, variable: 'TARGET_SSH_KNOWN_HOSTS'),
string(credentialsId: params.DEEPSEEK_CREDENTIAL_ID, variable: 'DEEPSEEK_API_KEY'),
string(credentialsId: params.BAIDU_ASR_API_KEY_CREDENTIAL_ID, variable: 'BAIDU_ASR_API_KEY'),
string(credentialsId: params.BAIDU_ASR_SECRET_KEY_CREDENTIAL_ID, variable: 'BAIDU_ASR_SECRET_KEY'),
usernamePassword(credentialsId: params.PREVIEW_AUTH_CREDENTIAL_ID, usernameVariable: 'PREVIEW_USERNAME', passwordVariable: 'PREVIEW_PASSWORD')
]) {
sh 'bash deploy/publish-gray.sh'
script {
if (params.IMPORT_ORDER_SAMPLES) {
withCredentials([
file(credentialsId: params.ORDER_SNAPSHOT_CREDENTIAL_ID, variable: 'ORDER_SNAPSHOT_FILE'),
file(credentialsId: params.TRACKING_SNAPSHOT_CREDENTIAL_ID, variable: 'TRACKING_SNAPSHOT_FILE')
]) {
sh 'bash deploy/import-gray-order-samples.sh'
}
}
}
}
}
}
deploy/gray_config.py
+22 -6
View File
@@ -114,14 +114,30 @@ server {{
'''
def write_release(directory, config, key):
if not key.strip() or any(character in key for character in ('\r', '\n', '\x00')):
raise ValueError('The model credential must be a nonempty single line')
def runtime_environment(key, speech=None):
fields = {'DEEPSEEK_API_KEY': key}
speech = speech or {}
provider = (speech.get('ASR_PROVIDER') or '').strip()
if provider:
if provider != 'baidu':
raise ValueError('Gray speech provider must be baidu or disabled')
fields.update(ASR_PROVIDER=provider,
BAIDU_ASR_API_KEY=speech.get('BAIDU_ASR_API_KEY', ''),
BAIDU_ASR_SECRET_KEY=speech.get('BAIDU_ASR_SECRET_KEY', ''))
for value in fields.values():
if not isinstance(value, str) or not value.strip() or any(c in value for c in ('\r', '\n', '\x00')):
raise ValueError('Runtime credentials must be nonempty single lines')
def literal(value):
return "'" + value.strip().replace('\\', '\\\\').replace("'", "\\'") + "'"
return ''.join(name + '=' + literal(value) + '\n' for name, value in fields.items())
def write_release(directory, config, key, speech=None):
runtime = runtime_environment(key, speech)
directory.mkdir(parents=True, exist_ok=False)
(directory / 'release.env').write_text(config.release_environment(), encoding='utf-8')
(directory / 'site.conf').write_text(config.nginx_config(), encoding='utf-8')
escaped = key.strip().replace('\\', '\\\\').replace("'", "\\'")
(directory / 'runtime.env').write_text("DEEPSEEK_API_KEY='" + escaped + "'\n", encoding='utf-8')
(directory / 'runtime.env').write_text(runtime, encoding='utf-8')
(directory / 'runtime.env').chmod(0o600)
source = Path(__file__).resolve().parent
shutil.copyfile(source / 'compose.gray.yml', directory / 'compose.yml')
@@ -136,7 +152,7 @@ def main():
args = parser.parse_args()
try:
config = GrayConfig.from_environment(os.environ)
write_release(args.directory, config, os.environ['DEEPSEEK_API_KEY'])
write_release(args.directory, config, os.environ['DEEPSEEK_API_KEY'], os.environ)
except (KeyError, ValueError) as error:
parser.exit(2, 'Gray release configuration is incomplete or invalid.\n')
deploy/import-gray-order-samples.sh
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env bash
set +x
set -Eeuo pipefail
umask 077
: "${ORDER_SNAPSHOT_FILE:?}" "${TRACKING_SNAPSHOT_FILE:?}"
: "${TARGET_SSH_KEY:?}" "${TARGET_SSH_USER:?}" "${TARGET_SSH_HOST:?}" "${TARGET_SSH_KNOWN_HOSTS:?}"
: "${GRAY_DEPLOY_DIR:?}" "${SOURCE_COMMIT:?}" "${BUILD_NUMBER:?}"
[[ "$TARGET_SSH_USER" =~ ^[A-Za-z0-9_.-]+$ && "$TARGET_SSH_HOST" =~ ^[A-Za-z0-9.-]+$ ]] || exit 2
[[ "$SOURCE_COMMIT" =~ ^[a-f0-9]{40}$ && "$BUILD_NUMBER" =~ ^[1-9][0-9]*$ ]] || exit 2
root=$GRAY_DEPLOY_DIR
[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2
release_id="${SOURCE_COMMIT:0:12}-b$BUILD_NUMBER"
release="$root/releases/$release_id"
order_sha=$(sha256sum "$ORDER_SNAPSHOT_FILE" | awk '{print $1}')
tracking_sha=$(sha256sum "$TRACKING_SNAPSHOT_FILE" | awk '{print $1}')
ssh_options=(-i "$TARGET_SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=yes
-o "UserKnownHostsFile=$TARGET_SSH_KNOWN_HOSTS" -o ConnectTimeout=12)
target="$TARGET_SSH_USER@$TARGET_SSH_HOST"
# Names are fixed; credentials and snapshot contents never enter the shell log.
scp "${ssh_options[@]}" "$ORDER_SNAPSHOT_FILE" "$target:$release/order-input.json"
scp "${ssh_options[@]}" "$TRACKING_SNAPSHOT_FILE" "$target:$release/tracking-input.json"
ssh "${ssh_options[@]}" "$target" bash -s -- "$root" "$release_id" "$SOURCE_COMMIT" "$order_sha" "$tracking_sha" <<'IMPORT'
set -Eeuo pipefail
umask 077
root=$1
release_id=$2
source_commit=$3
order_sha=$4
tracking_sha=$5
[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2
[[ "$release_id" =~ ^[a-f0-9]{12}-b[0-9]+$ && "$source_commit" =~ ^[a-f0-9]{40}$ ]] || exit 2
[[ "$order_sha" =~ ^[a-f0-9]{64}$ && "$tracking_sha" =~ ^[a-f0-9]{64}$ ]] || exit 2
release="$root/releases/$release_id"
[[ "$(cat "$root/.project")" == baiyue-logistics-chat-gray && "$(cat "$root/current-release")" == "$release" ]] || exit 2
container=baiyue-logistics-chat-gray
[[ "$(docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' "$container")" == baiyue-logistics-chat-gray ]] || exit 2
[[ "$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container")" == "$source_commit" ]] || exit 2
printf '%s %s\n' "$order_sha" "$release/order-input.json" "$tracking_sha" "$release/tracking-input.json" | sha256sum --check --status
input="$root/data/imports/$release_id"
[[ ! -e "$input" ]] || { echo 'Existing import input requires inspection.' >&2; exit 2; }
cleanup() {
rm -f -- "$input/orders.json" "$input/tracking.json" "$release/order-input.json" "$release/tracking-input.json"
rmdir -- "$input" 2>/dev/null || true
}
trap cleanup EXIT
install -d -m 700 -o 10001 -g 10001 "$root/data/imports" "$input"
install -m 600 -o 10001 -g 10001 "$release/order-input.json" "$input/orders.json"
install -m 600 -o 10001 -g 10001 "$release/tracking-input.json" "$input/tracking.json"
docker exec "$container" python -c 'import pathlib,sqlite3,sys; p=pathlib.Path("/var/lib/baiyue-logistics-chat/backups")/("pre-order-import-"+sys.argv[1]+".db"); assert not p.exists(); source=sqlite3.connect("/var/lib/baiyue-logistics-chat/history.db"); target=sqlite3.connect(str(p)); source.backup(target); target.close(); source.close(); print("Chat SQLite backup completed before sample import.")' "$release_id"
docker exec "$container" python scripts/import_order_test_data.py \
--db /var/lib/baiyue-logistics-chat/history.db --snapshot "/var/lib/baiyue-logistics-chat/imports/$release_id/orders.json"
docker exec "$container" python scripts/import_order_test_data.py \
--db /var/lib/baiyue-logistics-chat/history.db --snapshot "/var/lib/baiyue-logistics-chat/imports/$release_id/tracking.json" --tracking-only
echo 'Private snapshots imported with existing order fields and nonempty tracking preserved.'
IMPORT
docs/deployment.md
+9 -1
View File
@@ -29,9 +29,17 @@
| `SSH_KNOWN_HOSTS_CREDENTIAL_ID` | 已核实的 known_hosts 文件引用 |
| `DEEPSEEK_CREDENTIAL_ID` | 本服务模型密钥的 Secret text 引用 |
| `PREVIEW_AUTH_CREDENTIAL_ID` | 专用验收用户名和密码引用 |
| `ASR_PROVIDER` | `baidu` 启用语音；空值关闭 |
| `BAIDU_ASR_API_KEY_CREDENTIAL_ID` / `BAIDU_ASR_SECRET_KEY_CREDENTIAL_ID` | 本服务百度语音专用 Secret text 引用 |
| `IMPORT_ORDER_SAMPLES` | 默认关闭；明确需要补样本时才启用 |
| `ORDER_SNAPSHOT_CREDENTIAL_ID` / `TRACKING_SNAPSHOT_CREDENTIAL_ID` | 本次导入的临时 Secret file 引用 |
运行密钥和验收账号保存在私有灰度配置与 Jenkins 凭据库，供后续发布持续复用；轮换时同步更新。不要将密钥放在构建参数、Git URL、镜像 ARG、日志或示例文件中。
语音运行配置只注入 `ASR_PROVIDER`、`BAIDU_ASR_API_KEY`、`BAIDU_ASR_SECRET_KEY`，与 DS 密钥一起写入权限 600 的 `runtime.env`。容器使用镜像内的 FFmpeg，不传本机 `FFMPEG_BIN`，也不使用 OCR 凭据。缺少任一百度凭据时拒绝生成发布配置。
私有订单样本不进入 Git、构建上下文或镜像。可在明确授权后临时上传为 Jenkins Secret file 并启用 `IMPORT_ORDER_SAMPLES`；发布脚本核验快照校验和与实际运行提交，在线备份聊天 SQLite，再调用 `scripts/import_order_test_data.py` 默认导入订单、以 `--tracking-only` 补充轨迹。两步均不使用 `--replace`，保留手工订单字段和已有非空轨迹。临时文件在导入后清除；验收后移除临时 Jenkins 文件凭据并关闭导入开关。此操作只访问聊天服务自己的 SQLite。
## 发布步骤
1. 完成检查并提交代码，创建 `gray-chat-YYYYMMDD-<commit12>` 一类不可变发布标签。不要重写已有标签或强推覆盖远端分支；分支有其他更新时先明确发布基线。
@@ -62,7 +70,7 @@
更新前执行 SQLite 在线备份。应用启动或 Nginx 检查失败时，仅恢复本服务之前的 Compose 和站点配置，保留数据；不自动用旧库覆盖新记录。失败发布目录保留供定位，数据库降级及删除数据需另行处理。
新环境自动创建八张业务表，不创建默认客服账号或演示数据。验收用全站密码直接进入工作台；Nginx 验收账号不是应用内客服账号。
新环境自动创建业务表，不创建默认客服账号或演示数据。验收用全站密码直接进入工作台；Nginx 验收账号不是应用内客服账号。
## HTTPS 及续期
tests/test_gray_config.py
+26
View File
@@ -65,6 +65,32 @@ class GrayConfigTests(unittest.TestCase):
write_release(path, config, 'fixture-key\nANOTHER_SETTING=bad')
self.assertFalse(path.exists())
def test_baidu_credentials_only_enter_private_runtime_file(self):
config = GrayConfig.from_environment(self.settings())
speech = {'ASR_PROVIDER': 'baidu', 'BAIDU_ASR_API_KEY': 'fixture-${asr-key}',
'BAIDU_ASR_SECRET_KEY': "fixture-asr-secret's", 'FFMPEG_BIN': r'C:\local\ffmpeg.exe'}
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / 'release'
write_release(path, config, 'fixture-model-key', speech)
runtime = dotenv_values(path / 'runtime.env', interpolate=False)
self.assertEqual(runtime['ASR_PROVIDER'], 'baidu')
for key in ('BAIDU_ASR_API_KEY', 'BAIDU_ASR_SECRET_KEY'):
self.assertEqual(runtime[key], speech[key])
for file in path.iterdir():
if file.name != 'runtime.env':
self.assertNotIn(speech[key], file.read_text(encoding='utf-8'))
self.assertNotIn('FFMPEG_BIN', runtime)
def test_incomplete_or_multiline_speech_credentials_are_rejected_before_writing(self):
config = GrayConfig.from_environment(self.settings())
for speech in ({'ASR_PROVIDER': 'baidu'}, {'ASR_PROVIDER': 'unknown'},
{'ASR_PROVIDER': 'baidu', 'BAIDU_ASR_API_KEY': 'key', 'BAIDU_ASR_SECRET_KEY': 'secret\nOTHER=bad'}):
with self.subTest(provider=speech['ASR_PROVIDER']), tempfile.TemporaryDirectory() as directory:
path = Path(directory) / 'release'
with self.assertRaises(ValueError):
write_release(path, config, 'fixture-key', speech)
self.assertFalse(path.exists())
if __name__ == '__main__':
unittest.main()
tests/test_gray_order_import.py
+102
View File
@@ -0,0 +1,102 @@
"""Run the real private-import guard against a disposable database and fake Docker."""
import hashlib
import json
import os
from pathlib import Path
import shutil
import sqlite3
import subprocess
import sys
import tempfile
import unittest
from logistics_chat.order_data import import_records
@unittest.skipUnless(os.name == 'posix' and shutil.which('bash'), 'Private import requires Linux Bash')
class GrayOrderImportTests(unittest.TestCase):
def run_import(self, corrupt=False):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
directory = Path(temporary.name)
root = directory / 'gray-logistics-chat'
release_id = 'a' * 12 + '-b7'
release = root / 'releases' / release_id
release.mkdir(parents=True)
(root / '.project').write_text('baiyue-logistics-chat-gray\n')
(root / 'current-release').write_text(str(release) + '\n')
data = root / 'data'
(data / 'backups').mkdir(parents=True)
database = data / 'history.db'
with sqlite3.connect(database) as connection:
connection.execute('CREATE TABLE messages (content TEXT)')
connection.execute("INSERT INTO messages VALUES ('keep chat history')")
import_records(connection, [{'order_no': 'EXISTING1', 'source_order_id': '1',
'tenant_id': 'test', 'gross_weight_kg': 'manual weight',
'tracking_json': [{'manual': True}]}])
orders = [{'order_no': 'EXISTING1', 'source_order_id': '1', 'tenant_id': 'test', 'gross_weight_kg': 'source weight'},
{'order_no': 'NEW00002', 'source_order_id': '2', 'tenant_id': 'test'}]
tracking = [{'order_no': row['order_no'], 'source_order_id': row['source_order_id'],
'tenant_id': 'test', 'tracking_json': [{'code': 'IN_TRANSIT', 'actual_at': '2026-10-30'}]}
for row in orders]
hashes = []
for name, records in [('order-input.json', orders), ('tracking-input.json', tracking)]:
path = release / name
path.write_text(json.dumps({'orders': records}))
hashes.append(hashlib.sha256(path.read_bytes()).hexdigest())
if corrupt:
(release / 'order-input.json').write_text('changed in transit')
commands = directory / 'commands'
commands.mkdir()
docker = commands / 'docker'
docker.write_text('#!' + sys.executable + '\n' + '''import os, pathlib, subprocess, sys
args = sys.argv[1:]
if args[0] == 'inspect':
print('a'*40 if 'revision' in args[2] else 'baiyue-logistics-chat-gray')
elif args[0] == 'exec':
command = [sys.executable] + args[3:]
command = [value.replace('/var/lib/baiyue-logistics-chat', os.environ['FAKE_DATA']) for value in command]
if len(command) > 1 and command[1] != '-c':
command[1] = str(pathlib.Path(os.environ['FAKE_SOURCE']) / command[1])
sys.exit(subprocess.run(command).returncode)
else:
sys.exit(3)
''')
docker.chmod(0o755)
source = Path(__file__).resolve().parents[1]
script = (source / 'deploy/import-gray-order-samples.sh').read_text()
remote = script.split("<<'IMPORT'\n", 1)[1].split('\nIMPORT\n', 1)[0]
environment = dict(os.environ, PATH=str(commands) + os.pathsep + os.environ['PATH'],
FAKE_DATA=str(data), FAKE_SOURCE=str(source))
result = subprocess.run(['bash', '-s', '--', str(root), release_id, 'a' * 40, *hashes],
input=remote, env=environment, capture_output=True, text=True, timeout=30)
return result, data, release, release_id
def test_import_backs_up_and_preserves_manual_fields_tracking_and_chat_history(self):
result, data, release, release_id = self.run_import()
self.assertEqual(result.returncode, 0, result.stderr)
with sqlite3.connect(data / 'history.db') as connection:
self.assertEqual(connection.execute('SELECT content FROM messages').fetchall(), [('keep chat history',)])
row = connection.execute('SELECT gross_weight_kg,tracking_json FROM order_test_data WHERE order_no="EXISTING1"').fetchone()
self.assertEqual(row[0], 'manual weight')
self.assertEqual(json.loads(row[1]), [{'manual': True}])
self.assertEqual(connection.execute('SELECT COUNT(*) FROM order_test_data').fetchone()[0], 2)
row = connection.execute('SELECT tracking_json FROM order_test_data WHERE order_no="NEW00002"').fetchone()
self.assertEqual(json.loads(row[0])[0]['code'], 'IN_TRANSIT')
with sqlite3.connect(data / 'backups' / ('pre-order-import-' + release_id + '.db')) as backup:
self.assertEqual(backup.execute('SELECT COUNT(*) FROM order_test_data').fetchone()[0], 1)
self.assertFalse((data / 'imports' / release_id).exists())
self.assertFalse((release / 'order-input.json').exists())
self.assertFalse((release / 'tracking-input.json').exists())
def test_corrupt_snapshot_is_rejected_before_database_changes(self):
result, data, release, release_id = self.run_import(corrupt=True)
self.assertNotEqual(result.returncode, 0)
with sqlite3.connect(data / 'history.db') as connection:
self.assertEqual(connection.execute('SELECT COUNT(*) FROM order_test_data').fetchone()[0], 1)
self.assertFalse(list((data / 'backups').iterdir()))
if __name__ == '__main__':
unittest.main()