diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..19dbc91 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +** +!requirements.txt +!am.py +!rates_extra.py +!start.bat +!启动.bat +!logistics_chat/ +!logistics_chat/** +!templates/ +!templates/** +!static/ +!static/** +!deploy/ +!deploy/** +!scripts/ +!scripts/** +!tests/ +!tests/** +**/__pycache__/ +**/*.pyc +**/.env +**/.env.* +**/*.db +**/*.sqlite3 +**/runtime.env +**/release.env +**/*.htpasswd +**/*.pem +**/*.key +**/tls/ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0f2891d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +deploy/*.service text eol=lf +deploy/*.sh text eol=lf +deploy/*.example text eol=lf +deploy/*.yml text eol=lf +Jenkinsfile text eol=lf +Dockerfile text eol=lf +.dockerignore text eol=lf diff --git a/.gitignore b/.gitignore index ecc2069..fc8fe66 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,12 @@ venv/ # 自签证书与私钥(本地演示用,不要提交) cert.pem key.pem +*.pem +*.key +*.htpasswd +runtime.env +release.env +/work/ # 系统文件 Thumbs.db diff --git a/AGENTS.md b/AGENTS.md index 5a83285..2199939 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project Structure & Module Navigation -`am.py` is a compatible CLI/WSGI entry. Business code lives in `logistics_chat/`; templates load page assets from `static/css/` and `static/js/`. See [architecture](docs/architecture.md) for ownership, contracts, and extraction rules. +Use `am.py` for CLI/WSGI, `logistics_chat/` for business code, and `static/css/`/`static/js/` for page assets. [Architecture](docs/architecture.md) defines module contracts and extraction rules. | Area | Files and search anchors | |---|---| @@ -16,6 +16,7 @@ | UI | `templates/`; `static/css/theme.css`: tokens; page CSS; `static/js/chat.js`: `doSend`, `I18N`; `static/js/admin.js`: `poll`, `sendReply`, `setChatView` | | Startup / hooks | `logistics_chat/__init__.py`: `create_app`; `config.py`: `default_config`; `hooks.py`, `runtime.py` in that package | | CLI / tests | `start.bat`: `probe_python`; `scripts/`, `tests/`, `examples/requests/` | +| Deployment | `Jenkinsfile`, `Dockerfile`; `deploy/gray_config.py`: `GrayConfig`; publish/activate/TLS scripts; `docs/deployment.md` | ## Efficient Search & File Boundaries @@ -40,7 +41,7 @@ Copy-Item .env.example .env # First run only; set DEEPSEEK_API_KEY inside .env. .\.venv\Scripts\python.exe -m unittest discover -s tests -v ``` -Optional `start.bat` prefers active `VIRTUAL_ENV`; verify selection with `start.bat --check-python`. No frontend build is required. +`start.bat` prefers active `VIRTUAL_ENV`; verify with `start.bat --check-python`. No frontend build is required. ## Coding Style & Naming Conventions @@ -50,7 +51,7 @@ Use four-space Python indentation, `snake_case`, `UPPER_SNAKE_CASE` constants, a Use `tests/test_.py` and `test_`. Mock DeepSeek; configure temporary database/upload/backup/log paths before `create_app`. Pytest: `.\.venv\Scripts\python.exe -m pytest -v`; Linux/macOS: `.venv/bin/python -m pytest -v`. -With Node.js installed, tests also run `tests/frontend/admin.test.cjs`; no npm dependencies are required. +Tests run `tests/frontend/admin.test.cjs` when Node.js is available; no npm dependencies. ## Commit & Pull Request Guidelines @@ -58,4 +59,4 @@ History uses concise Chinese summaries. Keep commits focused. PRs explain behavi ## Security & Configuration Tips -Never commit credentials or runtime/customer data. Revoke exposed keys. Keep `uploads/.gitkeep`. Startup reads the root `.env`; environment variables override it. Restart after changes. Imports are inert; `create_app` initializes runtime storage unless `INITIALIZE_RUNTIME=False`. Default HOST remains `0.0.0.0`. Missing `agents/orders/tickets` initialization and complete admin authentication remain unresolved. +Never commit credentials or runtime/customer data. Revoke exposed keys. Keep `uploads/.gitkeep`. Startup reads root `.env`; environment variables override it. Restart after changes. `create_app` initializes storage unless `INITIALIZE_RUNTIME=False`. Default HOST is `0.0.0.0`; Linux deployment binds loopback. Admin authentication remains incomplete; protect all acceptance routes through Nginx. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6d15821 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +ARG PYTHON_BASE_IMAGE=python:3.12-slim +FROM ${PYTHON_BASE_IMAGE} AS runtime + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 +WORKDIR /app +COPY requirements.txt ./ +RUN python -m pip install --no-cache-dir -r requirements.txt \ + && groupadd --gid 10001 baiyue \ + && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin baiyue \ + && mkdir -p /var/lib/baiyue-logistics-chat /var/log/baiyue-logistics-chat \ + && chown 10001:10001 /var/lib/baiyue-logistics-chat /var/log/baiyue-logistics-chat +COPY am.py rates_extra.py ./ +COPY logistics_chat/ ./logistics_chat/ +COPY templates/ ./templates/ +COPY static/ ./static/ +COPY deploy/__init__.py deploy/wsgi.py ./deploy/ +USER 10001:10001 +EXPOSE 5000 +HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=6 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:5000/healthz', timeout=2).close()"] +CMD ["python", "-m", "waitress", "--listen=0.0.0.0:5000", "--threads=8", "--channel-timeout=300", "--max-request-body-size=12582912", "--url-scheme=https", "--call", "deploy.wsgi:create_app"] + +FROM runtime AS test +COPY deploy/ ./deploy/ +COPY tests/ ./tests/ +COPY scripts/ ./scripts/ +COPY start.bat 启动.bat ./ +RUN python -m pytest -q -p no:cacheprovider + +FROM ${PYTHON_BASE_IMAGE} AS certbot +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 +RUN python -m pip install --no-cache-dir certbot==5.9.0 +ENTRYPOINT ["certbot"] + +FROM runtime AS production diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..2444574 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,69 @@ +pipeline { + agent { label 'java-builder' } + options { + timestamps() + disableConcurrentBuilds() + skipDefaultCheckout(true) + timeout(time: 30, unit: 'MINUTES') + } + stages { + stage('Checkout selected gray revision') { + steps { + script { + if (env.JOB_BASE_NAME != 'baiyue-logistics-chat-gray' || params.ENVIRONMENT != 'gray') { + error('This pipeline is restricted to the logistics chat gray job') + } + checkout scm + env.SOURCE_COMMIT = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() + env.CHAT_IMAGE = "baiyue-logistics-chat-gray:${env.SOURCE_COMMIT.take(12)}-b${env.BUILD_NUMBER}" + env.CERTBOT_IMAGE = "baiyue-logistics-chat-certbot:${env.SOURCE_COMMIT.take(12)}-b${env.BUILD_NUMBER}" + if (!(params.PYTHON_BASE_IMAGE ==~ /[A-Za-z0-9][A-Za-z0-9._\/:@-]+/)) { + error('Invalid Python base image reference') + } + } + sh '''set -eu + for file in deploy/*.sh; do bash -n "$file"; done + node --test tests/frontend/admin.test.cjs + ''' + } + } + stage('Build and test image') { + steps { + withEnv(["PYTHON_BASE_IMAGE=${params.PYTHON_BASE_IMAGE}"]) { + sh '''set -eu + docker build --target test --build-arg PYTHON_BASE_IMAGE="$PYTHON_BASE_IMAGE" -t "$CHAT_IMAGE-test" . + docker build --target production --build-arg PYTHON_BASE_IMAGE="$PYTHON_BASE_IMAGE" \ + --label "org.opencontainers.image.revision=$SOURCE_COMMIT" -t "$CHAT_IMAGE" . + docker build --target certbot --build-arg PYTHON_BASE_IMAGE="$PYTHON_BASE_IMAGE" \ + --label "org.opencontainers.image.revision=$SOURCE_COMMIT" -t "$CERTBOT_IMAGE" . + ''' + } + script { + env.CHAT_IMAGE_ID = sh(script: 'docker image inspect --format "{{.Id}}" "$CHAT_IMAGE"', returnStdout: true).trim() + env.CERTBOT_IMAGE_ID = sh(script: 'docker image inspect --format "{{.Id}}" "$CERTBOT_IMAGE"', returnStdout: true).trim() + } + } + } + stage('Deploy isolated gray service') { + steps { + lock(resource: 'baiyue-logistics-chat-gray-release') { + withEnv([ + "PUBLIC_ORIGIN=${params.PUBLIC_ORIGIN}", "HOST_PORT=${params.HOST_PORT}", + "GRAY_DEPLOY_DIR=${params.GRAY_DEPLOY_DIR}", "GRAY_PROXY_CONF_DIR=${params.GRAY_PROXY_CONF_DIR}", + "GRAY_TLS_CERT_FILE=${params.GRAY_TLS_CERT_FILE}", "GRAY_TLS_KEY_FILE=${params.GRAY_TLS_KEY_FILE}" + ]) { + withCredentials([ + sshUserPrivateKey(credentialsId: params.SSH_CREDENTIAL_ID, keyFileVariable: 'TARGET_SSH_KEY', usernameVariable: 'TARGET_SSH_USER'), + string(credentialsId: params.SSH_HOST_CREDENTIAL_ID, variable: 'TARGET_SSH_HOST'), + file(credentialsId: params.SSH_KNOWN_HOSTS_CREDENTIAL_ID, variable: 'TARGET_SSH_KNOWN_HOSTS'), + string(credentialsId: params.DEEPSEEK_CREDENTIAL_ID, variable: 'DEEPSEEK_API_KEY'), + usernamePassword(credentialsId: params.PREVIEW_AUTH_CREDENTIAL_ID, usernameVariable: 'PREVIEW_USERNAME', passwordVariable: 'PREVIEW_PASSWORD') + ]) { + sh 'bash deploy/publish-gray.sh' + } + } + } + } + } + } +} diff --git a/README.md b/README.md index ce7bd0a..750aa22 100644 --- a/README.md +++ b/README.md @@ -112,11 +112,18 @@ cp .env.example .env # 仅首次运行;已有 .env 时跳过 - 当前服务监听 `0.0.0.0:5000`,局域网设备可能访问。仅供本机开发时,将 `logistics_chat/config.py` 中默认的 `HOST` 设置为 `"127.0.0.1"`。 - 历史启动脚本曾提交明文密钥。当前文件已移除该配置,但 Git 历史仍保留旧内容,应在平台作废旧密钥,并在 `.env` 或部署环境中配置新密钥。 +### Jenkins 灰度部署 + +现有灰度环境按 [Jenkins 部署指南](docs/deployment.md) 发布:构建和测试镜像,SSH 发送至专用 Compose 服务,再配置独立域名的 HTTPS 和全站访问密码。密钥由 Jenkins 注入,数据和证书保存在独立宿主机目录;启动失败时恢复之前的应用和站点。面向客户开放前仍需补齐鉴权和数据隔离。另附 [Linux 单机参考](docs/deployment-linux.md)。 + ## 目录结构 ```text baiyue-logistics-chat/ ├── am.py CLI/WSGI 兼容入口 +├── Jenkinsfile 灰度构建、测试及受管发布 +├── Dockerfile 应用、测试及证书运维镜像 +├── .dockerignore 显式限制构建上下文 ├── rates_extra.py 扩展数据旧导入路径的兼容转发 ├── logistics_chat/ │ ├── __init__.py create_app 应用工厂 @@ -141,6 +148,7 @@ baiyue-logistics-chat/ │ ├── js/ 对应经典页面脚本 │ └── fa.jpg ├── scripts/ 人工会话检查/关闭 CLI +├── deploy/ 灰度 Compose、发布/回退、证书及单机参考 ├── tests/ 规则、存储、HTTP/SSE、启动与维护回归 ├── docs/ architecture.md、knowledge.md ├── examples/requests/ 人工接口示例 @@ -222,14 +230,14 @@ data: {"d": 1} ## 数据库与运行数据 -首次启动会自动创建 `history.db` 中的 `chat_history`、`messages`、`feedback`、`human_sessions` 和 `human_messages` 五张表。运单卡片以 `__CARD__{json}` 形式保存,前端据此恢复展示。 +首次启动会自动创建 `chat_history`、`messages`、`feedback`、`human_sessions`、`human_messages`、`agents`、`orders` 和 `tickets` 八张表,不自动创建客服账号或演示数据。本地入口使用根目录 `history.db`,Linux 部署工厂另存至专用数据目录。运单卡片以 `__CARD__{json}` 形式保存,前端据此恢复展示。 普通导入 `am.py` 或 `logistics_chat` 不创建运行数据。显式调用 `create_app()` 或访问 WSGI 兼容入口 `am.app` 时,默认初始化 SQLite、创建正常运行日志、备份数据库并清理修改时间超过 7 天的上传文件。测试须事先配置临时数据库、附件、日志和备份目录;`INITIALIZE_RUNTIME=False` 可禁止自动初始化,此时调用方负责准备 schema。历史会话的自动清理仍已关闭。 ## 当前限制 - 运单轨迹为模拟数据,尚未接入真实物流查询接口。 -- `agents`、`orders`、`tickets` 被接口使用,但初始化代码尚未创建这三张表;新环境下客服登录、手机号订单查询、离线留言等功能需要先补齐数据库初始化。 +- 不预置客服账号;应用内客服登录仍沿用旧账号校验逻辑。公网自用验收可按部署指南使用全站访问密码进入工作台。 - 后台登录尚未建立完整会话鉴权,管理页面和接口缺少登录校验。 - 运费数字由 Python 计算后交给模型组织说明;附加费是否适用仍需结合具体货物和地址核对。 - 回答提示词目标为 120–350 字;代码压缩阈值为中文 800、英文 1400 字符,超过阈值最多尝试压缩两次。 diff --git a/deploy/__init__.py b/deploy/__init__.py new file mode 100644 index 0000000..d14f67d --- /dev/null +++ b/deploy/__init__.py @@ -0,0 +1 @@ +"""Linux acceptance deployment entry points.""" diff --git a/deploy/activate-gray.sh b/deploy/activate-gray.sh new file mode 100644 index 0000000..851d134 --- /dev/null +++ b/deploy/activate-gray.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set +x +set -Eeuo pipefail +umask 077 + +root=${1:?} +release=${2:?} +[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2 +[[ "$release" == "$root"/releases/* && "$release" != *..* ]] || exit 2 +[[ "$(cat "$root/.project")" == baiyue-logistics-chat-gray ]] || exit 2 + +# Parse a small allowlist of non-secret release fields; never source a credential file. +while IFS='=' read -r name value; do + case "$name" in + CHAT_IMAGE|CHAT_IMAGE_ID|CHAT_CERTBOT_IMAGE|CHAT_CERTBOT_IMAGE_ID|SOURCE_COMMIT|HOST_PORT|CHAT_ROOT|CHAT_RELEASE|CHAT_DATA_DIR|CHAT_LOG_DIR|CHAT_RUNTIME_ENV|CHAT_DOMAIN|CHAT_PROXY_DIR|CHAT_TLS_CERT|CHAT_TLS_KEY) ;; + *) echo 'Unexpected release setting.' >&2; exit 2 ;; + esac + [[ "$value" =~ ^[A-Za-z0-9_./:-]+$ ]] || exit 2 + printf -v "$name" '%s' "$value" +done < "$release/release.env" +[[ "$CHAT_ROOT" == "$root" && "$CHAT_RELEASE" == "$release" ]] || exit 2 +[[ "$CHAT_DATA_DIR" == "$root/data" && "$CHAT_LOG_DIR" == "$root/logs" && "$CHAT_RUNTIME_ENV" == "$release/runtime.env" ]] || exit 2 +[[ "$SOURCE_COMMIT" =~ ^[a-f0-9]{40}$ && "$CHAT_IMAGE_ID" =~ ^sha256:[a-f0-9]{64}$ ]] || exit 2 +[[ "$CHAT_IMAGE" =~ ^baiyue-logistics-chat-gray:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2 +[[ "$HOST_PORT" =~ ^[0-9]+$ ]] && ((HOST_PORT >= 1024 && HOST_PORT <= 65535)) || exit 2 +[[ -d "$CHAT_PROXY_DIR" && -r "$CHAT_TLS_CERT" && -r "$CHAT_TLS_KEY" ]] || exit 2 +[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_IMAGE")" == "$CHAT_IMAGE_ID" ]] || exit 2 +[[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CHAT_IMAGE")" == "$SOURCE_COMMIT" ]] || exit 2 +openssl x509 -in "$CHAT_TLS_CERT" -checkhost "$CHAT_DOMAIN" -noout >/dev/null + +container=baiyue-logistics-chat-gray +if docker container inspect "$container" >/dev/null 2>&1; then + [[ "$(docker inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' "$container")" == baiyue-logistics-chat-gray ]] || exit 2 + old_port=$(docker port "$container" 5000/tcp || true) +else + old_port='' +fi +if [[ -n "$(ss -H -ltn "sport = :$HOST_PORT")" && "$old_port" != "127.0.0.1:$HOST_PORT" ]]; then + echo 'The requested host port belongs to another service.' >&2 + exit 2 +fi + +site="$CHAT_PROXY_DIR/baiyue-logistics-chat-gray.conf" +auth="$CHAT_PROXY_DIR/.baiyue-logistics-chat.htpasswd" +if [[ -f "$site" ]] && ! head -n 1 "$site" | grep -Fq 'Managed for baiyue-logistics-chat-gray'; then + echo 'The proxy file is not owned by this application.' >&2 + exit 2 +fi +for candidate in "$CHAT_PROXY_DIR/"*.conf; do + [[ -f "$candidate" && "$candidate" != "$site" ]] || continue + if grep -Fq "$CHAT_DOMAIN" "$candidate"; then + echo 'The requested domain is already configured by another site.' >&2 + exit 2 + fi +done +previous='' +if [[ -f "$root/current-release" ]]; then + previous=$(cat "$root/current-release") + [[ "$previous" == "$root"/releases/* && "$previous" != *..* && -f "$previous/release.env" ]] || exit 2 +fi +[[ ! -f "$site" ]] || cp -p "$site" "$release/previous-site.conf" +[[ ! -f "$auth" ]] || cp -p "$auth" "$release/previous-site.htpasswd" +chmod 600 "$release/runtime.env" + +compose() { docker compose --env-file "$1/release.env" -f "$1/compose.yml" "${@:2}"; } +activated=0 +proxy_changed=0 +rollback() { + result=$? + trap - ERR + set +e + if ((proxy_changed)); then + if [[ -f "$release/previous-site.conf" ]]; then cp -p "$release/previous-site.conf" "$site"; else rm -f -- "$site"; fi + if [[ -f "$release/previous-site.htpasswd" ]]; then cp -p "$release/previous-site.htpasswd" "$auth"; else rm -f -- "$auth"; fi + nginx -t && nginx -s reload + fi + if ((activated)); then + if [[ -n "$previous" ]]; then + compose "$previous" up -d --wait --wait-timeout 120 + else + compose "$release" stop chat + compose "$release" rm -f chat + fi + fi + echo 'Gray activation failed; previous application/proxy configuration was restored where available. Persistent data was retained.' >&2 + exit "$result" +} +trap rollback ERR + +if [[ -f "$root/data/history.db" ]] && docker container inspect "$container" >/dev/null 2>&1; then + docker exec "$container" python -c 'import pathlib,sqlite3,sys; p=pathlib.Path("/var/lib/baiyue-logistics-chat/backups");p.mkdir(exist_ok=True); source=sqlite3.connect("/var/lib/baiyue-logistics-chat/history.db"); target=sqlite3.connect(str(p/("pre-release-"+sys.argv[1]+".db"))); source.backup(target);target.close();source.close()' "$(basename "$release")" +fi +compose "$release" config --quiet +activated=1 +compose "$release" up -d --wait --wait-timeout 120 +curl --fail --silent --show-error "http://127.0.0.1:$HOST_PORT/healthz" >/dev/null +proxy_changed=1 +install -m 644 "$release/site.htpasswd" "$auth" +install -m 644 "$release/site.conf" "$site" +nginx -t +nginx -s reload +for path in / /api/export_chat /api/get_history; do + code=$(curl --silent --show-error --resolve "$CHAT_DOMAIN:443:127.0.0.1" -o /dev/null -w '%{http_code}' "https://$CHAT_DOMAIN$path") + [[ "$code" == 401 ]] || { echo 'Gray access protection verification failed.' >&2; false; } +done +printf '%s\n' "$release" > "$root/.current-release.new" +mv -f "$root/.current-release.new" "$root/current-release" +trap - ERR +printf 'Gray service activated at revision %s.\n' "$SOURCE_COMMIT" diff --git a/deploy/baiyue-logistics-chat.service b/deploy/baiyue-logistics-chat.service new file mode 100644 index 0000000..3906221 --- /dev/null +++ b/deploy/baiyue-logistics-chat.service @@ -0,0 +1,28 @@ +[Unit] +Description=Baiyue Logistics Chat acceptance service +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +User=baiyue +Group=baiyue +WorkingDirectory=/opt/baiyue-logistics-chat +Environment=PYTHONUNBUFFERED=1 +Environment=PYTHONDONTWRITEBYTECODE=1 +ExecStart=/opt/baiyue-logistics-chat/.venv/bin/python -m waitress --listen=127.0.0.1:5000 --threads=8 --channel-timeout=300 --max-request-body-size=12582912 --url-scheme=https --call deploy.wsgi:create_app +Restart=on-failure +RestartSec=3 +TimeoutStopSec=90 +UMask=0077 +StateDirectory=baiyue-logistics-chat +StateDirectoryMode=0700 +LogsDirectory=baiyue-logistics-chat +LogsDirectoryMode=0700 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict + +[Install] +WantedBy=multi-user.target diff --git a/deploy/compose.gray.yml b/deploy/compose.gray.yml new file mode 100644 index 0000000..d0a4e17 --- /dev/null +++ b/deploy/compose.gray.yml @@ -0,0 +1,23 @@ +name: baiyue-logistics-chat-gray +services: + chat: + image: "${CHAT_IMAGE:?CHAT_IMAGE is required}" + container_name: baiyue-logistics-chat-gray + pull_policy: never + restart: unless-stopped + init: true + ports: + - "127.0.0.1:${HOST_PORT:?HOST_PORT is required}:5000" + env_file: + - "${CHAT_RUNTIME_ENV:?CHAT_RUNTIME_ENV is required}" + volumes: + - "${CHAT_DATA_DIR:?CHAT_DATA_DIR is required}:/var/lib/baiyue-logistics-chat" + - "${CHAT_LOG_DIR:?CHAT_LOG_DIR is required}:/var/log/baiyue-logistics-chat" + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + stop_grace_period: 90s diff --git a/deploy/gray_config.py b/deploy/gray_config.py new file mode 100644 index 0000000..14c9e69 --- /dev/null +++ b/deploy/gray_config.py @@ -0,0 +1,143 @@ +"""Validate gray release inputs and render files without logging credentials.""" + +import argparse +from dataclasses import dataclass +import os +from pathlib import Path, PurePosixPath +import re +import shutil +from urllib.parse import urlsplit + + +def absolute_path(value): + path = PurePosixPath(value) + if not re.fullmatch(r'/[A-Za-z0-9_./-]+', value) or '..' in path.parts or str(path) == '/': + raise ValueError('A deployment path is invalid') + return str(path) + + +@dataclass(frozen=True) +class GrayConfig: + domain: str + port: int + root: str + release: str + proxy_dir: str + certificate: str + private_key: str + commit: str + image: str + image_id: str + certbot_image_id: str + + @classmethod + def from_environment(cls, environment): + origin = urlsplit(environment['PUBLIC_ORIGIN']) + if (origin.scheme != 'https' or origin.username or origin.password or origin.port not in (None, 443) + or origin.path not in ('', '/') or origin.query or origin.fragment + or not re.fullmatch(r'[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?', origin.hostname or '')): + raise ValueError('PUBLIC_ORIGIN must be an HTTPS domain without a path') + port = int(environment['HOST_PORT']) + if not 1024 <= port <= 65535: + raise ValueError('HOST_PORT must be an unprivileged TCP port') + root = absolute_path(environment['GRAY_DEPLOY_DIR']) + if PurePosixPath(root).name != 'gray-logistics-chat': + raise ValueError('The deployment must use its dedicated gray-logistics-chat directory') + commit = environment['SOURCE_COMMIT'] + image = environment['CHAT_IMAGE'] + image_id = environment['CHAT_IMAGE_ID'] + certbot_image_id = environment['CERTBOT_IMAGE_ID'] + build = environment['BUILD_NUMBER'] + if not re.fullmatch(r'[0-9a-f]{40}', commit) or not re.fullmatch(r'[1-9][0-9]*', build): + raise ValueError('Source commit or build number is invalid') + if image != f'baiyue-logistics-chat-gray:{commit[:12]}-b{build}': + raise ValueError('Image coordinates must identify the selected source and build') + if any(not re.fullmatch(r'sha256:[0-9a-f]{64}', value) for value in (image_id, certbot_image_id)): + raise ValueError('Docker image ID is invalid') + return cls(origin.hostname, port, root, f'{root}/releases/{commit[:12]}-b{build}', + absolute_path(environment['GRAY_PROXY_CONF_DIR']), + absolute_path(environment['GRAY_TLS_CERT_FILE']), + absolute_path(environment['GRAY_TLS_KEY_FILE']), commit, image, image_id, certbot_image_id) + + def release_environment(self): + fields = { + 'CHAT_IMAGE': self.image, 'CHAT_IMAGE_ID': self.image_id, + 'CHAT_CERTBOT_IMAGE': self.image.replace('baiyue-logistics-chat-gray:', 'baiyue-logistics-chat-certbot:'), + 'CHAT_CERTBOT_IMAGE_ID': self.certbot_image_id, + 'SOURCE_COMMIT': self.commit, 'HOST_PORT': str(self.port), + 'CHAT_ROOT': self.root, 'CHAT_RELEASE': self.release, + 'CHAT_DATA_DIR': self.root + '/data', 'CHAT_LOG_DIR': self.root + '/logs', + 'CHAT_RUNTIME_ENV': self.release + '/runtime.env', + 'CHAT_DOMAIN': self.domain, 'CHAT_PROXY_DIR': self.proxy_dir, + 'CHAT_TLS_CERT': self.certificate, 'CHAT_TLS_KEY': self.private_key, + } + return ''.join(f'{key}={value}\n' for key, value in fields.items()) + + def nginx_config(self): + return f'''# Managed for baiyue-logistics-chat-gray; all application routes require authentication. +server {{ + listen 80; + server_name {self.domain}; + location ^~ /.well-known/acme-challenge/ {{ + root /var/lib/baiyue-logistics-chat-acme; + try_files $uri =404; + }} + location / {{ return 301 https://$host$request_uri; }} +}} +server {{ + listen 443 ssl; + server_name {self.domain}; + ssl_certificate {self.certificate}; + ssl_certificate_key {self.private_key}; + ssl_protocols TLSv1.2 TLSv1.3; + auth_basic "Baiyue gray acceptance"; + auth_basic_user_file {self.proxy_dir}/.baiyue-logistics-chat.htpasswd; + client_max_body_size 12m; + location / {{ + proxy_pass http://127.0.0.1:{self.port}; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Authorization ""; + proxy_buffering off; + proxy_cache off; + gzip off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + }} +}} +''' + + +def write_release(directory, config, key): + if not key.strip() or any(character in key for character in ('\r', '\n', '\x00')): + raise ValueError('The model credential must be a nonempty single line') + directory.mkdir(parents=True, exist_ok=False) + (directory / 'release.env').write_text(config.release_environment(), encoding='utf-8') + (directory / 'site.conf').write_text(config.nginx_config(), encoding='utf-8') + escaped = key.strip().replace('\\', '\\\\').replace("'", "\\'") + (directory / 'runtime.env').write_text("DEEPSEEK_API_KEY='" + escaped + "'\n", encoding='utf-8') + (directory / 'runtime.env').chmod(0o600) + source = Path(__file__).resolve().parent + shutil.copyfile(source / 'compose.gray.yml', directory / 'compose.yml') + shutil.copyfile(source / 'activate-gray.sh', directory / 'activate.sh') + shutil.copyfile(source / 'prepare-gray-tls.sh', directory / 'prepare-tls.sh') + shutil.copyfile(source / 'renew-gray-tls.sh', directory / 'renew-tls.sh') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('directory', type=Path) + args = parser.parse_args() + try: + config = GrayConfig.from_environment(os.environ) + write_release(args.directory, config, os.environ['DEEPSEEK_API_KEY']) + except (KeyError, ValueError) as error: + parser.exit(2, 'Gray release configuration is incomplete or invalid.\n') + + +if __name__ == '__main__': + main() diff --git a/deploy/nginx-bootstrap.conf.example b/deploy/nginx-bootstrap.conf.example new file mode 100644 index 0000000..8c1b37d --- /dev/null +++ b/deploy/nginx-bootstrap.conf.example @@ -0,0 +1,16 @@ +# Replace chat.example.com with your actual domain before installing. +# HTTP certificate bootstrap only; the application is not exposed here. +server { + listen 80; + listen [::]:80; + server_name chat.example.com; + + location ^~ /.well-known/acme-challenge/ { + root /var/www/letsencrypt; + try_files $uri =404; + } + + location / { + return 404; + } +} diff --git a/deploy/nginx.conf.example b/deploy/nginx.conf.example new file mode 100644 index 0000000..b99a5c2 --- /dev/null +++ b/deploy/nginx.conf.example @@ -0,0 +1,51 @@ +# Replace every chat.example.com with your actual domain before installing. +# Install only after the certificate and .baiyue-preview.htpasswd exist. +server { + listen 80; + listen [::]:80; + server_name chat.example.com; + + location ^~ /.well-known/acme-challenge/ { + root /var/www/letsencrypt; + try_files $uri =404; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + server_name chat.example.com; + + ssl_certificate /etc/letsencrypt/live/chat.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/chat.example.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + # Guard all pages, API routes, attachments and history/export operations. + auth_basic "Baiyue private acceptance"; + auth_basic_user_file /etc/nginx/.baiyue-preview.htpasswd; + + # Allow multipart overhead around the application's 10 MiB file limit. + client_max_body_size 12m; + + location / { + proxy_pass http://127.0.0.1:5000; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Authorization ""; + + # SSE must reach the browser as chunks arrive. + proxy_buffering off; + proxy_cache off; + gzip off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + } +} diff --git a/deploy/prepare-gray-tls.sh b/deploy/prepare-gray-tls.sh new file mode 100644 index 0000000..b52eeb1 --- /dev/null +++ b/deploy/prepare-gray-tls.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set +x +set -Eeuo pipefail +umask 022 +root=${1:?} +release=${2:?} +[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2 +[[ "$release" == "$root"/releases/* && "$release" != *..* ]] || exit 2 +[[ "$(cat "$root/.project")" == baiyue-logistics-chat-gray ]] || exit 2 +while IFS='=' read -r name value; do + case "$name" in + CHAT_ROOT|CHAT_RELEASE|CHAT_DOMAIN|CHAT_PROXY_DIR|CHAT_TLS_CERT|CHAT_TLS_KEY|CHAT_CERTBOT_IMAGE|CHAT_CERTBOT_IMAGE_ID|SOURCE_COMMIT) + [[ "$value" =~ ^[A-Za-z0-9_./:-]+$ ]] || exit 2 + printf -v "$name" '%s' "$value" ;; + esac +done < "$release/release.env" +[[ "$CHAT_ROOT" == "$root" && "$CHAT_RELEASE" == "$release" ]] || exit 2 +[[ "$CHAT_TLS_CERT" == "$root/tls/config/live/$CHAT_DOMAIN/fullchain.pem" ]] || exit 2 +[[ "$CHAT_TLS_KEY" == "$root/tls/config/live/$CHAT_DOMAIN/privkey.pem" ]] || exit 2 +[[ "$CHAT_CERTBOT_IMAGE" =~ ^baiyue-logistics-chat-certbot:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2 +[[ "$(docker image inspect --format '{{.Id}}' "$CHAT_CERTBOT_IMAGE")" == "$CHAT_CERTBOT_IMAGE_ID" ]] || exit 2 +[[ "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CHAT_CERTBOT_IMAGE")" == "$SOURCE_COMMIT" ]] || exit 2 + +site="$CHAT_PROXY_DIR/baiyue-logistics-chat-gray.conf" +if [[ -f "$site" ]]; then + head -n 1 "$site" | grep -Fq 'Managed for baiyue-logistics-chat-gray' || exit 2 +else + for candidate in "$CHAT_PROXY_DIR/"*.conf; do + [[ -f "$candidate" ]] || continue + if grep -Fq "$CHAT_DOMAIN" "$candidate"; then echo 'Domain already belongs to another site.' >&2; exit 2; fi + done + cat > "$release/bootstrap.conf" </dev/null; then + docker run --rm --name baiyue-logistics-chat-certbot-issue \ + -v "$root/tls/config:/etc/letsencrypt" -v "$root/tls/work:/var/lib/letsencrypt" \ + -v "$root/tls/logs:/var/log/letsencrypt" -v /var/lib/baiyue-logistics-chat-acme:/webroot \ + "$CHAT_CERTBOT_IMAGE" certonly --non-interactive --webroot -w /webroot \ + --cert-name "$CHAT_DOMAIN" -d "$CHAT_DOMAIN" --agree-tos --register-unsafely-without-email +fi +openssl x509 -in "$CHAT_TLS_CERT" -checkhost "$CHAT_DOMAIN" -noout >/dev/null + +install -m 700 "$release/renew-tls.sh" "$root/tls/renew.sh" +unit=/etc/systemd/system/baiyue-logistics-chat-tls-renew.service +timer=/etc/systemd/system/baiyue-logistics-chat-tls-renew.timer +for file in "$unit" "$timer"; do + if [[ -e "$file" ]]; then head -n 1 "$file" | grep -Fq 'Managed for baiyue-logistics-chat-gray' || exit 2; fi +done +cat > "$unit" < "$timer" < "$temporary/release/site.htpasswd" +printf '%s\n' "$PREVIEW_PASSWORD" | openssl passwd -apr1 -stdin >> "$temporary/release/site.htpasswd" + +ssh_options=(-i "$TARGET_SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=$TARGET_SSH_KNOWN_HOSTS" -o ConnectTimeout=12) +target="$TARGET_SSH_USER@$TARGET_SSH_HOST" +run_ssh() { ssh "${ssh_options[@]}" "$target" "$@"; } + +# Only an owned, dedicated directory may receive releases or runtime data. +run_ssh bash -s -- "$root" "$release" <<'PREPARE' +set -Eeuo pipefail +root=$1 +release=$2 +[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2 +[[ "$release" == "$root"/releases/* ]] || exit 2 +[[ "$(id -u)" == 0 ]] || { echo 'The managed gray deployment account must have the required host permissions.' >&2; exit 2; } +for program in docker nginx openssl curl ss systemctl; do command -v "$program" >/dev/null; done +if [[ -e "$root" && ! -f "$root/.project" ]] && [[ -n "$(find "$root" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then + echo 'Refusing to reuse an unowned deployment directory.' >&2 + exit 2 +fi +if [[ -f "$root/.project" ]]; then + [[ "$(cat "$root/.project")" == baiyue-logistics-chat-gray ]] || exit 2 +fi +install -d -m 700 "$root" "$root/releases" "$root/data" "$root/logs" +printf 'baiyue-logistics-chat-gray\n' > "$root/.project" +chown 10001:10001 "$root/data" "$root/logs" +[[ ! -e "$release" ]] || { echo 'Release directory already exists; inspect its state before retrying.' >&2; exit 2; } +install -d -m 700 "$release" +PREPARE + +# Transport only tested images built without runtime credentials. +docker save "$CHAT_IMAGE" "$CERTBOT_IMAGE" | gzip | run_ssh "bash -o pipefail -c 'gzip -dc | docker load'" +scp "${ssh_options[@]}" "$temporary/release/"* "$target:$release/" +run_ssh bash "$release/prepare-tls.sh" "$root" "$release" +run_ssh bash "$release/activate.sh" "$root" "$release" + +# Verify the public TLS endpoint and the actual acceptance password without putting it in arguments. +python3 - <<'VERIFY' +import json +import os +from urllib.request import HTTPBasicAuthHandler, HTTPPasswordMgrWithDefaultRealm, build_opener +origin = os.environ['PUBLIC_ORIGIN'].rstrip('/') +manager = HTTPPasswordMgrWithDefaultRealm() +manager.add_password(None, origin, os.environ['PREVIEW_USERNAME'], os.environ['PREVIEW_PASSWORD']) +opener = build_opener(HTTPBasicAuthHandler(manager)) +with opener.open(origin + '/healthz', timeout=30) as response: + assert response.status == 200 and json.load(response) == {'status': 'ok'} +print('Authenticated gray HTTPS health check passed.') +VERIFY diff --git a/deploy/reload-nginx.sh b/deploy/reload-nginx.sh new file mode 100644 index 0000000..d9136b3 --- /dev/null +++ b/deploy/reload-nginx.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +/usr/sbin/nginx -t +/usr/bin/systemctl reload nginx diff --git a/deploy/renew-gray-tls.sh b/deploy/renew-gray-tls.sh new file mode 100644 index 0000000..4da0d58 --- /dev/null +++ b/deploy/renew-gray-tls.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set +x +set -Eeuo pipefail +root=${1:?} +domain=${2:?} +image=${3:?} +[[ "$root" == /*/gray-logistics-chat && "$root" != *..* ]] || exit 2 +[[ "$domain" =~ ^[a-z0-9.-]+$ && "$image" =~ ^baiyue-logistics-chat-certbot:[a-f0-9]{12}-b[0-9]+$ ]] || exit 2 +[[ "$(cat "$root/.project")" == baiyue-logistics-chat-gray ]] || exit 2 +certificate="$root/tls/config/live/$domain/fullchain.pem" +previous=$(sha256sum "$certificate") +docker run --rm --name baiyue-logistics-chat-certbot-renew \ + -v "$root/tls/config:/etc/letsencrypt" -v "$root/tls/work:/var/lib/letsencrypt" \ + -v "$root/tls/logs:/var/log/letsencrypt" -v /var/lib/baiyue-logistics-chat-acme:/webroot \ + "$image" renew --non-interactive --cert-name "$domain" +if [[ "$(sha256sum "$certificate")" != "$previous" ]]; then nginx -t; nginx -s reload; fi diff --git a/deploy/wsgi.py b/deploy/wsgi.py new file mode 100644 index 0000000..3b38b80 --- /dev/null +++ b/deploy/wsgi.py @@ -0,0 +1,19 @@ +"""Keep Linux service data outside the read-only application checkout.""" + +from pathlib import Path + +from logistics_chat import create_app as create_application + + +DATA_DIR = Path('/var/lib/baiyue-logistics-chat') +LOG_DIR = Path('/var/log/baiyue-logistics-chat') + + +def create_app(): + return create_application({ + 'DATABASE': DATA_DIR / 'history.db', + 'UPLOAD_DIR': DATA_DIR / 'uploads', + 'BACKUP_DIR': DATA_DIR / 'backups', + 'LOG_DIR': LOG_DIR, + 'HOST': '127.0.0.1', + }) diff --git a/docs/architecture.md b/docs/architecture.md index 96c27ca..c19068e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -20,6 +20,7 @@ PowerShell 下把目录和 `-g '*.py'` 分开传给 `rg`。报价字典可能整 | 业务范围 | 实际文件与搜索锚点 | 联动位置 | |---|---|---| | 应用入口与配置 | `am.py`:`main`、延迟访问的 `app`;`logistics_chat/__init__.py`:`create_app`;`config.py`:`default_config`、`get_setting`;`runtime.py`:`serve_app` | `start.bat`、`.env.example` | +| Jenkins 灰度部署 | `Jenkinsfile`、`Dockerfile`;`deploy/gray_config.py`:`GrayConfig`;`publish-gray.sh`、`activate-gray.sh`、`prepare-gray-tls.sh`、`renew-gray-tls.sh` | [部署指南](deployment.md);独立持久化数据、证书和回环端口;`wsgi.py` 提供容器内路径 | | 请求钩子 | `logistics_chat/hooks.py`:`rate_limit`、`no_cache_html` | 工厂注册钩子;限流状态在 `app.extensions['rate_hits']` | | SQLite 与维护 | `logistics_chat/db.py`:`connect`、`close_connections`、`init_db`、`backup_db`、`save_message`、`last_user_question`、`load_context` | 维护 CLI 在 `scripts/`;旧会话自动清理仍未启用 | | 历史接口 | `logistics_chat/history_service.py`:`new_chat`、`get_history`、`get_messages`、`del_history`、`clear_all`;对应 `routes/history.py` | `static/js/chat.js`:`ensureConv`、`loadHistory`、`openConversation` | @@ -31,7 +32,7 @@ PowerShell 下把目录和 `-g '*.py'` 分开传给 `rg`。报价字典可能整 | 附件 | `logistics_chat/file_service.py`:`upload_file`、`cleanup_uploads`;`routes/files.py`:`uploaded_file` | 扩展名/大小/目录由配置提供;客户端 `addMsg`,客服端 `appendMsg` | | 登录、统计、订单、留言、导出 | `logistics_chat/business_service.py`:`admin_login`、`admin_stats`、`my_orders`、`add_ticket`、`list_tickets`、`export_chat` | `routes/admin.py`、`routes/business.py`;登录页 `static/js/admin_login.js` | | 页面 | `templates/index.html`、`templates/admin.html`、`templates/admin_login.html` | 先加载共享 `static/css/theme.css`,再加载对应 `chat`、`admin`、`admin_login` 的 CSS/JS;保留全局事件入口 | -| 回归 | `tests/test_rules.py`、`test_storage.py`、`test_api.py`、`test_config.py`、`test_startup.py`、`test_maintenance_scripts.py`、`test_frontend.py`;`tests/frontend/admin.test.cjs` | 配置测试隔离 `.env` 和环境变量;数据、HTTP 与 SSE 使用隔离配置和模拟模型;Node 内置测试验证草稿、异步发送和服务资料抽屉;浏览器使用临时测试库 | +| 回归 | `tests/test_rules.py`、`test_storage.py`、`test_api.py`、`test_config.py`、`test_deployment.py`、`test_gray_config.py`、`test_gray_activation.py`、`test_startup.py`、`test_maintenance_scripts.py`、`test_frontend.py`;`tests/frontend/admin.test.cjs` | Linux 回退模拟基础设施、实际操作临时文件;模型、数据库及上传隔离;Node 验证异步交互 | 除标明的根目录文件外,表内缩写的 Python 文件均相对 `logistics_chat/`。 @@ -76,6 +77,7 @@ tests/ # 按业务模块组织的回归 - 路由只解析请求并包装响应;服务组织业务;规则模块提供确定性计算。依赖方向为 `routes → services → rules / db / ai_client`,下层不导入路由或根入口。 - `db.py` 统一连接、schema 和历史持久化。人工与管理业务的 SQL 放在所属服务,HTTP 路由不执行 SQL。请求及流式请求结束时关闭连接,未提交事务回滚。 +- `init_db` 包括客服账号、订单与留言表,首次启动只建空表;不内置账号。重复初始化保留已有数据,业务测试从真实初始化结果插入夹具,不另建缺失表。 - 所有模型 HTTP 经 `ai_client.post_completion`。保留流式问答 `(5, 90)`、压缩 `(5, 30)`、追问 `(5, 15)` 三组连接/读取超时;不能因共用客户端而合并期限。流结束或取消时释放 HTTP 响应。 - 模块导入不建库、不备份、不清理附件、不访问模型。`create_app` 默认显式初始化运行目录、数据库、备份与旧附件清理;`INITIALIZE_RUNTIME=False` 可禁止这些动作,此时调用方负责提供已有 schema。访问兼容入口 `am.app` 视为显式创建应用。 - `config.ROOT_DIR` 指向仓库根目录,模板/静态目录显式提供给 Flask。数据库、上传、备份、日志使用配置中的绝对路径,避免包文件移动改变数据位置。 @@ -105,7 +107,7 @@ tests/ # 按业务模块组织的回归 - 人工申请使用 `conv_id`,返回 `session_id`,后续用 `sid`。状态为 `waiting/active/closed`,消息方向为 `customer/agent`。 - 计费档位、单位换算和舍入保持原样。`calc_freight.total` 仅含基础运费;附加费尚未统一并入。知识文档与 `knowledge_list` 同步维护。 - 上传 URL 保留 `/uploads/`,默认 10MB,扩展名配置保持不变。原附件保留本地、继续忽略,目录迁移不另建一份数据。 -- `agents`、`orders`、`tickets` 缺少初始化、后台鉴权不完整、轨迹为模拟数据,仍是已知限制。临时 UI/接口测试中补表只属于测试夹具,不代表修复生产初始化。 +- 后台鉴权和会话隔离仍不完整,轨迹为模拟数据。Linux 公网验收用 Nginx 全站密码隔离访问,不代表已修复面向客户开放所需的应用权限。 ## 验证与维护 diff --git a/docs/deployment-linux.md b/docs/deployment-linux.md new file mode 100644 index 0000000..8f33bc0 --- /dev/null +++ b/docs/deployment-linux.md @@ -0,0 +1,154 @@ +# Linux 单机手动部署参考 + +现行灰度环境使用 [Jenkins 部署流程](deployment.md)。本页保留独立主机的安装方法,不用于覆盖已有灰度服务。 + +适用场景:Ubuntu / Debian、已有域名、仅自己验收。使用系统 Python 3.10+,无需前端构建或 GPU。入口为 HTTPS → Nginx 全站访问密码 → 本机 Waitress → Flask / SQLite。 + +当前应用尚无完整登录会话、权限和会话数据隔离。验收期间必须保留全站访问密码,不能只保护 `/admin`,因为历史、导出和人工接口也能直接调用。此配置不代表已具备面向客户开放的权限体系。 + +## 1. 域名与网络 + +- 将验收子域名的 A 记录解析到服务器公网 IPv4;只有正确配置服务器 IPv6 后才添加 AAAA。 +- 服务器安全组和防火墙允许 TCP 80、443,保留实际使用的 SSH 管理端口。不要开放应用端口 5000。 +- 服务器需要能够出站访问 Python 包源及 `https://api.deepseek.com`。 +- 下文的 `chat.example.com` 均需替换成真实域名,包括 Nginx 配置内的证书路径。 + +## 2. 上传源码并安装依赖 + +首次部署使用 `baiyue-logistics-chat-linux-acceptance.tar.gz`。部署包包含本次工作区的部署文件和缺表修复,不含 `.git`、`.env`、虚拟环境、日志、数据库或本机附件。之前 Git 推送因认证失败未完成;若改用 Git 拉取,须先确认远端版本包含这些变更。 + +将部署包上传到服务器 `/tmp/`,以下命令均在服务器执行。首次部署假设 `/opt/baiyue-logistics-chat` 尚不存在;已有安装请使用文末更新流程。 + +```bash +sudo apt update +sudo apt install -y python3 python3-venv python3-pip nginx apache2-utils certbot +python3 --version +sudo tar --no-same-owner -xzf /tmp/baiyue-logistics-chat-linux-acceptance.tar.gz -C /opt +sudo adduser --system --group --home /nonexistent --no-create-home baiyue +sudo chown root:baiyue /opt/baiyue-logistics-chat +sudo chmod 750 /opt/baiyue-logistics-chat +cd /opt/baiyue-logistics-chat +sudo python3 -m venv .venv +sudo .venv/bin/python -m pip install -r requirements.txt +``` + +若系统已有专用 `baiyue` 用户,跳过 `adduser`。代码与虚拟环境归 root 所有;服务数据另存到 `/var/lib/`,无需让 Web 进程修改源码。 + +## 3. 配置 DS 密钥 + +仅首次复制配置,已有 `.env` 时跳过复制;不要把本机配置打进部署包。 + +```bash +sudo install -o root -g baiyue -m 640 .env.example .env +sudoedit /opt/baiyue-logistics-chat/.env +``` + +在编辑器内填写 `DEEPSEEK_API_KEY=你的密钥`,无需加 `export`,不要在终端命令或日志中输出密钥。文件由应用工厂读取;本方案的 systemd 单元不再注入同名环境变量。更换密钥后重启服务。 + +## 4. 启动应用服务 + +```bash +sudo install -m 644 deploy/baiyue-logistics-chat.service /etc/systemd/system/baiyue-logistics-chat.service +sudo systemd-analyze verify /etc/systemd/system/baiyue-logistics-chat.service +sudo systemctl daemon-reload +sudo systemctl enable --now baiyue-logistics-chat +sudo systemctl status baiyue-logistics-chat --no-pager +curl -I http://127.0.0.1:5000/ +``` + +本机检查应返回 HTTP 200。服务使用独立系统用户、单进程 8 线程并仅监听 `127.0.0.1:5000`,不走 `am.py` 的 Flask 开发服务器回退。单进程保留当前应用的内存输入状态和限流行为;暂不配置多进程或多个副本。 + +systemd 自动创建并授权以下目录,重启与更新源码不会替换其中的数据: + +| 内容 | 服务器路径 | +|---|---| +| SQLite | `/var/lib/baiyue-logistics-chat/history.db` | +| 上传附件 | `/var/lib/baiyue-logistics-chat/uploads/` | +| 启动备份 | `/var/lib/baiyue-logistics-chat/backups/` | +| 应用日志 | `/var/log/baiyue-logistics-chat/app.log` | +| 密钥配置 | `/opt/baiyue-logistics-chat/.env` | + +首次启动创建八张业务表,不插入演示数据、默认客服账号或密码。已有表和记录保持兼容。应用当前仍会在启动时清理超过 7 天的上传文件,并执行启动备份;启动备份不能替代定时备份。 + +## 5. 申请 HTTPS 证书 + +先安装只提供证书验证路径的 HTTP 配置,此时不会转发到应用: + +```bash +sudo install -d -m 755 /var/www/letsencrypt +sudo install -m 644 deploy/nginx-bootstrap.conf.example /etc/nginx/sites-available/baiyue-logistics-chat +sudoedit /etc/nginx/sites-available/baiyue-logistics-chat +sudo ln -s /etc/nginx/sites-available/baiyue-logistics-chat /etc/nginx/sites-enabled/baiyue-logistics-chat +sudo nginx -t +sudo systemctl enable --now nginx +sudo systemctl reload nginx +sudo certbot certonly --webroot -w /var/www/letsencrypt --cert-name chat.example.com -d chat.example.com +``` + +编辑配置时替换域名,申请证书时也替换两处域名;按 Certbot 提示填写邮箱和确认条款。若申请失败,先核对 DNS 和外网 80 端口,保留引导配置排查,暂不安装下一步需要证书的 TLS 配置。 + +## 6. 设置访问密码并开放 HTTPS 验收 + +创建验收账号 `preview`,密码由命令交互输入,不写入项目配置。`-c` 仅首次创建密码文件时使用;以后修改密码应去掉 `-c`。 + +```bash +sudo htpasswd -cB /etc/nginx/.baiyue-preview.htpasswd preview +sudo chown root:www-data /etc/nginx/.baiyue-preview.htpasswd +sudo chmod 640 /etc/nginx/.baiyue-preview.htpasswd +sudo install -m 644 deploy/nginx.conf.example /etc/nginx/sites-available/baiyue-logistics-chat +sudoedit /etc/nginx/sites-available/baiyue-logistics-chat +sudo nginx -t +sudo systemctl reload nginx +``` + +替换 TLS 配置中所有域名及证书路径;`nginx -t` 失败时先修正,不要继续 reload。模板的访问密码覆盖页面、附件、静态资源和全部 API,后端 5000 端口继续只对本机开放。不能只在导航层隐藏后台页面。 + +Nginx 已关闭代理响应缓冲及压缩,并将等待上游数据的超时设为 300 秒,供 AI SSE 分段输出;上传入口允许 12 MiB 请求体以容纳应用 10 MiB 文件上限外的表单开销。 + +## 7. 验收和证书续期 + +```bash +curl -I https://chat.example.com/ +curl -I https://chat.example.com/api/export_chat +curl -u preview -I https://chat.example.com/ +``` + +前两条应返回 401,证明页面和数据接口都受保护;第三条交互输入访问密码后应返回 200。浏览器也会先弹出访问密码: + +- 客户页:`https://chat.example.com/`,发送消息检查 DS 实际回复及流式显示。 +- 验收工作台:`https://chat.example.com/admin/dashboard`,从客户页申请人工,再检查接待和双向消息。 +- 当前未预置应用内客服登录账号;Nginx 的 `preview` 是整站验收密码,不是 `/admin` 页里的客服账号。 + +安装证书续期后的 Nginx 重载钩子,并检查续期定时器: + +```bash +sudo install -d -m 755 /etc/letsencrypt/renewal-hooks/deploy +sudo install -m 755 deploy/reload-nginx.sh /etc/letsencrypt/renewal-hooks/deploy/baiyue-reload-nginx +sudo systemctl enable --now certbot.timer +sudo certbot renew --dry-run --run-deploy-hooks +``` + +最终配置保留 HTTP 的 `/.well-known/acme-challenge/` 路径,供 webroot 续期使用;其他 HTTP 请求跳转到 HTTPS。不要删除验证路径或只对自己的 IP 开放证书验证端口,否则后续续期会失败。 + +## 维护、更新和问题定位 + +```bash +sudo journalctl -u baiyue-logistics-chat -n 100 --no-pager +sudo tail -n 100 /var/log/baiyue-logistics-chat/app.log +sudo systemctl restart baiyue-logistics-chat +sudo nginx -t +``` + +- **502**:先查应用状态、journal 日志、依赖安装及 5000 端口本机检查。 +- **一直弹访问密码**:检查 Nginx 密码文件和 `www-data` 读权限;不要用应用内客服密码替代验收密码。 +- **AI 无回复**:确认服务器 `.env` 已填写并重启,检查服务器能否访问 DS;本机验收成功不代表服务器网络已验证。 +- **输出集中到最后**:检查实际生效的 Nginx 配置和 CDN 是否重新启用了缓冲。 +- **429**:目前反向代理后的请求共享应用默认限流桶,适合单人验收;多人开放前需正确配置可信代理和业务限流。 + +更新前停止应用并备份 `/var/lib/baiyue-logistics-chat/`,另行保护 `.env`。源码更新只替换项目文件,不复制本机数据库或 `work/`,不覆盖服务器 `.env`;按需重新安装依赖后重启。若修改 systemd 单元,重新安装单元并执行 `daemon-reload`;若修改 Nginx,先检查再 reload。回退源码时仍保留数据副本,本项目尚无数据库降级迁移工具。 + +本地已通过应用工厂和全新数据库回归;目标服务器的 `systemd-analyze verify`、`nginx -t`、DNS、证书签发及真实 DS 请求仍需在部署时验证。 + +## 参考 + +工厂调用与监听参数依据 [Waitress 官方文档](https://docs.pylonsproject.org/projects/waitress/en/stable/runner.html)。流式转发配置依据 [Nginx 代理模块](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering),全站密码依据 [Nginx Basic Auth 模块](https://nginx.org/en/docs/http/ngx_http_auth_basic_module.html)。证书和续期使用 [Certbot webroot](https://eff-certbot.readthedocs.io/en/stable/using.html#webroot);服务目录与只读保护依据 [systemd.exec](https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html)。 diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..44f2c93 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,96 @@ +# Jenkins 灰度部署 + +现有灰度环境使用 `Jenkinsfile`、`Dockerfile` 和 `deploy/` 发布独立物流聊天服务。流水线构建并测试镜像,通过严格校验主机指纹的 SSH 发送到专用 Docker Compose 服务,Nginx 提供 HTTPS 和全站访问密码。 + +应用使用单个 Waitress 进程、8 个线程和 UID 10001,宿主机端口仅绑定回环地址。运行密钥、聊天数据、附件和证书不进入镜像。后台鉴权与会话隔离仍不完整,灰度验收须保留全站密码;此流程不代表已具备面向多个客户开放的权限体系。 + +## Jenkins 任务 + +任务名为 `baiyue-logistics-chat-gray`。定义选择 Pipeline script from SCM,本项目 Git 地址为仓库,分支字段为 `${SOURCE_REF}`,脚本路径为 `Jenkinsfile`。公开仓库可匿名读取;转为私有后配置只读 SCM 凭据。 + +`java-builder` 标签代理须有 Docker、Bash、Python 3、Node.js 18+、OpenSSL、SSH,并能下载所选 Python 基础镜像和 PyPI 依赖。标签不表示固定物理节点。 + +以下参数由私有灰度配置提供,实际地址和凭据值不写入源码: + +| 参数 | 含义 | +|---|---| +| `ENVIRONMENT` | 单选 `gray`,流水线拒绝其他值 | +| `SOURCE_REF` | 本次发布的不可变标签或提交,也可明确选择开发分支 | +| `PYTHON_BASE_IMAGE` | 已验证的官方 Python 3.12 基础镜像 | +| `PUBLIC_ORIGIN` | 独立 HTTPS 域名,不带子路径 | +| `HOST_PORT` | 已确认空闲的宿主机回环端口,如 `29013` | +| `GRAY_DEPLOY_DIR` | 专用目录,末级必须为 `gray-logistics-chat` | +| `GRAY_PROXY_CONF_DIR` | 宿主机 Nginx 实际加载的站点目录 | +| `GRAY_TLS_CERT_FILE` / `GRAY_TLS_KEY_FILE` | 本服务独立证书和私钥路径 | +| `SSH_CREDENTIAL_ID` | SSH 用户与私钥凭据引用 | +| `SSH_HOST_CREDENTIAL_ID` | SSH 目标的 Secret text 引用 | +| `SSH_KNOWN_HOSTS_CREDENTIAL_ID` | 已核实的 known_hosts 文件引用 | +| `DEEPSEEK_CREDENTIAL_ID` | 本服务模型密钥的 Secret text 引用 | +| `PREVIEW_AUTH_CREDENTIAL_ID` | 专用验收用户名和密码引用 | + +运行密钥和验收账号保存在私有灰度配置与 Jenkins 凭据库,供后续发布持续复用;轮换时同步更新。不要将密钥放在构建参数、Git URL、镜像 ARG、日志或示例文件中。 + +## 发布步骤 + +1. 完成检查并提交代码,创建 `gray-chat-YYYYMMDD-` 一类不可变发布标签。不要重写已有标签或强推覆盖远端分支;分支有其他更新时先明确发布基线。 +2. 推送标签,在任务 `SOURCE_REF` 选择该标签。Jenkins 按此引用加载流水线并检出代码,镜像记录完整提交 SHA。 +3. 流水线检查 Bash 语法,运行 Node 回归,再构建 `test` 目标执行 Python 回归;通过后构建 `production` 应用镜像及 `certbot` 运维镜像。 +4. 构建完成后才绑定运行凭据。发布脚本校验路径、域名、源提交和镜像 ID,生成私有配置并发送镜像及发布文件。 +5. 首次签发域名证书,等待应用健康,再安装本服务的 Nginx 配置。已有业务的容器和站点配置不参与本次发布。 +6. Jenkins 使用验收账号验证公网 HTTPS `/healthz`;之后在浏览器验证 AI 流式回复、上传、转人工和工作台。 + +应用镜像必须选择 `production` 构建目标;`certbot` 仅用于证书运维。每个构建生成独立镜像标签,接收主机再次核验镜像 ID,不依赖 `latest`。 + +## 数据与失败回退 + +`GRAY_DEPLOY_DIR` 如 `/data/services/gray-logistics-chat`,包含: + +| 路径 | 内容 | +|---|---| +| `.project` | 服务所有权标记,拒绝复用未知目录 | +| `data/` | SQLite、附件和备份,跨发布保留 | +| `logs/` | 应用日志 | +| `releases/-b/` | Compose、镜像坐标、私有运行配置和站点候选 | +| `current-release` | 已通过应用及入口检查的发布目录 | +| `tls/` | 独立 ACME 账号、证书、日志和续期脚本 | + +模型密钥写入权限为 600 的 `runtime.env`,由 Compose 注入进程。验收密码只在 Jenkins 生成 Nginx 哈希,服务器保存哈希文件。 + +更新前执行 SQLite 在线备份。应用启动或 Nginx 检查失败时,仅恢复本服务之前的 Compose 和站点配置,保留数据;不自动用旧库覆盖新记录。失败发布目录保留供定位,数据库降级及删除数据需另行处理。 + +新环境自动创建八张业务表,不创建默认客服账号或演示数据。验收用全站密码直接进入工作台;Nginx 验收账号不是应用内客服账号。 + +## HTTPS 及续期 + +域名须解析到已核实的灰度入口,公网 80、443 端口可达。其他域名的证书不能直接用于新域名。 + +独立 Certbot 镜像通过 HTTP-01 webroot 签发证书;首次只新增本服务的 HTTP 验证站点,业务接口尚不公开。验证目录为 `/var/lib/baiyue-logistics-chat-acme`,最终配置持续保留该 HTTP 路径。 + +证书参数须指向: + +```text +/tls/config/live/<域名>/fullchain.pem +/tls/config/live/<域名>/privkey.pem +``` + +ACME 账号和私钥仅保存在服务器私有目录,当前使用无邮箱账号。专用 `baiyue-logistics-chat-tls-renew.timer` 每日检查本服务证书,内容变化后检查并重载 Nginx,不接管其他证书任务。 + +## 排查 + +先从 Jenkins 构建记录核对阶段及源 SHA。服务器检查: + +```bash +docker ps --filter name=baiyue-logistics-chat-gray +docker logs --tail 100 baiyue-logistics-chat-gray +systemctl status baiyue-logistics-chat-tls-renew.timer --no-pager +nginx -t +``` + +- 构建失败:核对镜像和 PyPI 网络,不跳过测试发布。 +- SSH 失败:核对受管凭据和 known_hosts,不禁用指纹检查。 +- 证书失败:核对 DNS 和 HTTP 验证路径,保留本服务的验证站点与私有 ACME 状态。 +- 容器不健康:检查独立目录的 UID 10001 权限、日志和 `/healthz`;健康检查不计入业务限流。 +- 入口 401:未登录时为预期,使用专用验收密码后应返回 200。 +- 流式显示延迟:检查实际 Nginx 或 CDN 是否重新启用了响应缓冲。 + +独立主机安装另见 [Linux 手动参考](deployment-linux.md)。凭据与 API 依据 [Jenkins 官方文档](https://www.jenkins.io/doc/pipeline/steps/credentials-binding/) 和 [Remote API](https://www.jenkins.io/doc/book/using/remote-access-api/),证书验证与续期依据 [Certbot 文档](https://eff-certbot.readthedocs.io/en/stable/using.html#webroot)。 diff --git a/logistics_chat/db.py b/logistics_chat/db.py index e2090ee..d31672f 100644 --- a/logistics_chat/db.py +++ b/logistics_chat/db.py @@ -87,6 +87,31 @@ def init_db(): hm_cols = [r[1] for r in cur.execute('PRAGMA table_info(human_messages)').fetchall()] if 'read_at' not in hm_cols: cur.execute('ALTER TABLE human_messages ADD COLUMN read_at TEXT') + cur.execute(''' + CREATE TABLE IF NOT EXISTS agents ( + username TEXT PRIMARY KEY, + password TEXT NOT NULL, + name TEXT NOT NULL + ) + ''') + cur.execute(''' + CREATE TABLE IF NOT EXISTS orders ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + phone TEXT, + track_no TEXT, + status TEXT, + created TEXT + ) + ''') + cur.execute(''' + CREATE TABLE IF NOT EXISTS tickets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content TEXT, + phone TEXT, + ctime TEXT, + status TEXT + ) + ''') old = cur.execute('SELECT id, title FROM chat_history WHERE id NOT IN (SELECT DISTINCT conv_id FROM messages)').fetchall() for cid, title in old: cur.execute('INSERT INTO messages(conv_id, role, content, ctime) VALUES (?,?,?,?)', (cid, 'user', title, now_str())) diff --git a/logistics_chat/hooks.py b/logistics_chat/hooks.py index c52d023..10757ab 100644 --- a/logistics_chat/hooks.py +++ b/logistics_chat/hooks.py @@ -7,7 +7,7 @@ from flask import current_app, jsonify, request def rate_limit(): path = request.path - if path.startswith('/static') or path.startswith('/api/human/'): + if path == '/healthz' or path.startswith('/static') or path.startswith('/api/human/'): return None ip = request.remote_addr or 'unknown' now = time.time() diff --git a/logistics_chat/routes/pages.py b/logistics_chat/routes/pages.py index 77c7bb5..e767626 100644 --- a/logistics_chat/routes/pages.py +++ b/logistics_chat/routes/pages.py @@ -5,6 +5,11 @@ from flask import Blueprint, render_template bp = Blueprint('pages', __name__) + +@bp.route('/healthz') +def health(): + return {'status': 'ok'} + @bp.route('/') def index(): return render_template('index.html') diff --git a/tests/test_api.py b/tests/test_api.py index fdbad59..de41a4a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -176,6 +176,14 @@ class ApiTests(unittest.TestCase): downloaded.close() self.assertEqual(self.client.post('/api/upload', data={'file': (io.BytesIO(b'x'), 'x.exe')}).status_code, 400) + def test_health_check_remains_available_when_business_requests_are_rate_limited(self): + self.app.config['RATE_LIMIT_PER_MIN'] = 1 + self.assertEqual(self.client.get('/').status_code, 200) + self.assertEqual(self.client.get('/api/get_history').status_code, 429) + response = self.client.get('/healthz') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {'status': 'ok'}) + def test_feedback_stats_and_csv(self): cid = self.conversation() with self.app.app_context(): @@ -189,14 +197,12 @@ class ApiTests(unittest.TestCase): self.client.delete('/api/clear_all') self.assertEqual(self.client.get('/api/get_history').json, []) - def test_business_routes_with_fixture_tables_and_legacy_auth(self): + def test_business_routes_with_initialized_tables_and_legacy_auth(self): with self.app.app_context(): connection = db.connect() tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")} - self.assertTrue({'agents', 'orders', 'tickets'}.isdisjoint(tables)) - connection.executescript('CREATE TABLE agents(username TEXT, password TEXT, name TEXT);' - 'CREATE TABLE orders(id INTEGER PRIMARY KEY, phone TEXT, track_no TEXT, status TEXT, created TEXT);' - 'CREATE TABLE tickets(id INTEGER PRIMARY KEY, content TEXT, phone TEXT, ctime TEXT, status TEXT);' + self.assertTrue({'agents', 'orders', 'tickets'}.issubset(tables)) + connection.executescript( "INSERT INTO agents VALUES('demo', 'test-password', 'Agent');" "INSERT INTO orders VALUES(1, '123', '15252125', 'active', '2026-10-28');") connection.commit() diff --git a/tests/test_deployment.py b/tests/test_deployment.py new file mode 100644 index 0000000..9e047b5 --- /dev/null +++ b/tests/test_deployment.py @@ -0,0 +1,47 @@ +"""Exercise the server factory using disposable state/log paths and no model key.""" + +import io +import logging +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from deploy import wsgi + + +class DeploymentTests(unittest.TestCase): + def test_fresh_server_persists_business_data_and_uploads_outside_checkout(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + root = Path(directory.name) + state, logs = root / 'state', root / 'logs' + with patch.object(wsgi, 'DATA_DIR', state), patch.object(wsgi, 'LOG_DIR', logs), \ + patch.dict(os.environ, {'DEEPSEEK_API_KEY': ''}): + app = wsgi.create_app() + + def close_handlers(): + for handler in list(app.logger.handlers): + if isinstance(handler, logging.FileHandler): + handler.close() + app.logger.removeHandler(handler) + + self.addCleanup(close_handlers) + client = app.test_client() + for path in ('/', '/admin/dashboard', '/api/tickets', '/api/my_orders?phone=123'): + self.assertEqual(client.get(path).status_code, 200, path) + self.assertEqual(client.post('/admin/login', json={'username': 'demo', 'password': 'test-password'}).status_code, 401) + self.assertEqual(client.post('/api/ticket', json={'content': 'Acceptance ticket', 'phone': '123'}).status_code, 200) + self.assertEqual(client.get('/api/tickets').json[0]['content'], 'Acceptance ticket') + uploaded = client.post('/api/upload', data={'file': (io.BytesIO(b'acceptance'), 'note.txt')}) + self.assertEqual(uploaded.status_code, 200) + self.assertEqual((state / 'uploads' / uploaded.json['url'].rsplit('/', 1)[1]).read_bytes(), b'acceptance') + self.assertTrue((state / 'history.db').is_file()) + self.assertEqual(len(list((state / 'backups').glob('history_*.db'))), 1) + self.assertTrue((logs / 'app.log').is_file()) + self.assertEqual(app.config['HOST'], '127.0.0.1') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_gray_activation.py b/tests/test_gray_activation.py new file mode 100644 index 0000000..e9f4cde --- /dev/null +++ b/tests/test_gray_activation.py @@ -0,0 +1,113 @@ +"""Exercise real Bash release/rollback file operations with all infrastructure faked.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +from deploy.gray_config import GrayConfig, write_release + + +FAKE_COMMAND = '''import json, os, pathlib, sys +name = pathlib.Path(sys.argv[0]).name +args = sys.argv[1:] +log = pathlib.Path(os.environ['FAKE_LOG']) +with log.open('a') as output: + output.write(json.dumps([name, args]) + '\\n') +scenario = os.environ['FAKE_SCENARIO'] +if name == 'docker': + if args[:2] == ['image', 'inspect']: + print('sha256:' + 'b'*64 if args[3] == '{{.Id}}' else 'a'*40) + elif args[:2] == ['container', 'inspect']: + sys.exit(0) + elif args[0] == 'inspect': + print('baiyue-logistics-chat-gray') + elif args[0] == 'port': + print('127.0.0.1:29013') + elif args[0] == 'compose' and 'up' in args and scenario == 'app-fail' and os.environ['FAKE_NEW'] + '/release.env' in args: + sys.exit(17) +elif name == 'nginx' and '-t' in args and scenario == 'proxy-fail': + calls = [json.loads(line) for line in log.read_text().splitlines()] + if sum(item[0] == 'nginx' and '-t' in item[1] for item in calls) == 1: + sys.exit(19) +elif name == 'curl' and '-w' in args: + print('401', end='') +''' + + +@unittest.skipUnless(os.name == 'posix' and shutil.which('bash'), 'Gray activation requires Linux Bash') +class GrayActivationTests(unittest.TestCase): + def run_activation(self, scenario): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + directory = Path(temporary.name) + root = directory / 'gray-logistics-chat' + proxy = directory / 'proxy' + for path in (root / 'data', root / 'logs', root / 'releases', proxy): + path.mkdir(parents=True, exist_ok=True) + (root / '.project').write_text('baiyue-logistics-chat-gray\n') + (root / 'data/history.db').write_bytes(b'preserve-existing-data') + certificate, private_key = directory / 'cert.pem', directory / 'key.pem' + certificate.touch() + private_key.touch() + previous = root / 'releases/previous' + previous.mkdir() + (previous / 'release.env').write_text('CHAT_IMAGE=previous\n') + (previous / 'compose.yml').write_text('services: {}\n') + (root / 'current-release').write_text(str(previous) + '\n') + old_site = '# Managed for baiyue-logistics-chat-gray\n# previous site\n' + (proxy / 'baiyue-logistics-chat-gray.conf').write_text(old_site) + (proxy / '.baiyue-logistics-chat.htpasswd').write_text('previous-auth-hash\n') + unrelated = proxy / 'unrelated.conf' + unrelated.write_text('# unrelated site\n') + config = GrayConfig('chat.example.com', 29013, str(root), str(root / 'releases/new'), + str(proxy), str(certificate), str(private_key), 'a'*40, + 'baiyue-logistics-chat-gray:' + 'a'*12 + '-b7', 'sha256:' + 'b'*64, 'sha256:' + 'c'*64) + release = Path(config.release) + write_release(release, config, 'fixture-model-key') + (release / 'site.htpasswd').write_text('new-auth-hash\n') + commands = directory / 'commands' + commands.mkdir() + for name in ('docker', 'nginx', 'openssl', 'ss', 'curl'): + command = commands / name + command.write_text('#!' + sys.executable + '\n' + FAKE_COMMAND) + command.chmod(0o755) + environment = os.environ.copy() + log = directory / 'commands.jsonl' + environment.update(PATH=str(commands) + os.pathsep + environment['PATH'], + FAKE_LOG=str(log), FAKE_SCENARIO=scenario, FAKE_NEW=str(release)) + result = subprocess.run(['bash', str(release / 'activate.sh'), str(root), str(release)], + env=environment, capture_output=True, text=True, timeout=30) + self.assertEqual((root / 'data/history.db').read_bytes(), b'preserve-existing-data') + self.assertEqual(unrelated.read_text(), '# unrelated site\n') + calls = [json.loads(line) for line in log.read_text().splitlines()] + return result, root, proxy, previous, release, old_site, calls + + def test_success_publishes_only_the_owned_site_and_release_pointer(self): + result, root, proxy, previous, release, old_site, calls = self.run_activation('success') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((root / 'current-release').read_text().strip(), str(release)) + self.assertIn('server_name chat.example.com', (proxy / 'baiyue-logistics-chat-gray.conf').read_text()) + + def assert_rolled_back(self, scenario): + result, root, proxy, previous, release, old_site, calls = self.run_activation(scenario) + self.assertNotEqual(result.returncode, 0) + self.assertEqual((root / 'current-release').read_text().strip(), str(previous)) + self.assertEqual((proxy / 'baiyue-logistics-chat-gray.conf').read_text(), old_site) + self.assertEqual((proxy / '.baiyue-logistics-chat.htpasswd').read_text(), 'previous-auth-hash\n') + self.assertTrue(any(name == 'docker' and 'up' in args and str(previous / 'release.env') in args + for name, args in calls), result.stderr) + + def test_failed_application_restores_previous_release(self): + self.assert_rolled_back('app-fail') + + def test_invalid_nginx_configuration_restores_previous_site_and_application(self): + self.assert_rolled_back('proxy-fail') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_gray_config.py b/tests/test_gray_config.py new file mode 100644 index 0000000..c16a63e --- /dev/null +++ b/tests/test_gray_config.py @@ -0,0 +1,70 @@ +"""Reject unsafe release coordinates and keep secrets out of public release files.""" + +from dataclasses import replace +from pathlib import Path +import tempfile +import unittest + +from dotenv import dotenv_values + +from deploy.gray_config import GrayConfig, write_release + + +class GrayConfigTests(unittest.TestCase): + def settings(self): + return { + 'PUBLIC_ORIGIN': 'https://chat.example.com', 'HOST_PORT': '29013', + 'GRAY_DEPLOY_DIR': '/data/services/gray-logistics-chat', + 'GRAY_PROXY_CONF_DIR': '/etc/nginx/conf.d', + 'GRAY_TLS_CERT_FILE': '/etc/ssl/example/fullchain.pem', + 'GRAY_TLS_KEY_FILE': '/etc/ssl/example/privkey.pem', + 'SOURCE_COMMIT': 'a' * 40, 'BUILD_NUMBER': '7', + 'CHAT_IMAGE': 'baiyue-logistics-chat-gray:' + 'a' * 12 + '-b7', + 'CHAT_IMAGE_ID': 'sha256:' + 'b' * 64, + 'CERTBOT_IMAGE_ID': 'sha256:' + 'c' * 64, + } + + def test_unsafe_origins_paths_and_mismatched_images_are_rejected(self): + cases = ( + ('PUBLIC_ORIGIN', 'http://chat.example.com'), + ('PUBLIC_ORIGIN', 'https://user:password@chat.example.com'), + ('PUBLIC_ORIGIN', 'https://chat.example.com/admin'), + ('GRAY_DEPLOY_DIR', '/data/services/../gray-logistics-chat'), + ('GRAY_DEPLOY_DIR', '/data/shared-services'), + ('GRAY_PROXY_CONF_DIR', '/etc/nginx; echo unsafe'), + ('HOST_PORT', '443'), ('CHAT_IMAGE', 'another-service:latest'), + ('SOURCE_COMMIT', 'main'), ('CHAT_IMAGE_ID', 'unverified-image'), + ) + for key, value in cases: + with self.subTest(setting=key, value=value): + settings = self.settings() + settings[key] = value + with self.assertRaises(ValueError): + GrayConfig.from_environment(settings) + + def test_data_paths_remain_stable_across_releases_and_credentials_stay_separate(self): + config = GrayConfig.from_environment(self.settings()) + updated = replace(config, release=config.root + '/releases/next-build') + for release in (config, updated): + self.assertIn('CHAT_DATA_DIR=' + config.root + '/data\n', release.release_environment()) + self.assertIn('CHAT_LOG_DIR=' + config.root + '/logs\n', release.release_environment()) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'release' + key = "fixture-${literal}-don't-expand" + write_release(path, config, key) + self.assertEqual(dotenv_values(path / 'runtime.env', interpolate=False)['DEEPSEEK_API_KEY'], key) + for name in ('release.env', 'site.conf', 'compose.yml', 'activate.sh'): + self.assertNotIn(key, (path / name).read_text(encoding='utf-8')) + self.assertIn('auth_basic_user_file', (path / 'site.conf').read_text(encoding='utf-8')) + + def test_invalid_model_credential_does_not_create_a_release_directory(self): + config = GrayConfig.from_environment(self.settings()) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'release' + with self.assertRaises(ValueError): + write_release(path, config, 'fixture-key\nANOTHER_SETTING=bad') + self.assertFalse(path.exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_storage.py b/tests/test_storage.py index c630b09..bdcd115 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -42,6 +42,25 @@ class StorageTests(unittest.TestCase): self.assertEqual(connection.execute('SELECT role, content FROM messages').fetchone(), ('user', '旧会话')) connection.close() + def test_business_tables_start_empty_and_reinitialization_preserves_records(self): + connection = db.connect() + for table in ('agents', 'orders', 'tickets'): + self.assertEqual(connection.execute(f'SELECT COUNT(*) FROM {table}').fetchone()[0], 0) + connection.executescript( + "INSERT INTO agents VALUES('fixture-agent', 'fixture-password', 'Fixture Agent');" + "INSERT INTO orders(phone, track_no, status, created) VALUES('123', 'fixture-order', 'active', 'now');" + "INSERT INTO tickets(content, phone, ctime, status) VALUES('fixture-ticket', '123', 'now', 'pending');" + ) + connection.commit() + connection.close() + db.init_db() + connection = db.connect() + self.assertEqual(connection.execute('SELECT username, password, name FROM agents').fetchall(), + [('fixture-agent', 'fixture-password', 'Fixture Agent')]) + self.assertEqual(connection.execute('SELECT track_no FROM orders').fetchall(), [('fixture-order',)]) + self.assertEqual(connection.execute('SELECT content FROM tickets').fetchall(), [('fixture-ticket',)]) + connection.close() + def test_user_retry_is_idempotent_and_card_context_is_readable(self): connection = db.connect() connection.execute('INSERT INTO chat_history(title, ctime, utime) VALUES(?,?,?)', ('新对话', 'now', 'now'))