From 3f1cee8006a59f1a1a0488b1c015fe4af2e08a5b Mon Sep 17 00:00:00 2001 From: ouyangxq <775153912@qq.com> Date: Fri, 30 Oct 2026 11:09:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9EH5=E9=95=BF=E6=8C=89=E5=BD=95?= =?UTF-8?q?=E9=9F=B3=E5=B9=B6=E6=8E=A5=E5=85=A5=E7=99=BE=E5=BA=A6=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E8=BD=AC=E6=96=87=E5=AD=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 10 ++ AGENTS.md | 3 +- Dockerfile | 2 + README.md | 4 + docs/architecture.md | 1 + logistics_chat/__init__.py | 2 + logistics_chat/config.py | 8 ++ logistics_chat/routes/__init__.py | 4 +- logistics_chat/routes/speech.py | 47 +++++++ logistics_chat/speech_service.py | 174 ++++++++++++++++++++++++ static/css/chat.css | 12 ++ static/js/chat.js | 78 +++++++++-- static/js/voice.js | 219 ++++++++++++++++++++++++++++++ templates/index.html | 7 +- tests/frontend/chat.test.cjs | 62 ++++++++- tests/frontend/voice.test.cjs | 162 ++++++++++++++++++++++ tests/test_frontend.py | 2 +- tests/test_speech.py | 173 +++++++++++++++++++++++ 18 files changed, 954 insertions(+), 16 deletions(-) create mode 100644 logistics_chat/routes/speech.py create mode 100644 logistics_chat/speech_service.py create mode 100644 static/js/voice.js create mode 100644 tests/frontend/voice.test.cjs create mode 100644 tests/test_speech.py diff --git a/.env.example b/.env.example index 43feefd..e89ca2a 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,13 @@ DEEPSEEK_API_KEY= # Optional fixed tenant scope for the internal order test dataset. Empty = all imported test rows. ORDER_DATA_TENANT_ID= +# Short-recording ASR is independent of DEEPSEEK_API_KEY. No credentials = voice input unavailable. +ASR_PROVIDER=baidu +BAIDU_ASR_API_KEY= +BAIDU_ASR_SECRET_KEY= +# Optional absolute ffmpeg executable path. Empty = find ffmpeg on PATH. Docker includes it. +FFMPEG_BIN= +# Optional alternative: ASR_PROVIDER=dashscope, with a separate Alibaba Cloud Model Studio key. +ASR_API_KEY= +ASR_URL=https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions +ASR_MODEL=qwen3-asr-flash diff --git a/AGENTS.md b/AGENTS.md index c809fd9..7d57d5d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,7 @@ Use `am.py` for CLI/WSGI, `logistics_chat/` for business code, and `static/css/` | Area | Files and search anchors | |---|---| | AI | `logistics_chat/chat_service.py`: `create_chat_stream`; `logistics_chat/ai_client.py`: `post_completion` | +| Voice input | `static/js/voice.js`: `ChatVoice.create`; `logistics_chat/speech_service.py`: `transcribe_recording`, `baidu_transcribe`; `logistics_chat/routes/speech.py` | | Knowledge | `logistics_chat/knowledge.py`: `knowledge_list`, `search_knowledge`; `docs/knowledge.md` | | Pricing | `logistics_chat/pricing.py`: `RATES`, `calc_freight`; `logistics_chat/rates_extra.py` | | Tracking | `logistics_chat/tracking.py`: `extract_tracking_no`, `build_track`, `CARD_PREFIX` | @@ -52,7 +53,7 @@ Use four-space Python indentation, `snake_case`, `UPPER_SNAKE_CASE` constants, a Use `tests/test_.py` and `test_`. Mock DeepSeek; configure temporary database/upload/backup/log paths before `create_app`. Pytest: `.\.venv\Scripts\python.exe -m pytest -v`; Linux/macOS: `.venv/bin/python -m pytest -v`. -Tests run `tests/frontend/admin.test.cjs` and `chat.test.cjs` when Node.js is available; no npm dependencies. +Tests run `tests/frontend/admin.test.cjs`, `chat.test.cjs`, and `voice.test.cjs` when Node.js is available; no npm dependencies. Baidu ASR uses FFmpeg on PATH or `FFMPEG_BIN`; provider calls are mocked in tests. ## Commit & Pull Request Guidelines diff --git a/Dockerfile b/Dockerfile index c30063d..871e413 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ ENV PYTHONUNBUFFERED=1 \ PIP_DISABLE_PIP_VERSION_CHECK=1 WORKDIR /app COPY requirements.txt ./ +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* RUN python -m pip install --no-cache-dir -r requirements.txt \ && groupadd --gid 10001 baiyue \ && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin baiyue \ diff --git a/README.md b/README.md index 1e582b4..167c906 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,10 @@ cp .env.example .env # 仅首次运行；已有 .env 时跳过 ### 配置说明 +手机语音输入：点击输入区上方的麦克风按钮，再按住说话，松开后转文字并发送；上滑或取消触摸不发送，最长 60 秒。微信和手机浏览器使用同一服务端转写链路；页面必须通过 HTTPS 或本机 localhost 打开，并允许麦克风。旧版内置浏览器不支持录音时仍可文字输入。手机订单信息默认折叠，物流轨迹优先展示。 + +在根目录 `.env` 配置 `ASR_PROVIDER=baidu`、`BAIDU_ASR_API_KEY` 和 `BAIDU_ASR_SECRET_KEY`，使用已开通短语音识别的百度语音应用凭据，不能混用 OCR 或 DeepSeek 凭据。服务端须安装 FFmpeg 并加入 PATH，也可用 `FFMPEG_BIN` 指定可执行文件绝对路径；Docker 应用镜像已包含 FFmpeg。浏览器 WebM/MP4 录音通过内存管道转为 16 kHz 单声道 PCM，录音不写入上传目录或历史，只发送识别文字。修改后重启服务。可选的 `ASR_PROVIDER=dashscope` 使用 `.env.example` 中独立的 ASR 配置。 + - 本地密钥放在项目根目录的 `.env` 中：首次运行复制 [`.env.example`](.env.example)，填写 `DEEPSEEK_API_KEY=你的密钥`。`.env` 已被 Git 忽略，示例文件中的值应保持为空。 - 应用启动时自动读取根目录 `.env`，不依赖终端的当前目录；`am.py`、Windows 启动脚本和 WSGI 应用工厂均使用同一配置入口。修改密钥后须重启服务。 - 配置优先级为：显式传给 `create_app` 的配置 > 进程环境变量 > `.env` > 空值。环境变量即使为空也会覆盖 `.env`；本地填写后仍提示未配置时，请检查并移除旧的空环境变量。 diff --git a/docs/architecture.md b/docs/architecture.md index eb699af..9e50dcb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -115,6 +115,7 @@ tests/ # 按业务模块组织的回归 - 计费档位、单位换算和舍入保持原样。`shipping_intent_with_context` 用 `is_shipping_followup` 识别渠道、重量、尺寸、单独目的地回复及重试，保留报价条件；新订单、另外一票、理赔及新的完整问题形成回溯边界，不继承上一票尺寸。尺寸支持带小数和空格，换算单位后计价。中英混写的 UK/US 按 ASCII 边界识别；目的地未确认时先询问目的国家，不默认给出欧洲报价。英国使用英国渠道和默认比价组合。明确提到美国等未覆盖目的地时不套用欧洲费率，英国按方报价不套用欧洲仓价；指定国家或仓点没有匹配时 `calc_warehouse_card` 返回空。`calc_freight.total` 仅含基础运费；附加费尚未统一并入。知识文档与 `knowledge_list` 同步维护。 - 聊天请求的会话编号仅接受 SQLite 整数范围内的 ASCII 十进制整数或空值；字符串转换前限制长度，非法 Unicode 数字和超长数字返回 SSE 输入错误，不进入模型调用。 - 上传 URL 保留 `/uploads/`，默认 10MB，扩展名配置保持不变。原附件保留本地、继续忽略，目录迁移不另建一份数据。 +- 手机语音由 `voice.js` 管理按住、松开、上滑取消、60 秒上限及媒体轨道释放；切换会话、离开页面或权限/转写超时取消录音与待处理请求，迟到结果不能进入新会话，已有草稿不自动提交。`/api/speech/config` 仅返回可用性及限制，`/api/speech/transcribe` 有独立 5MB 请求限制并以 `MemoryAudioRequest` 将音频文件保持在内存中；音频不进入 uploads 或聊天历史。百度适配通过 FFmpeg 管道转换 16 kHz 单声道 PCM，校验真实解码长度后调用百度短语音识别，Token 按应用隔离缓存。可选 DashScope 适配和 DS 文字聊天凭据独立，密钥及上游原始错误不得返回客户端。识别结果走既有 AI/人工消息入口。H5 订单详情默认折叠；可视高度变化保持最新回复可见，阅读旧消息时不强制拉到底部。 - `agents`、`orders`、`tickets` 在应用初始化时建空表；兼容已有独立 id 列的 agents 表，不预置账号或演示数据。 - 后台鉴权和会话隔离仍不完整，订单表为内部测试副本，不提供实时物流轨迹。Linux 公网验收用 Nginx 全站密码隔离访问，不代表已修复面向客户开放所需的应用权限。 diff --git a/logistics_chat/__init__.py b/logistics_chat/__init__.py index 1491d68..ea6520b 100644 --- a/logistics_chat/__init__.py +++ b/logistics_chat/__init__.py @@ -11,9 +11,11 @@ def create_app(config=None): from .config import ROOT_DIR, default_config from . import db, file_service, hooks from .routes import register_routes + from .routes.speech import MemoryAudioRequest app = Flask(__name__, template_folder=str(ROOT_DIR/'templates'), static_folder=str(ROOT_DIR/'static')) + app.request_class = MemoryAudioRequest app.config.update(default_config()) if config: app.config.update(config) diff --git a/logistics_chat/config.py b/logistics_chat/config.py index a850cdf..f7e014a 100644 --- a/logistics_chat/config.py +++ b/logistics_chat/config.py @@ -20,6 +20,14 @@ def default_config(): 'BACKUP_DIR': ROOT_DIR / 'backups', 'LOG_DIR': ROOT_DIR / 'logs', 'DEEPSEEK_API_KEY': (api_key or '').strip(), + 'ASR_API_KEY': str(os.environ.get('ASR_API_KEY', local_config.get('ASR_API_KEY')) or '').strip(), + 'ASR_PROVIDER': str(os.environ.get('ASR_PROVIDER', local_config.get('ASR_PROVIDER')) or 'baidu').strip().lower(), + 'BAIDU_ASR_API_KEY': str(os.environ.get('BAIDU_ASR_API_KEY', local_config.get('BAIDU_ASR_API_KEY')) or '').strip(), + 'BAIDU_ASR_SECRET_KEY': str(os.environ.get('BAIDU_ASR_SECRET_KEY', local_config.get('BAIDU_ASR_SECRET_KEY')) or '').strip(), + 'FFMPEG_BIN': str(os.environ.get('FFMPEG_BIN', local_config.get('FFMPEG_BIN')) or '').strip(), + 'ASR_URL': str(os.environ.get('ASR_URL', local_config.get('ASR_URL')) or + 'https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions').strip(), + 'ASR_MODEL': str(os.environ.get('ASR_MODEL', local_config.get('ASR_MODEL')) or 'qwen3-asr-flash').strip(), 'ORDER_DATA_TENANT_ID': str(os.environ.get('ORDER_DATA_TENANT_ID', local_config.get('ORDER_DATA_TENANT_ID')) or '').strip(), 'MODEL_URL': 'https://api.deepseek.com/v1/chat/completions', 'MODEL_NAME': 'deepseek-chat', diff --git a/logistics_chat/routes/__init__.py b/logistics_chat/routes/__init__.py index 4ebd026..4dad024 100644 --- a/logistics_chat/routes/__init__.py +++ b/logistics_chat/routes/__init__.py @@ -2,6 +2,6 @@ def register_routes(app): - from . import admin, business, chat, files, history, human, pages - for module in (admin, business, chat, files, history, human, pages): + from . import admin, business, chat, files, history, human, pages, speech + for module in (admin, business, chat, files, history, human, pages, speech): app.register_blueprint(module.bp) diff --git a/logistics_chat/routes/speech.py b/logistics_chat/routes/speech.py new file mode 100644 index 0000000..72916b3 --- /dev/null +++ b/logistics_chat/routes/speech.py @@ -0,0 +1,47 @@ +"""HTTP adapters for short voice input; no audio is stored or added to history.""" + +from io import BytesIO + +from flask import Blueprint, Request, request +from werkzeug.exceptions import RequestEntityTooLarge + +from .. import speech_service + + +bp = Blueprint('speech', __name__) + + +class MemoryAudioRequest(Request): + def _get_file_stream(self, total_content_length, content_type, filename=None, content_length=None): + if self.path == '/api/speech/transcribe': + return BytesIO() + return super()._get_file_stream(total_content_length, content_type, filename, content_length) + + +@bp.after_request +def no_cache(response): + response.headers['Cache-Control'] = 'no-store' + return response + + +@bp.get('/api/speech/config') +def speech_config(): + return {'enabled': speech_service.speech_enabled(), 'max_seconds': speech_service.MAX_DURATION_SECONDS, + 'max_bytes': speech_service.MAX_AUDIO_BYTES}, {'Cache-Control': 'no-store'} + + +@bp.post('/api/speech/transcribe') +def speech_transcribe(): + request.max_content_length = speech_service.MAX_AUDIO_BYTES + 64 * 1024 + if not speech_service.speech_enabled(): + return {'ok': False, 'error': speech_service.UNAVAILABLE}, 503 + try: + files = request.files + try: + return speech_service.transcribe_recording(files.get('audio'), request.form.get('duration'), + request.form.get('language')) + finally: + for _, upload in files.items(multi=True): + upload.close() + except RequestEntityTooLarge: + return {'ok': False, 'error': '录音文件过大，请缩短后重试。'}, 413 diff --git a/logistics_chat/speech_service.py b/logistics_chat/speech_service.py new file mode 100644 index 0000000..a5c7a7a --- /dev/null +++ b/logistics_chat/speech_service.py @@ -0,0 +1,174 @@ +"""Bounded, in-memory recording transcription; separate from text chat credentials.""" + +import base64 +import json +import math +import hashlib +import os +from pathlib import Path +import shutil +import subprocess +import threading +import time +from urllib.parse import urlsplit + +import requests +from flask import current_app + +from .config import get_setting + + +MAX_AUDIO_BYTES = 5 * 1024 * 1024 +MAX_DURATION_SECONDS = 60 +UNAVAILABLE = '语音输入暂不可用，请使用文字输入。' + + +class SpeechFailure(Exception): + def __init__(self, message, status=502): + self.message, self.status = message, status + + +def ffmpeg_binary(): + configured = get_setting('FFMPEG_BIN') + return str(configured) if configured and Path(configured).is_file() else shutil.which('ffmpeg') + + +def speech_enabled(): + provider = get_setting('ASR_PROVIDER') + if provider == 'baidu': + return bool(get_setting('BAIDU_ASR_API_KEY') and get_setting('BAIDU_ASR_SECRET_KEY') and ffmpeg_binary()) + if provider != 'dashscope': + return False + try: + url = urlsplit(get_setting('ASR_URL')) + except ValueError: + return False + return bool(get_setting('ASR_API_KEY') and get_setting('ASR_MODEL') + and url.scheme == 'https' and url.hostname and not url.username and not url.password) + + +def post_json(url, **options): + response = requests.post(url, timeout=(5, 30), stream=True, allow_redirects=False, **options) + try: + if response.status_code != 200: + raise SpeechFailure('语音转文字失败，请稍后重试。') + data = bytearray() + for chunk in response.iter_content(4096): + data.extend(chunk) + if len(data) > 64 * 1024: + raise SpeechFailure('语音识别结果无效，请重试。') + return json.loads(data) + finally: + response.close() + + +def baidu_token(): + key, secret = get_setting('BAIDU_ASR_API_KEY'), get_setting('BAIDU_ASR_SECRET_KEY') + identity = hashlib.sha256((key + '\0' + secret).encode()).hexdigest() + lock = current_app.extensions.setdefault('speech_token_lock', threading.Lock()) + with lock: + cached = current_app.extensions.get('speech_token') + if cached and cached['identity'] == identity and cached['expires'] > time.time(): + return cached['token'] + data = post_json('https://aip.baidubce.com/oauth/2.0/token', data={ + 'grant_type':'client_credentials', 'client_id':key, 'client_secret':secret}) + token = data.get('access_token') + if not isinstance(token, str) or not token: + raise SpeechFailure(UNAVAILABLE, 503) + current_app.extensions['speech_token'] = {'identity':identity, 'token':token, + 'expires':time.time() + max(0, min(float(data.get('expires_in', 0)), 2592000) - 60)} + return token + + +def to_pcm(content): + """Decode browser WebM/MP4 in pipes to mono 16 kHz, without temporary audio files.""" + binary = ffmpeg_binary() + if not binary: + raise SpeechFailure(UNAVAILABLE, 503) + result = subprocess.run([binary, '-hide_banner', '-loglevel', 'error', '-protocol_whitelist', 'pipe', + '-i', 'pipe:0', '-map', '0:a:0', '-vn', '-sn', '-dn', '-ac', '1', '-ar', '16000', + '-t', '61', '-f', 's16le', 'pipe:1'], input=content, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=10, creationflags=subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0) + if result.returncode != 0 or len(result.stdout) < 12800: + raise SpeechFailure('录音无法识别，请重新录制。', 422) + if len(result.stdout) > 16000 * 2 * MAX_DURATION_SECONDS: + raise SpeechFailure('录音超过 60 秒，请缩短后重试。', 400) + return result.stdout + + +def baidu_transcribe(content, language): + pcm = to_pcm(content) + result = post_json('https://vop.baidu.com/server_api', json={'format':'pcm', 'rate':16000, 'channel':1, + 'cuid':'baiyue-logistics-chat', 'token':baidu_token(), 'dev_pid':1737 if language == 'en' else 1537, + 'speech':base64.b64encode(pcm).decode('ascii'), 'len':len(pcm)}) + code = result.get('err_no') + if code == 3301: + raise SpeechFailure('没有识别到文字，请靠近麦克风重试。', 422) + if code != 0: + current_app.logger.warning('Baidu ASR request failed: code=%s', code) + raise SpeechFailure(UNAVAILABLE if code == 3302 else '语音转文字失败，请稍后重试。', 503 if code == 3302 else 502) + parts = result.get('result') + if not isinstance(parts, list) or any(not isinstance(part, str) for part in parts): + raise SpeechFailure('语音识别结果无效，请重试。') + return ''.join(parts) + + +def audio_format(content, mime): + """Check container signatures, never trust the uploaded filename.""" + mime = mime.split(';', 1)[0].lower() + if mime == 'audio/webm' and content.startswith(b'\x1aE\xdf\xa3'): + return 'audio/webm' + if mime in ('audio/mp4', 'audio/x-m4a') and content[4:8] == b'ftyp': + return 'audio/mp4' + if mime in ('audio/ogg', 'audio/opus') and content.startswith(b'OggS'): + return 'audio/ogg' + if mime in ('audio/wav', 'audio/x-wav') and content[:4] == b'RIFF' and content[8:12] == b'WAVE': + return 'audio/wav' + return None + + +def transcribe_recording(upload, duration, language): + if not speech_enabled(): + return {'ok': False, 'error': UNAVAILABLE}, 503 + if not upload: + return {'ok': False, 'error': '没有收到录音，请重新按住说话。'}, 400 + try: + duration = float(duration) + except (TypeError, ValueError): + duration = 0 + if not math.isfinite(duration) or not 0.4 <= duration <= MAX_DURATION_SECONDS + 1: + return {'ok': False, 'error': '录音时长无效，请录制 1 至 60 秒语音。'}, 400 + content = upload.read(MAX_AUDIO_BYTES + 1) + if len(content) > MAX_AUDIO_BYTES: + return {'ok': False, 'error': '录音文件过大，请缩短后重试。'}, 413 + mime = audio_format(content, upload.mimetype) + if len(content) < 16 or not mime: + return {'ok': False, 'error': '录音格式不受支持，请使用文字输入或更换浏览器。'}, 415 + language = 'en' if language == 'en' else 'zh' + try: + if get_setting('ASR_PROVIDER') == 'baidu': + text = baidu_transcribe(content, language) + else: + body = {'model': get_setting('ASR_MODEL'), 'stream': False, + 'messages': [{'role': 'user', 'content': [{'type': 'input_audio', 'input_audio': { + 'data': 'data:' + mime + ';base64,' + base64.b64encode(content).decode('ascii')}}]}], + 'asr_options': {'language': language, 'enable_itn': True}} + result = post_json(get_setting('ASR_URL'), json=body, + headers={'Authorization': 'Bearer ' + get_setting('ASR_API_KEY')}) + text = result['choices'][0]['message']['content'] + if not isinstance(text, str) or not text.strip(): + return {'ok': False, 'error': '没有识别到文字，请靠近麦克风重试。'}, 422 + text = text.strip() + if len(text) > 2000: + return {'ok': False, 'error': '识别文字过长，请缩短录音后重试。'}, 422 + return {'ok': True, 'text': text}, 200 + except requests.Timeout: + return {'ok': False, 'error': '语音转文字超时，请重试。'}, 504 + except SpeechFailure as error: + return {'ok': False, 'error': error.message}, error.status + except subprocess.TimeoutExpired: + return {'ok': False, 'error': '录音处理超时，请缩短后重试。'}, 504 + except OSError: + return {'ok': False, 'error': UNAVAILABLE}, 503 + except (requests.RequestException, ValueError, KeyError, IndexError, TypeError): + return {'ok': False, 'error': '语音转文字失败，请稍后重试。'}, 502 diff --git a/static/css/chat.css b/static/css/chat.css index 90ab73b..881e73c 100644 --- a/static/css/chat.css +++ b/static/css/chat.css @@ -86,6 +86,14 @@ body { padding: 20px; height: 100dvh; overflow: hidden; } .recent-item:hover { color: var(--brand-text); } .recent-date { white-space: nowrap; color: var(--muted); font-size: 10px; } .composer { flex-shrink: 0; align-self: center; width: min(820px, calc(100% - 76px)); padding: 12px 0 15px; background: var(--surface); } +.composer-toolbar { display: flex; gap: 8px; align-items: flex-start; } +.composer-toolbar .hot-bar { flex: 1; min-width: 0; } +.voice-toggle, .voice-hold { display: none; } +.voice-panel { position: absolute; z-index: 35; left: 50%; bottom: calc(var(--composer-height, 150px) + 14px); transform: translateX(-50%); width: min(330px, calc(100% - 32px)); padding: 20px; border: 1px solid var(--border); border-radius: 14px; background: var(--surface); color: var(--brand-text); box-shadow: var(--shadow); text-align: center; pointer-events: none; } +.voice-panel p { margin: 9px 0; font-size: 13px; } +.voice-panel.is-cancelling { color: var(--danger); border-color: var(--danger); } +.voice-wave { display: block; font-size: 26px; letter-spacing: 7px; } +#voice-clock { font-size: 12px; color: var(--muted); } .hot-bar { display: flex; flex-wrap: wrap; align-items: center; gap: 7px; margin-bottom: 10px; } .human-session { display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 10px 12px; margin-bottom: 10px; border: 1px solid var(--border); border-radius: 8px; background: var(--brand-soft); color: var(--secondary); font-size: 12px; } .human-session button { flex-shrink: 0; padding: 6px 9px; border: 1px solid var(--border); border-radius: 6px; background: var(--surface); color: var(--brand-text); font-size: 12px; } @@ -210,6 +218,10 @@ body { padding: 20px; height: 100dvh; overflow: hidden; } .composer { width: calc(100% - 40px); } .user-message, .robot-message { max-width: 94%; } .del-history-btn { opacity: 1; } + .voice-toggle { display: grid; place-items: center; flex: 0 0 40px; height: 36px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--brand-text); font-size: 20px; } + .input-area.voice-mode #input-text, .input-area.voice-mode .emoji-control, .input-area.voice-mode #send-button { display: none; } + .input-area.voice-mode .voice-hold { display: block; flex: 1; min-width: 0; min-height: 44px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface-soft); color: var(--text); font-size: 15px; font-weight: 600; user-select: none; -webkit-user-select: none; -webkit-touch-callout: none; touch-action: none; } + .voice-hold.is-recording { color: var(--brand-text); background: var(--brand-soft); } } @media (max-width: 520px) { .header-kicker { font-size: 9px; } diff --git a/static/js/chat.js b/static/js/chat.js index f0903b4..286d769 100644 --- a/static/js/chat.js +++ b/static/js/chat.js @@ -10,6 +10,7 @@ let uploadVersion = 0; let uploadController = null; let historyRequestVersion = 0; let followLatest = true; +let voiceInput = null; let creatingConv = null; let sendPreparing = false; let conversationVersion = 0; @@ -239,6 +240,7 @@ const I18N = { copy: "复制", copied: "已复制", retry: "重答", + retryAIOnly: "请先返回 AI 问答，再重答这条消息。", clickRetry: "点击重试", welcomeTitle: "物流问题，随时为您解答", welcomeSub: "查询订单状态、了解专线报价，或联系人工客服。", @@ -256,6 +258,18 @@ const I18N = { uploading: "附件正在上传，请稍候…", uploadFailed: "附件上传失败，请重新选择文件", serviceUnavailable: "服务暂时不可用，请稍后重试，或联系人工客服。", + voiceInput: "语音输入", voiceKeyboard: "键盘输入", voiceHold: "按住说话", + voiceRelease: "松开发送，上滑取消", voicePreparing: "正在准备麦克风…", + voiceCancelTip: "正在录音，松开发送，上滑取消", voiceCancelRelease: "松开取消发送", + voiceTranscribing: "正在转文字…", voiceCancelled: "已取消语音", + voiceUnavailable: "语音输入暂不可用，请使用文字输入。", + voiceUnsupported: "当前浏览器不支持录音，请使用文字输入或在其他浏览器打开。", + voiceHttps: "语音输入需要 HTTPS 安全连接，请使用安全访问地址。", + voicePermission: "未获得麦克风权限，请允许录音后重试。", voiceNoMic: "没有找到麦克风，请使用文字输入。", + voiceTooShort: "按住时间太短，请按住按钮说话后再松开。", voiceTooLarge: "录音过大，请缩短后重试。", + voiceEmpty: "没有识别到文字，请靠近麦克风重试。", voiceFailed: "语音输入失败，请重试或使用文字输入。", + voiceTimeout: "语音处理超时，请重试。", voiceWait: "请等待当前操作完成后，再使用语音输入。", + voiceDraft: "输入框还有未发送内容，请先发送或清空后再使用语音。", attachmentReady: "已准备好，发送后将附在消息中", removeAttachment: "移除待发送附件", you: "您", @@ -336,6 +350,7 @@ const I18N = { copy: "Copy", copied: "Copied", retry: "Regenerate", + retryAIOnly: "Return to AI chat before regenerating this answer.", clickRetry: "Retry", welcomeTitle: "Your next shipment, made clearer.", welcomeSub: "Track a parcel, explore freight options, or speak to our team.", @@ -353,6 +368,20 @@ const I18N = { uploading: "Uploading your attachment…", uploadFailed: "Upload failed. Please select the file again.", serviceUnavailable: "Service is temporarily unavailable. Please try again later or contact support.", + voiceInput: "Voice input", voiceKeyboard: "Keyboard input", voiceHold: "Hold to speak", + voiceRelease: "Release to send, slide up to cancel", voicePreparing: "Preparing the microphone…", + voiceCancelTip: "Recording · Release to send · Slide up to cancel", voiceCancelRelease: "Release to cancel", + voiceTranscribing: "Converting speech to text…", voiceCancelled: "Voice input cancelled", + voiceUnavailable: "Voice input is unavailable. Please type your message.", + voiceUnsupported: "This browser cannot record audio. Please type or open another browser.", + voiceHttps: "Voice input requires a secure HTTPS connection.", + voicePermission: "Microphone permission is required. Please allow recording and try again.", + voiceNoMic: "No microphone found. Please type your message.", + voiceTooShort: "Hold the button while speaking, then release.", voiceTooLarge: "The recording is too large. Please record a shorter message.", + voiceEmpty: "No speech was recognized. Please try again closer to the microphone.", + voiceFailed: "Voice input failed. Please try again or type your message.", voiceTimeout: "Voice input timed out. Please try again.", + voiceWait: "Please wait for the current operation to finish before recording.", + voiceDraft: "Please send or clear your existing draft before recording.", attachmentReady: "Ready to attach to your next message", removeAttachment: "Remove pending attachment", you: "You", @@ -428,6 +457,7 @@ function applyLanguage(lang){ const tg = document.querySelector(".toggle-divider"); if(tg) tg.title = tg.getAttribute('aria-expanded') === 'false' ? dict.sideExpand : dict.sideCollapse; updateHumanMode(); + if(voiceInput) voiceInput.refresh(); // 欢迎页（若当前显示的是欢迎页则重绘） if(chatDom.querySelector(".ai-welcome-card")){ @@ -788,6 +818,7 @@ function ensureConv(){ } function resetConversation(id){ + if(voiceInput) voiceInput.cancel(); if(cancelActive){ cancelActive(); cancelActive = null; } if(sseInstance){ sseInstance.close(); sseInstance = null; } conversationVersion += 1; @@ -1083,6 +1114,7 @@ async function humanRequest(path, data){ } window.showHuman = async function(){ + if(voiceInput) voiceInput.cancel(); if(openingConv === 'clearing') return; if(humanRequestToken || humanClosing) return; if(window.humanSid && humanConvId === currentConvId){ updateHumanMode(); return; } @@ -1152,10 +1184,12 @@ async function sendHumanMessage(content){ if(window.__pendingImg === attachment) clearAttachment(); setComposerStatus(t(humanStatus === 'active' ? 'humanSent' : 'humanWaiting')); inputDom.focus(); + return true; }catch(error){ if(version !== conversationVersion || humanSendingToken !== token) return; if(error.status === 404 || error.status === 409) stopHumanMode(); setComposerStatus(t('humanSendFailed'), true); + return false; }finally{ if(humanSendingToken === token){ humanSendingToken = null; updateHumanMode(); } } @@ -1348,6 +1382,14 @@ function mdToPlain(md){ .replace(/“|”/g, '"'); } +async function retryAnswer(bubble, question){ + if(window.humanSid || humanRequestToken){ setComposerStatus(t('retryAIOnly')); return; } + if(await doSend(question)){ + const row = bubble.closest('.msg-row'); + if(row) row.remove(); + } +} + // 给一条回答挂上「复制 / 重答」按钮 function attachActions(bubble, rawText, question, showRetry){ const old = bubble.querySelector(".msg-actions"); @@ -1390,11 +1432,7 @@ function attachActions(bubble, rawText, question, showRetry){ retryBtn.className = "msg-action-btn"; retryBtn.dataset.action = "retry"; retryBtn.textContent = t("retry"); - retryBtn.onclick = function(){ - const row = bubble.closest(".msg-row"); - if(row) row.remove(); - doSend(question); // 传字符串 = 只重发，不重复插入提问气泡 - }; + retryBtn.onclick = function(){ return retryAnswer(bubble, question); }; bar.appendChild(retryBtn); } @@ -1443,6 +1481,7 @@ function renderFollowups(bubble){ } async function doSend(retryText){ + if(voiceInput && voiceInput.busy()){ setComposerStatus(t('voiceWait')); return false; } const isRetry = typeof retryText === "string"; const sendWord = (isRetry ? retryText : inputDom.value).trim(); var imgUrl = isRetry ? "" : (window.__pendingImg || ""); @@ -1540,9 +1579,7 @@ async function doSend(retryText){ setBusy(false); if(activeBubble === aiBubble) activeBubble = null; renderError(aiBubble, text || t("busyRetry"), function(){ - const row = aiBubble.closest(".msg-row"); - if(row) row.remove(); - doSend(fullContent); + return retryAnswer(aiBubble, fullContent); }); } @@ -1672,6 +1709,7 @@ async function doSend(retryText){ if(settled) return; fail(t("networkFail")); }); + return true; } // ===== 发送 / 停止 按钮状态 ===== @@ -1792,6 +1830,30 @@ window.addEventListener("resize", function(){ if(window.visualViewport) window.visualViewport.addEventListener('resize', syncChatViewport); syncChatViewport(); +voiceInput = ChatVoice.create({ + toggle:document.getElementById('voice-toggle'), hold:document.getElementById('voice-hold'), + panel:document.getElementById('voice-panel'), label:document.getElementById('voice-label'), + clock:document.getElementById('voice-clock'), inputArea:document.querySelector('.input-area'), input:inputDom, + text:t, status:setComposerStatus, language:() => LANG, + context:() => conversationVersion + ':' + currentConvId + ':' + window.humanSid, + canStart:function(){ + if(uploadPending || sendPreparing || window.__busy || openingConv !== null || humanRequestToken || humanSendingToken || humanClosing) return t('voiceWait'); + return inputDom.value.trim() ? t('voiceDraft') : ''; + }, + onText:async function(text){ + if(inputDom.value.trim()){ + inputDom.value += '\n' + text; + autoGrow(); voiceInput.setMode(false); return; + } + inputDom.value = text; + autoGrow(); + if(!await doSend()) voiceInput.setMode(false); + } +}); +window.addEventListener('resize', function(){ + if(!isNarrow()) voiceInput.setMode(false); +}); + document.querySelectorAll('[data-question-index]').forEach(function(button){ button.addEventListener('click', function(){ window.askNewTopic(t('quick')[Number(button.dataset.questionIndex)]); diff --git a/static/js/voice.js b/static/js/voice.js new file mode 100644 index 0000000..bd54a56 --- /dev/null +++ b/static/js/voice.js @@ -0,0 +1,219 @@ +// Short recordings use the same server ASR in mobile browsers and embedded browsers. +var ChatVoice = { + create(options){ + const {toggle, hold, panel, label, clock, inputArea, input, text, status} = options; + let session = null, mode = false; + const active = current => session === current; + function stopTracks(current){ + if(current.stream) current.stream.getTracks().forEach(track => track.stop()); + } + function clearTimers(current){ + clearTimeout(current.limitTimer); + clearTimeout(current.permissionTimer); + clearInterval(current.clockTimer); + } + function render(){ + toggle.setAttribute('aria-pressed', String(mode)); + toggle.setAttribute('aria-label', text(mode ? 'voiceKeyboard' : 'voiceInput')); + toggle.title = text(mode ? 'voiceKeyboard' : 'voiceInput'); + toggle.textContent = mode ? '⌨' : '🎙'; + inputArea.classList.toggle('voice-mode', mode); + hold.textContent = text(session ? 'voiceRelease' : 'voiceHold'); + hold.classList.toggle('is-recording', Boolean(session)); + panel.hidden = !session; + } + function cancel(message){ + const current = session; + session = null; + if(current){ + current.controller.abort(); + clearTimers(current); + if(current.recorder && current.recorder.state !== 'inactive'){ + try{ current.recorder.stop(); }catch(error){} + } + stopTracks(current); + } + render(); + if(message) status(text(message), true); + } + function setMode(on){ + cancel(); + mode = on; + render(); + if(on && input.blur) input.blur(); + } + function supported(){ + if(window.isSecureContext === false){ status(text('voiceHttps'), true); return false; } + if(!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia || !window.MediaRecorder){ + status(text('voiceUnsupported'), true); + return false; + } + return true; + } + async function jsonRequest(current, url, init, timeout){ + const timer = setTimeout(function(){ current.timedOut = true; current.controller.abort(); }, timeout); + try{ + const response = await fetch(url, Object.assign({}, init, {signal:current.controller.signal})); + const data = await response.json(); + if(!response.ok) throw new Error(data.error || data.e || text('voiceFailed')); + return data; + }finally{ clearTimeout(timer); } + } + async function transcribe(current){ + if(!active(current)) return; + if(!current.chunks.length){ cancel('voiceEmpty'); return; } + current.phase = 'transcribing'; + label.textContent = text('voiceTranscribing'); + const blob = new Blob(current.chunks, {type:current.recorder.mimeType || current.mime}); + if(!blob.size || blob.size > current.maxBytes){ cancel('voiceTooLarge'); return; } + const form = new FormData(); + form.append('audio', blob, 'recording'); + form.append('duration', String(Math.min(60, current.duration / 1000))); + form.append('language', options.language()); + try{ + const data = await jsonRequest(current, '/api/speech/transcribe', {method:'POST',body:form}, 40000); + if(!active(current)) return; + if(current.context !== options.context()){ cancel(); return; } + if(!data.ok || typeof data.text !== 'string' || !data.text.trim()){ + cancel('voiceEmpty'); return; + } + session = null; + render(); + await options.onText(data.text.trim()); + }catch(error){ + if(!active(current)) return; + const message = current.timedOut ? text('voiceTimeout') : error.message || text('voiceFailed'); + cancel(); + status(message, true); + } + } + function finish(current, cancelled){ + if(!active(current) || !current.holding) return; + current.holding = false; + if(cancelled || current.cancelled){ cancel(); status(text('voiceCancelled')); return; } + if(current.phase !== 'recording'){ cancel('voiceTooShort'); return; } + current.duration = Date.now() - current.startedAt; + if(current.duration < 400){ cancel('voiceTooShort'); return; } + current.phase = 'stopping'; + label.textContent = text('voiceTranscribing'); + clearTimers(current); + try{ current.recorder.stop(); } + catch(error){ cancel('voiceFailed'); } + finally{ stopTracks(current); } + } + async function start(id, y){ + if(session || !mode) return; + const reason = options.canStart(); + if(reason){ status(reason, true); return; } + if(!supported()) return; + const current = {id:id,startY:y,holding:true,chunks:[],bytes:0,phase:'preparing', + controller:new AbortController(),context:options.context()}; + session = current; + label.textContent = text('voicePreparing'); + clock.textContent = ''; + panel.classList.remove('is-cancelling'); + status(''); + render(); + try{ + const config = await jsonRequest(current, '/api/speech/config', {}, 5000); + if(!active(current)) return; + if(!config.enabled){ cancel('voiceUnavailable'); return; } + current.maxBytes = Math.min(Number(config.max_bytes) || 5242880, 5242880); + current.permissionTimer = setTimeout(function(){ if(active(current)) cancel('voiceTimeout'); }, 15000); + const stream = await navigator.mediaDevices.getUserMedia({audio:{echoCancellation:true,noiseSuppression:true},video:false}); + clearTimeout(current.permissionTimer); + if(!active(current)){ stream.getTracks().forEach(track => track.stop()); return; } + current.stream = stream; + const formats = ['audio/webm;codecs=opus','audio/mp4','audio/webm','audio/ogg;codecs=opus']; + current.mime = window.MediaRecorder.isTypeSupported + ? formats.find(mime => window.MediaRecorder.isTypeSupported(mime)) || '' : ''; + current.recorder = new window.MediaRecorder(stream, Object.assign({audioBitsPerSecond:64000}, + current.mime ? {mimeType:current.mime} : {})); + current.recorder.ondataavailable = function(event){ + if(!active(current) || !event.data || !event.data.size) return; + current.bytes += event.data.size; + if(current.bytes > current.maxBytes){ cancel('voiceTooLarge'); return; } + current.chunks.push(event.data); + }; + current.recorder.onerror = function(){ if(active(current)) cancel('voiceFailed'); }; + current.recorder.onstop = function(){ + if(!active(current)) return; + if(current.phase === 'stopping') transcribe(current); + else cancel('voiceFailed'); + }; + current.recorder.start(250); + current.startedAt = Date.now(); + current.phase = 'recording'; + label.textContent = text('voiceCancelTip'); + clock.textContent = '0 / 60 s'; + current.clockTimer = setInterval(function(){ + clock.textContent = Math.min(60, Math.floor((Date.now()-current.startedAt)/1000)) + ' / 60 s'; + }, 250); + current.limitTimer = setTimeout(function(){ finish(current, false); }, 60000); + }catch(error){ + if(!active(current)) return; + const key = current.timedOut ? 'voiceTimeout' + : ['NotAllowedError','PermissionDeniedError'].includes(error.name) ? 'voicePermission' + : ['NotFoundError','DevicesNotFoundError'].includes(error.name) ? 'voiceNoMic' : 'voiceFailed'; + cancel(key); + } + } + function move(id, y){ + if(!session || !session.holding || session.id !== id) return; + session.cancelled = session.startY - y > 72; + panel.classList.toggle('is-cancelling', session.cancelled); + if(session.phase === 'recording') label.textContent = text(session.cancelled ? 'voiceCancelRelease' : 'voiceCancelTip'); + } + toggle.onclick = function(){ + if(!mode && !supported()) return; + setMode(!mode); + if(!mode) input.focus(); + }; + if(window.PointerEvent){ + hold.addEventListener('pointerdown', function(event){ + if(event.button !== 0) return; + event.preventDefault(); + if(hold.setPointerCapture) try{ hold.setPointerCapture(event.pointerId); }catch(error){} + start(event.pointerId, event.clientY); + }); + hold.addEventListener('pointermove', event => move(event.pointerId, event.clientY)); + hold.addEventListener('pointerup', function(event){ + event.preventDefault(); + if(session && session.id === event.pointerId) finish(session, false); + }); + ['pointercancel','lostpointercapture'].forEach(name => hold.addEventListener(name, function(){ + if(session && session.holding) finish(session, true); + })); + }else{ + hold.addEventListener('touchstart', function(event){ + event.preventDefault(); + if(event.touches.length !== 1) return; + const touch=event.touches[0]; start(touch.identifier, touch.clientY); + }, {passive:false}); + hold.addEventListener('touchmove', function(event){ + event.preventDefault(); + for(const touch of event.changedTouches) move(touch.identifier, touch.clientY); + }, {passive:false}); + hold.addEventListener('touchend', function(event){ + event.preventDefault(); + for(const touch of event.changedTouches) if(session && session.id === touch.identifier) finish(session, false); + }, {passive:false}); + hold.addEventListener('touchcancel', function(){ if(session) finish(session, true); }); + } + hold.addEventListener('contextmenu', event => event.preventDefault()); + hold.addEventListener('keydown', function(event){ + if(![' ','Enter'].includes(event.key) || event.repeat) return; + event.preventDefault(); start('keyboard', 0); + }); + hold.addEventListener('keyup', function(event){ + if([' ','Enter'].includes(event.key) && session && session.id === 'keyboard'){ + event.preventDefault(); finish(session, false); + } + }); + document.addEventListener('keydown', event => { if(event.key === 'Escape') cancel(); }); + document.addEventListener('visibilitychange', function(){ if(document.hidden) cancel(); }); + window.addEventListener('pagehide', function(){ cancel(); }); + render(); + return {cancel:cancel,setMode:setMode,refresh:render,busy:() => Boolean(session)}; + } +}; diff --git a/templates/index.html b/templates/index.html index 2d5ce35..10a1a53 100644 --- a/templates/index.html +++ b/templates/index.html @@ -50,27 +50,30 @@
-
+
快捷咨询 -
+
+
AI 回答仅供参考，具体以人工客服回复为准Enter 发送 · Shift + Enter 换行
+ + diff --git a/tests/frontend/chat.test.cjs b/tests/frontend/chat.test.cjs index 412afbd..16288a6 100644 --- a/tests/frontend/chat.test.cjs +++ b/tests/frontend/chat.test.cjs @@ -8,6 +8,7 @@ const flush = () => new Promise(setImmediate); async function customerPage(pageOptions = {}){ const elements = new Map(), listeners = {}, windowListeners = {}, viewportListeners = {}, intervals = new Map(), timers = new Map(), calls = [], hooks = {}; let intervalId = 0, timerId = 0, nextConvId = 1; + let voiceOptions; class Element { constructor(){ this.children = []; this.attributes = {}; this.value = ''; this.style = {}; @@ -97,14 +98,18 @@ async function customerPage(pageOptions = {}){ return {ok:true, json:async () => data}; }, }); - for(const file of ['support.js', 'chat.js']){ + for(const file of ['support.js', 'voice.js', 'chat.js']){ vm.runInContext(fs.readFileSync(path.join(__dirname, '../../static/js', file), 'utf8'), context, {filename:file}); + if(file === 'voice.js'){ + const createVoice=context.ChatVoice.create; + context.ChatVoice.create=options => {voiceOptions=options; return createVoice(options);}; + } } listeners.DOMContentLoaded(); await flush(); if(pageOptions.human !== false) await context.window.showHuman(); await flush(); - return {context, element, calls, hooks, quickButtons, + return {context, element, calls, hooks, quickButtons, voiceOptions, resize:async height => { if(context.window.visualViewport) context.window.visualViewport.height=height; (windowListeners.resize || []).forEach(callback => callback()); @@ -628,3 +633,56 @@ test('clearing a sent draft restores a single-row composer instead of sizing to assert.equal(input.value,''); assert.equal(input.style.height,''); }); + +test('recognized voice uses the normal AI and human send routes with a visible user message', async () => { + for(const human of [false,true]){ + const app=await customerPage({human,width:390}); + await app.voiceOptions.onText('语音识别的物流问题'); + await flush(); + const call=app.calls.find(call=>call.url===(human ? '/api/human/send' : '/stream_chat')); + assert.ok(call); + assert.equal(human ? call.body.content : call.body.msg,'语音识别的物流问题'); + assert.match(app.element('chatDom').querySelector('.user-message').textContent,/语音识别的物流问题/); + assert.equal(app.element('input-text').value,''); + } +}); + +test('a failed voice send keeps the transcript editable and a newer draft is not auto-submitted', async () => { + const app=await customerPage({human:false,width:390}); + app.hooks['/api/new_chat']=()=>Promise.reject(new Error('offline')); + await app.voiceOptions.onText('语音内容'); + assert.equal(app.element('input-text').value,'语音内容'); + assert.equal(app.element('.input-area').classList.contains('voice-mode'),false); + app.element('input-text').value='新草稿'; + await app.voiceOptions.onText('后到的识别文字'); + assert.equal(app.element('input-text').value,'新草稿\n后到的识别文字'); + assert.equal(app.calls.filter(call=>call.url==='/stream_chat').length,0); +}); + +test('a blocked retry retains its previous answer until a replacement can start', async () => { + const app=await customerPage({human:false}); + await app.context.window.fillInput('之前的问题');await flush(); + const log=app.element('chatDom'), retry=log.querySelectorAll('.msg-action-btn').find(button=>button.dataset.action==='retry'); + app.hooks['/api/upload']=options=>new Promise((resolve,reject)=>{ + options.signal.addEventListener('abort',()=>reject(Object.assign(new Error(),{name:'AbortError'}))); + }); + app.element('upload-input').files=[{name:'pending.pdf'}]; + app.element('upload-input').onchange();await flush(); + await app.click(retry); + assert.match(log.textContent,/测试 AI 回复/); + assert.equal(app.calls.filter(call=>call.url==='/stream_chat').length,1); + await app.runTimers(15000); + await app.click(retry);await flush(); + assert.equal(app.calls.filter(call=>call.url==='/stream_chat').length,2); + assert.equal(log.querySelectorAll('.msg-action-btn').filter(button=>button.dataset.action==='retry').length,1); +}); + +test('an AI regeneration is not forwarded to a human session', async () => { + const app=await customerPage({human:false}); + await app.context.window.fillInput('之前的问题');await flush(); + const retry=app.element('chatDom').querySelectorAll('.msg-action-btn').find(button=>button.dataset.action==='retry'); + await app.context.window.showHuman();await app.click(retry); + assert.equal(app.calls.filter(call=>call.url==='/api/human/send').length,0); + assert.match(app.element('chatDom').textContent,/测试 AI 回复/); + assert.match(app.element('composer-status').textContent,/先返回 AI/); +}); diff --git a/tests/frontend/voice.test.cjs b/tests/frontend/voice.test.cjs new file mode 100644 index 0000000..7d42987 --- /dev/null +++ b/tests/frontend/voice.test.cjs @@ -0,0 +1,162 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const flush = () => new Promise(setImmediate); + +function voicePage(options={}){ + const elements={}, timers=new Map(), docEvents={}, pageEvents={}, calls=[], messages=[], statuses=[]; + let now=1000, timerId=0, contextId='conversation-1', recorder, permissionCalls=0, trackStops=0; + class Element { + constructor(){ this.listeners={}; this.attributes={}; this.classes=new Set(); this.value=''; + this.classList={toggle:(name,on) => on ? this.classes.add(name) : this.classes.delete(name),remove:name=>this.classes.delete(name)}; } + setAttribute(name,value){this.attributes[name]=value;} + addEventListener(name,callback){this.listeners[name]=callback;} + setPointerCapture(){} + focus(){} + blur(){} + } + for(const key of ['toggle','hold','panel','label','clock','inputArea','input']) elements[key]=new Element(); + const tracks=[{stop(){trackStops++;}}], stream={getTracks:()=>tracks}; + class Recorder { + constructor(source, settings){ recorder=this; this.state='inactive'; this.mimeType=settings.mimeType || 'audio/mp4'; } + static isTypeSupported(type){return options.mp4 ? type==='audio/mp4' : type.startsWith('audio/webm');} + start(){this.state='recording';} + stop(){this.state='inactive'; this.ondataavailable({data:new Blob(['audio fixture'],{type:this.mimeType})}); this.onstop();} + } + class ClockDate extends Date {static now(){return now;}} + const response = data => ({ok:true,json:async()=>data}); + const sandbox=vm.createContext({console,Blob,AbortController,Date:ClockDate, + FormData:class{constructor(){this.values={};}append(key,value){this.values[key]=value;}}, + navigator:{mediaDevices:{getUserMedia:async()=>{permissionCalls++; if(options.permission) return options.permission(); return stream;}}}, + document:{hidden:false,addEventListener:(name,callback)=>{docEvents[name]=callback;}}, + window:{isSecureContext:options.secure!==false,MediaRecorder:options.unsupported ? undefined : Recorder, + PointerEvent:options.touch ? undefined : function(){},addEventListener:(name,callback)=>{pageEvents[name]=callback;}}, + setTimeout:(callback,delay)=>{const id=++timerId;timers.set(id,{callback,delay});return id;}, + clearTimeout:id=>timers.delete(id),setInterval:()=>++timerId,clearInterval(){}, + fetch:async(url,init)=>{calls.push({url,init}); + if(url==='/api/speech/config') return response({enabled:options.enabled!==false,max_bytes:5242880,max_seconds:60}); + if(options.transcribe) return options.transcribe(init); + return response({ok:true,text:'查询订单 FL20261030001'}); + }, + }); + vm.runInContext(fs.readFileSync(path.join(__dirname,'../../static/js/voice.js'),'utf8'),sandbox); + const voice=sandbox.ChatVoice.create({...elements,text:key=>key,status:(...args)=>statuses.push(args), + language:()=> 'zh',context:()=>contextId,canStart:()=>options.reason || '',onText:async text=>messages.push(text)}); + const event=(extra={})=>({button:0,pointerId:7,clientY:300,preventDefault(){},...extra}); + return {voice,elements,calls,messages,statuses,stream, + get permissionCalls(){return permissionCalls;},get trackStops(){return trackStops;},get recorder(){return recorder;}, + context:value=>{contextId=value;}, + open:()=>elements.toggle.onclick(), + down:async()=>{elements.hold.listeners.pointerdown(event());await flush();}, + up:async(duration=1000)=>{now+=duration;elements.hold.listeners.pointerup(event());await flush();}, + move:y=>elements.hold.listeners.pointermove(event({clientY:y})), + event,docEvents,pageEvents,sandbox, + expire:async delay=>{for(const [id,timer] of [...timers]) if(timer.delay===delay){timers.delete(id);now+=delay;timer.callback();}await flush();}, + }; +} + +test('hold and release transcribes audio and sends text once',async()=>{ + const app=voicePage();app.open();await app.down(); + assert.equal(app.recorder.state,'recording'); + assert.equal(app.elements.panel.hidden,false); + await app.up(); + assert.deepEqual(app.messages,['查询订单 FL20261030001']); + assert.equal(app.calls.filter(call=>call.url==='/api/speech/transcribe').length,1); + assert.ok(app.trackStops>0); + assert.equal(app.elements.panel.hidden,true); + await app.up(); + assert.equal(app.messages.length,1); +}); + +test('sliding up cancels without uploading and supports another recording',async()=>{ + const app=voicePage();app.open();await app.down();app.move(200);await app.up(); + assert.equal(app.messages.length,0); + assert.equal(app.calls.filter(call=>call.url==='/api/speech/transcribe').length,0); + assert.ok(app.trackStops>0); + await app.down();await app.up(); + assert.equal(app.messages.length,1); +}); + +test('short presses and unavailable service do not upload recordings',async()=>{ + const app=voicePage();app.open();await app.down();await app.up(50); + assert.equal(app.messages.length,0); + assert.equal(app.statuses.at(-1)[0],'voiceTooShort'); + const disabled=voicePage({enabled:false});disabled.open();await disabled.down(); + assert.equal(disabled.permissionCalls,0); + assert.equal(disabled.statuses.at(-1)[0],'voiceUnavailable'); +}); + +test('late microphone permission after cancellation immediately stops its tracks',async()=>{ + let grant;const app=voicePage({permission:()=>new Promise(resolve=>{grant=resolve;})}); + app.open();await app.down();await app.up();grant(app.stream);await flush(); + assert.equal(app.messages.length,0); + assert.ok(app.trackStops>0); + assert.equal(app.voice.busy(),false); +}); + +test('switching conversation or explicitly cancelling ignores a late transcript',async()=>{ + for(const cancelled of [false,true]){ + let resolve;const app=voicePage({transcribe:()=>new Promise(done=>{resolve=done;})}); + app.open();await app.down();await app.up(); + if(cancelled) app.voice.cancel(); else app.context('conversation-2'); + resolve({ok:true,json:async()=>({ok:true,text:'must not enter the new conversation'})});await flush(); + assert.equal(app.messages.length,0); + assert.equal(app.voice.busy(),false); + } +}); + +test('permission denial and unsupported or insecure browsers leave typing available',async()=>{ + const denied=voicePage({permission:()=>Promise.reject(Object.assign(new Error(),{name:'NotAllowedError'}))}); + denied.open();await denied.down(); + assert.equal(denied.voice.busy(),false); + assert.equal(denied.statuses.at(-1)[0],'voicePermission'); + for(const options of [{unsupported:true},{secure:false}]){ + const app=voicePage(options);app.open(); + assert.equal(app.elements.inputArea.classes.has('voice-mode'),false); + assert.equal(app.permissionCalls,0); + } +}); + +test('pending drafts and busy chat do not request microphone access',async()=>{ + const app=voicePage({reason:'wait for current chat'});app.open();await app.down(); + assert.equal(app.permissionCalls,0); + assert.equal(app.calls.length,0); + assert.equal(app.statuses.at(-1)[0],'wait for current chat'); +}); + +test('maximum recording length stops the microphone and sends once',async()=>{ + const app=voicePage({mp4:true});app.open();await app.down();await app.expire(60000); + assert.equal(app.messages.length,1); + assert.ok(app.trackStops>0); + assert.equal(app.calls.find(call=>call.url==='/api/speech/transcribe').init.body.values.audio.type,'audio/mp4'); + await app.up();assert.equal(app.messages.length,1); +}); + +test('touch fallback and browser cancellation do not submit accidentally',async()=>{ + const app=voicePage({touch:true});app.open(); + app.elements.hold.listeners.touchstart(app.event({touches:[{identifier:1,clientY:300}]}));await flush(); + app.elements.hold.listeners.touchcancel();await flush(); + assert.equal(app.messages.length,0); + assert.ok(app.trackStops>0); +}); + +test('a stalled transcription times out and releases the recorder state',async()=>{ + const app=voicePage({transcribe:init=>new Promise((resolve,reject)=>{ + init.signal.addEventListener('abort',()=>reject(Object.assign(new Error(),{name:'AbortError'}))); + })});app.open();await app.down();await app.up();await app.expire(40000); + assert.equal(app.voice.busy(),false); + assert.equal(app.statuses.at(-1)[0],'voiceTimeout'); + assert.equal(app.messages.length,0); +}); + +test('an unanswered microphone permission prompt times out and a late stream is closed',async()=>{ + let grant;const app=voicePage({permission:()=>new Promise(resolve=>{grant=resolve;})}); + app.open();await app.down();await app.expire(15000); + assert.equal(app.voice.busy(),false); + assert.equal(app.statuses.at(-1)[0],'voiceTimeout'); + grant(app.stream);await flush(); + assert.ok(app.trackStops>0); + assert.equal(app.messages.length,0); +}); diff --git a/tests/test_frontend.py b/tests/test_frontend.py index 3cd20bd..7a4f874 100644 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -11,7 +11,7 @@ class FrontendTests(unittest.TestCase): def test_reply_workflows(self): root = Path(__file__).resolve().parents[1] result = subprocess.run( - ['node', '--test', 'tests/frontend/admin.test.cjs', 'tests/frontend/chat.test.cjs'], cwd=root, + ['node', '--test', 'tests/frontend/admin.test.cjs', 'tests/frontend/chat.test.cjs', 'tests/frontend/voice.test.cjs'], cwd=root, text=True, encoding='utf-8', capture_output=True, timeout=30, ) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) diff --git a/tests/test_speech.py b/tests/test_speech.py new file mode 100644 index 0000000..cc6d89f --- /dev/null +++ b/tests/test_speech.py @@ -0,0 +1,173 @@ +"""Short recordings are validated and sent only to the separately configured ASR.""" + +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import subprocess + +import requests + +from logistics_chat import create_app +from logistics_chat.speech_service import MAX_AUDIO_BYTES + + +class SpeechResponse: + def __init__(self, payload=None, status=200, raw=None): + self.status_code = status + self.raw = raw if raw is not None else json.dumps(payload or {'choices': [{'message': {'content': '查询订单状态'}}]}).encode() + self.closed = False + + def iter_content(self, size): + yield self.raw + + def close(self): + self.closed = True + + +class SpeechTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + self.app = create_app({'TESTING': True, 'DATABASE': self.root/'history.db', + 'UPLOAD_DIR': self.root/'uploads', 'BACKUP_DIR': self.root/'backups', 'LOG_DIR': self.root/'logs', + 'DEEPSEEK_API_KEY': 'separate-chat-key', 'ASR_PROVIDER':'dashscope', 'ASR_API_KEY': 'asr-test-key', + 'ASR_URL': 'https://asr.example.test/chat/completions', 'ASR_MODEL': 'test-asr', 'RATE_LIMIT_PER_MIN':1000}) + self.client = self.app.test_client() + + def recording(self, content=b'\x1aE\xdf\xa3' + b'x' * 40, mime='audio/webm', duration='2.5', **extra): + response = self.client.post('/api/speech/transcribe', data={'audio': (io.BytesIO(content), 'private-filename.webm', mime), + 'duration': duration, 'language':'zh', **extra}) + self.addCleanup(response.close) + self.addCleanup(response.request.environ['wsgi.input'].close) + return response + + def test_missing_asr_key_does_not_reuse_chat_key_or_accept_audio(self): + self.app.config['ASR_API_KEY'] = '' + response = self.client.get('/api/speech/config') + self.assertFalse(response.json['enabled']) + self.assertIn('no-store', response.headers['Cache-Control']) + self.assertNotIn('key', response.get_data(as_text=True).lower()) + with patch('logistics_chat.speech_service.requests.post') as post: + self.assertEqual(self.recording().status_code, 503) + post.assert_not_called() + + def test_valid_recording_uses_asr_only_and_never_persists_audio(self): + upstream = SpeechResponse() + with patch('logistics_chat.speech_service.requests.post', return_value=upstream) as post: + response = self.recording() + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {'ok': True, 'text': '查询订单状态'}) + url = post.call_args.args[0] + options = post.call_args.kwargs + self.assertEqual(url, self.app.config['ASR_URL']) + self.assertEqual(options['headers']['Authorization'], 'Bearer asr-test-key') + self.assertEqual(options['json']['model'], 'test-asr') + self.assertEqual(options['json']['asr_options']['language'], 'zh') + self.assertTrue(options['json']['messages'][0]['content'][0]['input_audio']['data'].startswith('data:audio/webm;base64,')) + self.assertFalse(options['allow_redirects']) + self.assertTrue(upstream.closed) + self.assertEqual(list(self.root.glob('uploads/**/*')), []) + self.assertEqual(self.client.get('/api/get_history').json, []) + + def test_browser_audio_containers_are_supported_without_trusting_filename(self): + samples = [(b'\x00\x00\x00\x18ftypisom' + b'x'*20, 'audio/mp4'), + (b'OggS'+b'x'*32, 'audio/ogg'), (b'RIFF'+b'x'*4+b'WAVE'+b'x'*20, 'audio/wav')] + for content, mime in samples: + with self.subTest(mime=mime), patch('logistics_chat.speech_service.requests.post', return_value=SpeechResponse()): + self.assertEqual(self.recording(content, mime).status_code, 200) + + def test_invalid_audio_size_duration_and_container_never_reach_provider(self): + with patch('logistics_chat.speech_service.requests.post') as post: + self.assertEqual(self.recording(b'html masquerading as audio'*2).status_code, 415) + self.assertEqual(self.recording(b'\x1aE\xdf\xa3'+b'x'*MAX_AUDIO_BYTES).status_code, 413) + for duration in ('NaN', 'inf', '-1', '0.1', '62', 'broken'): + with self.subTest(duration=duration): + self.assertEqual(self.recording(duration=duration).status_code, 400) + self.assertEqual(self.client.post('/api/speech/transcribe', data={}).status_code, 400) + post.assert_not_called() + + def test_request_body_limit_returns_json_instead_of_an_html_error(self): + response = self.recording(b'x'*(MAX_AUDIO_BYTES+128*1024)) + self.assertEqual(response.status_code, 413) + self.assertFalse(response.json['ok']) + + def test_upstream_failures_timeout_and_malformed_results_are_recoverable(self): + for upstream in (SpeechResponse(status=401, raw=b'secret upstream message'), + SpeechResponse(raw=b'invalid'), SpeechResponse(payload={'choices': []}), + SpeechResponse(raw=b'x' * 65537)): + with self.subTest(raw=upstream.raw[:20]), patch('logistics_chat.speech_service.requests.post', return_value=upstream): + response = self.recording() + self.assertEqual(response.status_code, 502) + self.assertNotIn('secret', response.get_data(as_text=True)) + self.assertTrue(upstream.closed) + with patch('logistics_chat.speech_service.requests.post', side_effect=requests.Timeout): + self.assertEqual(self.recording().status_code, 504) + + def test_silence_and_excessively_long_transcripts_are_not_sent_to_chat(self): + for text in ('', ' ', '字' * 2001): + with self.subTest(length=len(text)), patch('logistics_chat.speech_service.requests.post', + return_value=SpeechResponse(payload={'choices': [{'message': {'content': text}}]})): + self.assertEqual(self.recording().status_code, 422) + + def test_baidu_converts_browser_audio_and_caches_a_separate_access_token(self): + self.app.config.update(ASR_PROVIDER='baidu', BAIDU_ASR_API_KEY='baidu-test-key', BAIDU_ASR_SECRET_KEY='baidu-test-secret') + responses=[SpeechResponse(payload={'access_token':'baidu-test-token','expires_in':3600}), + SpeechResponse(payload={'err_no':0,'result':['查询订单。']}), + SpeechResponse(payload={'err_no':0,'result':['查询订单。']})] + with patch('logistics_chat.speech_service.ffmpeg_binary',return_value='ffmpeg'), \ + patch('logistics_chat.speech_service.to_pcm',return_value=b'\0'*32000), \ + patch('logistics_chat.speech_service.requests.post',side_effect=responses) as post: + for _ in range(2): + response=self.recording() + self.assertEqual(response.status_code,200) + self.assertEqual(response.json['text'],'查询订单。') + self.assertEqual(post.call_count,3) + self.assertEqual(post.call_args_list[0].kwargs['data']['client_id'],'baidu-test-key') + for call in post.call_args_list[1:]: + self.assertEqual(call.args[0],'https://vop.baidu.com/server_api') + self.assertEqual(call.kwargs['json']['format'],'pcm') + self.assertEqual(call.kwargs['json']['rate'],16000) + self.assertEqual(call.kwargs['json']['channel'],1) + self.assertEqual(call.kwargs['json']['token'],'baidu-test-token') + self.assertTrue(all(response.closed for response in responses)) + + def test_baidu_reports_permission_failure_and_never_returns_upstream_secrets(self): + self.app.config.update(ASR_PROVIDER='baidu', BAIDU_ASR_API_KEY='baidu-test-key', BAIDU_ASR_SECRET_KEY='baidu-test-secret') + with patch('logistics_chat.speech_service.ffmpeg_binary',return_value='ffmpeg'), \ + patch('logistics_chat.speech_service.to_pcm',return_value=b'\0'*32000), \ + patch('logistics_chat.speech_service.requests.post',side_effect=[ + SpeechResponse(payload={'access_token':'token','expires_in':3600}), + SpeechResponse(payload={'err_no':3302,'err_msg':'private provider details'})]): + response=self.recording() + self.assertEqual(response.status_code,503) + self.assertNotIn('private',response.get_data(as_text=True)) + + def test_decoder_rejects_real_duration_beyond_limit_and_handles_failure_or_timeout(self): + self.app.config.update(ASR_PROVIDER='baidu', BAIDU_ASR_API_KEY='test-key', BAIDU_ASR_SECRET_KEY='test-secret') + cases=[(subprocess.CompletedProcess([],0,stdout=b'\0'*(32000*61),stderr=b''),400), + (subprocess.CompletedProcess([],1,stdout=b'',stderr=b'private decoder error'),422)] + with patch('logistics_chat.speech_service.ffmpeg_binary',return_value='ffmpeg'), \ + patch('logistics_chat.speech_service.requests.post') as post: + for result,status in cases: + with patch('logistics_chat.speech_service.subprocess.run',return_value=result): + self.assertEqual(self.recording(duration='1').status_code,status) + with patch('logistics_chat.speech_service.subprocess.run',side_effect=subprocess.TimeoutExpired('ffmpeg',10)): + self.assertEqual(self.recording().status_code,504) + post.assert_not_called() + + def test_large_recordings_stay_in_memory_and_the_upload_stream_is_closed(self): + from logistics_chat import speech_service + original=speech_service.transcribe_recording + streams=[] + def inspect(upload,*args): + self.assertIsInstance(upload.stream,io.BytesIO) + streams.append(upload.stream) + return original(upload,*args) + with patch('logistics_chat.speech_service.transcribe_recording',side_effect=inspect), \ + patch('logistics_chat.speech_service.requests.post',return_value=SpeechResponse()): + self.assertEqual(self.recording(b'\x1aE\xdf\xa3'+b'x'*600000).status_code,200) + self.assertTrue(streams[0].closed)