105 lines
4.8 KiB
Python
105 lines
4.8 KiB
Python
"""Model-selected clarifications must survive grounded rendering and persistence."""
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
|
|
from logistics_chat import config, create_app, db
|
|
from logistics_chat.chat_tools import validate_calls
|
|
|
|
|
|
def clarification_plan(**extra):
|
|
return {'role': 'assistant', 'content': None, 'tool_calls': [{
|
|
'id': 'clarify_synthetic', 'type': 'function', 'function': {
|
|
'name': 'read_service_info',
|
|
'arguments': json.dumps({'question': '还是按刚才说的那个办吧', **extra}, ensure_ascii=False),
|
|
},
|
|
}]}
|
|
|
|
|
|
def completion_response(message):
|
|
response = Mock(status_code=200)
|
|
response.json.return_value = {'choices': [{
|
|
'finish_reason': 'tool_calls' if message.get('tool_calls') else 'stop',
|
|
'message': message,
|
|
}]}
|
|
return response
|
|
|
|
|
|
class ModelClarificationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
directory = tempfile.TemporaryDirectory()
|
|
self.addCleanup(directory.cleanup)
|
|
root = Path(directory.name)
|
|
root_patch = patch.object(config, 'ROOT_DIR', root)
|
|
root_patch.start()
|
|
self.addCleanup(root_patch.stop)
|
|
environment = patch.dict(os.environ, {}, clear=True)
|
|
environment.start()
|
|
self.addCleanup(environment.stop)
|
|
network = patch('requests.sessions.Session.request',
|
|
side_effect=AssertionError('Clarification tests must not access the network'))
|
|
network.start()
|
|
self.addCleanup(network.stop)
|
|
self.app = create_app({
|
|
'TESTING': True, 'INITIALIZE_RUNTIME': False,
|
|
'DATABASE': root / 'history.db', 'UPLOAD_DIR': root / 'uploads',
|
|
'BACKUP_DIR': root / 'backups', 'LOG_DIR': root / 'logs',
|
|
'CHAT_API_KEY': 'synthetic-key', 'DEEPSEEK_API_KEY': '',
|
|
'CHAT_USE_MODEL_TOOLS': True, 'MODEL_PROVIDER': 'compatible',
|
|
'MODEL_URL': 'https://model.example.invalid/chat/completions',
|
|
'MODEL_NAME': 'synthetic-model', 'CUSTOMER_DEMO_ENABLED': False,
|
|
})
|
|
with self.app.app_context():
|
|
db.init_db()
|
|
self.client = self.app.test_client()
|
|
self.client.post('/api/visitor/session', json={'key': '8' * 64})
|
|
self.cid = self.client.post('/api/new_chat').json['id']
|
|
with self.app.app_context():
|
|
db.save_message(self.cid, 'user', '请比较此前的两种方案')
|
|
db.save_message(self.cid, 'assistant', '已展示方案甲和方案乙，请明确选择哪一种。')
|
|
|
|
def ask(self, final_content):
|
|
responses = [completion_response(clarification_plan(clarification='quote_option')),
|
|
completion_response({'role': 'assistant', 'content': final_content})]
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model:
|
|
response = self.client.post('/stream_chat', json={
|
|
'msg': '还是按刚才说的那个办吧', 'conv_id': self.cid})
|
|
events = [json.loads(frame[6:]) for frame in response.get_data(as_text=True).split('\n\n')
|
|
if frame.startswith('data: ')]
|
|
self.assertEqual(events[-1], {'d': 1})
|
|
self.assertEqual(model.call_count, 2)
|
|
evidence = json.loads(model.call_args_list[1].args[0]['messages'][-1]['content'])
|
|
return events[0], evidence
|
|
|
|
def test_model_selects_quote_clarification_and_renders_its_fact(self):
|
|
answer, evidence = self.ask('{"facts":["fact_0"],"question":""}')
|
|
self.assertEqual(evidence['status'], 'needs_clarification')
|
|
self.assertEqual(answer['t'], evidence['result'])
|
|
self.assertIn('报价方案编号或渠道', answer['t'])
|
|
self.assertNotIn('您好', answer['t'])
|
|
self.assertNotIn('fact_0', answer['t'])
|
|
history = self.client.get('/api/get_messages', query_string={'conv_id': self.cid}).json
|
|
self.assertEqual(history[-1]['content'], answer['t'])
|
|
|
|
def test_untrusted_rewrite_falls_back_to_the_same_clarification(self):
|
|
answer, evidence = self.ask('已选择方案甲，运费9999元。')
|
|
self.assertEqual(answer['t'], evidence['result'])
|
|
self.assertNotIn('已选择', answer['t'])
|
|
self.assertNotIn('9999', answer['t'])
|
|
|
|
def test_clarification_is_optional_and_rejects_unknown_values(self):
|
|
self.assertEqual(validate_calls(clarification_plan())[0][1], 'read_service_info')
|
|
valid = validate_calls(clarification_plan(clarification='quote_option'))
|
|
self.assertEqual(valid[0][2]['clarification'], 'quote_option')
|
|
for value in ('choose_first', '', None):
|
|
with self.subTest(value=value), self.assertRaises(ValueError):
|
|
validate_calls(clarification_plan(clarification=value))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|