312 lines
15 KiB
Python
312 lines
15 KiB
Python
"""Bounded, read-only evidence retrieval from reviewed FAQ and one quote snapshot.
|
||
|
||
The caller supplies the quote book and authorization context. This module never
|
||
opens files, queries the database, or follows a model-supplied reference as a URL.
|
||
Each FAQ entry / workbook cell stays intact, including its conditions and exceptions.
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import unicodedata
|
||
|
||
from .knowledge import knowledge_list
|
||
|
||
|
||
MAX_QUERY = 600
|
||
MAX_PASSAGE = 2200
|
||
MAX_RESULTS = 5
|
||
MAX_TOTAL_TEXT = 7000
|
||
MAX_DOCUMENTS = 5000
|
||
|
||
# These existing FAQ entries describe general meanings / boundaries, not rates.
|
||
# New unclassified entries deliberately do not inherit this universal scope.
|
||
GENERAL_KEYS = {
|
||
'查询订单', '订单号和运单号', '付款后没发货', '手机号查订单', '联系人填错',
|
||
'订单负责人', '收货地址怎么填', '最终送到哪里', '修改收货地址',
|
||
'节点是什么意思', '已预报是什么意思', '到港是什么意思', '完整物流轨迹',
|
||
'换单号后怎么查', '实时位置', '履约中是什么意思', '货物是否完好',
|
||
'已签收但没收到', '清关了吗', '已申报是什么意思', '清关完为什么没派送',
|
||
'人工客服', '电话',
|
||
}
|
||
REGIONAL_KEYS = {
|
||
'海运': ('EU',), '铁路': ('EU',), '卡航': ('EU',), '空运': ('EU',),
|
||
'英国海运': ('GB',), '英国空运': ('GB',),
|
||
'报价': ('EU', 'GB'), '时效': ('EU', 'GB'), '揽收': ('EU', 'GB'),
|
||
'超轻费': ('EU', 'GB'), '超重': ('EU', 'GB'), '清关': ('EU', 'GB'),
|
||
'亚马逊': ('EU',), '退件': ('EU', 'GB'),
|
||
}
|
||
EU_COUNTRIES = {'DE', 'FR', 'ES', 'IT', 'PL', 'CZ', 'NL', 'BE', 'AT', 'RO',
|
||
'SE', 'FI', 'DK', 'IE', 'PT', 'HU', 'SK', 'SI', 'HR', 'GR',
|
||
'BG', 'EE', 'LV', 'LT', 'LU', 'MT', 'CY'}
|
||
COUNTRIES = {
|
||
'美国': 'US', '美利坚': 'US', 'USA': 'US', 'UNITED STATES': 'US',
|
||
'UNITED STATES OF AMERICA': 'US', '英国': 'GB', 'UK': 'GB',
|
||
'UNITED KINGDOM': 'GB', '欧洲': 'EU', '欧盟': 'EU', 'EUROPE': 'EU',
|
||
'德国': 'DE', '法国': 'FR', '意大利': 'IT', '西班牙': 'ES',
|
||
'波兰': 'PL', '荷兰': 'NL', '捷克': 'CZ', '罗马尼亚': 'RO',
|
||
}
|
||
ALIASES = (
|
||
('贴标', '标签', '唛头', '条码', 'label', 'labels', 'marking'),
|
||
('揽收', '上门取件', '上门提货', '上门收货', '上门取货', 'pickup', 'pick up'),
|
||
('交仓', '送仓', '送货', '交货', 'warehouse delivery'),
|
||
('发货资料', '出货资料', '发货文件', 'shipping documents'),
|
||
('报关', '报关资料', 'customs declaration'),
|
||
('清关', 'customs clearance'),
|
||
('保险', '投保', 'insurance'),
|
||
('发票', '开票', 'invoice', 'invoicing'),
|
||
('付款', '支付', '结算', 'payment'),
|
||
('材积', '体积重', '泡重', 'dimensional weight', 'volumetric weight'),
|
||
('时效', '多久', '几天', 'transit time'),
|
||
('费用', '运费', '报价', '多少钱', 'price', 'cost', 'freight'),
|
||
('营业时间', '工作时间', '几点上班', '周末上班', 'opening hours', 'business hours'),
|
||
('禁运', '禁品', '违禁', '不能寄', '不能发', '禁止运输', 'prohibited'),
|
||
('电池', '带电', 'battery', 'batteries'),
|
||
)
|
||
STOP_TERMS = {'请问', '一下', '可以', '需要', '怎么', '什么', '哪些', '是否',
|
||
'我们', '你们', '货物', '美国', '英国', '欧洲', '资料', '提供', '确认',
|
||
'这个', '那个', '我的', '我要', '一下', '当前'}
|
||
SENSITIVE = re.compile(
|
||
r'(?i)(?:[A-Z]:[\\/]|\\\\|/(?:Users|home|mnt|tmp|var|srv|opt|etc|data|uploads)/|'
|
||
r'https?://|file://|[\w.+-]+@[\w.-]+\.[A-Z]{2,}|(?<!\d)1[3-9]\d{9}(?!\d)|'
|
||
r'(?:联系人|联系电话|联系电话号码|电话号码|电话|手机号|微信|TEL|CONTACT)\s*[:：]|'
|
||
r'(?<!\d)0\d{2,3}[- ]\d{7,8}(?!\d)|'
|
||
r'(?<!\d)(?:\+?1[- .]?)?\(?\d{3}\)?[- .]\d{3}[- .]\d{4}(?!\d)|'
|
||
r'\.(?:xlsx?|docx?|pdf|csv|txt)(?:\b|$))'
|
||
)
|
||
|
||
|
||
def _digest(value):
|
||
raw = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
||
return hashlib.sha256(raw.encode('utf-8')).hexdigest()[:24]
|
||
|
||
|
||
def _normalize(value):
|
||
return unicodedata.normalize('NFKC', value).lower().strip()
|
||
|
||
|
||
def _country(value):
|
||
if not isinstance(value, str) or len(value) > 80:
|
||
return None
|
||
key = unicodedata.normalize('NFKC', value).strip().upper()
|
||
return COUNTRIES.get(key, key)
|
||
|
||
|
||
def _search_country(query, country):
|
||
supplied = _country(country)
|
||
if supplied:
|
||
return supplied
|
||
found = set()
|
||
for term, code in COUNTRIES.items():
|
||
if (re.search(r'\b' + re.escape(term) + r'\b', query, re.I)
|
||
if term.isascii() else term in query):
|
||
found.add(code)
|
||
if re.search(r'\bUS\b', query, re.I):
|
||
found.add('US')
|
||
return next(iter(found)) if len(found) == 1 else ''
|
||
|
||
|
||
def _services(values):
|
||
if values is None:
|
||
return ()
|
||
if not isinstance(values, (list, tuple)) or len(values) > 12:
|
||
return None
|
||
if any(not isinstance(value, str) or not 0 < len(value.strip()) <= 100 for value in values):
|
||
return None
|
||
return tuple(sorted(set(value.strip() for value in values)))
|
||
|
||
|
||
def _safe_text(value, maximum):
|
||
return (isinstance(value, str) and 0 < len(value.strip()) <= maximum
|
||
and not SENSITIVE.search(value) and not re.search(r'[\x00-\x08\x0b\x0c\x0e-\x1f]', value))
|
||
|
||
|
||
def _book_id(book):
|
||
value = book.get('id') if isinstance(book, dict) else None
|
||
return value if type(value) is int and value > 0 else None
|
||
|
||
|
||
def _service_names(book):
|
||
result = {}
|
||
rates = book.get('rates', []) if isinstance(book, dict) else []
|
||
if not isinstance(rates, list):
|
||
return result
|
||
for row in rates[:10000]:
|
||
if isinstance(row, dict) and _safe_text(row.get('service'), 100):
|
||
result[row['service']] = row.get('kind')
|
||
return result
|
||
|
||
|
||
def _matches_service(requested, scope, service_names):
|
||
if not requested:
|
||
return True
|
||
if not scope:
|
||
return False
|
||
for wanted in requested:
|
||
known = [name for name in service_names if name == wanted or wanted in name]
|
||
for value in scope:
|
||
if wanted == value or (len(wanted) >= 2 and wanted in value):
|
||
return True
|
||
if value == '海派' and any(service_names[name] == 'parcel' for name in known):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _entries(country, services, book):
|
||
"""Build only the permitted view; direct reads use the identical filter."""
|
||
result = []
|
||
rejected = 0
|
||
for item in knowledge_list[:500]:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
keys = item.get('keys')
|
||
if not isinstance(keys, list) or not keys or not all(isinstance(key, str) for key in keys):
|
||
continue
|
||
first = keys[0]
|
||
general = first in GENERAL_KEYS
|
||
countries = () if general else REGIONAL_KEYS.get(first)
|
||
if countries is None or (not general and country not in countries
|
||
and not (country in EU_COUNTRIES and 'EU' in countries)):
|
||
continue
|
||
content = item.get('text')
|
||
if not _safe_text(content, MAX_PASSAGE):
|
||
rejected += 1
|
||
continue
|
||
if general and re.search(r'欧洲|欧盟|英国|美国|中欧|\d+(?:\.\d+)?\s*(?:元|USD|CNY|RMB)', content, re.I):
|
||
# A newly edited general FAQ must not silently acquire regional
|
||
# pricing authority just because its old lookup key is unchanged.
|
||
continue
|
||
# Legacy paragraphs describe several distinct channels. Never filter one
|
||
# into a requested channel based merely on a matching word in the body.
|
||
if services and not general:
|
||
continue
|
||
identity = _digest({'keys': keys, 'text': content, 'countries': countries})
|
||
result.append({'reference': 'kb1:k:' + identity, 'text': content.strip(),
|
||
'country': 'GLOBAL' if general else ','.join(countries), 'services': [],
|
||
'scope_confirmed': general, 'keys': keys,
|
||
'source': {'title': '客服知识 ' + first, 'kind': 'faq',
|
||
'data_version': 'builtin:' + identity}})
|
||
book_id = _book_id(book)
|
||
if country != 'US' or book_id is None:
|
||
return result, rejected
|
||
names = _service_names(book)
|
||
documents = book.get('documents', [])
|
||
if not isinstance(documents, list):
|
||
return result, rejected
|
||
for row in documents[:MAX_DOCUMENTS]:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
from .business_rules import reviewed_document
|
||
row = reviewed_document(row) if isinstance(row.get('content'), str) else row
|
||
if row.get('book_id', book_id) != book_id:
|
||
continue
|
||
sheet, cell, content = row.get('sheet'), row.get('source_cell'), row.get('content')
|
||
if (not _safe_text(sheet, 100) or not isinstance(cell, str)
|
||
or not re.fullmatch(r'[A-Z]{1,3}[1-9][0-9]{0,6}', cell)
|
||
or not _safe_text(content, MAX_PASSAGE)):
|
||
rejected += 1
|
||
continue
|
||
# The book importer is US-specific. A contradictory regional paragraph
|
||
# must not become US evidence merely by residing inside that workbook.
|
||
if re.search(r'欧洲|欧盟|英国|中欧|\b(?:UK|Europe|United Kingdom)\b', content, re.I):
|
||
continue
|
||
scope = [sheet] if sheet in names or sheet == '海派' else []
|
||
if not _matches_service(services, scope, names):
|
||
continue
|
||
identity = _digest({'book': book_id, 'sheet': sheet, 'cell': cell, 'text': content})
|
||
result.append({'reference': f'kb1:b:{book_id}:{identity}', 'text': content.strip(),
|
||
'country': 'US', 'services': scope, 'scope_confirmed': bool(scope), 'keys': [],
|
||
'source': {'title': '美国报价资料 ' + sheet, 'kind': 'quote_document',
|
||
'sheet': sheet, 'cell': cell, 'data_version': book_id,
|
||
**{key: row[key] for key in ('business_rule_revision', 'original_content_sha256') if key in row}}})
|
||
return result, rejected
|
||
|
||
|
||
def _terms(text):
|
||
normalized = _normalize(text)
|
||
terms = set(re.findall(r'[a-z][a-z0-9_-]{1,}', normalized))
|
||
for word in re.findall(r'[\u4e00-\u9fff]+', normalized):
|
||
terms.update(word[index:index + 2] for index in range(len(word) - 1))
|
||
return terms - STOP_TERMS
|
||
|
||
|
||
def _concepts(text):
|
||
value = _normalize(text)
|
||
return {index for index, group in enumerate(ALIASES) if any(
|
||
re.search(r'\b' + re.escape(alias) + r'\b', value) if alias.isascii() else alias in value
|
||
for alias in group)}
|
||
|
||
|
||
def _score(query, entry):
|
||
text = entry['text']
|
||
aliases = _concepts(query) & _concepts(text + ' ' + ' '.join(entry['keys']))
|
||
overlaps = _terms(query) & _terms(text)
|
||
direct = [key for key in entry['keys'] if len(key) >= 2 and _normalize(key) in _normalize(query)]
|
||
if len(overlaps) < 2 and not direct:
|
||
# A generic mention of cost / transit time in an unrelated warning is
|
||
# not a price / timing policy. Specific concepts such as pickup still
|
||
# support synonyms with no literal overlap.
|
||
aliases.difference_update({10, 11})
|
||
# A lone common bigram is too weak to establish an answerable match.
|
||
if not aliases and not direct and len(overlaps) < 2:
|
||
return 0
|
||
return len(aliases) * 20 + len(overlaps) * 2 + max([len(key) * 2 for key in direct] or [0])
|
||
|
||
|
||
def _response(status, matches, book, message='', rejected=0, country=''):
|
||
sources = [{'reference': item['reference'], **item['source']} for item in matches]
|
||
public = [{key: value for key, value in item.items() if key != 'keys'} for item in matches]
|
||
if matches:
|
||
lines = []
|
||
for item in matches:
|
||
scope = '、'.join(item['services']) or ('通用概念与服务说明' if item['country'] == 'GLOBAL'
|
||
else '渠道适用范围需结合原条款确认')
|
||
lines.append(f"【{item['source']['title']}；适用范围：{scope}】\n{item['text']}")
|
||
message = '\n\n'.join(lines)
|
||
warnings = ['部分资料过长或包含未公开联系信息，未纳入本次检索。'] if rejected else []
|
||
return {'t': message, 'status': status, 'data': {'matches': public, 'warnings': warnings,
|
||
'resolved_country': country},
|
||
'sources': sources, 'data_version': _book_id(book)}
|
||
|
||
|
||
def search_business_knowledge(query, *, country='', services=None, topic='', book=None, limit=5):
|
||
"""Return complete relevant evidence, not computed prices or live order facts."""
|
||
requested = _services(services)
|
||
if (not isinstance(query, str) or not 0 < len(query.strip()) <= MAX_QUERY
|
||
or not isinstance(topic, str) or len(topic) > 100 or requested is None
|
||
or _country(country) is None or type(limit) is not int or limit < 1):
|
||
return _response('error', [], book, '检索参数无效，请缩短问题并提供明确的国家和渠道。')
|
||
selected_country = _search_country(query, country)
|
||
entries, rejected = _entries(selected_country, requested, book)
|
||
ranked = [(score, entry) for entry in entries if (score := _score(query + ' ' + topic, entry))]
|
||
ranked.sort(key=lambda pair: (-pair[0], pair[1]['reference']))
|
||
selected, seen, total = [], set(), 0
|
||
for _, entry in ranked:
|
||
identity = (_normalize(entry['text']), entry['country'], tuple(entry['services']))
|
||
if identity in seen:
|
||
continue
|
||
size = len(entry['text']) + len(entry['source']['title']) + 100
|
||
if total + size > MAX_TOTAL_TEXT:
|
||
continue
|
||
seen.add(identity)
|
||
selected.append(entry)
|
||
total += size
|
||
if len(selected) >= min(limit, MAX_RESULTS):
|
||
break
|
||
status = 'ok' if selected else 'not_found'
|
||
message = '未查到适用于本次条件的资料，请补充国家、渠道或具体问题，必要时联系人工确认。'
|
||
return _response(status, selected, book, message, rejected, selected_country)
|
||
|
||
|
||
def read_business_knowledge(reference, *, country='', services=None, book=None):
|
||
"""Re-resolve an opaque content reference within the current permitted snapshot."""
|
||
requested = _services(services)
|
||
selected_country = _country(country)
|
||
if (not isinstance(reference, str) or not re.fullmatch(r'kb1:(?:k:|b:[1-9][0-9]{0,18}:)[0-9a-f]{24}', reference)
|
||
or selected_country is None or requested is None):
|
||
return _response('not_found', [], book, '资料引用无效或不适用于当前条件，请重新搜索。')
|
||
entries, rejected = _entries(selected_country, requested, book)
|
||
found = next((entry for entry in entries if entry['reference'] == reference), None)
|
||
return _response('ok' if found else 'not_found', [found] if found else [], book,
|
||
'资料版本或适用范围已变化，请按当前条件重新搜索。', rejected, selected_country)
|