310 lines
12 KiB
Python
310 lines
12 KiB
Python
"""SQLite schema, history persistence and backups; initialized explicitly by create_app."""
|
|
|
|
import sqlite3
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
from .config import get_setting
|
|
from .tracking import card_to_text
|
|
from .database_backup import check_database, daily_backup, prune_migration_snapshots, snapshot_database, unique_backup_path
|
|
|
|
SCHEMA_VERSION = 5
|
|
|
|
|
|
def connect():
|
|
connection = sqlite3.connect(get_setting('DATABASE'))
|
|
try:
|
|
from flask import g, has_app_context
|
|
except ImportError:
|
|
return connection
|
|
if has_app_context():
|
|
g.setdefault('_database_connections', []).append(connection)
|
|
return connection
|
|
|
|
|
|
def close_connections(error=None):
|
|
from flask import g
|
|
for connection in g.pop('_database_connections', []):
|
|
connection.close()
|
|
|
|
|
|
|
|
def now_str():
|
|
return datetime.now().strftime('%Y-%m-%d %H:%M')
|
|
|
|
|
|
def init_db():
|
|
Path(get_setting('DATABASE')).parent.mkdir(parents=True, exist_ok=True)
|
|
conn = connect()
|
|
try:
|
|
# A reserved write lock serializes snapshots and schema migration across workers.
|
|
conn.execute('BEGIN IMMEDIATE')
|
|
has_tables = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' LIMIT 1").fetchone()
|
|
version = 0
|
|
if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_meta'").fetchone():
|
|
row = conn.execute("SELECT value FROM schema_meta WHERE key='schema_version'").fetchone()
|
|
if row:
|
|
version = int(row[0])
|
|
if version > SCHEMA_VERSION:
|
|
raise RuntimeError('Database schema is newer than this application; refusing downgrade')
|
|
if version < 0:
|
|
raise RuntimeError('Invalid database schema version')
|
|
if [number for number, operation in MIGRATIONS] != list(range(1, SCHEMA_VERSION + 1)):
|
|
raise RuntimeError('Database migration registry has missing or unordered versions')
|
|
if has_tables:
|
|
snapshot_database(get_setting('DATABASE'), unique_backup_path(get_setting('BACKUP_DIR'), 'pre-migration'))
|
|
prune_migration_snapshots(get_setting('BACKUP_DIR'), keep=5)
|
|
conn.execute('CREATE TABLE IF NOT EXISTS schema_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
|
|
for target_version, migration in MIGRATIONS:
|
|
if version < target_version <= SCHEMA_VERSION:
|
|
migration(conn)
|
|
conn.execute("INSERT OR REPLACE INTO schema_meta VALUES('schema_version',?)", (str(target_version),))
|
|
if version == SCHEMA_VERSION:
|
|
# Retain legacy schema reconciliation for tables maintained by older tools.
|
|
_initialize_schema(conn)
|
|
_upgrade_ticket_workflow(conn)
|
|
_upgrade_ticket_history(conn)
|
|
_upgrade_staff_routing(conn)
|
|
conn.execute("INSERT OR REPLACE INTO schema_meta VALUES('schema_version',?)", (str(SCHEMA_VERSION),))
|
|
if conn.execute('PRAGMA quick_check').fetchall() != [('ok',)]:
|
|
raise sqlite3.DatabaseError('Database validation failed after migration')
|
|
conn.commit()
|
|
except Exception:
|
|
conn.rollback()
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _initialize_schema(conn):
|
|
"""Version 1 baseline reconciliation preserves supported legacy table layouts."""
|
|
cur = conn.cursor()
|
|
cur.execute('''
|
|
CREATE TABLE IF NOT EXISTS chat_history (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
title TEXT NOT NULL,
|
|
ctime TEXT NOT NULL
|
|
)
|
|
''')
|
|
cols = [r[1] for r in cur.execute('PRAGMA table_info(chat_history)').fetchall()]
|
|
if 'utime' not in cols:
|
|
cur.execute('ALTER TABLE chat_history ADD COLUMN utime TEXT')
|
|
cur.execute('UPDATE chat_history SET utime = ctime WHERE utime IS NULL')
|
|
if 'visitor_id' not in cols:
|
|
cur.execute('ALTER TABLE chat_history ADD COLUMN visitor_id TEXT')
|
|
if 'customer_id' not in cols:
|
|
cur.execute('ALTER TABLE chat_history ADD COLUMN customer_id INTEGER')
|
|
if 'customer_scope' not in cols:
|
|
cur.execute('ALTER TABLE chat_history ADD COLUMN customer_scope TEXT')
|
|
if 'customer_tenant_id' in cols:
|
|
# Preserve conversations created by the earlier local single-tenant login.
|
|
import hashlib
|
|
import json
|
|
for cid, tenant in cur.execute('SELECT id,customer_tenant_id FROM chat_history WHERE customer_id IS NOT NULL').fetchall():
|
|
scope = hashlib.sha256(json.dumps([tenant],ensure_ascii=False).encode()).hexdigest() if tenant else None
|
|
cur.execute('UPDATE chat_history SET customer_scope=? WHERE id=?',(scope,cid))
|
|
cur.execute('CREATE INDEX IF NOT EXISTS idx_chat_visitor ON chat_history(visitor_id, id)')
|
|
cur.execute('''CREATE TABLE IF NOT EXISTS visitor_uploads (
|
|
filename TEXT PRIMARY KEY, visitor_id TEXT NOT NULL, ctime TEXT NOT NULL
|
|
)''')
|
|
cur.execute('''
|
|
CREATE TABLE IF NOT EXISTS messages (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
conv_id INTEGER NOT NULL,
|
|
role TEXT NOT NULL,
|
|
content TEXT NOT NULL,
|
|
ctime TEXT NOT NULL
|
|
)
|
|
''')
|
|
cur.execute('CREATE INDEX IF NOT EXISTS idx_msg_conv ON messages(conv_id, id)')
|
|
cur.execute('''
|
|
CREATE TABLE IF NOT EXISTS feedback (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
question TEXT,
|
|
answer TEXT,
|
|
vote TEXT,
|
|
ctime TEXT
|
|
)
|
|
''')
|
|
cur.execute('''
|
|
CREATE TABLE IF NOT EXISTS human_sessions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
conv_id INTEGER NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'waiting',
|
|
ctime TEXT NOT NULL
|
|
)
|
|
''')
|
|
cur.execute('''
|
|
CREATE TABLE IF NOT EXISTS human_messages (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
session_id INTEGER NOT NULL,
|
|
side TEXT NOT NULL,
|
|
content TEXT NOT NULL,
|
|
ctime TEXT NOT NULL
|
|
)
|
|
''')
|
|
hm_cols = [r[1] for r in cur.execute('PRAGMA table_info(human_messages)').fetchall()]
|
|
if 'read_at' not in hm_cols:
|
|
cur.execute('ALTER TABLE human_messages ADD COLUMN read_at TEXT')
|
|
if 'recalled_id' not in hm_cols:
|
|
cur.execute('ALTER TABLE human_messages ADD COLUMN recalled_id INTEGER')
|
|
hs_cols = [r[1] for r in cur.execute('PRAGMA table_info(human_sessions)').fetchall()]
|
|
if 'note' not in hs_cols:
|
|
cur.execute("ALTER TABLE human_sessions ADD COLUMN note TEXT DEFAULT ''")
|
|
cur.execute('''
|
|
CREATE TABLE IF NOT EXISTS ratings (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
sid INTEGER NOT NULL,
|
|
stars INTEGER NOT NULL,
|
|
comment TEXT DEFAULT '',
|
|
ctime TEXT NOT NULL
|
|
)
|
|
''')
|
|
cur.execute('''
|
|
CREATE TABLE IF NOT EXISTS agents (
|
|
username TEXT PRIMARY KEY,
|
|
password TEXT NOT NULL,
|
|
name TEXT NOT NULL
|
|
)
|
|
''')
|
|
cur.execute('''
|
|
CREATE TABLE IF NOT EXISTS orders (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
phone TEXT,
|
|
track_no TEXT,
|
|
status TEXT,
|
|
created TEXT
|
|
)
|
|
''')
|
|
cur.execute('''
|
|
CREATE TABLE IF NOT EXISTS tickets (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
content TEXT,
|
|
phone TEXT,
|
|
ctime TEXT,
|
|
status TEXT
|
|
)
|
|
''')
|
|
from .order_data import init_order_table
|
|
init_order_table(conn)
|
|
from .customer_service import init_customer_tables
|
|
init_customer_tables(conn)
|
|
from .staff_service import init_staff_tables
|
|
init_staff_tables(conn)
|
|
old = cur.execute('SELECT id, title FROM chat_history WHERE id NOT IN (SELECT DISTINCT conv_id FROM messages)').fetchall()
|
|
for cid, title in old:
|
|
cur.execute('INSERT INTO messages(conv_id, role, content, ctime) VALUES (?,?,?,?)', (cid, 'user', title, now_str()))
|
|
|
|
|
|
def _upgrade_ticket_workflow(conn):
|
|
from .ticket_service import init_ticket_tables
|
|
init_ticket_tables(conn)
|
|
|
|
|
|
def _upgrade_ticket_history(conn):
|
|
from .ticket_service import init_ticket_history
|
|
init_ticket_history(conn)
|
|
|
|
|
|
def _upgrade_order_admin(conn):
|
|
from .order_admin_service import init_order_admin_tables
|
|
init_order_admin_tables(conn)
|
|
|
|
|
|
def _upgrade_staff_routing(conn):
|
|
from .routing_service import init_routing_tables
|
|
init_routing_tables(conn)
|
|
|
|
|
|
MIGRATIONS = ((1, _initialize_schema), (2, _upgrade_ticket_workflow),
|
|
(3, _upgrade_ticket_history), (4, _upgrade_order_admin), (5, _upgrade_staff_routing))
|
|
|
|
|
|
def backup_db():
|
|
"""Verified daily online snapshot; failures propagate rather than claiming success."""
|
|
return daily_backup(get_setting('DATABASE'), get_setting('BACKUP_DIR'))
|
|
|
|
|
|
def cleanup_old_conversations():
|
|
"""自动清理 30 天前的旧对话，防止数据库无限膨胀"""
|
|
try:
|
|
conn = connect()
|
|
c = conn.cursor()
|
|
cutoff = (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d %H:%M:%S')
|
|
rows = c.execute('SELECT id FROM chat_history WHERE utime < ?', (cutoff,)).fetchall()
|
|
ids = [r[0] for r in rows]
|
|
if not ids:
|
|
return
|
|
for cid in ids:
|
|
c.execute('DELETE FROM messages WHERE conv_id=?', (cid,))
|
|
c.execute(f"DELETE FROM chat_history WHERE id IN ({','.join('?' * len(ids))})", ids)
|
|
conn.commit()
|
|
conn.close()
|
|
print(f'[自动清理] 已删除 {len(ids)} 条 30 天前的旧对话')
|
|
except Exception as e:
|
|
print(f'[自动清理失败] {e}')
|
|
|
|
|
|
def make_title(content):
|
|
"""生成会话标题：优先用文字内容，只有附件时显示附件数量"""
|
|
lines = [l for l in content.split('\n') if not l.strip().startswith('[附件]')]
|
|
text = ' '.join(lines).strip()
|
|
if text and text != '（发送了附件）' and (text != '请帮我看看这个附件'):
|
|
return text[:24]
|
|
n = len([l for l in content.split('\n') if l.strip().startswith('[附件]')])
|
|
if n == 1:
|
|
return '📎 已上传附件'
|
|
if n > 1:
|
|
return f'📎 {n}个附件'
|
|
return content[:24]
|
|
|
|
|
|
def save_message(conv_id, role, content):
|
|
"""把一条消息写进会话；用户消息做幂等处理，重试不会重复入库。"""
|
|
if not conv_id or not content:
|
|
return
|
|
conn = connect()
|
|
cur = conn.cursor()
|
|
if role == 'user':
|
|
last = cur.execute('SELECT role, content FROM messages WHERE conv_id=? ORDER BY id DESC LIMIT 1', (conv_id,)).fetchone()
|
|
if last and last[0] == 'user' and (last[1] == content):
|
|
conn.close()
|
|
return 'duplicate'
|
|
inserted = cur.execute('INSERT INTO messages(conv_id, role, content, ctime) '
|
|
'SELECT ?,?,?,? WHERE EXISTS(SELECT 1 FROM chat_history WHERE id=?)',
|
|
(conv_id, role, content, now_str(), conv_id)).rowcount
|
|
if not inserted:
|
|
conn.close()
|
|
return 'deleted'
|
|
if role == 'user':
|
|
row = cur.execute('SELECT title FROM chat_history WHERE id=?', (conv_id,)).fetchone()
|
|
if row and (not row[0] or row[0] == '新对话'):
|
|
cur.execute('UPDATE chat_history SET title=? WHERE id=?', (make_title(content), conv_id))
|
|
cur.execute('UPDATE chat_history SET utime=? WHERE id=?', (now_str(), conv_id))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
def last_user_question(conv_id):
|
|
"""上一条用户提问（必须在当前这句入库之前调用）"""
|
|
if not conv_id:
|
|
return None
|
|
conn = connect()
|
|
row = conn.execute("SELECT content FROM messages WHERE conv_id=? AND role='user' ORDER BY id DESC LIMIT 1", (conv_id,)).fetchone()
|
|
conn.close()
|
|
return row[0] if row else None
|
|
|
|
|
|
def load_context(conv_id, msg):
|
|
"""取最近 MAX_CONTEXT 条消息，拼成模型要的 messages（不含 system）。"""
|
|
if not conv_id:
|
|
return [{'role': 'user', 'content': msg}]
|
|
conn = connect()
|
|
rows = conn.execute('SELECT role, content FROM messages WHERE conv_id=? ORDER BY id DESC LIMIT ?', (conv_id, get_setting('MAX_CONTEXT'))).fetchall()
|
|
conn.close()
|
|
rows.reverse()
|
|
history = [{'role': r, 'content': card_to_text(c)} for r, c in rows]
|
|
if not history or history[-1]['role'] != 'user' or history[-1]['content'] != msg:
|
|
history.append({'role': 'user', 'content': msg})
|
|
return history
|