298 lines
13 KiB
Python
298 lines
13 KiB
Python
"""Safe parsing and row validation for staff-managed order imports."""
|
||
|
||
import csv
|
||
from datetime import date, datetime
|
||
from decimal import Decimal, InvalidOperation
|
||
import io
|
||
import json
|
||
from pathlib import Path
|
||
import zipfile
|
||
|
||
from openpyxl import load_workbook
|
||
|
||
from .order_data import STATUS_LABELS
|
||
from .order_tracking import NODE_LABELS, parse_time
|
||
|
||
MAX_UPLOAD_BYTES = 5 * 1024 * 1024
|
||
MAX_ROWS = 5000
|
||
MAX_ERRORS = 100
|
||
MAX_XLSX_UNPACKED_BYTES = 25 * 1024 * 1024
|
||
|
||
IMPORT_COLUMNS = (
|
||
('source_order_id', '来源订单ID'), ('tenant_id', '租户ID'), ('order_no', '订单号'),
|
||
('inbound_code', '入仓码'), ('shipment_id', 'Shipment ID'), ('status', '业务状态'),
|
||
('country_code', '目的国'), ('transport_mode', '运输方式'), ('warehouse_code', '目的仓'),
|
||
('order_role', '订单层级'), ('parent_order_no', '父订单号'), ('package_count', '件数'),
|
||
('gross_weight_kg', '毛重(kg)'), ('volume_cbm', '体积(m³)'),
|
||
('history_json', '业务状态历史(JSON)'), ('tracking_json', '物流轨迹(JSON)'),
|
||
('data_note', '数据备注'), ('source_updated_at', '来源更新时间'), ('snapshot_at', '数据快照时间'),
|
||
)
|
||
HEADER_TO_FIELD = {header.casefold(): field for field, header in IMPORT_COLUMNS}
|
||
HEADER_TO_FIELD.update({field.casefold(): field for field, _ in IMPORT_COLUMNS})
|
||
REQUIRED_FIELDS = {'tenant_id', 'order_no'}
|
||
TEXT_LIMITS = {
|
||
'source_order_id': 128, 'tenant_id': 128, 'order_no': 128, 'inbound_code': 128,
|
||
'shipment_id': 128, 'status': 64, 'country_code': 16, 'transport_mode': 32,
|
||
'warehouse_code': 64, 'order_role': 32, 'parent_order_no': 128, 'data_note': 2000,
|
||
'source_updated_at': 64, 'snapshot_at': 64,
|
||
}
|
||
|
||
|
||
class ImportFileError(ValueError):
|
||
"""The uploaded file cannot be interpreted as an order table."""
|
||
|
||
|
||
def template_csv():
|
||
"""Return an Excel-friendly UTF-8 CSV template."""
|
||
output = io.StringIO(newline='')
|
||
writer = csv.writer(output)
|
||
writer.writerow([header for _, header in IMPORT_COLUMNS])
|
||
return output.getvalue().encode('utf-8-sig')
|
||
|
||
|
||
def _cell_text(value):
|
||
if value is None:
|
||
return ''
|
||
if isinstance(value, (datetime, date)):
|
||
return value.isoformat(sep=' ') if isinstance(value, datetime) else value.isoformat()
|
||
if isinstance(value, bool):
|
||
return 'TRUE' if value else 'FALSE'
|
||
if isinstance(value, float):
|
||
if not value.is_integer():
|
||
return format(value, '.15g')
|
||
return str(int(value))
|
||
return str(value).strip()
|
||
|
||
|
||
def _validate_row(raw, line):
|
||
errors = []
|
||
row = {field: raw.get(field, '') for field, _ in IMPORT_COLUMNS}
|
||
for field, limit in TEXT_LIMITS.items():
|
||
value = _cell_text(row[field])
|
||
row[field] = value or None
|
||
if field in REQUIRED_FIELDS and not value:
|
||
errors.append(f'{dict(IMPORT_COLUMNS)[field]}不能为空')
|
||
elif len(value) > limit:
|
||
errors.append(f'{dict(IMPORT_COLUMNS)[field]}不能超过{limit}个字符')
|
||
if not row['status']:
|
||
row['status'] = 'DRAFT'
|
||
row['status'] = row['status'].upper()
|
||
if row['status'] not in STATUS_LABELS:
|
||
errors.append('业务状态不在允许范围内')
|
||
if row['order_role']:
|
||
row['order_role'] = row['order_role'].upper()
|
||
if row['order_role'] not in ('MASTER', 'CHILD'):
|
||
errors.append('订单层级只能是MASTER或CHILD')
|
||
|
||
package_value = _cell_text(row['package_count'])
|
||
if package_value:
|
||
try:
|
||
number = Decimal(package_value)
|
||
if not number.is_finite() or number < 0 or number != number.to_integral_value():
|
||
raise InvalidOperation
|
||
row['package_count'] = int(number)
|
||
except (InvalidOperation, ValueError):
|
||
errors.append('件数必须是大于或等于0的整数')
|
||
else:
|
||
row['package_count'] = None
|
||
|
||
for field, label in (('gross_weight_kg', '毛重'), ('volume_cbm', '体积')):
|
||
value = _cell_text(row[field])
|
||
if not value:
|
||
row[field] = None
|
||
continue
|
||
try:
|
||
number = Decimal(value)
|
||
if not number.is_finite() or number < 0:
|
||
raise InvalidOperation
|
||
row[field] = format(number.normalize(), 'f')
|
||
except (InvalidOperation, ValueError):
|
||
errors.append(f'{label}必须是大于或等于0的数字')
|
||
|
||
for field, label in (('history_json', '业务状态历史'), ('tracking_json', '物流轨迹')):
|
||
value = _cell_text(row[field])
|
||
if not value:
|
||
row[field] = []
|
||
continue
|
||
try:
|
||
parsed = json.loads(value)
|
||
if not isinstance(parsed, list) or len(parsed) > 500:
|
||
raise ValueError
|
||
if field == 'history_json':
|
||
for event in parsed:
|
||
if (not isinstance(event, dict) or event.get('status') not in STATUS_LABELS
|
||
or not isinstance(event.get('time'), str) or not parse_time(event['time'])):
|
||
raise ValueError
|
||
else:
|
||
for event in parsed:
|
||
if not isinstance(event, dict) or event.get('code') not in NODE_LABELS:
|
||
raise ValueError
|
||
actual = event.get('actual_at')
|
||
estimated = event.get('estimated_at')
|
||
if not any(parse_time(item) for item in (actual, estimated)):
|
||
raise ValueError
|
||
for key in ('actual_at', 'estimated_at', 'observed_at', 'created_at'):
|
||
if event.get(key) is not None and not parse_time(event[key]):
|
||
raise ValueError
|
||
for key, limit in (('event_id', 128), ('shipment_key', 128), ('shipment_no', 128), ('place', 256)):
|
||
if event.get(key) is not None and (not isinstance(event[key], str) or len(event[key]) > limit):
|
||
raise ValueError
|
||
row[field] = parsed
|
||
except (ValueError, TypeError):
|
||
errors.append(f'{label}格式无效，请按模板要求填写有效JSON数组')
|
||
|
||
return {'row': line, **row}, errors
|
||
|
||
|
||
def _read_csv(content):
|
||
try:
|
||
text = content.decode('utf-8-sig')
|
||
except UnicodeDecodeError:
|
||
try:
|
||
text = content.decode('gb18030')
|
||
except UnicodeDecodeError as error:
|
||
raise ImportFileError('CSV编码无法识别，请另存为UTF-8格式。') from error
|
||
try:
|
||
rows = []
|
||
for row in csv.reader(io.StringIO(text, newline='')):
|
||
rows.append(row)
|
||
if len(rows) > MAX_ROWS + 1:
|
||
raise ImportFileError(f'单次最多导入{MAX_ROWS}条订单。')
|
||
return rows
|
||
except csv.Error as error:
|
||
raise ImportFileError('CSV格式无效，请检查引号或分隔符。') from error
|
||
|
||
|
||
def _read_xlsx(content):
|
||
try:
|
||
with zipfile.ZipFile(io.BytesIO(content)) as archive:
|
||
if len(archive.infolist()) > 100 or sum(item.file_size for item in archive.infolist()) > MAX_XLSX_UNPACKED_BYTES:
|
||
raise ImportFileError('Excel文件内容过大，无法安全读取。')
|
||
workbook = load_workbook(io.BytesIO(content), read_only=True, data_only=False)
|
||
sheet = workbook.active
|
||
if sheet.max_column > len(IMPORT_COLUMNS):
|
||
raise ImportFileError(f'Excel最多允许{len(IMPORT_COLUMNS)}列。')
|
||
rows = []
|
||
for cells in sheet.iter_rows(max_col=len(IMPORT_COLUMNS)):
|
||
values = []
|
||
for cell in cells:
|
||
if cell.data_type == 'f':
|
||
raise ImportFileError(f'Excel第{cell.row}行含公式，请粘贴为纯文本或数值后再上传。')
|
||
values.append(cell.value)
|
||
rows.append(values)
|
||
if len(rows) > MAX_ROWS + 1:
|
||
raise ImportFileError(f'单次最多导入{MAX_ROWS}条订单。')
|
||
return rows
|
||
except ImportFileError:
|
||
raise
|
||
except (zipfile.BadZipFile, OSError, ValueError, KeyError) as error:
|
||
raise ImportFileError('Excel文件无法读取，请检查文件是否损坏。') from error
|
||
finally:
|
||
if 'workbook' in locals():
|
||
workbook.close()
|
||
|
||
|
||
def parse_order_upload(filename, content):
|
||
"""Parse a CSV/XLSX file. Return valid rows with source line numbers and row errors."""
|
||
suffix = Path(filename or '').suffix.lower()
|
||
if suffix not in ('.csv', '.xlsx'):
|
||
raise ImportFileError('只支持CSV或XLSX文件。')
|
||
if not content:
|
||
raise ImportFileError('请选择非空文件。')
|
||
if len(content) > MAX_UPLOAD_BYTES:
|
||
raise ImportFileError('文件不能超过5 MB。')
|
||
table = _read_csv(content) if suffix == '.csv' else _read_xlsx(content)
|
||
if not table:
|
||
raise ImportFileError('文件内容为空。')
|
||
|
||
header_line, headers = None, None
|
||
for line, values in enumerate(table, start=1):
|
||
if any(_cell_text(value) for value in values):
|
||
header_line, headers = line, values
|
||
break
|
||
if headers is None:
|
||
raise ImportFileError('文件内容为空。')
|
||
fields = []
|
||
seen_fields = set()
|
||
for value in headers:
|
||
name = _cell_text(value).strip().casefold()
|
||
if not name:
|
||
fields.append(None)
|
||
continue
|
||
field = HEADER_TO_FIELD.get(name)
|
||
if not field:
|
||
raise ImportFileError(f'第{header_line}行包含未知列“{_cell_text(value)}”，请使用模板中的列名。')
|
||
if field in seen_fields:
|
||
raise ImportFileError(f'第{header_line}行的“{_cell_text(value)}”列重复。')
|
||
seen_fields.add(field)
|
||
fields.append(field)
|
||
missing = REQUIRED_FIELDS.difference(seen_fields)
|
||
if missing:
|
||
labels = dict(IMPORT_COLUMNS)
|
||
raise ImportFileError('缺少必填列：' + '、'.join(labels[item] for item in sorted(missing)) + '。')
|
||
|
||
rows, errors, seen = [], [], set()
|
||
total = 0
|
||
for line, values in enumerate(table[header_line:], start=header_line + 1):
|
||
if not any(_cell_text(value) for value in values):
|
||
continue
|
||
total += 1
|
||
if total > MAX_ROWS:
|
||
raise ImportFileError(f'单次最多导入{MAX_ROWS}条订单。')
|
||
raw = {field: values[index] if index < len(values) else '' for index, field in enumerate(fields) if field}
|
||
if any(_cell_text(value) for value in values[len(fields):]):
|
||
if len(errors) < MAX_ERRORS:
|
||
errors.append({'row': line, 'message': '数据列数多于表头，请检查是否有多余单元格。'})
|
||
continue
|
||
if any(_cell_text(values[index]) for index, field in enumerate(fields) if field is None and index < len(values)):
|
||
if len(errors) < MAX_ERRORS:
|
||
errors.append({'row': line, 'message': '存在未命名的数据列，请删除该列或填写有效表头。'})
|
||
continue
|
||
parsed, row_errors = _validate_row(raw, line)
|
||
if not row_errors:
|
||
key = (parsed['tenant_id'], parsed['order_no'].casefold())
|
||
if key in seen:
|
||
row_errors.append('文件中出现重复的租户ID和订单号')
|
||
else:
|
||
seen.add(key)
|
||
if row_errors:
|
||
if len(errors) < MAX_ERRORS:
|
||
errors.append({'row': line, 'message': '；'.join(row_errors)})
|
||
else:
|
||
rows.append(parsed)
|
||
if total == 0:
|
||
raise ImportFileError('文件中没有订单数据。')
|
||
return {'rows': rows, 'errors': errors, 'total_rows': total}
|
||
|
||
|
||
def mark_existing_rows(connection, rows):
|
||
"""Annotate preview rows already present; import_records preserves those rows."""
|
||
duplicate_count = 0
|
||
for row in rows:
|
||
exists = connection.execute('SELECT 1 FROM order_test_data WHERE tenant_id=? AND order_no=?',
|
||
(row['tenant_id'], row['order_no'])).fetchone()
|
||
row['duplicate'] = bool(exists)
|
||
duplicate_count += bool(exists)
|
||
return duplicate_count
|
||
|
||
|
||
def validate_tenants(connection, parsed):
|
||
"""Require an imported tenant to be registered by config or a customer account."""
|
||
known = {row[0] for row in connection.execute(
|
||
"SELECT tenant_id FROM order_test_data WHERE tenant_id<>'' "
|
||
"UNION SELECT tenant_id FROM customer_users WHERE tenant_id<>'' "
|
||
"UNION SELECT tenant_id FROM customer_tenant_grants WHERE tenant_id<>''")}
|
||
from .config import get_setting
|
||
for key in ('ORDER_DATA_TENANT_ID', 'CUSTOMER_DEMO_TENANT_ID'):
|
||
configured = get_setting(key)
|
||
if configured:
|
||
known.add(str(configured))
|
||
rows, errors = [], list(parsed['errors'])
|
||
for row in parsed['rows']:
|
||
if row['tenant_id'] not in known:
|
||
if len(errors) < MAX_ERRORS:
|
||
errors.append({'row': row['row'], 'message': '租户ID未登记，请先关联客户账号或配置租户。'})
|
||
else:
|
||
rows.append(row)
|
||
return {**parsed, 'rows': rows, 'errors': errors}
|