"""Safe parsing and row validation for staff-managed order imports.""" import csv from datetime import date, datetime from decimal import Decimal, InvalidOperation import io import json from pathlib import Path import zipfile from openpyxl import load_workbook from .order_data import STATUS_LABELS from .order_tracking import NODE_LABELS, parse_time MAX_UPLOAD_BYTES = 5 * 1024 * 1024 MAX_ROWS = 5000 MAX_ERRORS = 100 MAX_XLSX_UNPACKED_BYTES = 25 * 1024 * 1024 IMPORT_COLUMNS = ( ('source_order_id', '来源订单ID'), ('tenant_id', '租户ID'), ('order_no', '订单号'), ('inbound_code', '入仓码'), ('shipment_id', 'Shipment ID'), ('status', '业务状态'), ('country_code', '目的国'), ('transport_mode', '运输方式'), ('warehouse_code', '目的仓'), ('order_role', '订单层级'), ('parent_order_no', '父订单号'), ('package_count', '件数'), ('gross_weight_kg', '毛重(kg)'), ('volume_cbm', '体积(m³)'), ('history_json', '业务状态历史(JSON)'), ('tracking_json', '物流轨迹(JSON)'), ('data_note', '数据备注'), ('source_updated_at', '来源更新时间'), ('snapshot_at', '数据快照时间'), ) HEADER_TO_FIELD = {header.casefold(): field for field, header in IMPORT_COLUMNS} HEADER_TO_FIELD.update({field.casefold(): field for field, _ in IMPORT_COLUMNS}) REQUIRED_FIELDS = {'tenant_id', 'order_no'} TEXT_LIMITS = { 'source_order_id': 128, 'tenant_id': 128, 'order_no': 128, 'inbound_code': 128, 'shipment_id': 128, 'status': 64, 'country_code': 16, 'transport_mode': 32, 'warehouse_code': 64, 'order_role': 32, 'parent_order_no': 128, 'data_note': 2000, 'source_updated_at': 64, 'snapshot_at': 64, } class ImportFileError(ValueError): """The uploaded file cannot be interpreted as an order table.""" def template_csv(): """Return an Excel-friendly UTF-8 CSV template.""" output = io.StringIO(newline='') writer = csv.writer(output) writer.writerow([header for _, header in IMPORT_COLUMNS]) return output.getvalue().encode('utf-8-sig') def _cell_text(value): if value is None: return '' if isinstance(value, (datetime, date)): return value.isoformat(sep=' ') if isinstance(value, datetime) else value.isoformat() if isinstance(value, bool): return 'TRUE' if value else 'FALSE' if isinstance(value, float): if not value.is_integer(): return format(value, '.15g') return str(int(value)) return str(value).strip() def _validate_row(raw, line): errors = [] row = {field: raw.get(field, '') for field, _ in IMPORT_COLUMNS} for field, limit in TEXT_LIMITS.items(): value = _cell_text(row[field]) row[field] = value or None if field in REQUIRED_FIELDS and not value: errors.append(f'{dict(IMPORT_COLUMNS)[field]}不能为空') elif len(value) > limit: errors.append(f'{dict(IMPORT_COLUMNS)[field]}不能超过{limit}个字符') if not row['status']: row['status'] = 'DRAFT' row['status'] = row['status'].upper() if row['status'] not in STATUS_LABELS: errors.append('业务状态不在允许范围内') if row['order_role']: row['order_role'] = row['order_role'].upper() if row['order_role'] not in ('MASTER', 'CHILD'): errors.append('订单层级只能是MASTER或CHILD') package_value = _cell_text(row['package_count']) if package_value: try: number = Decimal(package_value) if not number.is_finite() or number < 0 or number != number.to_integral_value(): raise InvalidOperation row['package_count'] = int(number) except (InvalidOperation, ValueError): errors.append('件数必须是大于或等于0的整数') else: row['package_count'] = None for field, label in (('gross_weight_kg', '毛重'), ('volume_cbm', '体积')): value = _cell_text(row[field]) if not value: row[field] = None continue try: number = Decimal(value) if not number.is_finite() or number < 0: raise InvalidOperation row[field] = format(number.normalize(), 'f') except (InvalidOperation, ValueError): errors.append(f'{label}必须是大于或等于0的数字') for field, label in (('history_json', '业务状态历史'), ('tracking_json', '物流轨迹')): value = _cell_text(row[field]) if not value: row[field] = [] continue try: parsed = json.loads(value) if not isinstance(parsed, list) or len(parsed) > 500: raise ValueError if field == 'history_json': for event in parsed: if (not isinstance(event, dict) or event.get('status') not in STATUS_LABELS or not isinstance(event.get('time'), str) or not parse_time(event['time'])): raise ValueError else: for event in parsed: if not isinstance(event, dict) or event.get('code') not in NODE_LABELS: raise ValueError actual = event.get('actual_at') estimated = event.get('estimated_at') if not any(parse_time(item) for item in (actual, estimated)): raise ValueError for key in ('actual_at', 'estimated_at', 'observed_at', 'created_at'): if event.get(key) is not None and not parse_time(event[key]): raise ValueError for key, limit in (('event_id', 128), ('shipment_key', 128), ('shipment_no', 128), ('place', 256)): if event.get(key) is not None and (not isinstance(event[key], str) or len(event[key]) > limit): raise ValueError row[field] = parsed except (ValueError, TypeError): errors.append(f'{label}格式无效，请按模板要求填写有效JSON数组') return {'row': line, **row}, errors def _read_csv(content): try: text = content.decode('utf-8-sig') except UnicodeDecodeError: try: text = content.decode('gb18030') except UnicodeDecodeError as error: raise ImportFileError('CSV编码无法识别，请另存为UTF-8格式。') from error try: rows = [] for row in csv.reader(io.StringIO(text, newline='')): rows.append(row) if len(rows) > MAX_ROWS + 1: raise ImportFileError(f'单次最多导入{MAX_ROWS}条订单。') return rows except csv.Error as error: raise ImportFileError('CSV格式无效，请检查引号或分隔符。') from error def _read_xlsx(content): try: with zipfile.ZipFile(io.BytesIO(content)) as archive: if len(archive.infolist()) > 100 or sum(item.file_size for item in archive.infolist()) > MAX_XLSX_UNPACKED_BYTES: raise ImportFileError('Excel文件内容过大，无法安全读取。') workbook = load_workbook(io.BytesIO(content), read_only=True, data_only=False) sheet = workbook.active if sheet.max_column > len(IMPORT_COLUMNS): raise ImportFileError(f'Excel最多允许{len(IMPORT_COLUMNS)}列。') rows = [] for cells in sheet.iter_rows(max_col=len(IMPORT_COLUMNS)): values = [] for cell in cells: if cell.data_type == 'f': raise ImportFileError(f'Excel第{cell.row}行含公式，请粘贴为纯文本或数值后再上传。') values.append(cell.value) rows.append(values) if len(rows) > MAX_ROWS + 1: raise ImportFileError(f'单次最多导入{MAX_ROWS}条订单。') return rows except ImportFileError: raise except (zipfile.BadZipFile, OSError, ValueError, KeyError) as error: raise ImportFileError('Excel文件无法读取，请检查文件是否损坏。') from error finally: if 'workbook' in locals(): workbook.close() def parse_order_upload(filename, content): """Parse a CSV/XLSX file. Return valid rows with source line numbers and row errors.""" suffix = Path(filename or '').suffix.lower() if suffix not in ('.csv', '.xlsx'): raise ImportFileError('只支持CSV或XLSX文件。') if not content: raise ImportFileError('请选择非空文件。') if len(content) > MAX_UPLOAD_BYTES: raise ImportFileError('文件不能超过5 MB。') table = _read_csv(content) if suffix == '.csv' else _read_xlsx(content) if not table: raise ImportFileError('文件内容为空。') header_line, headers = None, None for line, values in enumerate(table, start=1): if any(_cell_text(value) for value in values): header_line, headers = line, values break if headers is None: raise ImportFileError('文件内容为空。') fields = [] seen_fields = set() for value in headers: name = _cell_text(value).strip().casefold() if not name: fields.append(None) continue field = HEADER_TO_FIELD.get(name) if not field: raise ImportFileError(f'第{header_line}行包含未知列“{_cell_text(value)}”，请使用模板中的列名。') if field in seen_fields: raise ImportFileError(f'第{header_line}行的“{_cell_text(value)}”列重复。') seen_fields.add(field) fields.append(field) missing = REQUIRED_FIELDS.difference(seen_fields) if missing: labels = dict(IMPORT_COLUMNS) raise ImportFileError('缺少必填列：' + '、'.join(labels[item] for item in sorted(missing)) + '。') rows, errors, seen = [], [], set() total = 0 for line, values in enumerate(table[header_line:], start=header_line + 1): if not any(_cell_text(value) for value in values): continue total += 1 if total > MAX_ROWS: raise ImportFileError(f'单次最多导入{MAX_ROWS}条订单。') raw = {field: values[index] if index < len(values) else '' for index, field in enumerate(fields) if field} if any(_cell_text(value) for value in values[len(fields):]): if len(errors) < MAX_ERRORS: errors.append({'row': line, 'message': '数据列数多于表头，请检查是否有多余单元格。'}) continue if any(_cell_text(values[index]) for index, field in enumerate(fields) if field is None and index < len(values)): if len(errors) < MAX_ERRORS: errors.append({'row': line, 'message': '存在未命名的数据列，请删除该列或填写有效表头。'}) continue parsed, row_errors = _validate_row(raw, line) if not row_errors: key = (parsed['tenant_id'], parsed['order_no'].casefold()) if key in seen: row_errors.append('文件中出现重复的租户ID和订单号') else: seen.add(key) if row_errors: if len(errors) < MAX_ERRORS: errors.append({'row': line, 'message': '；'.join(row_errors)}) else: rows.append(parsed) if total == 0: raise ImportFileError('文件中没有订单数据。') return {'rows': rows, 'errors': errors, 'total_rows': total} def mark_existing_rows(connection, rows): """Annotate preview rows already present; import_records preserves those rows.""" duplicate_count = 0 for row in rows: exists = connection.execute('SELECT 1 FROM order_test_data WHERE tenant_id=? AND order_no=?', (row['tenant_id'], row['order_no'])).fetchone() row['duplicate'] = bool(exists) duplicate_count += bool(exists) return duplicate_count def validate_tenants(connection, parsed): """Require an imported tenant to be registered by config or a customer account.""" known = {row[0] for row in connection.execute( "SELECT tenant_id FROM order_test_data WHERE tenant_id<>'' " "UNION SELECT tenant_id FROM customer_users WHERE tenant_id<>'' " "UNION SELECT tenant_id FROM customer_tenant_grants WHERE tenant_id<>''")} from .config import get_setting for key in ('ORDER_DATA_TENANT_ID', 'CUSTOMER_DEMO_TENANT_ID'): configured = get_setting(key) if configured: known.add(str(configured)) rows, errors = [], list(parsed['errors']) for row in parsed['rows']: if row['tenant_id'] not in known: if len(errors) < MAX_ERRORS: errors.append({'row': row['row'], 'message': '租户ID未登记，请先关联客户账号或配置租户。'}) else: rows.append(row) return {**parsed, 'rows': rows, 'errors': errors}