test: verify fee void ACK loss with original event recovery

This commit is contained in:
2026-10-03 09:02:25 +08:00
parent 912e69aeb2
commit 4968745764
7 changed files with 688 additions and 16 deletions
+139 -11
View File
@@ -6,6 +6,7 @@ Only the original DEAD event can be retried through its protected HTTP API.
"""
import base64
import copy
import decimal
import hashlib
import http.client
import json
@@ -103,21 +104,34 @@ def private_write(path, value):
def scope_directory(scope, direction='LOADING'):
if direction not in ('LOADING', 'UNLOAD', 'DELIVERY'):
if direction not in ('LOADING', 'UNLOAD', 'DELIVERY', 'FEE_VOID'):
raise RuntimeError('INVALID_DIRECTION')
marker = scope.get('marker', '')
if not re.fullmatch(r'[A-Za-z0-9_-]{1,96}', marker):
raise RuntimeError('INVALID_SCOPE')
keys = ('customerId', 'deliveryId', 'sourceInboundId', 'sourceOrderId', 'rootOrderId') if direction == 'DELIVERY' else ('customerId', 'containerBookingId') if direction == 'UNLOAD' else ('customerId', 'loadingPlanId', 'containerBookingId')
keys = ('customerId', 'sourceFeeId', 'sourceVersion', 'sourceType') if direction == 'FEE_VOID' else ('customerId', 'deliveryId', 'sourceInboundId', 'sourceOrderId', 'rootOrderId') if direction == 'DELIVERY' else ('customerId', 'containerBookingId') if direction == 'UNLOAD' else ('customerId', 'loadingPlanId', 'containerBookingId')
if direction != 'LOADING' and set(scope) != {'marker', *keys}:
raise RuntimeError('INVALID_SCOPE')
if direction == 'FEE_VOID' and scope.get('sourceType') not in FEE_SOURCE_TYPES:
raise RuntimeError('INVALID_FEE_SOURCE_TYPE')
if direction == 'FEE_VOID' and (not str(scope.get('sourceVersion', '')).isdigit() or not 0 < int(scope['sourceVersion']) <= 2147483647):
raise RuntimeError('INVALID_SOURCE_VERSION')
for key in keys:
if key == 'sourceType':
continue
if not re.fullmatch(r'[1-9][0-9]{0,19}', scope.get(key, '')):
raise RuntimeError('INVALID_SCOPE')
identity = direction + ':' + marker if direction != 'LOADING' else marker
return Path(tempfile.gettempdir()) / ('chongchong-event-' + hashlib.sha256(identity.encode()).hexdigest()[:24])
FEE_SOURCE_TYPES = ('OVERSEAS_INBOUND_FEE', 'OVERSEAS_PROCUREMENT_FEE', 'OVERSEAS_TRUCK_FEE', 'OVERSEAS_LOOSEBOARD_FEE', 'OVERSEAS_WAREHOUSE_FEE', 'OVERSEAS_CONTAINER_FEE', 'OVERSEAS_ORDER_FEE', 'OVERSEAS_DELIVERY_FEE', 'OVERSEAS_RELABEL_FEE')
def fee_void(state):
return state.get('direction') == 'FEE_VOID'
def unload(state):
return state.get('direction', 'LOADING') == 'UNLOAD'
@@ -127,7 +141,7 @@ def delivery(state):
def reverse_delivery(state):
return state.get('direction') in ('UNLOAD', 'DELIVERY')
return state.get('direction') in ('UNLOAD', 'DELIVERY', 'FEE_VOID')
def producer_name(state):
@@ -135,6 +149,8 @@ def producer_name(state):
def target_setting(state):
if fee_void(state):
return ('FINANCE_SERVICE_URL', 'http://finance:9203')
return ('FIRSTLEG_SERVICE_URL', 'http://firstleg:9202') if reverse_delivery(state) else ('LASTLEG_SERVICE_URL', 'http://lastleg-service:9303')
@@ -167,7 +183,7 @@ def read_sql(container, sql):
if re.search(r'\b(INSERT|UPDATE|DELETE|ALTER|DROP|TRUNCATE|CALL|INTO|OUTFILE)\b', sql, re.I):
raise RuntimeError('NON_READ_ONLY_SQL')
e = env_of(container)
prefix = {'/firstleg-test-business': 'FIRSTLEG', '/firstleg-test-lastleg': 'LASTLEG', '/firstleg-test-order': 'ORDER'}[container['Name']]
prefix = {'/firstleg-test-business': 'FIRSTLEG', '/firstleg-test-lastleg': 'LASTLEG', '/firstleg-test-order': 'ORDER', '/firstleg-test-finance': 'FINANCE'}[container['Name']]
url = urllib.parse.urlsplit(e[prefix + '_DB_URL'].removeprefix('jdbc:'))
host = '127.0.0.1' if url.hostname == 'host.docker.internal' else url.hostname
transaction = 'SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ; START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY;\n' + sql + '\nCOMMIT;'
@@ -182,6 +198,8 @@ def read_sql(container, sql):
def producer(scope, direction='LOADING'):
if direction == 'FEE_VOID':
return fee_event(scope, 'VOIDED')
if direction == 'DELIVERY':
rows = read_sql(inspect('firstleg-test-lastleg'),
"SELECT JSON_OBJECT('entryId',CAST(id AS CHAR),'eventId',JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.eventId')),'payloadHash',JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.payloadHash')),'eventType',event_type,'customerId',customer_id,'aggregateId',aggregate_id,'aggregateType',aggregate_type,'eventKey',event_key,'envelope',JSON_EXTRACT(payload_json,'$'),'publishStatus',status,'failedAttempts',retry_count) FROM ll_event_outbox WHERE customer_id='" + scope['customerId'] + "' AND event_type='OverseasDeliveryCompleted' AND JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.payload.deliveryOrderId'))='" + scope['deliveryId'] + "';")
@@ -212,7 +230,101 @@ def business_ids(scope):
return [row['id'] for row in rows]
def fee_canonical(value):
"""Match EventPayloadCanonicalizer's sorted JSON and stripped BigDecimal.toString."""
if isinstance(value, dict):
return '{' + ','.join(json.dumps(key, ensure_ascii=False) + ':' + fee_canonical(value[key]) for key in sorted(value)) + '}'
if isinstance(value, list):
return '[' + ','.join(fee_canonical(item) for item in value) + ']'
if isinstance(value, decimal.Decimal):
with decimal.localcontext() as context:
context.prec = max(28, len(value.as_tuple().digits))
number = value.normalize()
if not number:
return '0'
return format(number, 'f') if number.as_tuple().exponent <= 0 and number.adjusted() >= -6 else str(number)
return json.dumps(value, ensure_ascii=False, separators=(',', ':'))
def fee_event(scope, action):
if action not in ('PUBLISHED', 'ADJUSTED', 'VOIDED'):
raise RuntimeError('INVALID_FEE_ACTION')
event_key = 'BusinessFeeSourceChanged:LASTLEG:' + scope['sourceType'] + ':' + scope['sourceFeeId'] + ':' + scope['sourceVersion'] + ':' + action
rows = read_sql(inspect('firstleg-test-lastleg'),
"SELECT JSON_OBJECT('entryId',CAST(id AS CHAR),'eventId',JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.eventId')),'payloadHash',JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.payloadHash')),'eventType',event_type,'customerId',customer_id,'aggregateId',aggregate_id,'aggregateType',aggregate_type,'eventKey',event_key,'envelope',JSON_EXTRACT(payload_json,'$'),'envelopeJson',payload_json,'publishStatus',status,'failedAttempts',retry_count) FROM ll_event_outbox WHERE event_type='BusinessFeeSourceChanged' AND event_key='" + event_key + "';")
if len(rows) > 1:
raise RuntimeError('AMBIGUOUS_FEE_EVENT')
if not rows:
return None
row = rows[0]
payload = json.loads(row['envelopeJson'], parse_float=decimal.Decimal, parse_int=decimal.Decimal)['payload']
row['payloadCanonicalJson'] = fee_canonical(payload)
unsigned = {**payload, 'action': None, 'sourceStatus': 'ACTIVE', 'sourceHash': None}
row['sourceCanonicalJson'] = fee_canonical(unsigned)
if hashlib.sha256(row['payloadCanonicalJson'].encode()).hexdigest() != row['payloadHash'].removeprefix('sha256:') or payload.get('sourceHash') != 'sha256:' + hashlib.sha256(row['sourceCanonicalJson'].encode()).hexdigest():
raise RuntimeError('FEE_PAYLOAD_HASH_MISMATCH')
if row['customerId'] != scope['customerId'] or row['aggregateId'] != scope['sourceFeeId'] or row['aggregateType'] != scope['sourceType']:
raise RuntimeError('FEE_EVENT_SCOPE_MISMATCH')
if payload.get('sourceSystem') != 'LASTLEG' or payload.get('customerId') != scope['customerId'] or payload.get('sourceFeeId') != scope['sourceFeeId'] or str(payload.get('sourceVersion')) != scope['sourceVersion'] or payload.get('sourceType') != scope['sourceType'] or payload.get('action') != action:
raise RuntimeError('FEE_PAYLOAD_SCOPE_MISMATCH')
return row
def fee_inbox(scope, event_id):
if not re.fullmatch(r'[0-9a-fA-F-]{36}|[0-9A-HJKMNP-TV-Z]{26}', event_id):
raise RuntimeError('INVALID_EVENT_IDENTITY')
return read_sql(inspect('firstleg-test-finance'),
"SELECT JSON_OBJECT('id',CAST(id AS CHAR),'eventId',event_id,'sourceSystem',source_system,'eventType',event_type,'envelope',JSON_EXTRACT(payload,'$'),'payloadHash',payload_hash,'resultResourceId',CAST(result_resource_id AS CHAR),'status',status,'errorCode',error_code,'receivedAt',CAST(received_at AS CHAR),'processedAt',CAST(processed_at AS CHAR)) FROM fin_inbox_event WHERE source_system='LASTLEG' AND event_id='" + event_id + "';")
def fee_facts(scope, event_id):
lastleg, finance = inspect('firstleg-test-lastleg'), inspect('firstleg-test-finance')
require_test(lastleg, 'firstleg-test-lastleg')
require_test(finance, 'firstleg-test-finance')
# Scalar decimals remain strings. Original event decimals are retained in envelopeJson.
def columns(names):
return ','.join("'" + name + "',CAST(" + re.sub(r'([A-Z])', lambda match: '_' + match[1].lower(), name) + " AS CHAR)" for name in names.split())
local_names = 'id sourceFeeId sourceVersion customerId customerName sourceType sourceNo businessType businessId businessNo orderId inboundId inboundCode containerId containerBookingId containerNo warehouseId feeItemCode feeItemName direction supplierId supplierName supplierCategory settlementMode paidAt paymentMethod paymentReferenceNo businessDate currency amount originalCurrency originalAmount exchangeRate exchangeRateMonth chargeableQuantity quantityUnit unitPrice periodStart periodEnd reviewStatus publicationStatus sourceStatus reviewReason reviewerId reviewedAt createdBy updatedBy createdAt updatedAt'
local = read_sql(lastleg, "SELECT JSON_OBJECT(" + columns(local_names) + ",'paymentVoucherFileIds',payment_voucher_file_ids) FROM ll_finance_fee_source WHERE source_fee_id=" + scope['sourceFeeId'] + " ORDER BY source_version,id;")
where = "source_system='LASTLEG' AND source_type='" + scope['sourceType'] + "' AND source_fee_id=" + scope['sourceFeeId']
finance_names = 'id sourceSystem sourceFeeId sourceFeeVersion customerId sourceType sourceNo businessType businessId businessNo inboundCode warehouseId direction orderId inboundId containerId containerBookingId containerNo feeItemCode feeItemName amount currencyCode settlementMode paidAt paymentMethod paymentReferenceNo sourceHash reviewStatus reviewReason operationVersion walletHoldId status createdAt updatedAt'
ids = 'SELECT id FROM fin_fee_source WHERE ' + where
rows = read_sql(finance,
"SELECT JSON_OBJECT('kind','source'," + columns(finance_names) + ",'sourceSnapshot',source_snapshot,'paymentVoucherFileIds',payment_voucher_file_ids) FROM fin_fee_source WHERE " + where + " ORDER BY source_fee_version,id;\n"
+ "SELECT JSON_OBJECT('kind','bill'," + columns('id tenantId billId sourceId sourceVersion amount currencyCode createdAt') + ") FROM fin_bill_line WHERE source_id IN (" + ids + ") ORDER BY id;\n"
+ "SELECT JSON_OBJECT('kind','settlement'," + columns('id tenantId settlementId feeSourceId amount currencyCode createdAt') + ") FROM fin_settlement_line WHERE fee_source_id IN (" + ids + ") ORDER BY id;\n"
+ "SELECT JSON_OBJECT('kind','cost'," + columns('id tenantId sourceSystem sourceType sourceFeeId sourceFeeVersion sourceHash status version costAmount createdAt voidedAt') + ") FROM fin_cost WHERE " + where + " ORDER BY id;")
return {'lastleg': local, 'finance': {**{kind: [row for row in rows if row['kind'] == kind] for kind in ('source', 'bill', 'settlement', 'cost')}, 'inbox': fee_inbox(scope, event_id)}}
def fee_prerequisite(scope, wait=False):
if fee_event(scope, 'VOIDED') is not None:
raise RuntimeError('FEE_VOID_EVENT_ALREADY_EXISTS')
publication = fee_event(scope, 'PUBLISHED' if scope['sourceVersion'] == '1' else 'ADJUSTED')
if not publication or publication['publishStatus'] not in ('PENDING', 'PROCESSING', 'PUBLISHED'):
raise RuntimeError('FEE_PUBLICATION_NOT_READY')
facts = fee_facts(scope, publication['eventId'])
current = [row for row in facts['lastleg'] if row['sourceVersion'] == scope['sourceVersion']]
financial = [row for row in facts['finance']['source'] if row['sourceFeeVersion'] == scope['sourceVersion']]
if len(current) != 1 or len(financial) != 1 or any(row['customerId'] != scope['customerId'] or row['sourceType'] != scope['sourceType'] for row in facts['lastleg'] + facts['finance']['source']):
raise RuntimeError('FEE_SOURCE_SCOPE_MISMATCH')
if max(int(row['sourceVersion']) for row in facts['lastleg']) != int(scope['sourceVersion']) or current[0]['sourceStatus'] != 'ACTIVE' or current[0]['reviewStatus'] != 'APPROVED' or current[0]['publicationStatus'] != 'PUBLISHED' or current[0]['direction'] != 'RECEIVABLE':
raise RuntimeError('FEE_SOURCE_NOT_ACTIVE_RECEIVABLE')
if financial[0]['status'] != 'AVAILABLE' or financial[0]['walletHoldId'] is not None or any(facts['finance'][key] for key in ('bill', 'settlement', 'cost')):
raise RuntimeError('FEE_SOURCE_ALREADY_FINANCIALLY_LINKED')
inbox = facts['finance']['inbox']
if len(inbox) != 1 or inbox[0]['status'] != 'PROCESSED' or inbox[0]['payloadHash'] != publication['payloadHash'] or inbox[0]['resultResourceId'] != financial[0]['id']:
raise RuntimeError('FEE_PUBLICATION_INBOX_MISMATCH')
attempts = read_sql(inspect('firstleg-test-lastleg'), "SELECT JSON_OBJECT('entryId',entry_id,'eventId',event_id,'outcome',outcome) FROM event_delivery_attempt WHERE entry_id='" + publication['entryId'] + "' AND event_type='BusinessFeeSourceChanged' ORDER BY started_at,attempt_id;")
ready = publication['publishStatus'] == 'PUBLISHED' and any(row['eventId'] == publication['eventId'] and row['outcome'] in ('APPLIED', 'DUPLICATE') for row in attempts)
if not ready and not wait:
raise RuntimeError('FEE_PUBLICATION_ACK_NOT_READY')
return {'ready': ready, 'publication': publication, 'business': facts, 'attempts': attempts}
def business_snapshot(scope, state, row):
if fee_void(state):
return {'fee': fee_facts(scope, row['eventId'])}
if delivery(state):
return {'business': delivery_facts(scope, row['eventId'])}
return {'container': container_facts(scope)} if unload(state) else {'businessOrderIds': business_ids(scope)}
@@ -245,10 +357,11 @@ def delivery_facts(scope, event_id):
def prepare(scope, directory, config):
if delivery(config) and producer(scope, 'DELIVERY') is not None:
raise RuntimeError('DELIVERY_EVENT_ALREADY_EXISTS')
prerequisite = fee_prerequisite(scope) if fee_void(config) else None
directory.mkdir(mode=0o700)
original = inspect(producer_name(config))
require_test(original, producer_name(config))
consumer_name = 'firstleg-test-business' if reverse_delivery(config) else 'firstleg-test-lastleg'
consumer_name = 'firstleg-test-finance' if fee_void(config) else 'firstleg-test-business' if reverse_delivery(config) else 'firstleg-test-lastleg'
consumer = inspect(consumer_name)
require_test(consumer, consumer_name)
old_env = env_of(original)
@@ -268,10 +381,12 @@ def prepare(scope, directory, config):
proxy.write_bytes(base64.b64decode(config['proxySource']))
ip = consumer['NetworkSettings']['Networks']['firstleg-test-network']['IPAddress']
log = open(directory / 'proxy-output.log', 'wb')
upstream = ':9202/internal/firstleg/v1/container-events' if reverse_delivery(config) else ':9303/internal/lastleg/events'
selector = [] if delivery(config) else ['--event-type', 'OverseasContainerUnloaded', '--container-booking-id', scope['containerBookingId']] if unload(config) else ['--event-type', 'FirstLegLoadingPlanCompleted', '--loading-plan-id', scope['loadingPlanId']]
upstream = ':9203/internal/finance/v1/business-fee-source-events' if fee_void(config) else ':9202/internal/firstleg/v1/container-events' if reverse_delivery(config) else ':9303/internal/lastleg/events'
selector = [] if (delivery(config) or fee_void(config)) else ['--event-type', 'OverseasContainerUnloaded', '--container-booking-id', scope['containerBookingId']] if unload(config) else ['--event-type', 'FirstLegLoadingPlanCompleted', '--loading-plan-id', scope['loadingPlanId']]
if delivery(config):
selector = ['--event-type', 'OverseasDeliveryCompleted', '--delivery-id', scope['deliveryId']]
if fee_void(config):
selector = ['--event-type', 'BusinessFeeSourceChanged', '--fee-source-id', scope['sourceFeeId'], '--source-version', scope['sourceVersion']]
process = subprocess.Popen(['python3', str(proxy), '--upstream', 'http://' + ip + upstream, *selector,
'--bind', '0.0.0.0', '--port', str(ports[0]), '--control-port', str(ports[1]),
'--log', str(directory / 'proxy-evidence.jsonl')],
@@ -279,9 +394,9 @@ def prepare(scope, directory, config):
state = {'scope': scope, 'originalBody': original_body, 'originalId': original['Id'], 'image': original['Config']['Image'],
'direction': config.get('direction', 'LOADING'), 'originalImageId': original['Image'], 'originalFingerprint': fingerprint(original),
'proxyPid': process.pid, 'proxyPort': ports[0], 'controlPort': ports[1], 'controlToken': token,
'operationsRequestId': str(uuid.uuid4()), 'prepared': False, 'restored': False}
'prerequisite': prerequisite, 'operationsRequestId': str(uuid.uuid4()), 'prepared': False, 'restored': False}
private_write(directory / 'state.json', state)
return {'created': True, 'originalImageId': state['originalImageId'], 'originalFingerprint': state['originalFingerprint']}
return {'created': True, 'originalImageId': state['originalImageId'], 'originalFingerprint': state['originalFingerprint'], **({'prerequisite': prerequisite} if fee_void(config) else {})}
def activate(scope, directory, state):
@@ -316,6 +431,12 @@ def recover(scope, directory, state):
if pinned['eventId'] != row['eventId'] or pinned['payloadHash'] != row['payloadHash']:
raise RuntimeError('PINNED_EVENT_MISMATCH')
state['beforeRetry'] = {**row, **business_snapshot(scope, state, row)}
if fee_void(state):
facts = state['beforeRetry']['fee']
local = [value for value in facts['lastleg'] if value['sourceVersion'] == scope['sourceVersion']]
financial = [value for value in facts['finance']['source'] if value['sourceFeeVersion'] == scope['sourceVersion']]
if len(local) != 1 or local[0]['sourceStatus'] != 'ACTIVE' or len(financial) != 1 or financial[0]['status'] != 'VOIDED':
raise RuntimeError('FEE_VOID_DEAD_PHASE_MISMATCH')
private_write(directory / 'state.json', state)
control(state, 'PASS')
container = inspect(producer_name(state))
@@ -375,7 +496,7 @@ def proof(scope, state):
event_id, entry_id = row['eventId'], row['entryId']
if not re.fullmatch(r'[0-9a-fA-F-]{36}|[0-9A-HJKMNP-TV-Z]{26}', event_id) or not re.fullmatch(r'[1-9][0-9]{0,19}', entry_id):
raise RuntimeError('INVALID_EVENT_IDENTITY')
event_type = 'OverseasDeliveryCompleted' if delivery(state) else 'OverseasContainerUnloaded' if unload(state) else 'FirstLegLoadingPlanCompleted'
event_type = 'BusinessFeeSourceChanged' if fee_void(state) else 'OverseasDeliveryCompleted' if delivery(state) else 'OverseasContainerUnloaded' if unload(state) else 'FirstLegLoadingPlanCompleted'
attempts = read_sql(inspect(producer_name(state)),
"SELECT JSON_OBJECT('attemptId',attempt_id,'serviceName',service_name,'entryId',entry_id,'eventId',event_id,'targetName',target_name,'outcome',outcome) FROM event_delivery_attempt WHERE entry_id='"
+ entry_id + "' AND event_type='" + event_type + "' ORDER BY started_at,attempt_id;")
@@ -383,7 +504,9 @@ def proof(scope, state):
"SELECT JSON_OBJECT('requestId',request_id,'entryId',entry_id,'eventId',event_id,'payloadHash',payload_hash,'previousFailures',previous_failures) FROM event_delivery_retry_audit WHERE request_id='"
+ state['operationsRequestId'] + "';" if not reverse_delivery(state) else
"SELECT JSON_OBJECT('requestId',request_id,'entryId',entry_id,'eventId',event_id,'payloadHash',payload_hash,'previousFailures',previous_failures) FROM event_delivery_retry_audit WHERE entry_id='" + entry_id + "' AND event_type='" + event_type + "';")
if reverse_delivery(state):
if fee_void(state):
consumers = fee_inbox(scope, row['eventId'])
elif reverse_delivery(state):
consumers = read_sql(inspect('firstleg-test-business'),
"SELECT JSON_OBJECT('eventId',event_id,'payloadHash',payload_hash,'status',status,'eventType',event_type,'aggregateId',aggregate_id,'payload',JSON_EXTRACT(payload_json,'$')) FROM fl_container_event_inbox WHERE event_id='" + event_id + "';")
else:
@@ -395,6 +518,9 @@ def proof(scope, state):
'proxyRecords': control(state)['records'], 'beforeRetry': state['beforeRetry'],
'afterRetry': {**row, **business_snapshot(scope, state, row)},
'retryAudit': audit[0] if len(audit) == 1 else None, 'retryAuditCount': len(audit), 'retryRequestId': state['operationsRequestId'], 'restored': False}
if fee_void(state):
result['prerequisite'] = state['prerequisite']
result['publicationNow'] = fee_event(scope, 'PUBLISHED' if scope['sourceVersion'] == '1' else 'ADJUSTED')
private_write(scope_directory(scope, state.get('direction', 'LOADING')) / 'proof.json', result)
return result
@@ -440,6 +566,8 @@ def main(config):
scope, action = config['scope'], config['action']
direction = config.get('direction', 'LOADING')
directory = scope_directory(scope, direction)
if action == 'preflight' and direction == 'FEE_VOID':
return fee_prerequisite(scope, wait=True)
if action == 'prepare':
return prepare(scope, directory, config)
state = json.loads((directory / 'state.json').read_text())