java-project/baiyue-firstleg
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(fba): verify signed delivery ACK loss and original event recovery

Browse Source
This commit is contained in:
ouyangxq
2026-10-03 08:13:04 +08:00
parent ef1eef3278
commit 51bf4e5558
11 changed files with 499 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test-event-fault-runtime.py
+71 -26
View File
@@ -103,18 +103,18 @@ def private_write(path, value):
def scope_directory(scope, direction='LOADING'):
if direction not in ('LOADING', 'UNLOAD'):
if direction not in ('LOADING', 'UNLOAD', 'DELIVERY'):
raise RuntimeError('INVALID_DIRECTION')
marker = scope.get('marker', '')
if not re.fullmatch(r'[A-Za-z0-9_-]{1,96}', marker):
raise RuntimeError('INVALID_SCOPE')
keys = ('customerId', 'containerBookingId') if direction == 'UNLOAD' else ('customerId', 'loadingPlanId', 'containerBookingId')
if direction == 'UNLOAD' and set(scope) != {'marker', *keys}:
keys = ('customerId', 'deliveryId', 'sourceInboundId', 'sourceOrderId', 'rootOrderId') if direction == 'DELIVERY' else ('customerId', 'containerBookingId') if direction == 'UNLOAD' else ('customerId', 'loadingPlanId', 'containerBookingId')
if direction != 'LOADING' and set(scope) != {'marker', *keys}:
raise RuntimeError('INVALID_SCOPE')
for key in keys:
if not re.fullmatch(r'[1-9][0-9]{0,19}', scope.get(key, '')):
raise RuntimeError('INVALID_SCOPE')
identity = 'UNLOAD:' + marker if direction == 'UNLOAD' else marker
identity = direction + ':' + marker if direction != 'LOADING' else marker
return Path(tempfile.gettempdir()) / ('chongchong-event-' + hashlib.sha256(identity.encode()).hexdigest()[:24])
@@ -122,12 +122,20 @@ def unload(state):
return state.get('direction', 'LOADING') == 'UNLOAD'
def delivery(state):
return state.get('direction') == 'DELIVERY'
def reverse_delivery(state):
return state.get('direction') in ('UNLOAD', 'DELIVERY')
def producer_name(state):
return 'firstleg-test-lastleg' if unload(state) else 'firstleg-test-business'
return 'firstleg-test-lastleg' if reverse_delivery(state) else 'firstleg-test-business'
def target_setting(state):
return ('FIRSTLEG_SERVICE_URL', 'http://firstleg:9202') if unload(state) else ('LASTLEG_SERVICE_URL', 'http://lastleg-service:9303')
return ('FIRSTLEG_SERVICE_URL', 'http://firstleg:9202') if reverse_delivery(state) else ('LASTLEG_SERVICE_URL', 'http://lastleg-service:9303')
def fingerprint(container):
@@ -159,7 +167,7 @@ def read_sql(container, sql):
if re.search(r'\b(INSERT|UPDATE|DELETE|ALTER|DROP|TRUNCATE|CALL|INTO|OUTFILE)\b', sql, re.I):
raise RuntimeError('NON_READ_ONLY_SQL')
e = env_of(container)
prefix = 'FIRSTLEG' if container['Name'] == '/firstleg-test-business' else 'LASTLEG'
prefix = {'/firstleg-test-business': 'FIRSTLEG', '/firstleg-test-lastleg': 'LASTLEG', '/firstleg-test-order': 'ORDER'}[container['Name']]
url = urllib.parse.urlsplit(e[prefix + '_DB_URL'].removeprefix('jdbc:'))
host = '127.0.0.1' if url.hostname == 'host.docker.internal' else url.hostname
transaction = 'SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ; START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY;\n' + sql + '\nCOMMIT;'
@@ -174,7 +182,10 @@ def read_sql(container, sql):
def producer(scope, direction='LOADING'):
if direction == 'UNLOAD':
if direction == 'DELIVERY':
rows = read_sql(inspect('firstleg-test-lastleg'),
"SELECT JSON_OBJECT('entryId',CAST(id AS CHAR),'eventId',JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.eventId')),'payloadHash',JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.payloadHash')),'eventType',event_type,'customerId',customer_id,'aggregateId',aggregate_id,'aggregateType',aggregate_type,'eventKey',event_key,'envelope',JSON_EXTRACT(payload_json,'$'),'publishStatus',status,'failedAttempts',retry_count) FROM ll_event_outbox WHERE customer_id='" + scope['customerId'] + "' AND event_type='OverseasDeliveryCompleted' AND JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.payload.deliveryOrderId'))='" + scope['deliveryId'] + "';")
elif direction == 'UNLOAD':
rows = read_sql(inspect('firstleg-test-lastleg'),
"SELECT JSON_OBJECT('entryId',CAST(id AS CHAR),'eventId',JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.eventId')),'payloadHash',JSON_UNQUOTE(JSON_EXTRACT(payload_json,'$.payloadHash')),'eventType',event_type,'customerId',customer_id,'aggregateId',aggregate_id,'aggregateType',aggregate_type,'eventKey',event_key,'envelope',JSON_EXTRACT(payload_json,'$'),'publishStatus',status,'failedAttempts',retry_count) FROM ll_event_outbox WHERE customer_id='0' AND aggregate_id='"
+ scope['containerBookingId'] + "' AND event_type='OverseasContainerUnloaded' AND event_key='OverseasContainerUnloaded:" + scope['containerBookingId'] + "';")
@@ -201,18 +212,50 @@ def business_ids(scope):
return [row['id'] for row in rows]
def business_snapshot(scope, state, row):
if delivery(state):
return {'business': delivery_facts(scope, row['eventId'])}
return {'container': container_facts(scope)} if unload(state) else {'businessOrderIds': business_ids(scope)}
def delivery_facts(scope, event_id):
if not re.fullmatch(r'[0-9a-fA-F-]{36}|[0-9A-HJKMNP-TV-Z]{26}', event_id):
raise RuntimeError('INVALID_EVENT_IDENTITY')
customer, source, root = scope['customerId'], scope['sourceOrderId'], scope['rootOrderId']
firstleg = inspect('firstleg-test-business')
order = inspect('firstleg-test-order')
require_test(firstleg, 'firstleg-test-business')
require_test(order, 'firstleg-test-order')
# Each service is read in its own consistent read-only transaction; no cross-schema join.
inbound_sql = "tenant_id=" + customer + " AND deleted=0 AND order_id IN (" + source + "," + root + ")"
first_rows = read_sql(firstleg,
"SELECT JSON_OBJECT('kind','inbound','id',CAST(id AS CHAR),'customerId',CAST(tenant_id AS CHAR),'orderId',CAST(order_id AS CHAR),'orderNo',order_no,'inboundCode',inbound_code,'processStatus',process_status,'orderStatusSnapshot',order_status_snapshot,'version',version,'updatedAt',CAST(updated_at AS CHAR)) FROM fl_inbound_order WHERE " + inbound_sql + " ORDER BY id;\n"
+ "SELECT JSON_OBJECT('kind','inboundHistory','id',CAST(id AS CHAR),'inboundId',CAST(inbound_id AS CHAR),'action',action,'fromStatus',from_status,'toStatus',to_status,'occurredAt',CAST(occurred_at AS CHAR)) FROM fl_inbound_status_history WHERE tenant_id=" + customer + " AND inbound_id=" + scope['sourceInboundId'] + " ORDER BY id;\n"
+ "SELECT JSON_OBJECT('kind','inbox','eventId',event_id,'eventType',event_type,'aggregateId',aggregate_id,'aggregateVersion',aggregate_version,'payloadHash',payload_hash,'payload',JSON_EXTRACT(payload_json,'$'),'status',status,'receivedAt',CAST(received_at AS CHAR),'processedAt',CAST(processed_at AS CHAR)) FROM fl_container_event_inbox WHERE event_id='" + event_id + "';")
tree = "tenant_id=" + customer + " AND deleted=0 AND (id=" + root + " OR root_order_id=" + root + ")"
ids = "SELECT id FROM ord_order WHERE " + tree
order_rows = read_sql(order,
"SELECT JSON_OBJECT('kind','order','id',CAST(id AS CHAR),'customerId',CAST(tenant_id AS CHAR),'rootOrderId',CAST(root_order_id AS CHAR),'orderRole',order_role,'orderNo',order_no,'inboundCode',inbound_code,'status',status,'version',version,'completedAt',CAST(completed_at AS CHAR),'updatedAt',CAST(updated_at AS CHAR)) FROM ord_order WHERE " + tree + " ORDER BY id;\n"
+ "SELECT JSON_OBJECT('kind','orderHistory','id',CAST(id AS CHAR),'orderId',CAST(order_id AS CHAR),'action',action,'fromStatus',from_status,'toStatus',to_status,'occurredAt',CAST(occurred_at AS CHAR)) FROM ord_order_status_history WHERE tenant_id=" + customer + " AND order_id IN (" + ids + ") ORDER BY id;\n"
+ "SELECT JSON_OBJECT('kind','completionEvent','id',CAST(id AS CHAR),'orderId',CAST(aggregate_id AS CHAR),'eventType',event_type,'status',publish_status,'payload',JSON_EXTRACT(payload_json,'$')) FROM ord_outbox_event WHERE tenant_id=" + customer + " AND aggregate_id IN (" + ids + ") AND event_type='FIRST_LEG_ORDER_FULFILLMENT_COMPLETED' ORDER BY id;")
return {kind: [row for row in first_rows + order_rows if row['kind'] == kind] for kind in
('inbound', 'inboundHistory', 'inbox', 'order', 'orderHistory', 'completionEvent')}
def prepare(scope, directory, config):
if delivery(config) and producer(scope, 'DELIVERY') is not None:
raise RuntimeError('DELIVERY_EVENT_ALREADY_EXISTS')
directory.mkdir(mode=0o700)
original = inspect(producer_name(config))
require_test(original, producer_name(config))
consumer_name = 'firstleg-test-business' if unload(config) else 'firstleg-test-lastleg'
consumer_name = 'firstleg-test-business' if reverse_delivery(config) else 'firstleg-test-lastleg'
consumer = inspect(consumer_name)
require_test(consumer, consumer_name)
old_env = env_of(original)
setting, direct = target_setting(config)
if old_env.get(setting) != direct:
raise RuntimeError('PRODUCER_NOT_DIRECT')
original_body = recreation(original, old_env, full=unload(config))
original_body = recreation(original, old_env, full=reverse_delivery(config))
token = secrets.token_urlsafe(32)
# Reserve task-specific ports and let the existing proxy select exactly this plan.
sockets = [socket.socket() for _ in range(2)]
@@ -225,8 +268,10 @@ def prepare(scope, directory, config):
proxy.write_bytes(base64.b64decode(config['proxySource']))
ip = consumer['NetworkSettings']['Networks']['firstleg-test-network']['IPAddress']
log = open(directory / 'proxy-output.log', 'wb')
upstream = ':9202/internal/firstleg/v1/container-events' if unload(config) else ':9303/internal/lastleg/events'
selector = ['--event-type', 'OverseasContainerUnloaded', '--container-booking-id', scope['containerBookingId']] if unload(config) else ['--event-type', 'FirstLegLoadingPlanCompleted', '--loading-plan-id', scope['loadingPlanId']]
upstream = ':9202/internal/firstleg/v1/container-events' if reverse_delivery(config) else ':9303/internal/lastleg/events'
selector = [] if delivery(config) else ['--event-type', 'OverseasContainerUnloaded', '--container-booking-id', scope['containerBookingId']] if unload(config) else ['--event-type', 'FirstLegLoadingPlanCompleted', '--loading-plan-id', scope['loadingPlanId']]
if delivery(config):
selector = ['--event-type', 'OverseasDeliveryCompleted', '--delivery-id', scope['deliveryId']]
process = subprocess.Popen(['python3', str(proxy), '--upstream', 'http://' + ip + upstream, *selector,
'--bind', '0.0.0.0', '--port', str(ports[0]), '--control-port', str(ports[1]),
'--log', str(directory / 'proxy-evidence.jsonl')],
@@ -248,7 +293,7 @@ def activate(scope, directory, state):
setting, direct = target_setting(state)
if original['Id'] != state['originalId'] or original['Config']['Image'] != state['image'] or env_of(original).get(setting) != direct:
raise RuntimeError('CONCURRENT_PRODUCER_CHANGE')
if unload(state) and (original['Image'] != state['originalImageId'] or fingerprint(original) != state['originalFingerprint']):
if reverse_delivery(state) and (original['Image'] != state['originalImageId'] or fingerprint(original) != state['originalFingerprint']):
raise RuntimeError('CONCURRENT_PRODUCER_CHANGE')
body = copy.deepcopy(state['originalBody'])
e = env_of(original)
@@ -270,7 +315,7 @@ def recover(scope, directory, state):
pinned = control(state)
if pinned['eventId'] != row['eventId'] or pinned['payloadHash'] != row['payloadHash']:
raise RuntimeError('PINNED_EVENT_MISMATCH')
state['beforeRetry'] = {**row, **({'container': container_facts(scope)} if unload(state) else {'businessOrderIds': business_ids(scope)})}
state['beforeRetry'] = {**row, **business_snapshot(scope, state, row)}
private_write(directory / 'state.json', state)
control(state, 'PASS')
container = inspect(producer_name(state))
@@ -278,7 +323,7 @@ def recover(scope, directory, state):
ip = container['NetworkSettings']['Networks']['firstleg-test-network']['IPAddress']
body = {'requestId': state['operationsRequestId'], 'operator': 'admin-full-test',
'reason': scope['marker'] + ': scoped ACK-loss recovery', 'expectedPayloadHash': row['payloadHash']}
endpoint = ':9303/internal/lastleg/event-operations/' if unload(state) else ':9202/internal/firstleg/event-operations/'
endpoint = ':9303/internal/lastleg/event-operations/' if reverse_delivery(state) else ':9202/internal/firstleg/event-operations/'
url = 'http://' + ip + endpoint + row['entryId'] + '/retry'
request = urllib.request.Request(url, json.dumps(body).encode(),
{'X-Event-Operations-Token': e['EVENT_OPERATIONS_TOKEN'], 'Content-Type': 'application/json'})
@@ -295,7 +340,7 @@ def restore(directory, state):
try:
control(state, 'PASS')
except Exception:
if not unload(state):
if not reverse_delivery(state):
raise
# A dead proxy must not prevent restoring the producer's direct target.
if state['prepared']:
@@ -309,8 +354,8 @@ def restore(directory, state):
state['prepared'] = False
private_write(directory / 'state.json', state)
return {'restoreStarted': True}
if current['Id'] == state['originalId'] or (unload(state) and current['Image'] == state['originalImageId'] and fingerprint(current) == state['originalFingerprint']):
if unload(state) and not current['State'].get('Running'):
if current['Id'] == state['originalId'] or (reverse_delivery(state) and current['Image'] == state['originalImageId'] and fingerprint(current) == state['originalFingerprint']):
if reverse_delivery(state) and not current['State'].get('Running'):
docker('POST', '/containers/' + current['Id'] + '/start')
state['prepared'] = False
private_write(directory / 'state.json', state)
@@ -330,15 +375,15 @@ def proof(scope, state):
event_id, entry_id = row['eventId'], row['entryId']
if not re.fullmatch(r'[0-9a-fA-F-]{36}|[0-9A-HJKMNP-TV-Z]{26}', event_id) or not re.fullmatch(r'[1-9][0-9]{0,19}', entry_id):
raise RuntimeError('INVALID_EVENT_IDENTITY')
event_type = 'OverseasContainerUnloaded' if unload(state) else 'FirstLegLoadingPlanCompleted'
event_type = 'OverseasDeliveryCompleted' if delivery(state) else 'OverseasContainerUnloaded' if unload(state) else 'FirstLegLoadingPlanCompleted'
attempts = read_sql(inspect(producer_name(state)),
"SELECT JSON_OBJECT('attemptId',attempt_id,'serviceName',service_name,'entryId',entry_id,'eventId',event_id,'targetName',target_name,'outcome',outcome) FROM event_delivery_attempt WHERE entry_id='"
+ entry_id + "' AND event_type='" + event_type + "' ORDER BY started_at,attempt_id;")
audit = read_sql(inspect(producer_name(state)),
"SELECT JSON_OBJECT('requestId',request_id,'entryId',entry_id,'eventId',event_id,'payloadHash',payload_hash,'previousFailures',previous_failures) FROM event_delivery_retry_audit WHERE request_id='"
+ state['operationsRequestId'] + "';" if not unload(state) else
"SELECT JSON_OBJECT('requestId',request_id,'entryId',entry_id,'eventId',event_id,'payloadHash',payload_hash,'previousFailures',previous_failures) FROM event_delivery_retry_audit WHERE entry_id='" + entry_id + "' AND event_type='OverseasContainerUnloaded';")
if unload(state):
+ state['operationsRequestId'] + "';" if not reverse_delivery(state) else
"SELECT JSON_OBJECT('requestId',request_id,'entryId',entry_id,'eventId',event_id,'payloadHash',payload_hash,'previousFailures',previous_failures) FROM event_delivery_retry_audit WHERE entry_id='" + entry_id + "' AND event_type='" + event_type + "';")
if reverse_delivery(state):
consumers = read_sql(inspect('firstleg-test-business'),
"SELECT JSON_OBJECT('eventId',event_id,'payloadHash',payload_hash,'status',status,'eventType',event_type,'aggregateId',aggregate_id,'payload',JSON_EXTRACT(payload_json,'$')) FROM fl_container_event_inbox WHERE event_id='" + event_id + "';")
else:
@@ -348,7 +393,7 @@ def proof(scope, state):
consumer = {**(consumers[0] if len(consumers) == 1 else {}), 'count': len(consumers)}
result = {**scope, 'readOnly': True, 'producer': row, 'consumer': consumer, 'attempts': attempts,
'proxyRecords': control(state)['records'], 'beforeRetry': state['beforeRetry'],
'afterRetry': {**row, **({'container': container_facts(scope)} if unload(state) else {'businessOrderIds': business_ids(scope)})},
'afterRetry': {**row, **business_snapshot(scope, state, row)},
'retryAudit': audit[0] if len(audit) == 1 else None, 'retryAuditCount': len(audit), 'retryRequestId': state['operationsRequestId'], 'restored': False}
private_write(scope_directory(scope, state.get('direction', 'LOADING')) / 'proof.json', result)
return result
@@ -363,14 +408,14 @@ def finish(directory, state):
raise RuntimeError('RESTORED_SERVICE_NOT_HEALTHY')
if sorted(current['Config']['Env']) != sorted(state['originalBody']['Env']):
raise RuntimeError('RESTORED_CONFIGURATION_MISMATCH')
if unload(state) and (current['Image'] != state['originalImageId'] or fingerprint(current) != state['originalFingerprint']):
if reverse_delivery(state) and (current['Image'] != state['originalImageId'] or fingerprint(current) != state['originalFingerprint']):
raise RuntimeError('RESTORED_CONFIGURATION_MISMATCH')
cmdline = Path('/proc/' + str(state['proxyPid']) + '/cmdline')
if cmdline.exists():
if str(directory / 'proxy.py').encode() not in cmdline.read_bytes().split(b'\x00'):
raise RuntimeError('PROXY_PROCESS_OWNERSHIP_MISMATCH')
os.kill(state['proxyPid'], signal.SIGTERM)
if unload(state):
if reverse_delivery(state):
for _ in range(50):
if not cmdline.exists() or not cmdline.read_bytes():
break
@@ -378,7 +423,7 @@ def finish(directory, state):
else:
raise RuntimeError('PROXY_NOT_STOPPED')
result = {'restored': True}
if unload(state):
if reverse_delivery(state):
result.update(healthy=True, proxyStopped=True, producerName=producer_name(state),
originalImageId=state['originalImageId'], currentImageId=current['Image'],
originalFingerprint=state['originalFingerprint'], currentFingerprint=fingerprint(current))