411 lines
40 KiB
JavaScript
411 lines
40 KiB
JavaScript
// Explicit TEST-only row-lock fault; importing this file performs no I/O.
|
|
import fs from 'node:fs'
|
|
import { createHash } from 'node:crypto'
|
|
import { resolve } from 'node:path'
|
|
import { pathToFileURL } from 'node:url'
|
|
import { parseCrossCustomerArguments, runCrossCustomer } from './fleet-cross-customer-test-env.mjs'
|
|
import { verifyFleetCrossCustomer } from './admin-test-env-modules/fleet-cross-customer-acceptance.mjs'
|
|
|
|
const demand = (value, message) => { if (!value) throw Error(message) }
|
|
const identifier = value => typeof value === 'string' && /^[1-9][0-9]{0,19}$/.test(value)
|
|
const businessNo = value => typeof value === 'string' && /^[A-Za-z0-9_-]{1,128}$/.test(value)
|
|
const literal = value => `CONVERT(0x${Buffer.from(value).toString('hex')} USING utf8mb4) COLLATE utf8mb4_bin`
|
|
const encoded = value => '[' + Buffer.from(value).toString('base64').match(/.{1,4000}/g).map(x => `'${x}'`).join(',') + "].join('')"
|
|
const pause = ms => new Promise(done => setTimeout(done, ms))
|
|
const canonical = value => JSON.stringify(sortFields(value))
|
|
const sha = value => createHash('sha256').update(canonical(value)).digest('hex')
|
|
function sortFields(value) {
|
|
return Array.isArray(value) ? value.map(sortFields) : value && typeof value === 'object'
|
|
? Object.fromEntries(Object.keys(value).sort().map(key => [key, sortFields(value[key])])) : value
|
|
}
|
|
const rr = ['SET NAMES utf8mb4 COLLATE utf8mb4_bin;', 'SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;', 'START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY;']
|
|
const databaseRuntime = "SELECT JSON_OBJECT('collection','databaseRuntime','serverUuid',@@server_uuid,'databaseName',DATABASE(),'performanceSchema',@@performance_schema);"
|
|
// Every column is obtained from the actual schema; no partial projection can masquerade as a full rollback baseline.
|
|
const tables = ['ll_dispatch_order', 'll_dispatch_history', 'll_dispatch_item', 'll_dispatch_command', 'll_dispatch_pallet_lock', 'll_dispatch_physical_pallet_lock', 'll_physical_pallet_guard', 'll_dispatch_item_source',
|
|
'll_fleet_trip', 'll_fleet_trip_dispatch', 'll_fleet_trip_history', 'll_fleet_pool_order_link', 'll_outbound_order', 'll_delivery_order',
|
|
'll_order', 'll_loading_order_status_history', 'll_warehouse_transfer_order', 'll_dispatch_pda_task', 'll_dispatch_pda_pick',
|
|
'll_dispatch_pda_pick_order', 'll_inventory_balance', 'll_inventory_position', 'll_inventory_ledger']
|
|
const columnsSql = `SELECT JSON_OBJECT('collection','column','table',TABLE_NAME,'column',COLUMN_NAME,'ordinal',ORDINAL_POSITION,'type',COLUMN_TYPE,'key',COLUMN_KEY) FROM information_schema.columns WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (${tables.map(literal).join(',')}) ORDER BY TABLE_NAME,ORDINAL_POSITION;`
|
|
const projection = (collection, table, fields, where) => `SELECT JSON_OBJECT('collection','${collection}',${Object.entries(fields).map(([key, expression]) => `'${key}',CAST(${expression} AS CHAR)`).join(',')}) FROM ${table} WHERE ${where};`
|
|
const one = (rows, collection) => { const found = rows.filter(row => row.collection === collection); demand(found.length === 1, `EXACT_${collection.toUpperCase()}_REQUIRED`); return found[0] }
|
|
|
|
export function validateRollbackRequest(scope) {
|
|
demand(scope && businessNo(scope.marker) && businessNo(scope.tripNo) && businessNo(scope.dispatchNo)
|
|
&& businessNo(scope.commandKey) && identifier(scope.commandVersion) && identifier(scope.warehouseId)
|
|
&& /^sha256:[a-f0-9]{64}$/.test(scope.expectedImage || ''), 'NATIVE_ROLLBACK_REQUEST_REQUIRED')
|
|
demand(scope.members?.length === 2 && new Set(scope.members.map(m => m.customerId)).size === 2, 'TWO_DISTINCT_CUSTOMERS_REQUIRED')
|
|
for (const member of scope.members) {
|
|
demand(['customerId', 'dispatchItemId', 'palletId', 'positionId', 'balanceId'].every(field => identifier(member[field]))
|
|
&& ['incomingNo', 'outboundNo', 'deliveryNo'].every(field => businessNo(member[field])), 'OWNED_MEMBER_IDENTITIES_REQUIRED')
|
|
}
|
|
for (const field of ['incomingNo', 'dispatchItemId', 'palletId', 'positionId', 'balanceId', 'outboundNo', 'deliveryNo'])
|
|
demand(new Set(scope.members.map(m => m[field])).size === 2, 'MEMBER_IDENTITY_REUSED')
|
|
return scope
|
|
}
|
|
|
|
function resolutionSql(scope) {
|
|
validateRollbackRequest(scope)
|
|
const root = `(SELECT id FROM ll_dispatch_order WHERE dispatch_no=${literal(scope.dispatchNo)})`
|
|
const trip = `(SELECT id FROM ll_fleet_trip WHERE trip_no=${literal(scope.tripNo)})`
|
|
const items = `dispatch_order_id=${root}`
|
|
const sourceOrders = `(SELECT order_id FROM ll_dispatch_item WHERE ${items})`
|
|
return [...rr, databaseRuntime, columnsSql,
|
|
projection('root', 'll_dispatch_order', { id: 'id', dispatchNo: 'dispatch_no', warehouseId: 'overseas_warehouse_id', version: 'record_version', status: 'status', stage: 'stage' }, `dispatch_no=${literal(scope.dispatchNo)}`),
|
|
projection('trip', 'll_fleet_trip', { id: 'id', tripNo: 'trip_no', warehouseId: 'warehouse_id', status: 'status' }, `trip_no=${literal(scope.tripNo)}`),
|
|
projection('binding', 'll_fleet_trip_dispatch', { dispatchId: 'dispatch_id', dispatchNo: 'dispatch_no', tripId: 'trip_id', tripNo: 'trip_no' }, `trip_id=${trip} OR dispatch_id=${root}`),
|
|
projection('item', 'll_dispatch_item', { id: 'id', dispatchId: 'dispatch_order_id', customerId: 'customer_id', palletId: 'warehouse_pallet_id', orderId: 'order_id', sourceType: 'source_business_type', sourceNo: 'source_business_no', incomingNo: 'inbound_code', packages: 'package_count' }, items),
|
|
projection('transfer', 'll_warehouse_transfer_order', { id: 'id', customerId: 'customer_id', incomingNo: 'transfer_no', orderId: 'lastleg_order_id', warehouseId: 'operating_warehouse_id', needRelabel: 'need_relabel', relabelNo: 'relabel_no', outboundAt: 'outbound_at' }, `lastleg_order_id IN ${sourceOrders}`),
|
|
projection('outbound', 'll_outbound_order', { id: 'id', customerId: 'customer_id', dispatchId: 'dispatch_order_id', outboundNo: 'outbound_no', status: 'status' }, items),
|
|
projection('delivery', 'll_delivery_order', { id: 'id', customerId: 'customer_id', dispatchId: 'dispatch_order_id', deliveryNo: 'delivery_no', outboundNo: 'outbound_no', status: 'status' }, items),
|
|
projection('task', 'll_dispatch_pda_task', { id: 'id', dispatchId: 'dispatch_order_id', warehouseId: 'warehouse_id' }, items),
|
|
'COMMIT;'].join('\n')
|
|
}
|
|
|
|
export function resolveRollbackScope(request, rows) {
|
|
validateRollbackRequest(request)
|
|
const root = one(rows, 'root'), trip = one(rows, 'trip'), binding = one(rows, 'binding'), task = one(rows, 'task')
|
|
demand(identifier(root.id) && identifier(trip.id) && identifier(task.id) && root.dispatchNo === request.dispatchNo
|
|
&& root.warehouseId === request.warehouseId && root.version === request.commandVersion && root.status === 'ACTIVE' && root.stage === 'AFTER_LOADING'
|
|
&& trip.tripNo === request.tripNo && trip.warehouseId === request.warehouseId && trip.status === 'PLANNED'
|
|
&& binding.dispatchId === root.id && binding.tripId === trip.id && binding.dispatchNo === request.dispatchNo && binding.tripNo === request.tripNo
|
|
&& task.dispatchId === root.id && task.warehouseId === request.warehouseId, 'ONE_NATIVE_ROOT_AND_FLEET_REQUIRED')
|
|
for (const collection of ['item', 'transfer', 'outbound', 'delivery']) demand(rows.filter(r => r.collection === collection).length === 2, `TWO_${collection.toUpperCase()}_ROWS_REQUIRED`)
|
|
const members = request.members.map(member => {
|
|
const own = collection => one(rows.filter(r => r.customerId === member.customerId), collection)
|
|
const item = own('item'), transfer = own('transfer'), outbound = own('outbound'), delivery = own('delivery')
|
|
demand(item.id === member.dispatchItemId && item.dispatchId === root.id && item.palletId === member.palletId && identifier(item.orderId)
|
|
// Native ordinary cargo leaves source_business_type/no NULL; the actual transfer/order/receipt links prove its origin.
|
|
&& item.sourceType === null && item.sourceNo === null && item.incomingNo === member.incomingNo && item.packages === '1'
|
|
&& identifier(transfer.id) && transfer.orderId === item.orderId && transfer.incomingNo === member.incomingNo
|
|
&& transfer.warehouseId === request.warehouseId && transfer.needRelabel === '0' && transfer.relabelNo === null && transfer.outboundAt === null
|
|
&& identifier(outbound.id) && outbound.dispatchId === root.id && outbound.outboundNo === member.outboundNo && outbound.status === 'WAIT_OUTBOUND'
|
|
&& identifier(delivery.id) && delivery.dispatchId === root.id && delivery.deliveryNo === member.deliveryNo
|
|
&& delivery.outboundNo === member.outboundNo && delivery.status === 'WAIT_DELIVERY', 'REAL_ORDINARY_TRANSFER_AND_CHILD_LINKS_REQUIRED')
|
|
return { ...member, orderId: item.orderId, transferId: transfer.id, outboundId: outbound.id, deliveryId: delivery.id }
|
|
}).sort((a, b) => a.customerId < b.customerId ? -1 : a.customerId > b.customerId ? 1 : 0) // Java String.compareTo for decimal customer IDs; never numeric/locale order.
|
|
const scope = { ...request, dispatchId: root.id, tripId: trip.id, taskId: task.id, members,
|
|
columns: rows.filter(row => row.collection === 'column') }
|
|
return validateRollbackScope(scope)
|
|
}
|
|
|
|
export function validateRollbackScope(scope) {
|
|
validateRollbackRequest(scope)
|
|
demand(['dispatchId', 'tripId', 'taskId'].every(field => identifier(scope[field]))
|
|
&& scope.members.every(member => ['orderId', 'transferId', 'outboundId', 'deliveryId'].every(field => identifier(member[field])))
|
|
&& scope.members[0].customerId < scope.members[1].customerId, 'RESOLVED_SHARED_ROOT_AND_JAVA_CUSTOMER_ORDER_REQUIRED')
|
|
for (const field of ['orderId', 'transferId', 'outboundId', 'deliveryId']) demand(new Set(scope.members.map(m => m[field])).size === 2, 'RESOLVED_MEMBER_ID_REUSED')
|
|
if (scope.observedTransactions) demand(scope.observedTransactions.length === 2 && scope.observedTransactions.every(identifier)
|
|
&& new Set(scope.observedTransactions).size === 2, 'EXACT_OBSERVED_TRANSACTION_IDS_REQUIRED')
|
|
demand(Array.isArray(scope.columns) && scope.columns.every(c => c.collection === 'column' && tables.includes(c.table)
|
|
&& /^[a-z][a-z0-9_]*$/.test(c.column) && Number.isSafeInteger(c.ordinal) && c.ordinal > 0 && typeof c.type === 'string'), 'ACTUAL_SCHEMA_COLUMNS_REQUIRED')
|
|
for (const table of tables) {
|
|
const columns = scope.columns.filter(c => c.table === table)
|
|
demand(columns.length > 0 && columns.some(c => c.key === 'PRI') && new Set(columns.map(c => c.column)).size === columns.length
|
|
&& columns.every((c, i) => c.ordinal === i + 1), 'COMPLETE_ACTUAL_SCHEMA_REQUIRED')
|
|
}
|
|
const fields = table => new Set(scope.columns.filter(column => column.table === table).map(column => column.column))
|
|
const guard = scope.columns.filter(column => column.table === 'll_physical_pallet_guard')
|
|
demand(guard.length === 2 && guard.some(column => column.column === 'warehouse_pallet_id' && column.key === 'PRI' && column.type.toLowerCase() === 'bigint')
|
|
&& guard.some(column => column.column === 'created_at' && column.key !== 'PRI' && column.type.toLowerCase() === 'datetime(3)'), 'PERMANENT_GLOBAL_PHYSICAL_PALLET_GUARD_SCHEMA_REQUIRED')
|
|
demand(fields('ll_dispatch_physical_pallet_lock').has('active_warehouse_pallet_id')
|
|
&& !fields('ll_dispatch_physical_pallet_lock').has('customer_id')
|
|
&& fields('ll_dispatch_pallet_lock').has('physical_lock_id') && !fields('ll_dispatch_pallet_lock').has('active_warehouse_pallet_id')
|
|
&& fields('ll_dispatch_item_source').has('dispatch_item_id') && fields('ll_dispatch_pda_pick_order').has('dispatch_item_source_id'),
|
|
'CURRENT_PHYSICAL_AND_LOGICAL_SOURCE_SCHEMA_REQUIRED')
|
|
return scope
|
|
}
|
|
|
|
function snapshotSql(scope) {
|
|
const source = scope.members.map(m => `(customer_id=${literal(m.customerId)} AND id=${m.orderId})`).join(' OR ')
|
|
const histories = scope.members.map(m => `(customer_id=${literal(m.customerId)} AND order_id=${m.orderId})`).join(' OR ')
|
|
const customer = `customer_id IN (${scope.members.map(m => literal(m.customerId)).join(',')})`
|
|
const stock = `${customer} AND overseas_warehouse_id=${literal(scope.warehouseId)}`
|
|
const predicates = {
|
|
ll_dispatch_order: `id=${scope.dispatchId}`, ll_dispatch_history: `dispatch_id=${scope.dispatchId}`,
|
|
ll_dispatch_item: `dispatch_order_id=${scope.dispatchId}`, ll_dispatch_command: `dispatch_id=${scope.dispatchId}`,
|
|
ll_dispatch_pallet_lock: `dispatch_order_id=${scope.dispatchId}`,
|
|
ll_dispatch_physical_pallet_lock: `dispatch_order_id=${scope.dispatchId} OR warehouse_pallet_id IN (${scope.members.map(member => member.palletId).join(',')})`,
|
|
ll_physical_pallet_guard: `warehouse_pallet_id IN (${scope.members.map(member => member.palletId).join(',')})`,
|
|
ll_dispatch_item_source: `dispatch_order_id=${scope.dispatchId} OR dispatch_item_id IN (${scope.members.map(member => member.dispatchItemId).join(',')})`,
|
|
ll_fleet_trip: `id=${scope.tripId}`,
|
|
ll_fleet_trip_dispatch: `trip_id=${scope.tripId} OR dispatch_id=${scope.dispatchId}`, ll_fleet_trip_history: `trip_id=${scope.tripId}`,
|
|
ll_fleet_pool_order_link: `dispatch_no=${literal(scope.dispatchNo)}`, ll_outbound_order: `dispatch_order_id=${scope.dispatchId}`,
|
|
ll_delivery_order: `dispatch_order_id=${scope.dispatchId}`, ll_order: source, ll_loading_order_status_history: histories,
|
|
ll_warehouse_transfer_order: `lastleg_order_id IN (${scope.members.map(m => m.orderId).join(',')})`,
|
|
ll_dispatch_pda_task: `dispatch_order_id=${scope.dispatchId}`, ll_dispatch_pda_pick: `task_id=${scope.taskId}`,
|
|
ll_dispatch_pda_pick_order: `pick_id IN (SELECT id FROM ll_dispatch_pda_pick WHERE task_id=${scope.taskId})`,
|
|
ll_inventory_balance: stock, ll_inventory_position: stock, ll_inventory_ledger: stock,
|
|
}
|
|
return [columnsSql, ...tables.flatMap(table => {
|
|
const columns = scope.columns.filter(c => c.table === table)
|
|
const json = columns.map(c => `'${c.column}',CAST(\`${c.column}\` AS CHAR CHARACTER SET utf8mb4)`).join(',')
|
|
return [`SELECT JSON_OBJECT('collection','coverage','table','${table}','count',COUNT(*)) FROM ${table} WHERE ${predicates[table]};`,
|
|
`SELECT JSON_OBJECT('collection','row','table','${table}','row',JSON_OBJECT(${json})) FROM ${table} WHERE ${predicates[table]};`]
|
|
})].join('\n')
|
|
}
|
|
|
|
function lockSql(scope) {
|
|
const target = scope.members[1]
|
|
const held = `l.OBJECT_SCHEMA=DATABASE() AND l.OBJECT_NAME='ll_warehouse_transfer_order' AND l.INDEX_NAME='PRIMARY' AND l.LOCK_DATA='${target.transferId}'`
|
|
const lease = `SELECT JSON_OBJECT('collection','lease','connectionId',CAST(t.trx_mysql_thread_id AS CHAR),'transactionId',CAST(t.trx_id AS CHAR),'lockId',l.LOCK_DATA,'mode',l.LOCK_MODE,'ageSeconds',TIMESTAMPDIFF(SECOND,t.trx_started,NOW())) FROM performance_schema.data_locks l JOIN information_schema.innodb_trx t ON t.trx_id=l.ENGINE_TRANSACTION_ID WHERE ${held} AND l.LOCK_STATUS='GRANTED' AND t.trx_query LIKE ${literal('%ROLLBACK_' + scope.marker + '%')};`
|
|
const waits = `SELECT JSON_OBJECT('collection','wait','requester',CAST(r.trx_mysql_thread_id AS CHAR),'blocker',CAST(b.trx_mysql_thread_id AS CHAR),'requestingTransactionId',CAST(r.trx_id AS CHAR),'blockingTransactionId',CAST(b.trx_id AS CHAR),'lockId',l.LOCK_DATA,'blockingLockId',bl.LOCK_DATA,'objectName',l.OBJECT_NAME,'indexName',l.INDEX_NAME,'requestStatus',l.LOCK_STATUS,'blockingStatus',bl.LOCK_STATUS,'statement',r.trx_query,'sessionLockWaitSeconds',v.VARIABLE_VALUE,'ownsRoot',EXISTS(SELECT 1 FROM performance_schema.data_locks dl WHERE dl.ENGINE_TRANSACTION_ID=r.trx_id AND dl.OBJECT_SCHEMA=DATABASE() AND dl.OBJECT_NAME='ll_dispatch_order' AND dl.INDEX_NAME='PRIMARY' AND dl.LOCK_DATA='${scope.dispatchId}' AND dl.LOCK_STATUS='GRANTED' AND dl.LOCK_MODE LIKE 'X%')) FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE JOIN performance_schema.data_locks bl ON bl.ENGINE_LOCK_ID=w.BLOCKING_ENGINE_LOCK_ID AND bl.ENGINE=w.ENGINE JOIN information_schema.innodb_trx r ON r.trx_id=w.REQUESTING_ENGINE_TRANSACTION_ID JOIN information_schema.innodb_trx b ON b.trx_id=w.BLOCKING_ENGINE_TRANSACTION_ID JOIN performance_schema.threads th ON th.PROCESSLIST_ID=r.trx_mysql_thread_id JOIN performance_schema.variables_by_thread v ON v.THREAD_ID=th.THREAD_ID AND v.VARIABLE_NAME='innodb_lock_wait_timeout' WHERE ${held} AND bl.OBJECT_SCHEMA=DATABASE() AND bl.OBJECT_NAME=l.OBJECT_NAME AND bl.INDEX_NAME='PRIMARY' AND bl.LOCK_DATA=l.LOCK_DATA AND b.trx_query LIKE ${literal('%ROLLBACK_' + scope.marker + '%')};`
|
|
return [lease, waits]
|
|
}
|
|
|
|
export function rollbackSql(scope, mode) {
|
|
if (mode === 'RESOLVE') return resolutionSql(scope)
|
|
validateRollbackScope(scope)
|
|
const second = scope.members[1]
|
|
if (mode === 'LEASE') return `SET NAMES utf8mb4 COLLATE utf8mb4_bin;${databaseRuntime}SET SESSION innodb_lock_wait_timeout=5;START TRANSACTION;SELECT JSON_OBJECT('collection','held','lockId',CAST(id AS CHAR),'customerId',customer_id,'connectionId',CAST(CONNECTION_ID() AS CHAR)) FROM ll_warehouse_transfer_order WHERE id=${second.transferId} AND customer_id=${literal(second.customerId)} AND lastleg_order_id=${second.orderId} AND transfer_no=${literal(second.incomingNo)} AND need_relabel=0 AND relabel_no IS NULL AND outbound_at IS NULL FOR UPDATE;SELECT JSON_OBJECT('marker','ROLLBACK_${scope.marker}','elapsed',SLEEP(65));ROLLBACK;SELECT JSON_OBJECT('released',JSON_EXTRACT('true','$'));`
|
|
if (mode === 'LOCKS' || mode === 'RELEASED') return [...rr, databaseRuntime, ...lockSql(scope),
|
|
...(mode === 'RELEASED' ? [(() => {
|
|
demand(scope.observedTransactions, 'OBSERVED_TRANSACTIONS_REQUIRED_FOR_RELEASE_PROOF')
|
|
return `SELECT JSON_OBJECT('collection','remainingTransaction','transactionId',CAST(trx_id AS CHAR)) FROM information_schema.innodb_trx WHERE trx_id IN (${scope.observedTransactions.map(literal).join(',')});`
|
|
})()] : []), 'COMMIT;'].join('\n')
|
|
if (['BASELINE', 'RESTORED', 'OBSERVE'].includes(mode)) return [
|
|
...(mode === 'OBSERVE' ? ['SET NAMES utf8mb4 COLLATE utf8mb4_bin;', 'SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;', 'START TRANSACTION READ ONLY;'] : rr),
|
|
databaseRuntime, ...(mode === 'OBSERVE' ? lockSql(scope) : []), snapshotSql(scope), 'COMMIT;'].join('\n')
|
|
throw Error('ROLLBACK_MODE_INVALID')
|
|
}
|
|
|
|
export function buildRollbackRemoteScript(scope, mode, agent) {
|
|
demand(['RUNTIME', 'RESOLVE', 'LEASE', 'LOCKS', 'OBSERVE', 'BASELINE', 'RESTORED', 'RELEASED'].includes(mode)
|
|
&& /^[A-Za-z0-9_.-]{1,128}$/.test(agent || ''), 'ROLLBACK_REMOTE_MODE_REQUIRED')
|
|
const sql = mode === 'RUNTIME' ? [databaseRuntime,
|
|
...['data_locks', 'data_lock_waits', 'threads', 'variables_by_thread'].map(table => `SELECT JSON_OBJECT('collection','privilege','table','${table}','count',COUNT(*)) FROM performance_schema.${table} WHERE 1=0;`),
|
|
"SELECT JSON_OBJECT('collection','privilege','table','innodb_trx','count',COUNT(*)) FROM information_schema.innodb_trx WHERE 1=0;"].join('\n') : rollbackSql(scope, mode)
|
|
const imageGuard = mode === 'RUNTIME' ? '' : ` or c['Image']!='${scope.expectedImage}'`
|
|
const python = `import json,os,subprocess,urllib.parse,re
|
|
try:
|
|
c=json.loads(subprocess.check_output(['docker','inspect','firstleg-test-lastleg'],text=True,stderr=subprocess.DEVNULL))[0]
|
|
e=dict(x.split('=',1) for x in c['Config']['Env'])
|
|
if c['Name']!='/firstleg-test-lastleg' or c['Config'].get('Labels',{}).get('com.docker.compose.project')!='firstleg-test' or e.get('DEPLOY_ENV')!='test' or c['State'].get('Health',{}).get('Status')!='healthy'${imageGuard}:raise RuntimeError('TEST_RUNTIME_CHANGED')
|
|
u=urllib.parse.urlsplit(e['LASTLEG_DB_URL'].removeprefix('jdbc:'));host='127.0.0.1' if u.hostname=='host.docker.internal' else u.hostname
|
|
q=subprocess.run(['mysql','--host='+host,'--port='+str(u.port or 3306),'--user='+e['LASTLEG_DB_USERNAME'],'--batch','--raw','--skip-column-names',u.path.lstrip('/')],input=${JSON.stringify(sql)},capture_output=True,text=True,env=dict(os.environ,MYSQL_PWD=e['LASTLEG_DB_PASSWORD']),timeout=${mode === 'LEASE' ? 75 : 25})
|
|
if q.returncode:
|
|
code=re.search(r'ERROR ([0-9]{3,5})',q.stderr);print('TRIP_ROLLBACK_JSON='+json.dumps({'ok':False,'error':'MYSQL_'+(code.group(1) if code else 'UNKNOWN')}))
|
|
else:print('TRIP_ROLLBACK_JSON='+json.dumps({'ok':True,'imageId':c['Image'],'rows':[json.loads(x) for x in q.stdout.splitlines() if x.strip()]}))
|
|
except Exception as error:
|
|
print('TRIP_ROLLBACK_JSON='+json.dumps({'ok':False,'error':str(error) if re.fullmatch('[A-Z_]{1,64}',str(error)) else 'REMOTE_ROLLBACK_FAILED'}))
|
|
`
|
|
const groovy = `def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${encoded(python)}),'UTF-8')]).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)`
|
|
return `def c=jenkins.model.Jenkins.instance.getComputer('${agent}');if(c==null||c.channel==null)throw new RuntimeException('TEST agent unavailable');print(hudson.util.RemotingDiagnostics.executeGroovy(new String(java.util.Base64.decoder.decode(${encoded(groovy)}),'UTF-8'),c.channel))`
|
|
}
|
|
|
|
export function fullRollbackSnapshot(rows, scope) {
|
|
validateRollbackScope(scope)
|
|
demand(canonical(rows.filter(r => r.collection === 'column')) === canonical(scope.columns), 'ACTUAL_SCHEMA_CHANGED')
|
|
const result = {}
|
|
for (const table of tables) {
|
|
const coverage = one(rows.filter(r => r.table === table), 'coverage')
|
|
const values = rows.filter(r => r.collection === 'row' && r.table === table).map(r => r.row)
|
|
const columns = scope.columns.filter(c => c.table === table).map(c => c.column).sort()
|
|
demand(Number.isSafeInteger(coverage.count) && coverage.count === values.length
|
|
&& values.every(r => canonical(Object.keys(r).sort()) === canonical(columns) && Object.values(r).every(v => v === null || typeof v === 'string'))
|
|
&& new Set(values.map(canonical)).size === values.length, 'COMPLETE_FULL_FIELD_SNAPSHOT_REQUIRED')
|
|
result[table] = values.sort((a, b) => canonical(a) < canonical(b) ? -1 : canonical(a) > canonical(b) ? 1 : 0)
|
|
}
|
|
return result
|
|
}
|
|
const exactRow = (snapshot, table, id) => {
|
|
const found = snapshot[table].filter(row => row.id === id)
|
|
demand(found.length === 1, `EXACT_OWNED_ROW_REQUIRED_${table}`); return found[0]
|
|
}
|
|
const immutableTables = ['ll_physical_pallet_guard', 'll_dispatch_physical_pallet_lock', 'll_dispatch_item_source', 'll_dispatch_pda_task', 'll_dispatch_pda_pick', 'll_dispatch_pda_pick_order', 'll_inventory_balance', 'll_inventory_position', 'll_inventory_ledger']
|
|
export function assertRollbackBaseline(snapshot, scope) {
|
|
const root = exactRow(snapshot, 'll_dispatch_order', scope.dispatchId), trip = exactRow(snapshot, 'll_fleet_trip', scope.tripId)
|
|
demand(snapshot.ll_dispatch_order.length === 1 && snapshot.ll_fleet_trip.length === 1 && snapshot.ll_fleet_trip_dispatch.length === 1
|
|
&& root.status === 'ACTIVE' && root.stage === 'AFTER_LOADING' && root.record_version === scope.commandVersion && root.departed_at === null
|
|
&& root.shipped_package_count === '0' && trip.status === 'PLANNED' && trip.departed_at === null, 'READY_NATIVE_BASELINE_REQUIRED')
|
|
demand(snapshot.ll_fleet_pool_order_link.length === 0, 'ORDINARY_TRANSFER_FIXTURE_CANNOT_HAVE_POOL_LINKS')
|
|
demand(snapshot.ll_dispatch_item_source.length === 0, 'ORDINARY_TRANSFER_FIXTURE_CANNOT_HAVE_RELABEL_CONTRIBUTIONS')
|
|
for (const table of ['ll_dispatch_item', 'll_outbound_order', 'll_delivery_order', 'll_order', 'll_warehouse_transfer_order', 'll_dispatch_pallet_lock', 'll_dispatch_physical_pallet_lock', 'll_physical_pallet_guard', 'll_dispatch_pda_pick', 'll_dispatch_pda_pick_order'])
|
|
demand(snapshot[table].length === 2, 'EXACT_TWO_CUSTOMER_BASELINE_REQUIRED')
|
|
demand(snapshot.ll_dispatch_pda_task.length === 1 && snapshot.ll_dispatch_pda_task[0].id === scope.taskId, 'ORIGINAL_TASK_REQUIRED')
|
|
demand(!snapshot.ll_dispatch_command.some(r => r.action === 'DEPART' || r.idempotency_key === scope.commandKey)
|
|
&& !snapshot.ll_dispatch_history.some(r => r.action === 'DEPART') && !snapshot.ll_fleet_trip_history.some(r => r.action === 'DEPART')
|
|
&& !snapshot.ll_loading_order_status_history.some(r => r.event_id === 'DISPATCH_DEPART:' + scope.dispatchNo), 'FRESH_DEPARTURE_REQUIRED')
|
|
for (const member of scope.members) {
|
|
const guards = snapshot.ll_physical_pallet_guard.filter(row => row.warehouse_pallet_id === member.palletId)
|
|
demand(guards.length === 1 && /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d{1,3})?$/.test(guards[0].created_at || ''), 'OWNED_PERMANENT_PHYSICAL_PALLET_GUARD_REQUIRED')
|
|
const order = exactRow(snapshot, 'll_order', member.orderId), transfer = exactRow(snapshot, 'll_warehouse_transfer_order', member.transferId)
|
|
const outbound = exactRow(snapshot, 'll_outbound_order', member.outboundId), delivery = exactRow(snapshot, 'll_delivery_order', member.deliveryId)
|
|
demand([order, transfer, outbound, delivery].every(r => r.customer_id === member.customerId)
|
|
&& order.picked_package_count === '1' && order.shipped_package_count === '0'
|
|
&& transfer.lastleg_order_id === member.orderId && transfer.transfer_no === member.incomingNo && transfer.outbound_at === null
|
|
&& outbound.status === 'WAIT_OUTBOUND' && delivery.status === 'WAIT_DELIVERY' && delivery.dispatched_at === null, 'OWNED_SOURCE_AND_CHILD_BASELINE_REQUIRED')
|
|
const picks = snapshot.ll_dispatch_pda_pick.filter(r => r.customer_id === member.customerId)
|
|
const attribution = snapshot.ll_dispatch_pda_pick_order.filter(r => r.customer_id === member.customerId)
|
|
demand(picks.length === 1 && picks[0].task_id === scope.taskId && picks[0].dispatch_item_id === member.dispatchItemId
|
|
&& picks[0].inventory_position_id === member.positionId && picks[0].packages === '1' && picks[0].return_carton_id === null
|
|
&& attribution.length === 1 && attribution[0].pick_id === picks[0].id && attribution[0].order_id === member.orderId
|
|
&& identifier(attribution[0].receipt_id) && attribution[0].packages === '1' && attribution[0].return_receipt_id === null
|
|
&& attribution[0].return_carton_id === null && attribution[0].dispatch_item_source_id === null, 'COMMITTED_REAL_PICK_ATTRIBUTION_REQUIRED')
|
|
const ship = snapshot.ll_inventory_ledger.filter(r => r.customer_id === member.customerId && r.business_type === 'SHIP')
|
|
demand(ship.length === 1 && ship[0].business_order_no === scope.dispatchNo && ship[0].position_id === member.positionId
|
|
&& ship[0].balance_id === member.balanceId && ship[0].delta_packages === '-1', 'PRIOR_COMMITTED_SHIP_REQUIRED')
|
|
const claims = snapshot.ll_dispatch_pallet_lock.filter(r => r.customer_id === member.customerId)
|
|
demand(claims.length === 1 && claims[0].dispatch_order_id === scope.dispatchId && claims[0].warehouse_pallet_id === member.palletId
|
|
&& claims[0].status === 'ACTIVE' && identifier(claims[0].physical_lock_id), 'ACTIVE_OWNED_CLAIM_REQUIRED')
|
|
const physical = exactRow(snapshot, 'll_dispatch_physical_pallet_lock', claims[0].physical_lock_id)
|
|
demand(physical.warehouse_id === scope.warehouseId && physical.warehouse_pallet_id === member.palletId
|
|
&& physical.active_warehouse_pallet_id === member.palletId && physical.dispatch_order_id === scope.dispatchId
|
|
&& physical.status === 'ACTIVE' && physical.draft_token === null && physical.draft_container_id === null
|
|
&& physical.expires_at === null && physical.released_at === null, 'ACTIVE_NATIVE_PHYSICAL_PALLET_OWNER_REQUIRED')
|
|
}
|
|
return snapshot
|
|
}
|
|
|
|
export function assertTargetTransferUpdate(statement, scope) {
|
|
demand(typeof statement === 'string', 'TARGET_TRANSFER_UPDATE_NOT_PROVED')
|
|
const sql = statement.replace(/`/g, '').replace(/\s+/g, ' ').replace(/\s*([=,])\s*/g, '$1').trim()
|
|
const value = "(?:\\?|'[^']*')"
|
|
const match = sql.match(new RegExp(`^UPDATE ll_warehouse_transfer_order SET status='OUTBOUNDED',\\s*outbound_at=${value},\\s*updated_at=${value} WHERE customer_id=(\\?|'[0-9]+') AND lastleg_order_id=(\\?|[0-9]+) AND outbound_at IS NULL$`, 'i'))
|
|
const second = scope.members[1]
|
|
demand(match && (match[1] === '?' || match[1] === `'${second.customerId}'`) && (match[2] === '?' || match[2] === second.orderId), 'TARGET_TRANSFER_UPDATE_NOT_PROVED')
|
|
return true
|
|
}
|
|
|
|
export function assertMidTransactionObservation(rows, scope, baseline) {
|
|
const lease = one(rows, 'lease'), wait = one(rows, 'wait'), second = scope.members[1]
|
|
demand(identifier(lease.connectionId) && identifier(wait.requester) && wait.blocker === lease.connectionId && wait.requester !== wait.blocker
|
|
&& identifier(lease.transactionId) && identifier(wait.requestingTransactionId) && wait.blockingTransactionId === lease.transactionId
|
|
&& wait.requestingTransactionId !== lease.transactionId
|
|
&& lease.lockId === second.transferId && wait.lockId === lease.lockId && wait.blockingLockId === lease.lockId
|
|
&& /^X(?:,|$)/.test(lease.mode) && wait.objectName === 'll_warehouse_transfer_order' && wait.indexName === 'PRIMARY'
|
|
&& wait.requestStatus === 'WAITING' && wait.blockingStatus === 'GRANTED' && wait.ownsRoot === 1, 'OWNED_LATE_ROW_WAIT_NOT_PROVED')
|
|
assertTargetTransferUpdate(wait.statement, scope)
|
|
demand(wait.sessionLockWaitSeconds === '50', 'REQUESTING_SESSION_50_SECOND_LOCK_TIMEOUT_NOT_PROVED')
|
|
const dirty = fullRollbackSnapshot(rows, scope)
|
|
for (const member of scope.members) {
|
|
const before = exactRow(baseline, 'll_order', member.orderId), order = exactRow(dirty, 'll_order', member.orderId)
|
|
const outbound = exactRow(dirty, 'll_outbound_order', member.outboundId), delivery = exactRow(dirty, 'll_delivery_order', member.deliveryId)
|
|
demand(outbound.status === 'OUTBOUNDED' && delivery.status === 'DELIVERING' && delivery.dispatched_at !== null
|
|
&& BigInt(delivery.version) === BigInt(exactRow(baseline, 'll_delivery_order', member.deliveryId).version) + 1n
|
|
&& BigInt(order.picked_package_count) === BigInt(before.picked_package_count) - 1n
|
|
&& BigInt(order.shipped_package_count) === BigInt(before.shipped_package_count) + 1n
|
|
&& order.outbound_order_no === member.outboundNo, 'CUSTOMER_UNCOMMITTED_CHILD_AND_SOURCE_WRITE_NOT_PROVED')
|
|
const history = dirty.ll_loading_order_status_history.filter(r => r.customer_id === member.customerId && r.order_id === member.orderId && r.event_id === 'DISPATCH_DEPART:' + scope.dispatchNo)
|
|
demand(history.length === 1 && !baseline.ll_loading_order_status_history.some(r => r.id === history[0].id), 'UNCOMMITTED_SOURCE_HISTORY_NOT_PROVED')
|
|
}
|
|
const firstTransfer = exactRow(dirty, 'll_warehouse_transfer_order', scope.members[0].transferId)
|
|
demand(firstTransfer.status === 'OUTBOUNDED' && firstTransfer.outbound_at !== null
|
|
&& canonical(exactRow(dirty, 'll_warehouse_transfer_order', second.transferId)) === canonical(exactRow(baseline, 'll_warehouse_transfer_order', second.transferId)), 'FIRST_TRANSFER_WRITTEN_SECOND_TRANSFER_WAITING_NOT_PROVED')
|
|
for (const table of ['ll_dispatch_order', 'll_fleet_trip', 'll_dispatch_command', 'll_dispatch_history', 'll_fleet_trip_history', 'll_dispatch_pallet_lock', ...immutableTables])
|
|
demand(canonical(dirty[table]) === canonical(baseline[table]), 'LATE_PHASE_OR_COMMITTED_SCAN_FACTS_CHANGED')
|
|
return { leaseConnectionId: lease.connectionId, requestingConnectionId: wait.requester, leaseTransactionId: lease.transactionId,
|
|
requestingTransactionId: wait.requestingTransactionId, lockId: lease.lockId,
|
|
sessionLockWaitSeconds: wait.sessionLockWaitSeconds, statement: wait.statement, dirtyObservationOnly: true, firstCustomerId: scope.members[0].customerId, secondCustomerId: second.customerId }
|
|
}
|
|
|
|
export function assertRestoredSnapshot(baseline, restored) {
|
|
demand(canonical(baseline) === canonical(restored), 'FULL_FIELD_COMMITTED_ROLLBACK_SNAPSHOT_CHANGED')
|
|
return { beforeSha256: sha(baseline), afterSha256: sha(restored), fullFields: true, committedRead: true }
|
|
}
|
|
export function requestRollbackDeparture(context, path, body, token) {
|
|
// Existing 60-second, single-attempt channel: an unknown HTTP outcome is never retried here.
|
|
return context.requestWithBody('POST', path, body, token)
|
|
}
|
|
export function assertDatabaseLockConflict(response) {
|
|
demand(response?.http === 200 && String(response.body?.code) === 'A0409' && response.body?.message === '资源正在处理中，请稍后重试'
|
|
&& response.requestId && response.traceId, 'EXPECTED_DATABASE_LOCK_CONFLICT_REQUIRED')
|
|
}
|
|
async function remote(env, scope, mode) {
|
|
const script = buildRollbackRemoteScript(scope, mode, env.FBA_CORE_JENKINS_AGENT)
|
|
const response = await fetch(new URL('scriptText', env.JENKINS_URL.replace(/\/?$/, '/')), { method: 'POST', redirect: 'error',
|
|
signal: AbortSignal.timeout(mode === 'LEASE' ? 95000 : 45000), headers: { Authorization: 'Basic ' + Buffer.from(env.JENKINS_USER + ':' + env.JENKINS_API_TOKEN).toString('base64'),
|
|
'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ script }) })
|
|
demand(response.ok, 'ROLLBACK_EVIDENCE_HTTP_FAILED')
|
|
const lines = (await response.text()).split(/\r?\n/).filter(x => x.startsWith('TRIP_ROLLBACK_JSON='))
|
|
demand(lines.length === 1, 'ROLLBACK_REMOTE_RESULT_AMBIGUOUS')
|
|
const result = JSON.parse(lines[0].slice('TRIP_ROLLBACK_JSON='.length))
|
|
demand(result.ok === true, `ROLLBACK_${mode}_EVIDENCE_BLOCKED_${/^[A-Z_0-9]+$/.test(result.error || '') ? result.error : 'REMOTE_FAILED'}`)
|
|
const identity = one(result.rows, 'databaseRuntime')
|
|
demand(identity.performanceSchema === 1 && identity.serverUuid && identity.databaseName && /^sha256:[a-f0-9]{64}$/.test(result.imageId || ''), 'OBSERVABLE_DATABASE_RUNTIME_REQUIRED')
|
|
if (scope.runtime) demand(result.imageId === scope.expectedImage && canonical(identity) === canonical(scope.runtime), 'PINNED_DATABASE_RUNTIME_CHANGED')
|
|
return result
|
|
}
|
|
|
|
export function createDepartureRollbackVerifier(env, config, runtime, { read = remote, wait = pause } = {}) {
|
|
return async ({ context, fixture, ready, departPath, departBody, snapshot }) => {
|
|
const output = resolve('reports', `fleet-cross-${config.runId}`, 'rollback.json'), fd = fs.openSync(output, 'wx', 0o600)
|
|
const originalCommand = canonical({ path: departPath, body: departBody })
|
|
const report = { verified: false, scope: 'OWNED_TWO_CUSTOMER_DEPARTURE_ROLLBACK', startedAt: new Date().toISOString(), samples: [],
|
|
command: JSON.parse(originalCommand), runtime }
|
|
const save = () => { fs.ftruncateSync(fd, 0); fs.writeSync(fd, JSON.stringify(report, null, 2), 0, 'utf8'); fs.fsyncSync(fd) }
|
|
let leasePromise, pending
|
|
try {
|
|
demand(ready.trip.dispatches?.length === 1 && ready.trip.dispatches[0].dispatchNo === fixture.dispatchNo && fixture.members.length === 2
|
|
&& ready.dispatch.dispatchNo === fixture.dispatchNo && String(ready.task.recordVersion) === String(departBody.recordVersion), 'ONE_NATIVE_READY_SCOPE_REQUIRED')
|
|
const request = validateRollbackRequest({ marker: context.marker, tripNo: fixture.tripNo, dispatchNo: fixture.dispatchNo, warehouseId: fixture.warehouseId,
|
|
expectedImage: runtime.imageId, runtime: one(runtime.rows, 'databaseRuntime'), commandKey: departBody.idempotencyKey, commandVersion: String(departBody.recordVersion),
|
|
members: fixture.members.map(({ customerId, incomingNo, dispatchItemId, palletId, positionId, balanceId, outboundNo, deliveryNo }) =>
|
|
({ customerId, incomingNo, dispatchItemId, palletId, positionId, balanceId, outboundNo, deliveryNo })) })
|
|
const resolved = await read(env, request, 'RESOLVE')
|
|
const scope = report.target = resolveRollbackScope(request, resolved.rows)
|
|
const baseline = report.baseline = assertRollbackBaseline(fullRollbackSnapshot((await read(env, scope, 'BASELINE')).rows, scope), scope)
|
|
save()
|
|
leasePromise = read(env, scope, 'LEASE').then(value => ({ value }), error => ({ error: error.message }))
|
|
let lease
|
|
for (let i = 0; i < 12; i++) {
|
|
const rows = (await read(env, scope, 'LOCKS')).rows; report.samples.push({ phase: 'LEASE', rows }); save()
|
|
if (rows.filter(x => x.collection === 'lease').length === 1) {
|
|
lease = one(rows, 'lease')
|
|
demand(lease.lockId === scope.members[1].transferId && identifier(lease.connectionId) && /^X(?:,|$)/.test(lease.mode)
|
|
&& Number.isSafeInteger(lease.ageSeconds) && lease.ageSeconds >= 0 && lease.ageSeconds <= 5, 'FRESH_OWNED_65_SECOND_LEASE_REQUIRED')
|
|
break
|
|
}
|
|
demand(i !== 11, 'LIVE_LEASE_NOT_OBSERVED'); await wait(250)
|
|
}
|
|
pending = requestRollbackDeparture(context, departPath, departBody, fixture.pdaToken).then(value => ({ value }), error => ({ error: error.message }))
|
|
for (let i = 0; i < 20; i++) {
|
|
const rows = (await read(env, scope, 'OBSERVE')).rows; report.samples.push({ phase: 'MID_TRANSACTION', rows }); save()
|
|
if (rows.some(x => x.collection === 'wait')) {
|
|
report.observation = assertMidTransactionObservation(rows, scope, baseline)
|
|
demand(report.observation.leaseConnectionId === lease.connectionId && report.observation.leaseTransactionId === lease.transactionId, 'LEASE_CONNECTION_CHANGED')
|
|
scope.observedTransactions = [report.observation.leaseTransactionId, report.observation.requestingTransactionId]
|
|
save(); break
|
|
}
|
|
demand(i !== 19, 'MID_TRANSACTION_NOT_OBSERVED'); await wait(250)
|
|
}
|
|
report.response = await pending; save()
|
|
report.lease = await leasePromise; save()
|
|
demand(!report.lease.error && report.lease.value.rows.at(-1)?.released === true, 'LEASE_RELEASE_NOT_PROVED')
|
|
const held = one(report.lease.value.rows, 'held')
|
|
demand(held.lockId === lease.lockId && held.connectionId === lease.connectionId && held.customerId === scope.members[1].customerId, 'ORIGINAL_LEASE_RESULT_CHANGED')
|
|
const released = report.released = (await read(env, scope, 'RELEASED')).rows
|
|
demand(!released.some(r => ['lease', 'wait', 'remainingTransaction'].includes(r.collection)), 'OWNED_ROW_LOCK_OR_TRANSACTION_REMAINS')
|
|
// A new RR consistent snapshot after the server outcome and lease completion, never the dirty observer transaction.
|
|
report.restored = fullRollbackSnapshot((await read(env, scope, 'RESTORED')).rows, scope)
|
|
report.consistentSnapshot = assertRestoredSnapshot(baseline, report.restored)
|
|
demand(!report.response.error && report.response.value?.http !== 0, 'DEPARTURE_OUTCOME_UNKNOWN_RETAIN_ORIGINAL_COMMAND')
|
|
assertDatabaseLockConflict(report.response.value)
|
|
demand(canonical(ready) === canonical(await snapshot('ROLLBACK_AFTER_FAULT')), 'PUBLIC_AND_COMPLETE_INVENTORY_SNAPSHOT_CHANGED')
|
|
demand(canonical({ path: departPath, body: departBody }) === originalCommand, 'ORIGINAL_DEPART_COMMAND_CHANGED')
|
|
// The existing caller now sends this identical original command once, then verifies replay and both customer receipts.
|
|
report.verified = true; return report
|
|
} catch (error) { report.error = error.message; throw error }
|
|
finally {
|
|
if (pending && !report.response) report.response = await pending
|
|
if (leasePromise && !report.lease) report.lease = await leasePromise
|
|
report.finishedAt = new Date().toISOString(); save(); fs.closeSync(fd)
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function runTripRollback(config, { env = process.env } = {}) {
|
|
if (!config.execute) return { ...config, scope: 'OWNED_TWO_CUSTOMER_DEPARTURE_ROLLBACK', fault: 'One native dispatch; two ordinary WAREHOUSE_TRANSFER customers in Java string order; 65-second read-only row lease on the second transfer; exact UPDATE waiter and its own 50-second session timeout; uncommitted first customer writes; fresh full-field committed rollback baseline; same original command and normal signing chain', businessSqlWrites: false, deployment: false }
|
|
demand(config.environment === 'TEST' && env.CHONGCHONG_ENV === 'test' && ['JENKINS_URL', 'JENKINS_USER', 'JENKINS_API_TOKEN', 'FBA_CORE_JENKINS_AGENT'].every(key => env[key]), 'EXPLICIT_TEST_EVIDENCE_PROFILE_REQUIRED')
|
|
const runtime = await remote(env, {}, 'RUNTIME')
|
|
demand(runtime.rows.filter(row => row.collection === 'privilege').length === 5, 'LOCK_AND_SESSION_OBSERVABILITY_REQUIRED')
|
|
const sources = ['scripts/fleet-trip-rollback-test-env.mjs', 'scripts/admin-test-env-modules/fleet-cross-customer-acceptance.mjs'].map(path => ({ path, sha256: createHash('sha256').update(fs.readFileSync(path)).digest('hex') }))
|
|
return runCrossCustomer(config, { env, verify: async (context, fixture, options) => {
|
|
fs.writeFileSync(resolve('reports', `fleet-cross-${config.runId}`, 'rollback-source.json'), JSON.stringify({ runtime, sources }, null, 2), { flag: 'wx' })
|
|
return verifyFleetCrossCustomer(context, fixture, { ...options, verifyDepartureRollback: createDepartureRollbackVerifier(env, config, runtime) })
|
|
} })
|
|
}
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
|
try { const config = parseCrossCustomerArguments(process.argv.slice(2)), result = await runTripRollback(config); console.log(JSON.stringify(config.execute ? { success: result.success, error: result.error, summary: result.summary, requests: result.requestCount, goalComplete: false } : result, null, 2)); if (config.execute && !result.success) process.exitCode = 1 }
|
|
catch (error) { console.error(/^[A-Z_0-9]+$/.test(error.message) ? error.message : 'ROLLBACK_ENTRY_FAILED'); process.exitCode = 1 }
|
|
}
|