java-project/baiyue-firstleg
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(order): verify warehouse faults and await recovery projection

Browse Source
This commit is contained in:
ouyangxq
2026-10-04 00:04:36 +08:00
parent 3f6083ff02
commit e8cf5e70ba
8 changed files with 229 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/order-submit-dependency-fault-executor.mjs
+62 -21
View File
@@ -29,17 +29,19 @@ export function scopeFor(plan) {
}
assert(plan.commands.merchant.orderId !== plan.commands.platform.orderId, 'Two distinct new drafts required')
assert(plan.runtimeChangesExecuted === false, 'Only a reviewed unexecuted plan is accepted')
assert(Array.isArray(plan.phases) && plan.phases.length === 2, 'Exact two dependency phases required')
assert(/^[1-9]\d{0,19}$/.test(plan.receivingWarehouseId || ''), 'Bounded public receiving warehouse required')
assert(Array.isArray(plan.phases) && plan.phases.length === 3, 'Exact three dependency phases required')
for (const [dependency, property, path, messages] of [
['IAM','baiyue.clients.iam.url',`/internal/iam/tenants/${plan.customerId}/order-eligibility`,['IAM eligibility service is unavailable','IAM 下单资格响应无效']],
['FINANCE','baiyue.clients.finance.url',`/internal/finance/v1/wallet/accounts/${plan.customerId}`,['Finance account service is unavailable','Finance account response is invalid']],
['WAREHOUSE','baiyue.clients.warehouse.url',`/internal/warehouses/0/${plan.receivingWarehouseId}/active`,['主数据服务暂时不可用']],
]) {
const rows = plan.phases.filter(row => row.dependency === dependency)
assert(rows.length === 1 && rows[0].urlProperty === property && rows[0].targetPath === path && rows[0].targetMethod === 'GET'
&& rows[0].faultHttp === 503 && rows[0].expectedHttp === 200 && rows[0].expectedCode === 'S0503'
&& JSON.stringify(rows[0].expectedMessages) === JSON.stringify(messages), 'Dependency failure contract may not be weakened')
}
return { marker: plan.marker, customerId: plan.customerId,
return { marker: plan.marker, customerId: plan.customerId, receivingWarehouseId: plan.receivingWarehouseId,
orderIds: Object.fromEntries(['merchant', 'platform'].map(role => [role, plan.commands[role].orderId])), planSha256: sha(plan) }
}
@@ -50,7 +52,7 @@ export function renderOrderFaultAction(action, scope, agent, extra = {}) {
const routing = extra.routing
assert(routing && Object.keys(routing).every(key => ['configurationReviewed','sourceSha256','upstreams','nacos'].includes(key))
&& routing.configurationReviewed === true && routing.nacos && Object.keys(routing.nacos).every(key => ['baseUrl','loginPath','configPath'].includes(key))
&& routing.upstreams && Object.keys(routing.upstreams).sort().join(',') === 'FINANCE,IAM', 'Reviewed routing schema only; do not embed credentials')
&& routing.upstreams && Object.keys(routing.upstreams).sort().join(',') === 'FINANCE,IAM,WAREHOUSE', 'Reviewed routing schema only; do not embed credentials')
for (const address of [routing.nacos.baseUrl,...Object.values(routing.upstreams)]) {
const url = new URL(address)
assert(['http:','https:'].includes(url.protocol) && !url.username && !url.password && !url.search && !url.hash
@@ -60,8 +62,9 @@ export function renderOrderFaultAction(action, scope, agent, extra = {}) {
&& /^[0-9a-f]{64}$/.test(routing.sourceSha256 || ''), 'Reviewed image and Nacos source fingerprint required')
}
const data = encode(JSON.stringify({ ...extra, action, scope, commonSource, ...(action === 'prepare' ? { proxySource } : {}) }))
const expression = value => '[' + encode(value).match(/.{1,4000}/g).map(part => `'${part}'`).join(',') + "].join('')"
const groovy = `def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${expression(runtime)}),'UTF-8'),'${data}']).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)`
const literal = encoded => '[' + encoded.match(/.{1,4000}/g).map(part => `'${part}'`).join(',') + "].join('')"
const expression = value => literal(encode(value))
const groovy = `def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${expression(runtime)}),'UTF-8'),${literal(data)}]).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)`
return `def c=jenkins.model.Jenkins.instance.getComputer('${agent}');if(c==null || c.channel==null)throw new RuntimeException('TEST agent unavailable');print(hudson.util.RemotingDiagnostics.executeGroovy(new String(java.util.Base64.decoder.decode(${expression(groovy)}),'UTF-8'),c.channel))`
}
@@ -78,7 +81,10 @@ export function createOrderFaultRuntime(env, scope, fetcher = globalThis.fetch)
})
} catch { throw Error(`RUNTIME_${action.toUpperCase().replaceAll('-', '_')}_OUTCOME_UNKNOWN_USE_STATUS`) }
assert(response.ok, `RUNTIME_HTTP_${response.status}_NO_AUTO_RETRY`)
const rows = (await response.text()).split(/\r?\n/).filter(row => row.startsWith('ORDER_DEPENDENCY_JSON='))
const output = await response.text()
assert(!(output.includes('MultipleCompilationErrorsException') && output.includes('String too long')),
'RUNTIME_RENDERER_COMPILATION_FAILED_NO_ACTION')
const rows = output.split(/\r?\n/).filter(row => row.startsWith('ORDER_DEPENDENCY_JSON='))
assert(rows.length === 1, 'RUNTIME_RESPONSE_AMBIGUOUS_NO_AUTO_RETRY')
const result = JSON.parse(rows[0].slice('ORDER_DEPENDENCY_JSON='.length))
assert(result?.ok === true, /^[A-Z_0-9]{1,90}$/.test(result?.error || '') ? result.error : 'RUNTIME_ACTION_FAILED')
@@ -103,6 +109,7 @@ export async function runOrderFaultWindow({ plan, routing, expectedImageId, acti
assert(createHash('sha256').update(orderPersistenceFacts(before.detail)).digest('hex') === plan.commands[role].initialOrderSha256
&& before.detail.order.status === 'DRAFT' && String(before.detail.order.tenantId) === scope.customerId
&& String(before.detail.order.id) === scope.orderIds[role]
&& String(before.detail.order.receivingWarehouseId) === scope.receivingWarehouseId
&& ['cargoes','parties','internalNotes','statusHistory'].every(field => Array.isArray(before.detail[field]))
&& before.inboundPageComplete === true, 'PREPARED_DRAFT_CHANGED')
}
@@ -113,7 +120,7 @@ export async function runOrderFaultWindow({ plan, routing, expectedImageId, acti
assert(i < 14, 'PROXY_STARTUP_NOT_READY'); await wait(1000)
}
await action('activate'); await health()
for (const dependency of ['IAM','FINANCE']) {
for (const dependency of ['IAM','FINANCE','WAREHOUSE']) {
for (const role of ['merchant','platform']) {
const before = await snapshot(role), command = plan.commands[role]
const request = { method: command.method, path: command.path, body: structuredClone(command.body),
@@ -138,20 +145,51 @@ export async function runOrderFaultWindow({ plan, routing, expectedImageId, acti
&& report.restoredRuntime.currentImageId === expectedImageId
&& report.restoredRuntime.originalFingerprint === report.restoredRuntime.currentFingerprint
&& report.restoredRuntime.nacosSourcesUnchanged === true
&& report.restoredRuntime.upstreamReads?.IAM === true && report.restoredRuntime.upstreamReads?.FINANCE === true,
&& report.restoredRuntime.upstreamReads?.IAM === true && report.restoredRuntime.upstreamReads?.FINANCE === true
&& report.restoredRuntime.upstreamReads?.WAREHOUSE === true,
'RESTORED_IDENTITY_OR_ROUTES_NOT_PROVED')
const businessRecovery = await recovery()
report.businessRecovery = businessRecovery
report.restoration = await action('finish', { businessRecovery })
assert(report.restoration.restored && report.restoration.proxyStopped && report.restoration.orderRoutesVerified, 'RESTORATION_NOT_COMPLETE')
} catch (error) { report.recoveryError = safeError(error); failure ||= error }
} catch (error) {
report.recoveryError = safeError(error)
if (error.recoveryDiagnostics) report.recoveryDiagnostics = error.recoveryDiagnostics
failure ||= error
}
}
}
report.success = !failure && report.observations.length === 4 && Boolean(report.restoration?.restored)
report.success = !failure && report.observations.length === 6 && Boolean(report.restoration?.restored)
if (failure) report.error = safeError(failure)
return report
}
export async function waitForRecoveredSubmission({ plan, role, snapshot, wait = pause }) {
const command = plan.commands[role]
let previous, orderFacts
for (let i = 0; i <= 60; i++) {
const current = await snapshot(role), detail = current.detail
assert(current.inboundPageComplete === true && Array.isArray(current.inbounds) && current.inbounds.length <= 1
&& ['cargoes','parties','internalNotes','statusHistory'].every(field => Array.isArray(detail?.[field]))
&& String(detail.order.id) === command.orderId && String(detail.order.tenantId) === plan.customerId
&& detail.order.status === 'SUBMITTED' && Number(detail.order.version) === command.body.expectedVersion + 1
&& detail.statusHistory.filter(row => row.action === 'SUBMITTED').length === 1
&& detail.statusHistory.filter(row => row.action === 'CREATE').length === 1
&& (current.task === null || current.task && String(current.task.orderId) === command.orderId
&& String(current.task.tenantId) === plan.customerId)
&& current.inbounds.every(row => String(row.orderId) === command.orderId && row.orderNo === command.orderNo
&& String(row.tenantId) === plan.customerId), 'RECOVERY_SUBMISSION_FACTS_INVALID')
const facts = orderPersistenceFacts(detail)
orderFacts ??= facts
assert(facts === orderFacts, 'RECOVERY_ORDER_DRIFT')
const projected = current.inbounds.length === 1 && current.inbounds[0].orderStatusSnapshot === 'SUBMITTED'
if (projected && previous && sha(previous) === sha(current)) return current
previous = projected ? current : undefined
if (i < 60) await wait(500)
}
throw Error('RECOVERY_PROJECTION_TIMEOUT')
}
function businessClient(env, plan, fetcher = globalThis.fetch) {
const sessions = {}
const call = async (method, path, body, headers = {}, token) => {
@@ -191,16 +229,19 @@ function businessClient(env, plan, fetcher = globalThis.fetch) {
},
async recovery() {
const evidence={}
for(const role of ['merchant','platform']) {
const command=plan.commands[role]
ok(await submit(role,command))
let before=await snapshot(role)
for(let i=0;i<10;i++) { await pause(500);const next=await snapshot(role);if(sha(before)===sha(next)){before=next;break}before=next }
assert(before.detail.order.status==='SUBMITTED' && before.detail.statusHistory.filter(row=>row.action==='SUBMITTED').length===1
&& before.inbounds.length<=1, 'RECOVERY_SINGLE_SUBMISSION_NOT_PROVED')
ok(await submit(role,command));const after=await snapshot(role)
assert(sha(before)===sha(after),'RECOVERY_REPLAY_DRIFT')
evidence[role]={orderId:command.orderId,status:'SUBMITTED',sameKeyReplayVerified:true,snapshotSha256:sha(after)}
try {
for(const role of ['merchant','platform']) {
const command=plan.commands[role]
ok(await submit(role,command))
const before=await waitForRecoveredSubmission({plan,role,snapshot})
ok(await submit(role,command));const after=await snapshot(role)
evidence[role]={orderId:command.orderId,before,after,snapshotSha256:sha(after)}
assert(sha(before)===sha(after),'RECOVERY_REPLAY_DRIFT')
Object.assign(evidence[role],{status:'SUBMITTED',sameKeyReplayVerified:true})
}
} catch(error) {
error.recoveryDiagnostics=evidence
throw error
}
return evidence
},
@@ -217,7 +258,7 @@ async function main(argv) {
for(let i=0;i<rest.length;i+=2){assert(rest[i]?.startsWith('--') && rest[i+1] && !rest[i+1].startsWith('--'),'Use explicit --name value arguments');args[rest[i].slice(2)]=rest[i+1]}
assert(['plan','preflight','run','status','reconcile','restore'].includes(mode),'Use plan/preflight/run/status/reconcile/restore')
assert(args.plan,'--plan required');const plan=JSON.parse(readFileSync(args.plan,'utf8')),scope=scopeFor(plan)
if(mode==='plan'){console.log(JSON.stringify({scope,actions:['prepare','activate','IAM x 2','FINANCE x 2','finally restore','verify original routes','finish proxy'],requiresExclusiveOrderWindow:true},null,2));return}
if(mode==='plan'){console.log(JSON.stringify({scope,actions:['prepare','activate','IAM x 2','FINANCE x 2','WAREHOUSE x 2','finally restore','verify original routes','finish proxy'],requiresExclusiveOrderWindow:true},null,2));return}
assert(args['confirm-test-runtime']==='yes','--confirm-test-runtime yes required')
const action=createOrderFaultRuntime(process.env,scope)
if(mode==='preflight'){assert(args.routing,'--routing required');console.log(JSON.stringify(await action('preflight',{routing:JSON.parse(readFileSync(args.routing,'utf8'))}),null,2));return}