test(firstleg): retry lost evidence uploads with the same command

2026-10-02 19:32:57 +08:00
parent fa51d9235c
commit c9a1e93299
3 changed files with 123 additions and 14 deletions
@@ -4,6 +4,8 @@

 总入口是 `../admin-test-env-acceptance.mjs`。每个业务模块使用独立的同名 `.mjs` 文件，避免多个对话同时修改总入口。

+`phase3-loading.mjs` 的 PDA 原子证据上传在未取得 HTTP 响应时最多尝试三次，冻结原 URL、作业员身份、幂等键、文件名/内容、证据版本和封签号；不重试 HTTP/业务拒绝或已收到响应后的读取失败。`createdResources.loadingEvidenceTransport` 保存各次结果及白名单网络错误码，不保存凭据或原始错误对象。`phase3-evidence-transport-contract.test.mjs` 用持久幂等模型验证响应丢失重放仅一条证据、持续传输失败有界终止及拒绝不重试；离线模型不替代真实 HTTP。
+
 `phase5-root-fulfillment.mjs` 是显式 TEST 的 F40 两箱根/余量专项，通过总入口 `--modules phase5-root-fulfillment` 准备本轮资质客户；模块用正式下单、测量、国内 PDA 拆分装柜、船节点、海外 PDA 收货、拣货发车及文件签收链创建两柜真实来源。默认根先签，编程调用 `AcceptanceRun.context.f40SignatureOrder='CHILD_FIRST'` 切换子先签；分别核对第一片签收不得根完成、最后一片签收才根完成、真实签收时间、状态历史和重放零新增。

 `AcceptanceRun.context.f40SignatureOrder='ROOT_SIGNED_REMAINDER_CANCELLED'` 单独验证合法末票余量取消：本轮新根通过 `orderStopAtApproved=true` 停在原始 `APPROVED`，只装运签收首箱；回查真实履约树及未装子单原始 `APPROVED`、入仓 `WAIT_SPLIT` 和一箱余量后，调用公开取消接口。`PROCESSING` 子单不得作为取消样本。根完成事件必须唯一发布，取消子单完成事件显式为零；原取消请求重放须 `A0409`，且根子版本、历史、摘要、履约树及完成 Outbox 零漂移。两种双签模式仍须两片实际签收，不能用取消替代第二签。
@@ -0,0 +1,86 @@
+import test from 'node:test'
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+import { uploadEvidence } from './phase3-loading.mjs'
+
+const context = () => ({ marker: 'CURRENT-SEAL', options: { gateway: 'https://offline.invalid' },
+  pdaToken: 'secret-test-token', token: 'different-admin-token', createdResources: {} })
+const invoke = ctx => uploadEvidence(ctx, '2105979722985902081', '193000000000200001', 'SEAL', 0, 'SEAL', { sealNo: 'SEAL-UNIQUE' })
+async function withTransport(handler, check) {
+  const originalFetch = globalThis.fetch, originalTimeout = globalThis.setTimeout
+  const waits = []
+  globalThis.fetch = handler
+  globalThis.setTimeout = (callback, delay) => { waits.push(delay); queueMicrotask(callback); return 0 }
+  try { await check(waits) } finally { globalThis.fetch = originalFetch; globalThis.setTimeout = originalTimeout }
+}
+function lost(code = 'ECONNRESET') {
+  return new TypeError('fetch failed https://private-host.invalid secret-test-token', { cause: { code, address: 'private-host.invalid' } })
+}
+
+test('committed upload with lost response replays frozen content/key and creates exactly one modeled evidence row', async () => {
+  const ctx = context(), records = new Map(), calls = []
+  let mutations = 0
+  await withTransport(async (url, options) => {
+    const file = options.body.get('file')
+    const hash = createHash('sha256').update(`${url}|${file.name}|${file.type}|${file.size}|`)
+      .update(Buffer.from(await file.arrayBuffer())).digest('hex')
+    const key = options.headers['Idempotency-Key']
+    calls.push({ url, key, hash, auth: options.headers.Authorization, body: options.body })
+    if (records.has(key)) assert.equal(records.get(key).hash, hash, 'same-key different payload must be rejected')
+    else { records.set(key, { hash, evidenceId: 'persisted-evidence' }); mutations++ }
+    if (calls.length === 1) {
+      ctx.marker = 'CHANGED'; ctx.pdaToken = 'new-token'; ctx.options.gateway = 'https://changed.invalid'
+      throw lost()
+    }
+    return new Response(JSON.stringify({ code: '200', data: { evidenceId: records.get(key).evidenceId } }), { status: 200 })
+  }, async waits => {
+    const result = await invoke(ctx)
+    assert.equal(result.body.data.evidenceId, 'persisted-evidence')
+    assert.equal(mutations, 1); assert.equal(records.size, 1); assert.equal(calls.length, 2)
+    assert.deepEqual(calls[0], calls[1]); assert.deepEqual(waits, [250])
+    assert.deepEqual(result.transportAttempts.map(item => item.outcome), ['NO_HTTP_RESPONSE', 'HTTP_RESPONSE'])
+    assert.equal(result.transportAttempts[0].causeCode, 'ECONNRESET')
+    const evidence = JSON.stringify(ctx.createdResources)
+    for (const secret of ['secret-test-token', 'new-token', 'private-host', 'offline.invalid', 'changed.invalid']) assert.equal(evidence.includes(secret), false)
+  })
+})
+
+test('persistent no-response failures stop at three attempts with bounded backoff and sanitized cause', async () => {
+  let calls = 0
+  await withTransport(async () => { calls++; throw lost('UND_ERR_CONNECT_TIMEOUT') }, async waits => {
+    const result = await invoke(context())
+    assert.equal(calls, 3); assert.equal(result.http, 0); assert.equal(result.body, null)
+    assert.deepEqual(waits, [250, 500]); assert.equal(result.transportAttempts.length, 3)
+    assert.match(result.raw, /UND_ERR_CONNECT_TIMEOUT/)
+    assert.doesNotMatch(JSON.stringify(result), /private-host|secret-test-token|address/)
+  })
+})
+
+for (const [http, code] of [[200, 'A0409'], [200, 'A0422'], [503, 'S0503'], [403, 'A0403']]) {
+  test(`HTTP ${http} business ${code} is returned without retry`, async () => {
+    let calls = 0
+    await withTransport(async () => { calls++; return new Response(JSON.stringify({ code, message: 'rejected', requestId: 'request' }), { status: http }) }, async waits => {
+      const result = await invoke(context())
+      assert.equal(calls, 1); assert.deepEqual(waits, []); assert.equal(result.http, http)
+      assert.equal(result.body.code, code); assert.equal(result.requestId, 'request')
+    })
+  })
+}
+
+test('received HTTP response with unreadable body is not reclassified as no-response retry', async () => {
+  let calls = 0
+  await withTransport(async () => { calls++; return { status: 502, headers: new Headers({ 'x-request-id': 'edge-request' }),
+    text: async () => { throw lost() } } }, async waits => {
+    const result = await invoke(context())
+    assert.equal(calls, 1); assert.deepEqual(waits, []); assert.equal(result.http, 502)
+    assert.equal(result.requestId, 'edge-request'); assert.equal(result.transportAttempts[0].outcome, 'RESPONSE_READ_FAILED')
+  })
+})
+
+test('unrecognized cause code containing sensitive material is omitted', async () => {
+  await withTransport(async () => { throw lost('secret-test-token https://private-host.invalid') }, async () => {
+    const result = await invoke(context())
+    assert.ok(result.transportAttempts.every(item => item.causeCode === null))
+    assert.doesNotMatch(JSON.stringify(result), /secret-test-token|private-host/)
+  })
+})
@@ -108,20 +108,41 @@ export async function uploadEvidence(context, taskId, warehouseId, evidenceType,
  const png = Uint8Array.from([137,80,78,71,13,10,26,10,0,0,0,13,73,72,68,82,0,0,0,1,0,0,0,1,8,6,0,0,0,31,21,196,137,0,0,0,13,73,68,65,84,8,215,99,248,207,192,240,31,0,5,0,1,255,137,153,61,29,0,0,0,0,73,69,78,68,174,66,96,130])
  const fileName = `${context.marker}-${String(fileSuffix).toLowerCase()}.png`
  form.append('file', new Blob([png], { type: 'image/png' }), fileName)
-  try {
-    const response = await fetch(`${context.options.gateway}${path}`, {
-      method: 'POST', headers: { Accept: 'application/json', Authorization: `Bearer ${context.pdaToken || context.token}`,
-        'Idempotency-Key': `${context.marker}-p3-evidence-${keySuffix}`.slice(0, 128) },
-      body: form, signal: AbortSignal.timeout(60_000),
-    })
-    const raw = await response.text(); let payload = null
-    try { payload = raw ? JSON.parse(raw) : null } catch { /* bounded raw evidence */ }
-    return { method: 'POST', path, http: response.status, body: payload,
-      raw: payload === null ? raw.slice(0, 500) : null,
-      requestId: payload?.requestId || response.headers.get('x-request-id'),
-      traceId: payload?.traceId || response.headers.get('x-trace-id') }
-  } catch (error) {
-    return { method: 'POST', path, http: 0, body: null, raw: String(error), requestId: null, traceId: null }
+  const url = `${context.options.gateway}${path}`
+  const headers = Object.freeze({ Accept: 'application/json', Authorization: `Bearer ${context.pdaToken || context.token}`,
+    'Idempotency-Key': `${context.marker}-p3-evidence-${keySuffix}`.slice(0, 128) })
+  const attempts = []
+  context.createdResources ??= {}
+  context.createdResources.loadingEvidenceTransport ??= []
+  // No credentials, absolute endpoint or raw error/cause object enters the report.
+  context.createdResources.loadingEvidenceTransport.push({ path, fileName,
+    idempotencyKey: headers['Idempotency-Key'], attempts })
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    let response
+    try {
+      response = await fetch(url, { method: 'POST', headers, body: form, signal: AbortSignal.timeout(60_000) })
+      const raw = await response.text(); let payload = null
+      try { payload = raw ? JSON.parse(raw) : null } catch { /* bounded raw evidence */ }
+      const result = { method: 'POST', path, http: response.status, body: payload,
+        raw: payload === null ? raw.slice(0, 500) : null,
+        requestId: payload?.requestId || response.headers.get('x-request-id'),
+        traceId: payload?.traceId || response.headers.get('x-trace-id') }
+      attempts.push({ attempt, http: result.http, code: payload?.code ?? null, outcome: 'HTTP_RESPONSE',
+        requestId: result.requestId, traceId: result.traceId })
+      return { ...result, transportAttempts: attempts }
+    } catch (error) {
+      const candidate = error?.cause?.code ?? error?.code
+      const causeCode = typeof candidate === 'string' && /^(?:ECONNRESET|ECONNREFUSED|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|EPIPE|ENETUNREACH|EHOSTUNREACH|UND_ERR_[A-Z_]+)$/.test(candidate)
+        ? candidate : null
+      const errorType = ['TypeError', 'AbortError', 'TimeoutError'].includes(error?.name) ? error.name : 'TransportError'
+      const http = response?.status ?? 0
+      attempts.push({ attempt, http, code: null, outcome: response ? 'RESPONSE_READ_FAILED' : 'NO_HTTP_RESPONSE', errorType, causeCode })
+      if (response || attempt === 3) return { method: 'POST', path, http, body: null,
+        raw: `${errorType}: ${response ? 'response read failed' : 'fetch failed'}${causeCode ? ` (${causeCode})` : ''}`,
+        requestId: response?.headers?.get('x-request-id') || null,
+        traceId: response?.headers?.get('x-trace-id') || null, transportAttempts: attempts }
+      await new Promise(resolve => setTimeout(resolve, 250 * attempt))
+    }
  }
 }