优化

2026-09-29 18:47:37 +08:00
parent 2c60a29e0f
commit c2927ab9a6
291 changed files with 4008 additions and 6298 deletions
@@ -0,0 +1,56 @@
+package com.baiyue.firstleg.controller;
+
+import com.baiyue.common.core.api.ApiResponse;
+import com.baiyue.common.core.api.ResultCode;
+import com.baiyue.common.core.exception.BizException;
+import com.baiyue.common.core.outbox.OutboxOperations;
+import com.baiyue.common.core.outbox.EventOperationsTokenGuard;
+import com.baiyue.common.event.*;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.web.bind.annotation.*;
+import java.util.Set;
+
+/** Internal-only operator recovery. It cannot mutate the original event or call a consumer directly. */
+@RestController
+@RequestMapping("/internal/firstleg/event-operations")
+public class EventOperationsController {
+    private final EventOperationsTokenGuard guard;
+    private final OutboxOperations operations;
+
+    public EventOperationsController(JdbcTemplate jdbc, PlatformTransactionManager transactions,
+            ObjectMapper mapper, @Value("${baiyue.event-delivery.operations-token:}") String token) {
+        guard = new EventOperationsTokenGuard(token);
+        operations = new OutboxOperations(jdbc, transactions,
+            new OutboxOperations.Layout("fl_inbound_outbox", "id", "publish_status", Set.of(EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED, EventTypes.FIRST_LEG_CONTAINER_SHIPPING_NODE_RECORDED)),
+            json -> {
+                try {
+                    EventEnvelope envelope = mapper.readValue(json, EventEnvelope.class).requireValid();
+                    if (!EventPayloadHash.matches(EventPayloadCanonicalizer.canonicalize(envelope.payload()), envelope.payloadHash())) {
+                        throw new IllegalArgumentException("Hash mismatch");
+                    }
+                    return new OutboxOperations.Identity(envelope.eventId(), envelope.eventType(), envelope.payloadHash());
+                } catch (Exception invalid) {
+                    throw new BizException(ResultCode.CONFLICT, "原事件格式或摘要无效，禁止人工重试");
+                }
+            });
+    }
+
+    @GetMapping("/{entryId}")
+    public ApiResponse<OutboxOperations.Entry> inspect(
+            @RequestHeader(value = "X-Event-Operations-Token", required = false) String token,
+            @PathVariable String entryId) {
+        guard.require(token);
+        return ApiResponse.success(operations.inspect(entryId));
+    }
+
+    @PostMapping("/{entryId}/retry")
+    public ApiResponse<OutboxOperations.RetryResult> retry(
+            @RequestHeader(value = "X-Event-Operations-Token", required = false) String token,
+            @PathVariable String entryId, @RequestBody OutboxOperations.RetryRequest request) {
+        guard.require(token);
+        return ApiResponse.success(operations.retry(entryId, request));
+    }
+}
@@ -10,7 +10,6 @@ import com.baiyue.common.event.EventPayloadCanonicalizer;
 import com.baiyue.common.event.EventPayloadHash;
 import com.baiyue.common.event.EventTypes;
 import com.baiyue.common.event.FirstLegContainerShippingNodeRecordedV1;
-import com.baiyue.common.event.RegionCode;
 import com.baiyue.firstleg.entity.InboundOutboxEntity;
 import com.baiyue.firstleg.mapper.InboundOutboxMapper;
 import com.baiyue.firstleg.portschedule.entity.CarrierScheduleNodeFactEntity;
@@ -68,7 +67,7 @@ public class CarrierScheduleNodeOutboxService {
        long aggregateVersion = Math.max(0, version == null ? 0 : version);
        EventEnvelope envelope = new EventEnvelope(EventIdentifiers.newUuid(),
            EventTypes.FIRST_LEG_CONTAINER_SHIPPING_NODE_RECORDED, 1,
-            fact.getActualAt().atZone(zone).toInstant(), RegionCode.CN, RegionCode.OVERSEAS,
+            fact.getActualAt().atZone(zone).toInstant(),
            "0", "CONTAINER_BOOKING", string(containerBookingId),
            aggregateVersion, aggregateVersion + 1, EventIdentifiers.newUuid(), eventKey, null,
            payloadNode, payloadHash).requireValid();
@@ -29,35 +29,42 @@ public class LoadingCompletedOutboxRepository {
            .orderByAsc(InboundOutboxEntity::getCreatedAt).orderByAsc(InboundOutboxEntity::getId)
            .last("LIMIT " + Math.min(Math.max(limit, 1), 100)));
    }
-    public boolean claim(long id, String worker, LocalDateTime now, LocalDateTime expiredLeaseBefore) {
+    public boolean claim(long id, String worker, LocalDateTime now) {
        return mapper.update(null, new LambdaUpdateWrapper<InboundOutboxEntity>()
-        .eq(InboundOutboxEntity::getId, id)
-        .in(InboundOutboxEntity::getEventType, CROSS_REGION_EVENT_TYPES)
-        .and(status -> status.eq(InboundOutboxEntity::getPublishStatus, "PENDING")
-            .and(retry -> retry.isNull(InboundOutboxEntity::getNextRetryAt)
-                .or().le(InboundOutboxEntity::getNextRetryAt, now))
-            .or(stale -> stale.eq(InboundOutboxEntity::getPublishStatus, "PROCESSING")
-                .le(InboundOutboxEntity::getLockedAt, expiredLeaseBefore)))
-        .setSql("retry_count = retry_count + CASE WHEN publish_status='PROCESSING' THEN 1 ELSE 0 END")
-        .set(InboundOutboxEntity::getPublishStatus, "PROCESSING").set(InboundOutboxEntity::getLockedBy, worker)
-        .set(InboundOutboxEntity::getLockedAt, now)) == 1;
+            .eq(InboundOutboxEntity::getId, id).in(InboundOutboxEntity::getEventType, CROSS_REGION_EVENT_TYPES)
+            .eq(InboundOutboxEntity::getPublishStatus, "PENDING")
+            .and(retry -> retry.isNull(InboundOutboxEntity::getNextRetryAt).or().le(InboundOutboxEntity::getNextRetryAt, now))
+            .set(InboundOutboxEntity::getPublishStatus, "PROCESSING")
+            .set(InboundOutboxEntity::getLockedBy, worker).set(InboundOutboxEntity::getLockedAt, now)) == 1;
+    }
+
+    public int expire(long id, String worker, LocalDateTime cutoff, int failures, LocalDateTime next, String error) {
+        return mapper.update(null, new LambdaUpdateWrapper<InboundOutboxEntity>()
+            .eq(InboundOutboxEntity::getId, id).in(InboundOutboxEntity::getEventType, CROSS_REGION_EVENT_TYPES)
+            .eq(InboundOutboxEntity::getPublishStatus, "PROCESSING").eq(InboundOutboxEntity::getLockedBy, worker)
+            .le(InboundOutboxEntity::getLockedAt, cutoff)
+            .set(InboundOutboxEntity::getPublishStatus, next == null ? "DEAD" : "PENDING")
+            .set(InboundOutboxEntity::getRetryCount, failures).set(InboundOutboxEntity::getNextRetryAt, next)
+            .set(InboundOutboxEntity::getLockedBy, null).set(InboundOutboxEntity::getLockedAt, null)
+            .set(InboundOutboxEntity::getLastError, truncate(error)));
    }
    public InboundOutboxEntity claimed(long id, String worker) { return mapper.selectOne(new LambdaQueryWrapper<InboundOutboxEntity>()
        .eq(InboundOutboxEntity::getId, id).in(InboundOutboxEntity::getEventType, CROSS_REGION_EVENT_TYPES)
        .eq(InboundOutboxEntity::getPublishStatus, "PROCESSING").eq(InboundOutboxEntity::getLockedBy, worker)); }
-    public void published(long id, String worker, LocalDateTime now) { mapper.update(null, claimedBase(id, worker)
+    public int published(long id, String worker, LocalDateTime now) { return mapper.update(null, claimedBase(id, worker)
        .set(InboundOutboxEntity::getPublishStatus, "PUBLISHED").set(InboundOutboxEntity::getPublishedAt, now)
+        .set(InboundOutboxEntity::getNextRetryAt, null)
        .set(InboundOutboxEntity::getLockedBy, null).set(InboundOutboxEntity::getLockedAt, null)
        .set(InboundOutboxEntity::getLastError, null)); }
-    public void dead(long id, String worker, int failures, String error) {
-        mapper.update(null, claimedBase(id, worker)
+    public int dead(long id, String worker, int failures, String error) {
+        return mapper.update(null, claimedBase(id, worker)
            .set(InboundOutboxEntity::getPublishStatus, "DEAD")
            .set(InboundOutboxEntity::getRetryCount, failures)
            .set(InboundOutboxEntity::getNextRetryAt, null)
            .set(InboundOutboxEntity::getLockedBy, null).set(InboundOutboxEntity::getLockedAt, null)
            .set(InboundOutboxEntity::getLastError, truncate(error)));
    }
-    public void retry(long id, String worker, LocalDateTime retryAt, String error) { mapper.update(null, claimedBase(id, worker)
+    public int retry(long id, String worker, LocalDateTime retryAt, String error) { return mapper.update(null, claimedBase(id, worker)
        .setSql("retry_count = retry_count + 1").set(InboundOutboxEntity::getPublishStatus, "PENDING")
        .set(InboundOutboxEntity::getNextRetryAt, retryAt).set(InboundOutboxEntity::getLockedBy, null)
        .set(InboundOutboxEntity::getLockedAt, null).set(InboundOutboxEntity::getLastError, truncate(error))); }
@@ -1,6 +1,12 @@
 package com.baiyue.firstleg.service;

 import com.baiyue.common.core.api.ApiResponse;
+import com.baiyue.common.core.outbox.EventDeliveryJournal;
+import com.baiyue.common.core.outbox.EventDeliveryProducer;
+import com.baiyue.common.core.outbox.EventDeliveryJournal.Outcome;
+import com.baiyue.common.web.EventDeliveryMetrics;
+import com.baiyue.common.web.EventDeliveryMetrics.Stream;
+import com.baiyue.common.web.EventDeliveryObservation;
 import com.baiyue.common.event.EventEnvelope;
 import com.baiyue.common.web.EventDeliveryFailure;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -21,10 +27,12 @@ import java.util.UUID;

 /** 至少一次投递国内头程业务事件；Lastleg按 eventId + payloadHash 幂等接收。 */
@Component @Slf4j
-public class LoadingCompletedOutboxRelay {
+public class LoadingCompletedOutboxRelay implements EventDeliveryProducer {
    @Resource private LoadingCompletedOutboxRepository repository;
    @Resource private LastlegEventClient client;
    @Resource private ObjectMapper objectMapper;
+    @Resource private EventDeliveryJournal journal;
+    @Resource private EventDeliveryMetrics metrics;
    @Value("${baiyue.security.internal-service-token}") private String internalToken;
    @Value("${baiyue.event-delivery.lease-seconds:60}") private long leaseSeconds = 60;

@@ -37,45 +45,63 @@ public class LoadingCompletedOutboxRelay {
    public void relay() {
        if (leaseSeconds < 1) throw new IllegalArgumentException("Lease duration must be positive");
        var policy = new BoundedRetryPolicy(maxAttempts, baseDelaySeconds, maxDelaySeconds, retryJitter);
-        LocalDateTime now=LocalDateTime.now();
-        LocalDateTime expiredLeaseBefore=now.minusSeconds(leaseSeconds);
-        for (InboundOutboxEntity candidate:repository.pending(50,now,expiredLeaseBefore)) {
-            String workerId = UUID.randomUUID().toString();
-            now = LocalDateTime.now();
-            expiredLeaseBefore = now.minusSeconds(leaseSeconds);
-
-            if(!repository.claim(candidate.getId(),workerId,now,expiredLeaseBefore)) continue;
-            InboundOutboxEntity row=repository.claimed(candidate.getId(),workerId);
-            if(row==null) continue;
+        LocalDateTime scan = LocalDateTime.now();
+        for (InboundOutboxEntity candidate : repository.pending(50, scan, scan.minusSeconds(leaseSeconds))) {
+            LocalDateTime started = LocalDateTime.now();
+            if ("PROCESSING".equals(candidate.getPublishStatus())) {
+                int failures = (candidate.getRetryCount() == null ? 0 : candidate.getRetryCount()) + 1;
+                LocalDateTime next = policy.exhausted(failures) ? null : started.plus(policy.delayAfterFailure(failures));
+                journal.complete(candidate.getLockedBy(), Outcome.LEASE_EXPIRED, started, "LEASE_EXPIRED", next,
+                    () -> repository.expire(candidate.getId(), candidate.getLockedBy(), started.minusSeconds(leaseSeconds),
+                        failures, next, "投递租约超时，消费结果未知"));
+                continue; // Recovery always waits for the new backoff window; never sends in this pass.
+            }
+            String worker = UUID.randomUUID().toString();
+            String type = Stream.FIRSTLEG.type(candidate.getEventType());
+            if (!journal.begin(worker, Stream.FIRSTLEG.service, type, String.valueOf(candidate.getId()),
+                    EventDeliveryObservation.eventId(objectMapper, candidate.getPayloadJson()), Stream.FIRSTLEG.target(type), started,
+                    () -> repository.claim(candidate.getId(), worker, started))) continue;
+            InboundOutboxEntity row = repository.claimed(candidate.getId(), worker);
+            if (row == null) continue;
            int failures = row.getRetryCount() == null ? 0 : row.getRetryCount();
            if (policy.exhausted(failures)) {
-                repository.dead(row.getId(), workerId, failures, "自动投递次数已耗尽（含租约超时）");
+                journal.complete(worker, Outcome.DEAD, LocalDateTime.now(), "RETRY_EXHAUSTED", null,
+                    () -> repository.dead(row.getId(), worker, failures, "自动投递次数已耗尽（含租约超时）"));
                continue;
            }
+            Outcome outcome;
+            String diagnostic = null;
+            String auditError = null;
+            boolean permanent = false;
            boolean requestStarted = false;
            try {
-                EventEnvelope event=objectMapper.readValue(row.getPayloadJson(),EventEnvelope.class);
-                EventEnvelopeValidator.requireValid(event);
+                EventEnvelope event = objectMapper.readValue(row.getPayloadJson(), EventEnvelope.class).requireValid();
                requestStarted = true;
-                var response=client.receive(internalToken,event);
-                requireAcknowledgement(event, response);
-                repository.published(row.getId(),workerId,LocalDateTime.now());
-            } catch(Exception exception) {
-                boolean permanent = EventDeliveryFailure.isPermanent(exception)
+                var response = client.receive(internalToken, event);
+                EventDeliveryFailure.requireSuccessful(response);
+                response.data().requireMatches(event);
+                outcome = Outcome.valueOf(response.data().outcome().name());
+            } catch (Exception exception) {
+                permanent = EventDeliveryFailure.isPermanent(exception)
                    || (!requestStarted && (exception instanceof JsonProcessingException || exception instanceof IllegalArgumentException));
-                String error = EventDeliveryFailure.diagnostic(exception, internalToken);
-                if (permanent || policy.exhausted(failures + 1)) repository.dead(row.getId(), workerId, failures + 1, error);
-                else repository.retry(row.getId(), workerId, LocalDateTime.now().plus(policy.delayAfterFailure(failures + 1)), error);
-                log.warn("头程业务事件投递失败: eventType={}, eventKey={}, reason={}",
-                    row.getEventType(), row.getEventKey(), error);
+                diagnostic = EventDeliveryFailure.diagnostic(exception, internalToken);
+                auditError = EventDeliveryObservation.errorCode(exception, permanent, requestStarted);
+                outcome = permanent || policy.exhausted(failures + 1) ? Outcome.DEAD : Outcome.RETRY;
            }
+            // Persistence failures propagate. Do not reinterpret a failed commit as an HTTP failure.
+            LocalDateTime finished = LocalDateTime.now();
+            LocalDateTime next = outcome == Outcome.RETRY ? finished.plus(policy.delayAfterFailure(failures + 1)) : null;
+            Outcome result = outcome;
+            String error = diagnostic;
+            int retries = failures + 1;
+            journal.complete(worker, result, finished, auditError, next, () -> {
+                if (result == Outcome.DEAD) return repository.dead(row.getId(), worker, retries, error);
+                if (result == Outcome.RETRY) return repository.retry(row.getId(), worker, next, error);
+                return repository.published(row.getId(), worker, finished);
+            });
        }
    }

-    private void requireAcknowledgement(EventEnvelope event,
-                                        ApiResponse<com.baiyue.common.event.DeliveryAck> response) {
-        EventDeliveryFailure.requireSuccessful(response);
-        response.data().requireMatches(event);
-    }
-
+    @Scheduled(fixedDelayString = "${baiyue.event-delivery.metrics-sample-ms:30000}")
+    public void sampleMetrics() { metrics.sample(Stream.FIRSTLEG, LocalDateTime.now()); }
 }
@@ -11,7 +11,6 @@ import com.baiyue.common.event.EventPayloadHash;
 import com.baiyue.common.event.EventTypes;
 import com.baiyue.common.event.FirstLegLoadingPlanCompletedV2;
 import com.baiyue.common.event.FirstLegLoadingPlanCompletedV3;
-import com.baiyue.common.event.RegionCode;
 import com.baiyue.firstleg.entity.InboundOutboxEntity;
 import com.baiyue.firstleg.mapper.InboundOutboxMapper;
 import com.baiyue.firstleg.mapper.LoadingCompletedEventMapper;
@@ -124,7 +123,7 @@ public class LoadingCompletedOutboxService {
        String eventKey = "FIRSTLEG_LOADING_COMPLETED:" + first.getLoadingPlanId() + ':' + accountId + ':' + HANDOVER_VERSION;
        String traceId = EventIdentifiers.newUuid();
        return new EventEnvelope(EventIdentifiers.newUuid(), EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED, 3,
-            Instant.now(), RegionCode.CN, RegionCode.OVERSEAS, String.valueOf(accountId), "LOADING_PLAN",
+            Instant.now(), String.valueOf(accountId), "LOADING_PLAN",
            String.valueOf(first.getLoadingPlanId()), first.getAggregateVersion(), HANDOVER_VERSION,
            traceId, eventKey, null, payloadNode, hash).requireValid();
    }
@@ -8,7 +8,6 @@ import com.baiyue.common.event.EventPayloadCanonicalizer;
 import com.baiyue.common.event.EventPayloadHash;
 import com.baiyue.common.event.EventTypes;
 import com.baiyue.common.event.OverseasContainerUnloadedV1;
-import com.baiyue.common.event.RegionCode;
 import com.baiyue.firstleg.entity.ContainerBookingEntity;
 import com.baiyue.firstleg.entity.ContainerEventInboxEntity;
 import com.baiyue.firstleg.mapper.ContainerBookingMapper;
@@ -83,9 +82,8 @@ public class OverseasContainerEventConsumer {
        if (!EventTypes.OVERSEAS_CONTAINER_UNLOADED.equals(event.eventType()) || event.eventVersion() != 1) {
            throw new EventProtocolException("不支持的海外货柜事件");
        }
-        if (event.sourceRegion() != RegionCode.OVERSEAS || event.targetRegion() != RegionCode.CN
-            || !"CONTAINER_BOOKING".equals(event.aggregateType())) {
-            throw new EventProtocolException("海外货柜事件区域或聚合类型无效");
+        if (!"CONTAINER_BOOKING".equals(event.aggregateType())) {
+            throw new EventProtocolException("海外货柜事件聚合类型无效");
        }
        if (!EventPayloadHash.matches(EventPayloadCanonicalizer.canonicalize(event.payload()), event.payloadHash())) {
            throw new EventProtocolException("payloadHash 与事件负载不一致");
@@ -9,7 +9,6 @@ import com.baiyue.common.event.EventPayloadCanonicalizer;
 import com.baiyue.common.event.EventPayloadHash;
 import com.baiyue.common.event.EventTypes;
 import com.baiyue.common.event.OverseasDeliveryCompletedV1;
-import com.baiyue.common.event.RegionCode;
 import com.baiyue.firstleg.client.OrderFulfillmentCompletionClient;
 import com.baiyue.firstleg.entity.ContainerEventInboxEntity;
 import com.baiyue.firstleg.entity.InboundOrderEntity;
@@ -102,7 +101,6 @@ public class OverseasDeliveryCompletedConsumer {
    private void validate(EventEnvelope event) {
        event.requireValid();
        if (!EventTypes.OVERSEAS_DELIVERY_COMPLETED.equals(event.eventType()) || event.eventVersion() != 1
-            || event.sourceRegion() != RegionCode.OVERSEAS || event.targetRegion() != RegionCode.CN
            || !"INBOUND_ORDER".equals(event.aggregateType())) {
            throw new EventProtocolException("不支持的海外派送完成事件");
        }
@@ -9,7 +9,6 @@ import com.baiyue.common.event.EventPayloadCanonicalizer;
 import com.baiyue.common.event.EventPayloadHash;
 import com.baiyue.common.event.EventTypes;
 import com.baiyue.common.event.OverseasInboundOrderDetainedChangedV1;
-import com.baiyue.common.event.RegionCode;
 import com.baiyue.firstleg.entity.ContainerEventInboxEntity;
 import com.baiyue.firstleg.entity.InboundHoldRecordEntity;
 import com.baiyue.firstleg.entity.InboundOrderEntity;
@@ -87,8 +86,7 @@ public class OverseasInboundOrderDetainedConsumer {
    private void validateEnvelope(EventEnvelope event) {
        event.requireValid();
        if (!EventTypes.OVERSEAS_INBOUND_ORDER_DETAINED_CHANGED.equals(event.eventType())
-            || event.eventVersion() != 1 || event.sourceRegion() != RegionCode.OVERSEAS
-            || event.targetRegion() != RegionCode.CN || !"INBOUND_ORDER".equals(event.aggregateType())) {
+            || event.eventVersion() != 1 || !"INBOUND_ORDER".equals(event.aggregateType())) {
            throw new EventProtocolException("不支持的海外入仓单暂扣事件");
        }
        if (!EventPayloadHash.matches(EventPayloadCanonicalizer.canonicalize(event.payload()), event.payloadHash())) {