优化
This commit is contained in:
+18
@@ -2,6 +2,7 @@ package com.baiyue.eventbridge.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.metrics.InboxMetricsSnapshot;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
@@ -60,4 +61,21 @@ public interface BridgeInboxMapper extends BaseMapper<BridgeInbox> {
|
||||
WHERE event_id=#{eventId} AND status='DEAD'
|
||||
""")
|
||||
int replayDead(@Param("eventId") String eventId, @Param("reason") String reason);
|
||||
|
||||
/**
|
||||
* Scheduled observability aggregation. The status predicate uses idx_bridge_inbox_consume;
|
||||
* Prometheus collection never invokes this query directly.
|
||||
*/
|
||||
@Select("""
|
||||
SELECT target_region AS targetRegion, event_type AS eventType,
|
||||
SUM(CASE WHEN status IN ('RECEIVED','RETRY','PROCESSING') THEN 1 ELSE 0 END) AS backlogCount,
|
||||
COALESCE(TIMESTAMPDIFF(MICROSECOND,
|
||||
MIN(CASE WHEN status IN ('RECEIVED','RETRY','PROCESSING') THEN received_at END),
|
||||
CURRENT_TIMESTAMP(3)) / 1000000.0, 0) AS oldestPendingAgeSeconds,
|
||||
SUM(CASE WHEN status='DEAD' THEN 1 ELSE 0 END) AS deadCount
|
||||
FROM bridge_inbox
|
||||
WHERE status IN ('RECEIVED','RETRY','PROCESSING','DEAD')
|
||||
GROUP BY target_region, event_type
|
||||
""")
|
||||
List<InboxMetricsSnapshot> observabilitySnapshots();
|
||||
}
|
||||
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
package com.baiyue.eventbridge.metrics;
|
||||
|
||||
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.micrometer.core.instrument.MultiGauge;
|
||||
import io.micrometer.core.instrument.Tags;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/** Periodically samples indexed Inbox aggregates; Prometheus scrapes only the cached gauges. */
|
||||
@Component
|
||||
@Slf4j
|
||||
public class EventBridgeInboxMetricsSampler {
|
||||
public static final String BACKLOG_METRIC = "baiyue.event.bridge.inbox.backlog";
|
||||
public static final String OLDEST_AGE_METRIC = "baiyue.event.bridge.inbox.oldest.age.seconds";
|
||||
public static final String DEAD_METRIC = "baiyue.event.bridge.inbox.dead";
|
||||
|
||||
private final BridgeInboxMapper inboxMapper;
|
||||
private final MultiGauge backlog;
|
||||
private final MultiGauge oldestAge;
|
||||
private final MultiGauge dead;
|
||||
|
||||
public EventBridgeInboxMetricsSampler(BridgeInboxMapper inboxMapper, MeterRegistry registry) {
|
||||
this.inboxMapper = inboxMapper;
|
||||
this.backlog = MultiGauge.builder(BACKLOG_METRIC)
|
||||
.description("Events waiting for downstream acknowledgement").register(registry);
|
||||
this.oldestAge = MultiGauge.builder(OLDEST_AGE_METRIC)
|
||||
.description("Age in seconds of the oldest pending Inbox event").register(registry);
|
||||
this.dead = MultiGauge.builder(DEAD_METRIC)
|
||||
.description("Current number of DEAD Inbox events").register(registry);
|
||||
}
|
||||
|
||||
@Scheduled(initialDelayString = "${baiyue.event-bridge.metrics.initial-delay:5000}",
|
||||
fixedDelayString = "${baiyue.event-bridge.metrics.sample-delay:30000}")
|
||||
public void sample() {
|
||||
final List<InboxMetricsSnapshot> snapshots;
|
||||
try {
|
||||
snapshots = inboxMapper.observabilitySnapshots();
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Event Bridge Inbox 指标采样失败,保留上次成功快照", exception);
|
||||
return;
|
||||
}
|
||||
Map<MetricKey, Aggregate> aggregates = aggregate(snapshots);
|
||||
backlog.register(rows(aggregates, MetricValue.BACKLOG), true);
|
||||
oldestAge.register(rows(aggregates, MetricValue.OLDEST_AGE), true);
|
||||
dead.register(rows(aggregates, MetricValue.DEAD), true);
|
||||
}
|
||||
|
||||
private Map<MetricKey, Aggregate> aggregate(List<InboxMetricsSnapshot> snapshots) {
|
||||
Map<MetricKey, Aggregate> result = new LinkedHashMap<>();
|
||||
if (snapshots == null) {
|
||||
return result;
|
||||
}
|
||||
for (InboxMetricsSnapshot snapshot : snapshots) {
|
||||
MetricKey key = new MetricKey(EventBridgeMetrics.targetRegion(snapshot.getTargetRegion()),
|
||||
EventBridgeMetrics.eventType(snapshot.getEventType()));
|
||||
result.computeIfAbsent(key, ignored -> new Aggregate()).add(snapshot);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<MultiGauge.Row<?>> rows(Map<MetricKey, Aggregate> aggregates, MetricValue value) {
|
||||
List<MultiGauge.Row<?>> rows = new ArrayList<>();
|
||||
aggregates.forEach((key, aggregate) -> rows.add(MultiGauge.Row.of(
|
||||
Tags.of("target_region", key.targetRegion, "event_type", key.eventType),
|
||||
value.value(aggregate))));
|
||||
return rows;
|
||||
}
|
||||
|
||||
private record MetricKey(String targetRegion, String eventType) { }
|
||||
|
||||
private static final class Aggregate {
|
||||
private long backlog;
|
||||
private double oldestAge;
|
||||
private long dead;
|
||||
|
||||
void add(InboxMetricsSnapshot snapshot) {
|
||||
backlog += snapshot.getBacklogCount();
|
||||
oldestAge = Math.max(oldestAge, snapshot.getOldestPendingAgeSeconds());
|
||||
dead += snapshot.getDeadCount();
|
||||
}
|
||||
}
|
||||
|
||||
private enum MetricValue {
|
||||
BACKLOG {
|
||||
@Override Number value(Aggregate aggregate) { return aggregate.backlog; }
|
||||
},
|
||||
OLDEST_AGE {
|
||||
@Override Number value(Aggregate aggregate) { return aggregate.oldestAge; }
|
||||
},
|
||||
DEAD {
|
||||
@Override Number value(Aggregate aggregate) { return aggregate.dead; }
|
||||
};
|
||||
|
||||
abstract Number value(Aggregate aggregate);
|
||||
}
|
||||
}
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
package com.baiyue.eventbridge.metrics;
|
||||
|
||||
import com.baiyue.common.event.EventEnvelope;
|
||||
import com.baiyue.common.event.EventTypes;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
/** Low-cardinality transport metrics for the cross-region event bridge. */
|
||||
@Component
|
||||
@Slf4j
|
||||
public class EventBridgeMetrics {
|
||||
public static final String RECEIVE_METRIC = "baiyue.event.bridge.receive";
|
||||
public static final String TRANSPORT_METRIC = "baiyue.event.bridge.transport";
|
||||
public static final String DELIVERY_LATENCY_METRIC = "baiyue.event.bridge.delivery.latency";
|
||||
|
||||
private static final Set<String> KNOWN_EVENT_TYPES = Set.of(
|
||||
EventTypes.ORDER_APPROVED,
|
||||
EventTypes.ORDER_CANCELLED,
|
||||
EventTypes.FIRST_LEG_SHIPMENT_DEPARTED,
|
||||
EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED,
|
||||
EventTypes.FIRST_LEG_CONTAINER_SHIPPING_NODE_RECORDED,
|
||||
EventTypes.OVERSEAS_CONTAINER_UNLOADED,
|
||||
EventTypes.IAM_USER_STATUS_CHANGED,
|
||||
EventTypes.IAM_TENANT_STATUS_CHANGED,
|
||||
EventTypes.IAM_PERMISSION_VERSION_CHANGED,
|
||||
EventTypes.LAST_MILE_HANDOFF_ACCEPTED,
|
||||
EventTypes.LAST_MILE_HANDOFF_REJECTED,
|
||||
EventTypes.LAST_MILE_MILESTONE_CHANGED,
|
||||
EventTypes.LAST_MILE_COMPLETED,
|
||||
EventTypes.TRACKING_MILESTONE_PROJECTED);
|
||||
|
||||
private final MeterRegistry registry;
|
||||
private final Clock clock;
|
||||
|
||||
public EventBridgeMetrics(MeterRegistry registry) {
|
||||
this(registry, Clock.systemDefaultZone());
|
||||
}
|
||||
|
||||
EventBridgeMetrics(MeterRegistry registry, Clock clock) {
|
||||
this.registry = registry;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
public void recordReceive(EventEnvelope envelope, ReceiveResult result) {
|
||||
try {
|
||||
registry.counter(RECEIVE_METRIC,
|
||||
"target_region", targetRegion(envelope.targetRegion().name()),
|
||||
"event_type", eventType(envelope.eventType()),
|
||||
"result", result.name()).increment();
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Event Bridge 接收指标记录失败,不影响事件事务", exception);
|
||||
}
|
||||
}
|
||||
|
||||
public void recordTransport(BridgeInbox event, TransportResult result) {
|
||||
try {
|
||||
String targetRegion = targetRegion(event.getTargetRegion());
|
||||
String eventType = eventType(event.getEventType());
|
||||
registry.counter(TRANSPORT_METRIC,
|
||||
"target_region", targetRegion,
|
||||
"event_type", eventType,
|
||||
"result", result.name()).increment();
|
||||
if (result.acknowledged() && event.getReceivedAt() != null) {
|
||||
Duration latency = Duration.between(event.getReceivedAt(), LocalDateTime.now(clock));
|
||||
if (latency.isNegative()) {
|
||||
latency = Duration.ZERO;
|
||||
}
|
||||
Timer.builder(DELIVERY_LATENCY_METRIC)
|
||||
.description("Time from Event Bridge receipt to downstream acknowledgement")
|
||||
.serviceLevelObjectives(Duration.ofSeconds(60), Duration.ofMinutes(15))
|
||||
.tags("target_region", targetRegion, "event_type", eventType)
|
||||
.register(registry).record(latency);
|
||||
}
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Event Bridge 传输指标记录失败,不影响可靠性状态机", exception);
|
||||
}
|
||||
}
|
||||
|
||||
public static String targetRegion(String value) {
|
||||
String normalized = value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
|
||||
return "CN".equals(normalized) || "OVERSEAS".equals(normalized) ? normalized : "OTHER";
|
||||
}
|
||||
|
||||
public static String eventType(String value) {
|
||||
return value != null && KNOWN_EVENT_TYPES.contains(value) ? value : "OTHER";
|
||||
}
|
||||
|
||||
public enum ReceiveResult {
|
||||
RECEIVED,
|
||||
DUPLICATE,
|
||||
CONFLICT
|
||||
}
|
||||
|
||||
public enum TransportResult {
|
||||
DELIVERED(true),
|
||||
RETRY(false),
|
||||
DEAD(false),
|
||||
ACKNOWLEDGED_LEASE_LOST(true),
|
||||
FAILED_LEASE_LOST(false),
|
||||
MANUAL_REPLAY(false);
|
||||
|
||||
private final boolean acknowledged;
|
||||
|
||||
TransportResult(boolean acknowledged) {
|
||||
this.acknowledged = acknowledged;
|
||||
}
|
||||
|
||||
boolean acknowledged() {
|
||||
return acknowledged;
|
||||
}
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
package com.baiyue.eventbridge.metrics;
|
||||
|
||||
/** Result row of the bounded, scheduled Inbox observability aggregation. */
|
||||
public class InboxMetricsSnapshot {
|
||||
private String targetRegion;
|
||||
private String eventType;
|
||||
private long backlogCount;
|
||||
private double oldestPendingAgeSeconds;
|
||||
private long deadCount;
|
||||
|
||||
public String getTargetRegion() {
|
||||
return targetRegion;
|
||||
}
|
||||
|
||||
public void setTargetRegion(String targetRegion) {
|
||||
this.targetRegion = targetRegion;
|
||||
}
|
||||
|
||||
public String getEventType() {
|
||||
return eventType;
|
||||
}
|
||||
|
||||
public void setEventType(String eventType) {
|
||||
this.eventType = eventType;
|
||||
}
|
||||
|
||||
public long getBacklogCount() {
|
||||
return backlogCount;
|
||||
}
|
||||
|
||||
public void setBacklogCount(long backlogCount) {
|
||||
this.backlogCount = backlogCount;
|
||||
}
|
||||
|
||||
public double getOldestPendingAgeSeconds() {
|
||||
return oldestPendingAgeSeconds;
|
||||
}
|
||||
|
||||
public void setOldestPendingAgeSeconds(double oldestPendingAgeSeconds) {
|
||||
this.oldestPendingAgeSeconds = oldestPendingAgeSeconds;
|
||||
}
|
||||
|
||||
public long getDeadCount() {
|
||||
return deadCount;
|
||||
}
|
||||
|
||||
public void setDeadCount(long deadCount) {
|
||||
this.deadCount = deadCount;
|
||||
}
|
||||
}
|
||||
+5
@@ -5,6 +5,7 @@ import com.baiyue.common.event.EventPayloadCanonicalizer;
|
||||
import com.baiyue.common.event.EventPayloadHash;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.dto.EventReceiveResult;
|
||||
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -23,6 +24,7 @@ import java.util.Objects;
|
||||
public class EventInboxService {
|
||||
@Resource private InboxStore inboxStore;
|
||||
@Resource private ObjectMapper objectMapper;
|
||||
@Resource private EventBridgeMetrics metrics;
|
||||
|
||||
@Value("${baiyue.region:CN}")
|
||||
private String localRegion;
|
||||
@@ -44,6 +46,7 @@ public class EventInboxService {
|
||||
BridgeInbox inbox = toInbox(envelope);
|
||||
try {
|
||||
inboxStore.insert(inbox);
|
||||
metrics.recordReceive(envelope, EventBridgeMetrics.ReceiveResult.RECEIVED);
|
||||
return new EventReceiveResult(envelope.eventId(), "已接收", inbox.getReceivedAt());
|
||||
} catch (DuplicateKeyException exception) {
|
||||
BridgeInbox concurrent = inboxStore.findByEventId(envelope.eventId());
|
||||
@@ -56,8 +59,10 @@ public class EventInboxService {
|
||||
|
||||
private EventReceiveResult duplicateOrConflict(BridgeInbox existing, EventEnvelope envelope) {
|
||||
if (!sameEnvelope(existing, envelope)) {
|
||||
metrics.recordReceive(envelope, EventBridgeMetrics.ReceiveResult.CONFLICT);
|
||||
throw new IllegalArgumentException("相同 eventId 对应的事件信封或 payloadHash 不一致,拒绝覆盖既有事件");
|
||||
}
|
||||
metrics.recordReceive(envelope, EventBridgeMetrics.ReceiveResult.DUPLICATE);
|
||||
return new EventReceiveResult(envelope.eventId(), "重复", existing.getReceivedAt());
|
||||
}
|
||||
|
||||
|
||||
+26
@@ -1,13 +1,23 @@
|
||||
package com.baiyue.eventbridge.service;
|
||||
|
||||
import com.baiyue.eventbridge.domain.BridgeDeliveryAttempt;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
|
||||
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
|
||||
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
|
||||
import jakarta.annotation.Resource;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Service
|
||||
public class EventReplayService {
|
||||
@Resource private BridgeInboxMapper inboxMapper;
|
||||
@Resource private BridgeDeliveryAttemptMapper attemptMapper;
|
||||
@Resource private EventBridgeMetrics metrics;
|
||||
|
||||
@Transactional
|
||||
public void replayDead(String eventId, String operatorId, String reason) {
|
||||
if (operatorId == null || operatorId.isBlank()) {
|
||||
throw new IllegalArgumentException("人工补投必须提供操作人");
|
||||
@@ -19,6 +29,22 @@ public class EventReplayService {
|
||||
if (inboxMapper.replayDead(eventId, auditReason) != 1) {
|
||||
throw new IllegalStateException("事件不存在或未处于 DEAD 状态");
|
||||
}
|
||||
BridgeInbox event = inboxMapper.selectById(eventId);
|
||||
if (event == null) {
|
||||
throw new IllegalStateException("事件补投状态已更新但事件记录不存在");
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
BridgeDeliveryAttempt audit = new BridgeDeliveryAttempt();
|
||||
audit.setEventId(eventId);
|
||||
audit.setSourceRegion(event.getSourceRegion());
|
||||
audit.setTargetRegion(event.getTargetRegion());
|
||||
audit.setAttemptNo(inboxMapper.nextAttemptNo(eventId));
|
||||
audit.setStatus("MANUAL_REPLAY");
|
||||
audit.setErrorMessage(auditReason);
|
||||
audit.setStartedAt(now);
|
||||
audit.setFinishedAt(now);
|
||||
attemptMapper.insert(audit);
|
||||
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.MANUAL_REPLAY);
|
||||
}
|
||||
|
||||
private static String abbreviate(String value, int maxLength) {
|
||||
|
||||
+22
-2
@@ -42,11 +42,31 @@ public class HttpFirstlegEventClient implements FirstlegEventClient {
|
||||
.header("X-Internal-Service-Token", serviceToken)
|
||||
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(envelope))).build();
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
JsonNode responseBody = objectMapper.readTree(response.body());
|
||||
JsonNode responseBody;
|
||||
try {
|
||||
responseBody = objectMapper.readTree(response.body());
|
||||
} catch (Exception ignored) {
|
||||
responseBody = null;
|
||||
}
|
||||
if (response.statusCode() / 100 != 2 || responseBody == null
|
||||
|| !"200".equals(responseBody.path("code").asText())) {
|
||||
throw new IllegalStateException("头程消费失败: HTTP " + response.statusCode());
|
||||
throw new IllegalStateException(failureMessage(response.statusCode(), responseBody));
|
||||
}
|
||||
return String.valueOf(response.statusCode());
|
||||
}
|
||||
|
||||
static String failureMessage(int statusCode, JsonNode body) {
|
||||
String code = body == null ? "missing" : body.path("code").asText("missing");
|
||||
String message = body == null ? "non-json response" : body.path("message").asText("missing");
|
||||
String requestId = body == null ? "missing" : body.path("requestId").asText("missing");
|
||||
String traceId = body == null ? "missing" : body.path("traceId").asText("missing");
|
||||
return "头程消费失败: HTTP " + statusCode + ", code=" + bounded(code)
|
||||
+ ", message=" + bounded(message) + ", requestId=" + bounded(requestId)
|
||||
+ ", traceId=" + bounded(traceId);
|
||||
}
|
||||
|
||||
private static String bounded(String value) {
|
||||
String safe = value == null || value.isBlank() ? "missing" : value.replaceAll("[\\r\\n]+", " ");
|
||||
return safe.length() <= 240 ? safe : safe.substring(0, 240);
|
||||
}
|
||||
}
|
||||
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
package com.baiyue.eventbridge.service;
|
||||
|
||||
import com.baiyue.common.event.EventEnvelope;
|
||||
import com.baiyue.common.event.RegionCode;
|
||||
import com.baiyue.eventbridge.config.EventDispatcherProperties;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
|
||||
@Component
|
||||
public class HttpSecurityProjectionEventClient implements SecurityProjectionEventClient {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final EventDispatcherProperties properties;
|
||||
@Value("${baiyue.event-dispatcher.security-consumer-url}") private URI consumerUrl;
|
||||
@Value("${baiyue.security.internal-service-token:}") private String serviceToken;
|
||||
private HttpClient httpClient;
|
||||
|
||||
public HttpSecurityProjectionEventClient(ObjectMapper objectMapper, EventDispatcherProperties properties) {
|
||||
this.objectMapper = objectMapper;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
@PostConstruct void initialize() {
|
||||
httpClient = HttpClient.newBuilder().connectTimeout(properties.connectTimeout()).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String deliver(BridgeInbox row) throws Exception {
|
||||
EventEnvelope envelope = new EventEnvelope(row.getEventId(), row.getEventType(), row.getEventVersion(),
|
||||
row.getOccurredAt(), RegionCode.valueOf(row.getSourceRegion()), RegionCode.valueOf(row.getTargetRegion()),
|
||||
row.getCustomerId(), row.getAggregateType(), row.getAggregateId(), row.getAggregateVersion(),
|
||||
row.getHandoverVersion(), row.getTraceId(), row.getCorrelationId(), row.getCausationId(),
|
||||
objectMapper.readTree(row.getPayloadJson()), row.getPayloadHash());
|
||||
HttpRequest request = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-Internal-Service-Token", serviceToken)
|
||||
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(envelope))).build();
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
JsonNode body;
|
||||
try { body = objectMapper.readTree(response.body()); }
|
||||
catch (Exception exception) { body = null; }
|
||||
if (response.statusCode() / 100 != 2 || body == null || !"200".equals(body.path("code").asText())
|
||||
|| !row.getEventId().equals(body.path("data").path("eventId").asText())) {
|
||||
throw new IllegalStateException("海外安全投影消费失败: HTTP " + response.statusCode());
|
||||
}
|
||||
return String.valueOf(response.statusCode());
|
||||
}
|
||||
}
|
||||
+40
-9
@@ -1,10 +1,12 @@
|
||||
package com.baiyue.eventbridge.service;
|
||||
|
||||
import com.baiyue.common.event.EventTypes;
|
||||
import com.baiyue.eventbridge.config.EventDispatcherProperties;
|
||||
import com.baiyue.eventbridge.domain.BridgeDeliveryAttempt;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
|
||||
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
|
||||
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
|
||||
import jakarta.annotation.Resource;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
@@ -12,6 +14,7 @@ import org.springframework.stereotype.Component;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
@Component
|
||||
@@ -21,7 +24,9 @@ public class OverseasEventDispatcher {
|
||||
@Resource private BridgeDeliveryAttemptMapper attemptMapper;
|
||||
@Resource private LastlegEventClient eventClient;
|
||||
@Resource private FirstlegEventClient firstlegEventClient;
|
||||
@Resource private SecurityProjectionEventClient securityProjectionEventClient;
|
||||
@Resource private EventDispatcherProperties properties;
|
||||
@Resource private EventBridgeMetrics metrics;
|
||||
private final String leaseOwner = ManagementFactory.getRuntimeMXBean().getName() + ":" + UUID.randomUUID();
|
||||
|
||||
@Scheduled(fixedDelayString = "${baiyue.event-dispatcher.fixed-delay:3000}")
|
||||
@@ -40,26 +45,52 @@ public class OverseasEventDispatcher {
|
||||
int retryAttempt = event.getRetryCount() == null ? 1 : event.getRetryCount() + 1;
|
||||
int deliveryAttemptNo = inboxMapper.nextAttemptNo(eventId);
|
||||
BridgeDeliveryAttempt attempt = startAttempt(event, deliveryAttemptNo);
|
||||
String responseCode;
|
||||
try {
|
||||
String responseCode = deliver(event);
|
||||
if (inboxMapper.markProcessed(eventId, leaseOwner) != 1) {
|
||||
throw new IllegalStateException("事件租约已失效,无法确认投递结果");
|
||||
}
|
||||
finishAttempt(attempt, "DELIVERED", responseCode, null);
|
||||
responseCode = deliver(event);
|
||||
} catch (Exception exception) {
|
||||
String error = abbreviate(exception.getMessage(), 1000);
|
||||
inboxMapper.markFailed(eventId, leaseOwner, properties.maxAttempts(), backoffSeconds(retryAttempt), error);
|
||||
finishAttempt(attempt, retryAttempt >= properties.maxAttempts() ? "DEAD" : "RETRY", null, error);
|
||||
int marked = inboxMapper.markFailed(eventId, leaseOwner, properties.maxAttempts(),
|
||||
backoffSeconds(retryAttempt), error);
|
||||
String attemptStatus = marked == 1
|
||||
? (retryAttempt >= properties.maxAttempts() ? "DEAD" : "RETRY")
|
||||
: "FAILED_LEASE_LOST";
|
||||
finishAttempt(attempt, attemptStatus, null, error);
|
||||
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.valueOf(attemptStatus));
|
||||
log.warn("跨区事件投递失败 eventId={}, retryAttempt={}, deliveryAttempt={}, reason={}",
|
||||
eventId, retryAttempt, deliveryAttemptNo, error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (inboxMapper.markProcessed(eventId, leaseOwner) != 1) {
|
||||
String error = "下游已确认成功,但事件租约已失效;保留当前 Inbox 状态并由新租约持有者幂等续投";
|
||||
finishAttempt(attempt, "ACKNOWLEDGED_LEASE_LOST", responseCode, error);
|
||||
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.ACKNOWLEDGED_LEASE_LOST);
|
||||
log.warn("跨区事件成功确认时租约已失效 eventId={}, deliveryAttempt={}", eventId, deliveryAttemptNo);
|
||||
return;
|
||||
}
|
||||
finishAttempt(attempt, "DELIVERED", responseCode, null);
|
||||
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.DELIVERED);
|
||||
}
|
||||
|
||||
private String deliver(BridgeInbox event) throws Exception {
|
||||
return "CN".equals(event.getTargetRegion())
|
||||
? firstlegEventClient.deliver(event) : eventClient.deliver(event);
|
||||
String eventType = event.getEventType();
|
||||
if (eventType == null || eventType.isBlank()) {
|
||||
throw new IllegalArgumentException("事件类型不能为空,无法选择下游消费者");
|
||||
}
|
||||
return switch (event.getTargetRegion()) {
|
||||
case "CN" -> firstlegEventClient.deliver(event);
|
||||
case "OVERSEAS" -> SECURITY_EVENT_TYPES.contains(eventType)
|
||||
? securityProjectionEventClient.deliver(event) : eventClient.deliver(event);
|
||||
default -> throw new IllegalArgumentException("不支持的事件目标区域: " + event.getTargetRegion());
|
||||
};
|
||||
}
|
||||
|
||||
private static final Set<String> SECURITY_EVENT_TYPES = Set.of(
|
||||
EventTypes.IAM_USER_STATUS_CHANGED,
|
||||
EventTypes.IAM_TENANT_STATUS_CHANGED,
|
||||
EventTypes.IAM_PERMISSION_VERSION_CHANGED);
|
||||
|
||||
private BridgeDeliveryAttempt startAttempt(BridgeInbox event, int attemptNo) {
|
||||
BridgeDeliveryAttempt attempt = new BridgeDeliveryAttempt();
|
||||
attempt.setEventId(event.getEventId());
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
package com.baiyue.eventbridge.service;
|
||||
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
|
||||
public interface SecurityProjectionEventClient {
|
||||
String deliver(BridgeInbox event) throws Exception;
|
||||
}
|
||||
@@ -33,6 +33,7 @@ baiyue:
|
||||
event-dispatcher:
|
||||
consumer-url: ${EVENT_CONSUMER_URL:${LASTLEG_CONSUMER_URL:http://127.0.0.1:9303/internal/lastleg/events}}
|
||||
firstleg-consumer-url: ${FIRSTLEG_CONTAINER_EVENT_CONSUMER_URL:http://127.0.0.1:9202/internal/firstleg/v1/container-events}
|
||||
security-consumer-url: ${OVERSEAS_SECURITY_EVENT_CONSUMER_URL:http://127.0.0.1:9307/internal/overseas-security/v1/events}
|
||||
fixed-delay: ${EVENT_DISPATCH_FIXED_DELAY:3000}
|
||||
batch-size: ${EVENT_DISPATCH_BATCH_SIZE:50}
|
||||
max-attempts: ${EVENT_DISPATCH_MAX_ATTEMPTS:10}
|
||||
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.baiyue.eventbridge.metrics;
|
||||
|
||||
import com.baiyue.common.event.EventTypes;
|
||||
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class EventBridgeInboxMetricsSamplerTest {
|
||||
|
||||
@Test
|
||||
void samplesDatabaseOnceAndServesCachedLowCardinalityGauges() {
|
||||
BridgeInboxMapper mapper = mock(BridgeInboxMapper.class);
|
||||
SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
InboxMetricsSnapshot known = snapshot("OVERSEAS", EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED,
|
||||
7, 121.5, 2);
|
||||
InboxMetricsSnapshot unknown = snapshot("region-123", "dynamic-event-123", 3, 900.0, 1);
|
||||
when(mapper.observabilitySnapshots()).thenReturn(List.of(known, unknown));
|
||||
EventBridgeInboxMetricsSampler sampler = new EventBridgeInboxMetricsSampler(mapper, registry);
|
||||
|
||||
sampler.sample();
|
||||
|
||||
verify(mapper).observabilitySnapshots();
|
||||
assertThat(gauge(registry, EventBridgeInboxMetricsSampler.BACKLOG_METRIC,
|
||||
"OVERSEAS", EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED)).isEqualTo(7.0);
|
||||
assertThat(gauge(registry, EventBridgeInboxMetricsSampler.OLDEST_AGE_METRIC,
|
||||
"OVERSEAS", EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED)).isEqualTo(121.5);
|
||||
assertThat(gauge(registry, EventBridgeInboxMetricsSampler.DEAD_METRIC,
|
||||
"OTHER", "OTHER")).isEqualTo(1.0);
|
||||
verifyNoMoreInteractions(mapper);
|
||||
}
|
||||
|
||||
private InboxMetricsSnapshot snapshot(String region, String eventType, long backlog,
|
||||
double oldestAge, long dead) {
|
||||
InboxMetricsSnapshot snapshot = new InboxMetricsSnapshot();
|
||||
snapshot.setTargetRegion(region);
|
||||
snapshot.setEventType(eventType);
|
||||
snapshot.setBacklogCount(backlog);
|
||||
snapshot.setOldestPendingAgeSeconds(oldestAge);
|
||||
snapshot.setDeadCount(dead);
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
private double gauge(SimpleMeterRegistry registry, String name, String region, String eventType) {
|
||||
return registry.get(name).tags("target_region", region, "event_type", eventType).gauge().value();
|
||||
}
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package com.baiyue.eventbridge.metrics;
|
||||
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.Arrays;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class EventBridgeMetricsTest {
|
||||
|
||||
@Test
|
||||
void collapsesUnboundedRegionAndEventTypeLabelsToOther() {
|
||||
SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
EventBridgeMetrics metrics = new EventBridgeMetrics(registry);
|
||||
BridgeInbox event = new BridgeInbox();
|
||||
event.setTargetRegion("customer-controlled-region");
|
||||
event.setEventType("dynamic-event-123456");
|
||||
|
||||
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.RETRY);
|
||||
|
||||
assertThat(registry.get(EventBridgeMetrics.TRANSPORT_METRIC)
|
||||
.tags("target_region", "OTHER", "event_type", "OTHER", "result", "RETRY")
|
||||
.counter().count()).isEqualTo(1.0);
|
||||
assertThat(registry.getMeters()).allSatisfy(meter -> assertThat(meter.getId().getTags())
|
||||
.noneMatch(tag -> tag.getValue().contains("dynamic-event")));
|
||||
assertThat(EventBridgeMetrics.eventType(null)).isEqualTo("OTHER");
|
||||
}
|
||||
|
||||
@Test
|
||||
void everyFrozenEventTypeHasABoundedTagValue() {
|
||||
assertThat(Arrays.stream(com.baiyue.common.event.EventTypes.class.getFields())
|
||||
.filter(field -> Modifier.isStatic(field.getModifiers()) && field.getType() == String.class)
|
||||
.map(field -> {
|
||||
try {
|
||||
return (String) field.get(null);
|
||||
} catch (IllegalAccessException exception) {
|
||||
throw new AssertionError(exception);
|
||||
}
|
||||
})).allMatch(value -> EventBridgeMetrics.eventType(value).equals(value));
|
||||
}
|
||||
|
||||
@Test
|
||||
void metricsFailureDoesNotChangeTransportControlFlow() {
|
||||
EventBridgeMetrics metrics = new EventBridgeMetrics(mock(MeterRegistry.class));
|
||||
BridgeInbox event = new BridgeInbox();
|
||||
event.setTargetRegion("CN");
|
||||
event.setEventType(com.baiyue.common.event.EventTypes.OVERSEAS_CONTAINER_UNLOADED);
|
||||
|
||||
assertThatCode(() -> metrics.recordTransport(event, EventBridgeMetrics.TransportResult.RETRY))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package com.baiyue.eventbridge.metrics;
|
||||
|
||||
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class EventBridgeObservabilityContractTest {
|
||||
|
||||
@Test
|
||||
void aggregationIsScheduledCacheInputAndUsesExistingStatusIndexBasis() throws Exception {
|
||||
Select annotation = BridgeInboxMapper.class.getMethod("observabilitySnapshots")
|
||||
.getAnnotation(Select.class);
|
||||
String query = String.join(" ", annotation.value()).replaceAll("\\s+", " ");
|
||||
String schema = Files.readString(Path.of("src/main/resources/db/event-bridge.sql"));
|
||||
|
||||
assertThat(query)
|
||||
.contains("WHERE status IN ('RECEIVED','RETRY','PROCESSING','DEAD')")
|
||||
.contains("GROUP BY target_region, event_type")
|
||||
.doesNotContain("event_id AS", "customer_id AS", "last_error AS");
|
||||
assertThat(schema).contains(
|
||||
"KEY idx_bridge_inbox_consume (status, next_retry_at, lease_until)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void developmentSloAndAlertThresholdsAreFrozenWithoutDeploymentClaim() throws Exception {
|
||||
String contract = Files.readString(Path.of("docs/development-observability-slo.md"));
|
||||
|
||||
assertThat(contract)
|
||||
.contains("99% 的新接收跨区事件在 60 秒内")
|
||||
.contains("99.9% 的新接收跨区事件在 15 分钟内")
|
||||
.contains("result=\"DEAD\"", "> 120", "> 900")
|
||||
.contains("不表示任何环境已经部署")
|
||||
.contains("idx_bridge_inbox_consume(status, next_retry_at, lease_until)")
|
||||
.doesNotContain("customer_id\"", "event_id\"");
|
||||
}
|
||||
}
|
||||
+14
@@ -5,7 +5,9 @@ import com.baiyue.common.event.EventPayloadCanonicalizer;
|
||||
import com.baiyue.common.event.EventPayloadHash;
|
||||
import com.baiyue.common.event.RegionCode;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
@@ -24,6 +26,8 @@ import static org.mockito.Mockito.when;
|
||||
class EventInboxServiceTest {
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final InboxStore store = mock(InboxStore.class);
|
||||
private final SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
private final EventBridgeMetrics metrics = new EventBridgeMetrics(registry);
|
||||
private final EventInboxService service = service("OVERSEAS");
|
||||
|
||||
@Test
|
||||
@@ -33,6 +37,7 @@ class EventInboxServiceTest {
|
||||
|
||||
assertThat(service.receive(envelope).result()).isEqualTo("已接收");
|
||||
verify(store).insert(any(BridgeInbox.class));
|
||||
assertThat(receiveCount("RECEIVED")).isEqualTo(1.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -57,6 +62,7 @@ class EventInboxServiceTest {
|
||||
when(store.findByEventId(envelope.eventId())).thenReturn(existing);
|
||||
|
||||
assertThat(service.receive(envelope).result()).isEqualTo("重复");
|
||||
assertThat(receiveCount("DUPLICATE")).isEqualTo(1.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -67,6 +73,7 @@ class EventInboxServiceTest {
|
||||
assertThatThrownBy(() -> service.receive(envelope))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.hasMessageContaining("拒绝覆盖");
|
||||
assertThat(receiveCount("CONFLICT")).isEqualTo(1.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -113,10 +120,17 @@ class EventInboxServiceTest {
|
||||
EventInboxService service = new EventInboxService();
|
||||
ReflectionTestUtils.setField(service, "inboxStore", store);
|
||||
ReflectionTestUtils.setField(service, "objectMapper", objectMapper);
|
||||
ReflectionTestUtils.setField(service, "metrics", metrics);
|
||||
ReflectionTestUtils.setField(service, "localRegion", region);
|
||||
return service;
|
||||
}
|
||||
|
||||
private double receiveCount(String result) {
|
||||
return registry.get(EventBridgeMetrics.RECEIVE_METRIC)
|
||||
.tags("target_region", "OVERSEAS", "event_type", "OrderApproved", "result", result)
|
||||
.counter().count();
|
||||
}
|
||||
|
||||
private EventEnvelope withHash(EventEnvelope source) {
|
||||
String hash = EventPayloadHash.sha256(EventPayloadCanonicalizer.canonicalize(source.payload()));
|
||||
return new EventEnvelope(source.eventId(), source.eventType(), source.eventVersion(), source.occurredAt(),
|
||||
|
||||
+23
@@ -1,6 +1,11 @@
|
||||
package com.baiyue.eventbridge.service;
|
||||
|
||||
import com.baiyue.eventbridge.domain.BridgeDeliveryAttempt;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
|
||||
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
|
||||
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
@@ -11,15 +16,31 @@ import static org.mockito.Mockito.*;
|
||||
|
||||
class EventReplayServiceTest {
|
||||
private final BridgeInboxMapper mapper = mock(BridgeInboxMapper.class);
|
||||
private final BridgeDeliveryAttemptMapper attemptMapper = mock(BridgeDeliveryAttemptMapper.class);
|
||||
private final SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
private final EventBridgeMetrics metrics = new EventBridgeMetrics(registry);
|
||||
private final EventReplayService service = service();
|
||||
|
||||
@Test
|
||||
void requeuesDeadEventWithAuditReason() {
|
||||
when(mapper.replayDead(eq("event-1"), contains("incident-42"))).thenReturn(1);
|
||||
BridgeInbox event = new BridgeInbox();
|
||||
event.setEventId("event-1"); event.setSourceRegion("CN"); event.setTargetRegion("OVERSEAS");
|
||||
event.setEventType(com.baiyue.common.event.EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED);
|
||||
when(mapper.selectById("event-1")).thenReturn(event);
|
||||
when(mapper.nextAttemptNo("event-1")).thenReturn(4);
|
||||
|
||||
service.replayDead("event-1", "operator-7", "incident-42 approved");
|
||||
|
||||
verify(mapper).replayDead(eq("event-1"), contains("operator-7"));
|
||||
var captor = org.mockito.ArgumentCaptor.forClass(BridgeDeliveryAttempt.class);
|
||||
verify(attemptMapper).insert(captor.capture());
|
||||
assertThat(captor.getValue().getAttemptNo()).isEqualTo(4);
|
||||
assertThat(captor.getValue().getStatus()).isEqualTo("MANUAL_REPLAY");
|
||||
assertThat(captor.getValue().getErrorMessage()).contains("operator-7", "incident-42");
|
||||
assertThat(registry.get(EventBridgeMetrics.TRANSPORT_METRIC)
|
||||
.tags("target_region", "OVERSEAS", "event_type", event.getEventType(),
|
||||
"result", "MANUAL_REPLAY").counter().count()).isEqualTo(1.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -43,6 +64,8 @@ class EventReplayServiceTest {
|
||||
private EventReplayService service() {
|
||||
EventReplayService service = new EventReplayService();
|
||||
ReflectionTestUtils.setField(service, "inboxMapper", mapper);
|
||||
ReflectionTestUtils.setField(service, "attemptMapper", attemptMapper);
|
||||
ReflectionTestUtils.setField(service, "metrics", metrics);
|
||||
return service;
|
||||
}
|
||||
}
|
||||
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package com.baiyue.eventbridge.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class HttpFirstlegEventClientTest {
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void businessFailureIsDiagnosableAndMustNotBeTreatedAsAcknowledged() throws Exception {
|
||||
var body = objectMapper.readTree("""
|
||||
{"code":"A0409","message":"事件摘要冲突","requestId":"request-cn-1","traceId":"trace-cn-1"}
|
||||
""");
|
||||
|
||||
assertThat(HttpFirstlegEventClient.failureMessage(200, body))
|
||||
.isEqualTo("头程消费失败: HTTP 200, code=A0409, message=事件摘要冲突, requestId=request-cn-1, traceId=trace-cn-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void nonJsonFailureIsStillDiagnosableWithoutRawBody() {
|
||||
assertThat(HttpFirstlegEventClient.failureMessage(503, null))
|
||||
.isEqualTo("头程消费失败: HTTP 503, code=missing, message=non-json response, requestId=missing, traceId=missing");
|
||||
}
|
||||
}
|
||||
+125
-3
@@ -1,10 +1,13 @@
|
||||
package com.baiyue.eventbridge.service;
|
||||
|
||||
import com.baiyue.common.event.EventTypes;
|
||||
import com.baiyue.eventbridge.config.EventDispatcherProperties;
|
||||
import com.baiyue.eventbridge.domain.BridgeDeliveryAttempt;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
|
||||
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
|
||||
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
@@ -12,6 +15,7 @@ import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -21,10 +25,14 @@ import static org.mockito.Mockito.*;
|
||||
class OverseasEventDispatcherTest {
|
||||
private final BridgeInboxMapper inboxMapper = mock(BridgeInboxMapper.class);
|
||||
private final BridgeDeliveryAttemptMapper attemptMapper = mock(BridgeDeliveryAttemptMapper.class);
|
||||
private final LastlegEventClient eventClient = mock(LastlegEventClient.class);
|
||||
private final FirstlegEventClient firstlegEventClient = mock(FirstlegEventClient.class);
|
||||
private final com.baiyue.eventbridge.service.LastlegEventClient eventClient =
|
||||
mock(com.baiyue.eventbridge.service.LastlegEventClient.class);
|
||||
private final com.baiyue.eventbridge.service.FirstlegEventClient firstlegEventClient =
|
||||
mock(com.baiyue.eventbridge.service.FirstlegEventClient.class);
|
||||
private final SecurityProjectionEventClient securityProjectionEventClient = mock(SecurityProjectionEventClient.class);
|
||||
private OverseasEventDispatcher dispatcher;
|
||||
private BridgeInbox event;
|
||||
private SimpleMeterRegistry registry;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
@@ -33,16 +41,21 @@ class OverseasEventDispatcherTest {
|
||||
Duration.ofSeconds(30), Duration.ofSeconds(5), Duration.ofMinutes(1),
|
||||
Duration.ofSeconds(3), Duration.ofSeconds(10));
|
||||
dispatcher = new OverseasEventDispatcher();
|
||||
registry = new SimpleMeterRegistry();
|
||||
ReflectionTestUtils.setField(dispatcher, "inboxMapper", inboxMapper);
|
||||
ReflectionTestUtils.setField(dispatcher, "attemptMapper", attemptMapper);
|
||||
ReflectionTestUtils.setField(dispatcher, "eventClient", eventClient);
|
||||
ReflectionTestUtils.setField(dispatcher, "firstlegEventClient", firstlegEventClient);
|
||||
ReflectionTestUtils.setField(dispatcher, "securityProjectionEventClient", securityProjectionEventClient);
|
||||
ReflectionTestUtils.setField(dispatcher, "properties", properties);
|
||||
ReflectionTestUtils.setField(dispatcher, "metrics", new EventBridgeMetrics(registry));
|
||||
event = new BridgeInbox();
|
||||
event.setEventId("event-1");
|
||||
event.setEventType(EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED);
|
||||
event.setSourceRegion("CN");
|
||||
event.setTargetRegion("OVERSEAS");
|
||||
event.setRetryCount(0);
|
||||
event.setReceivedAt(LocalDateTime.now().minusSeconds(5));
|
||||
when(inboxMapper.claim(eq("event-1"), anyString(), anyLong())).thenReturn(1);
|
||||
when(inboxMapper.selectById("event-1")).thenReturn(event);
|
||||
when(inboxMapper.nextAttemptNo("event-1")).thenReturn(1);
|
||||
@@ -56,27 +69,50 @@ class OverseasEventDispatcherTest {
|
||||
dispatcher.dispatchOne("event-1");
|
||||
|
||||
verify(inboxMapper).markProcessed(eq("event-1"), anyString());
|
||||
verify(eventClient).deliver(event);
|
||||
verifyNoInteractions(securityProjectionEventClient);
|
||||
assertThat(updatedAttempt().getStatus()).isEqualTo("DELIVERED");
|
||||
assertThat(transportCount("DELIVERED")).isEqualTo(1.0);
|
||||
assertThat(registry.get(EventBridgeMetrics.DELIVERY_LATENCY_METRIC).timer().count()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedDeliveryIsScheduledForRetry() throws Exception {
|
||||
when(eventClient.deliver(event)).thenThrow(new IllegalStateException("network down"));
|
||||
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("network down")))
|
||||
.thenReturn(1);
|
||||
|
||||
dispatcher.dispatchOne("event-1");
|
||||
|
||||
verify(inboxMapper).markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("network down"));
|
||||
verify(inboxMapper).markFailed(eq("event-1"), anyString(), eq(3), eq(5L), eq("network down"));
|
||||
assertThat(updatedAttempt().getStatus()).isEqualTo("RETRY");
|
||||
assertThat(transportCount("RETRY")).isEqualTo(1.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void finalFailureIsMovedToDeadState() throws Exception {
|
||||
event.setRetryCount(2);
|
||||
when(eventClient.deliver(event)).thenThrow(new IllegalStateException("still down"));
|
||||
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("still down")))
|
||||
.thenReturn(1);
|
||||
|
||||
dispatcher.dispatchOne("event-1");
|
||||
|
||||
verify(inboxMapper).markFailed(eq("event-1"), anyString(), eq(3), eq(20L), eq("still down"));
|
||||
assertThat(updatedAttempt().getStatus()).isEqualTo("DEAD");
|
||||
assertThat(transportCount("DEAD")).isEqualTo(1.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void exponentialBackoffIsCappedAtConfiguredMaximum() throws Exception {
|
||||
event.setRetryCount(4);
|
||||
when(eventClient.deliver(event)).thenThrow(new IllegalStateException("still down"));
|
||||
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("still down")))
|
||||
.thenReturn(1);
|
||||
|
||||
dispatcher.dispatchOne("event-1");
|
||||
|
||||
verify(inboxMapper).markFailed(eq("event-1"), anyString(), eq(3), eq(60L), eq("still down"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -92,6 +128,7 @@ class OverseasEventDispatcherTest {
|
||||
|
||||
@Test
|
||||
void cnTargetIsDeliveredToFirstleg() throws Exception {
|
||||
event.setEventType(EventTypes.OVERSEAS_CONTAINER_UNLOADED);
|
||||
event.setSourceRegion("OVERSEAS");
|
||||
event.setTargetRegion("CN");
|
||||
when(firstlegEventClient.deliver(event)).thenReturn("200");
|
||||
@@ -101,9 +138,87 @@ class OverseasEventDispatcherTest {
|
||||
|
||||
verify(firstlegEventClient).deliver(event);
|
||||
verifyNoInteractions(eventClient);
|
||||
verifyNoInteractions(securityProjectionEventClient);
|
||||
assertThat(updatedAttempt().getStatus()).isEqualTo("DELIVERED");
|
||||
}
|
||||
|
||||
@Test
|
||||
void iamSecurityEventsAreDeliveredToSecurityProjectionAndNeverLastleg() throws Exception {
|
||||
for (String eventType : List.of(EventTypes.IAM_USER_STATUS_CHANGED,
|
||||
EventTypes.IAM_TENANT_STATUS_CHANGED, EventTypes.IAM_PERMISSION_VERSION_CHANGED)) {
|
||||
reset(inboxMapper, attemptMapper, eventClient, firstlegEventClient, securityProjectionEventClient);
|
||||
event.setEventType(eventType);
|
||||
when(inboxMapper.claim(eq("event-1"), anyString(), anyLong())).thenReturn(1);
|
||||
when(inboxMapper.selectById("event-1")).thenReturn(event);
|
||||
when(inboxMapper.nextAttemptNo("event-1")).thenReturn(1);
|
||||
when(inboxMapper.markProcessed(eq("event-1"), anyString())).thenReturn(1);
|
||||
when(securityProjectionEventClient.deliver(event)).thenReturn("200");
|
||||
|
||||
dispatcher.dispatchOne("event-1");
|
||||
|
||||
verify(securityProjectionEventClient).deliver(event);
|
||||
verifyNoInteractions(eventClient);
|
||||
verifyNoInteractions(firstlegEventClient);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void unsupportedTargetRegionIsNeverMisroutedToLastleg() {
|
||||
event.setTargetRegion("UNKNOWN");
|
||||
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), contains("不支持")))
|
||||
.thenReturn(1);
|
||||
|
||||
dispatcher.dispatchOne("event-1");
|
||||
|
||||
verifyNoInteractions(eventClient);
|
||||
verifyNoInteractions(firstlegEventClient);
|
||||
verifyNoInteractions(securityProjectionEventClient);
|
||||
assertThat(updatedAttempt().getStatus()).isEqualTo("RETRY");
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingEventTypeFailsWithExplicitDiagnosticAndIsNeverDelivered() {
|
||||
event.setEventType(null);
|
||||
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(),
|
||||
eq("事件类型不能为空,无法选择下游消费者"))).thenReturn(1);
|
||||
|
||||
dispatcher.dispatchOne("event-1");
|
||||
|
||||
verifyNoInteractions(eventClient);
|
||||
verifyNoInteractions(firstlegEventClient);
|
||||
verifyNoInteractions(securityProjectionEventClient);
|
||||
BridgeDeliveryAttempt attempt = updatedAttempt();
|
||||
assertThat(attempt.getStatus()).isEqualTo("RETRY");
|
||||
assertThat(attempt.getErrorMessage()).isEqualTo("事件类型不能为空,无法选择下游消费者");
|
||||
}
|
||||
|
||||
@Test
|
||||
void acknowledgedDeliveryWithLostLeaseIsNotRecordedAsRetryOrDead() throws Exception {
|
||||
when(eventClient.deliver(event)).thenReturn("200");
|
||||
when(inboxMapper.markProcessed(eq("event-1"), anyString())).thenReturn(0);
|
||||
|
||||
dispatcher.dispatchOne("event-1");
|
||||
|
||||
verify(inboxMapper, never()).markFailed(anyString(), anyString(), anyInt(), anyLong(), anyString());
|
||||
BridgeDeliveryAttempt attempt = updatedAttempt();
|
||||
assertThat(attempt.getStatus()).isEqualTo("ACKNOWLEDGED_LEASE_LOST");
|
||||
assertThat(attempt.getResponseCode()).isEqualTo("200");
|
||||
assertThat(attempt.getErrorMessage()).contains("租约已失效");
|
||||
assertThat(transportCount("ACKNOWLEDGED_LEASE_LOST")).isEqualTo(1.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedDeliveryAfterLeaseTakeoverDoesNotOverwriteNewOwnerState() throws Exception {
|
||||
when(eventClient.deliver(event)).thenThrow(new IllegalStateException("network down"));
|
||||
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("network down")))
|
||||
.thenReturn(0);
|
||||
|
||||
dispatcher.dispatchOne("event-1");
|
||||
|
||||
assertThat(updatedAttempt().getStatus()).isEqualTo("FAILED_LEASE_LOST");
|
||||
assertThat(transportCount("FAILED_LEASE_LOST")).isEqualTo(1.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void downstream503RetriesThenDeliversWithDiagnosticAttemptsInBothDirections() throws Exception {
|
||||
for (String target : List.of("OVERSEAS", "CN")) {
|
||||
@@ -162,4 +277,11 @@ class OverseasEventDispatcherTest {
|
||||
verify(attemptMapper).updateById(captor.capture());
|
||||
return captor.getValue();
|
||||
}
|
||||
|
||||
private double transportCount(String result) {
|
||||
return registry.get(EventBridgeMetrics.TRANSPORT_METRIC)
|
||||
.tags("target_region", event.getTargetRegion(),
|
||||
"event_type", event.getEventType(), "result", result)
|
||||
.counter().count();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user