This commit is contained in:
2026-08-25 14:46:06 +08:00
parent 5abc7543ae
commit f325d57fc1
106 changed files with 3682 additions and 450 deletions
+3 -1
View File
@@ -12,11 +12,12 @@
|---|---|
| 接收、状态、人工补投 API | `src/main/java/com/baiyue/eventbridge/controller/EventBridgeController.java` |
| Inbox 接收与幂等 | `service/EventInboxService.java` |
| 按目标区域调度与重试 | `service/OverseasEventDispatcher.java`;`targetRegion=OVERSEAS` 投 Lastleg,`targetRegion=CN` 回投 Firstleg |
| 按目标区域调度与重试 | `service/OverseasEventDispatcher.java`;`targetRegion=CN` 回投 Firstleg;海外业务事件投 Lastleg,三类 IAM 安全事件按 eventType 投 `overseas-security-service`,传输层不解析 payload |
| 尾程 HTTP 客户端 | `service/LastlegEventClient.java`、`HttpLastlegEventClient.java` |
| 国内头程 HTTP 客户端 | `service/FirstlegEventClient.java`、`HttpFirstlegEventClient.java`;默认内部入口 `/internal/firstleg/v1/container-events` |
| 人工补投 | `service/EventReplayService.java`;只允许 `DEAD` 转回 `RETRY`,记录操作人与原因,并保留原 `eventId`、payload 与 `payloadHash` |
| Inbox 与投递记录 | `domain/BridgeInbox.java`、`BridgeDeliveryAttempt.java` 及对应 Mapper |
| 传输指标与 Inbox 采样 | `metrics/EventBridgeMetrics.java`、`EventBridgeInboxMetricsSampler.java`;SLO 与告警合同 `docs/development-observability-slo.md` |
| 内部服务令牌 | `security/InternalServiceTokenGuard.java` |
| 数据库 | `src/main/resources/db/event-bridge.sql` |
| 测试 | `src/test/java/com/baiyue/eventbridge/` |
@@ -27,6 +28,7 @@
- 每次投递保存 attempt、错误和时间;达到上限进入 `DEAD`,经授权人工补投后重新进入重试队列。
- 领取任务必须有数据库租约/并发保护,避免多实例重复发送;尾程幂等仍是最终保障。
- 内部接口使用 `X-Internal-Service-Token`,生产还应置于受信网络或 mTLS 后。
- 指标标签只允许有限 `target_region`、`event_type`、`result`,未知区域/类型折叠为 `OTHER`,禁止 eventId、customerId、错误消息等高基数标签。Inbox 聚合按现有状态索引定时采样,Prometheus scrape 不得查询数据库;开发阶段 SLO/告警只代表代码合同,不代表监控平台已部署。
## 修改与验证
@@ -0,0 +1,67 @@
# Event Bridge 开发阶段可观测性与 SLO 合同
## 状态与边界
本文冻结代码侧指标、SLI/SLO 和告警表达式模板。它不表示任何环境已经部署、接入或启用了
Prometheus、Alertmanager 或 Grafana,也不授权部署、访问环境或修改环境配置。
指标标签只允许 `target_region`、`event_type`、`result`:
- `target_region` 仅为 `CN`、`OVERSEAS`、`OTHER`。
- `event_type` 仅为 `EventTypes` 已登记类型或 `OTHER`。
- `result` 仅为代码中冻结的接收或传输结果枚举。
- 严禁把 `eventId`、`customerId`、聚合 ID、错误消息、请求 URL 或操作人写入指标标签。
## 指标合同
| Prometheus 指标 | 含义 |
|---|---|
| `baiyue_event_bridge_receive_total` | 接收结果;`result=RECEIVED|DUPLICATE|CONFLICT` |
| `baiyue_event_bridge_transport_total` | 传输结果;`result=DELIVERED|RETRY|DEAD|ACKNOWLEDGED_LEASE_LOST|FAILED_LEASE_LOST|MANUAL_REPLAY` |
| `baiyue_event_bridge_delivery_latency_seconds_bucket` | 从 Event Bridge `received_at` 到下游首次成功确认;固定包含 60 秒和 900 秒 SLO 桶 |
| `baiyue_event_bridge_inbox_backlog` | 当前 `RECEIVED`、`RETRY`、`PROCESSING` 数量 |
| `baiyue_event_bridge_inbox_oldest_age_seconds` | 当前最老待处理事件自 `received_at` 起的年龄 |
| `baiyue_event_bridge_inbox_dead` | 当前 `DEAD` 数量 |
`DELIVERED` 与 `ACKNOWLEDGED_LEASE_LOST` 都表示下游已经确认,因此均记录确认延迟;后者仍单独计数,
不得冒充本地 Inbox 已成功更新。`RETRY`、`DEAD`、`FAILED_LEASE_LOST` 不记录成功延迟。
Inbox Gauge 由服务每 30 秒执行一次受控聚合查询并缓存;Prometheus scrape 只读取 Micrometer Gauge,
不得触发数据库查询。查询仅处理 `RECEIVED`、`RETRY`、`PROCESSING`、`DEAD`,状态过滤以现有
`idx_bridge_inbox_consume(status, next_retry_at, lease_until)` 为索引依据;本合同不新增或修改 SQL 结构。
## 开发阶段 SLI 与 SLO
SLI 从事件首次得到 `result=RECEIVED` 开始,到下游首次成功确认结束;`DUPLICATE` 和 `CONFLICT`
不重复进入分母。滚动 30 天目标:
- 99% 的新接收跨区事件在 60 秒内得到下游确认。
- 99.9% 的新接收跨区事件在 15 分钟内得到下游确认。
代码暴露固定延迟桶,目标查询模板如下;按 `target_region` 汇总,必要时可保留有限 `event_type` 分组:
```promql
sum(rate(baiyue_event_bridge_delivery_latency_seconds_bucket{le="60.0"}[30d]))
/
sum(rate(baiyue_event_bridge_receive_total{result="RECEIVED"}[30d])) >= 0.99
```
```promql
sum(rate(baiyue_event_bridge_delivery_latency_seconds_bucket{le="900.0"}[30d]))
/
sum(rate(baiyue_event_bridge_receive_total{result="RECEIVED"}[30d])) >= 0.999
```
## 告警合同
规则由实际监控平台按 30 秒或更短评估周期配置;以下只是表达式合同:
- 新增 DEAD:下一次评估立即告警,不设置持续等待。
`sum(increase(baiyue_event_bridge_transport_total{result="DEAD"}[1m])) > 0`
- 最老待处理年龄预警:任一有限标签序列超过 120 秒。
`max(baiyue_event_bridge_inbox_oldest_age_seconds) > 120`
- 最老待处理年龄严重:任一有限标签序列超过 900 秒。
`max(baiyue_event_bridge_inbox_oldest_age_seconds) > 900`
- 当前 DEAD 存量用于持续可见性:`sum(baiyue_event_bridge_inbox_dead) > 0`;它不替代“新增 DEAD”告警。
告警通知中可通过日志和业务查询再定位 event ID;禁止为了通知便利把高基数标识加入指标标签。
@@ -2,6 +2,7 @@ package com.baiyue.eventbridge.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.metrics.InboxMetricsSnapshot;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
@@ -60,4 +61,21 @@ public interface BridgeInboxMapper extends BaseMapper<BridgeInbox> {
WHERE event_id=#{eventId} AND status='DEAD'
""")
int replayDead(@Param("eventId") String eventId, @Param("reason") String reason);
/**
* Scheduled observability aggregation. The status predicate uses idx_bridge_inbox_consume;
* Prometheus collection never invokes this query directly.
*/
@Select("""
SELECT target_region AS targetRegion, event_type AS eventType,
SUM(CASE WHEN status IN ('RECEIVED','RETRY','PROCESSING') THEN 1 ELSE 0 END) AS backlogCount,
COALESCE(TIMESTAMPDIFF(MICROSECOND,
MIN(CASE WHEN status IN ('RECEIVED','RETRY','PROCESSING') THEN received_at END),
CURRENT_TIMESTAMP(3)) / 1000000.0, 0) AS oldestPendingAgeSeconds,
SUM(CASE WHEN status='DEAD' THEN 1 ELSE 0 END) AS deadCount
FROM bridge_inbox
WHERE status IN ('RECEIVED','RETRY','PROCESSING','DEAD')
GROUP BY target_region, event_type
""")
List<InboxMetricsSnapshot> observabilitySnapshots();
}
@@ -0,0 +1,103 @@
package com.baiyue.eventbridge.metrics;
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.MultiGauge;
import io.micrometer.core.instrument.Tags;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.LinkedHashMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/** Periodically samples indexed Inbox aggregates; Prometheus scrapes only the cached gauges. */
@Component
@Slf4j
public class EventBridgeInboxMetricsSampler {
public static final String BACKLOG_METRIC = "baiyue.event.bridge.inbox.backlog";
public static final String OLDEST_AGE_METRIC = "baiyue.event.bridge.inbox.oldest.age.seconds";
public static final String DEAD_METRIC = "baiyue.event.bridge.inbox.dead";
private final BridgeInboxMapper inboxMapper;
private final MultiGauge backlog;
private final MultiGauge oldestAge;
private final MultiGauge dead;
public EventBridgeInboxMetricsSampler(BridgeInboxMapper inboxMapper, MeterRegistry registry) {
this.inboxMapper = inboxMapper;
this.backlog = MultiGauge.builder(BACKLOG_METRIC)
.description("Events waiting for downstream acknowledgement").register(registry);
this.oldestAge = MultiGauge.builder(OLDEST_AGE_METRIC)
.description("Age in seconds of the oldest pending Inbox event").register(registry);
this.dead = MultiGauge.builder(DEAD_METRIC)
.description("Current number of DEAD Inbox events").register(registry);
}
@Scheduled(initialDelayString = "${baiyue.event-bridge.metrics.initial-delay:5000}",
fixedDelayString = "${baiyue.event-bridge.metrics.sample-delay:30000}")
public void sample() {
final List<InboxMetricsSnapshot> snapshots;
try {
snapshots = inboxMapper.observabilitySnapshots();
} catch (RuntimeException exception) {
log.warn("Event Bridge Inbox 指标采样失败,保留上次成功快照", exception);
return;
}
Map<MetricKey, Aggregate> aggregates = aggregate(snapshots);
backlog.register(rows(aggregates, MetricValue.BACKLOG), true);
oldestAge.register(rows(aggregates, MetricValue.OLDEST_AGE), true);
dead.register(rows(aggregates, MetricValue.DEAD), true);
}
private Map<MetricKey, Aggregate> aggregate(List<InboxMetricsSnapshot> snapshots) {
Map<MetricKey, Aggregate> result = new LinkedHashMap<>();
if (snapshots == null) {
return result;
}
for (InboxMetricsSnapshot snapshot : snapshots) {
MetricKey key = new MetricKey(EventBridgeMetrics.targetRegion(snapshot.getTargetRegion()),
EventBridgeMetrics.eventType(snapshot.getEventType()));
result.computeIfAbsent(key, ignored -> new Aggregate()).add(snapshot);
}
return result;
}
private List<MultiGauge.Row<?>> rows(Map<MetricKey, Aggregate> aggregates, MetricValue value) {
List<MultiGauge.Row<?>> rows = new ArrayList<>();
aggregates.forEach((key, aggregate) -> rows.add(MultiGauge.Row.of(
Tags.of("target_region", key.targetRegion, "event_type", key.eventType),
value.value(aggregate))));
return rows;
}
private record MetricKey(String targetRegion, String eventType) { }
private static final class Aggregate {
private long backlog;
private double oldestAge;
private long dead;
void add(InboxMetricsSnapshot snapshot) {
backlog += snapshot.getBacklogCount();
oldestAge = Math.max(oldestAge, snapshot.getOldestPendingAgeSeconds());
dead += snapshot.getDeadCount();
}
}
private enum MetricValue {
BACKLOG {
@Override Number value(Aggregate aggregate) { return aggregate.backlog; }
},
OLDEST_AGE {
@Override Number value(Aggregate aggregate) { return aggregate.oldestAge; }
},
DEAD {
@Override Number value(Aggregate aggregate) { return aggregate.dead; }
};
abstract Number value(Aggregate aggregate);
}
}
@@ -0,0 +1,121 @@
package com.baiyue.eventbridge.metrics;
import com.baiyue.common.event.EventEnvelope;
import com.baiyue.common.event.EventTypes;
import com.baiyue.eventbridge.domain.BridgeInbox;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.time.Clock;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Locale;
import java.util.Set;
/** Low-cardinality transport metrics for the cross-region event bridge. */
@Component
@Slf4j
public class EventBridgeMetrics {
public static final String RECEIVE_METRIC = "baiyue.event.bridge.receive";
public static final String TRANSPORT_METRIC = "baiyue.event.bridge.transport";
public static final String DELIVERY_LATENCY_METRIC = "baiyue.event.bridge.delivery.latency";
private static final Set<String> KNOWN_EVENT_TYPES = Set.of(
EventTypes.ORDER_APPROVED,
EventTypes.ORDER_CANCELLED,
EventTypes.FIRST_LEG_SHIPMENT_DEPARTED,
EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED,
EventTypes.FIRST_LEG_CONTAINER_SHIPPING_NODE_RECORDED,
EventTypes.OVERSEAS_CONTAINER_UNLOADED,
EventTypes.IAM_USER_STATUS_CHANGED,
EventTypes.IAM_TENANT_STATUS_CHANGED,
EventTypes.IAM_PERMISSION_VERSION_CHANGED,
EventTypes.LAST_MILE_HANDOFF_ACCEPTED,
EventTypes.LAST_MILE_HANDOFF_REJECTED,
EventTypes.LAST_MILE_MILESTONE_CHANGED,
EventTypes.LAST_MILE_COMPLETED,
EventTypes.TRACKING_MILESTONE_PROJECTED);
private final MeterRegistry registry;
private final Clock clock;
public EventBridgeMetrics(MeterRegistry registry) {
this(registry, Clock.systemDefaultZone());
}
EventBridgeMetrics(MeterRegistry registry, Clock clock) {
this.registry = registry;
this.clock = clock;
}
public void recordReceive(EventEnvelope envelope, ReceiveResult result) {
try {
registry.counter(RECEIVE_METRIC,
"target_region", targetRegion(envelope.targetRegion().name()),
"event_type", eventType(envelope.eventType()),
"result", result.name()).increment();
} catch (RuntimeException exception) {
log.warn("Event Bridge 接收指标记录失败,不影响事件事务", exception);
}
}
public void recordTransport(BridgeInbox event, TransportResult result) {
try {
String targetRegion = targetRegion(event.getTargetRegion());
String eventType = eventType(event.getEventType());
registry.counter(TRANSPORT_METRIC,
"target_region", targetRegion,
"event_type", eventType,
"result", result.name()).increment();
if (result.acknowledged() && event.getReceivedAt() != null) {
Duration latency = Duration.between(event.getReceivedAt(), LocalDateTime.now(clock));
if (latency.isNegative()) {
latency = Duration.ZERO;
}
Timer.builder(DELIVERY_LATENCY_METRIC)
.description("Time from Event Bridge receipt to downstream acknowledgement")
.serviceLevelObjectives(Duration.ofSeconds(60), Duration.ofMinutes(15))
.tags("target_region", targetRegion, "event_type", eventType)
.register(registry).record(latency);
}
} catch (RuntimeException exception) {
log.warn("Event Bridge 传输指标记录失败,不影响可靠性状态机", exception);
}
}
public static String targetRegion(String value) {
String normalized = value == null ? "" : value.trim().toUpperCase(Locale.ROOT);
return "CN".equals(normalized) || "OVERSEAS".equals(normalized) ? normalized : "OTHER";
}
public static String eventType(String value) {
return value != null && KNOWN_EVENT_TYPES.contains(value) ? value : "OTHER";
}
public enum ReceiveResult {
RECEIVED,
DUPLICATE,
CONFLICT
}
public enum TransportResult {
DELIVERED(true),
RETRY(false),
DEAD(false),
ACKNOWLEDGED_LEASE_LOST(true),
FAILED_LEASE_LOST(false),
MANUAL_REPLAY(false);
private final boolean acknowledged;
TransportResult(boolean acknowledged) {
this.acknowledged = acknowledged;
}
boolean acknowledged() {
return acknowledged;
}
}
}
@@ -0,0 +1,50 @@
package com.baiyue.eventbridge.metrics;
/** Result row of the bounded, scheduled Inbox observability aggregation. */
public class InboxMetricsSnapshot {
private String targetRegion;
private String eventType;
private long backlogCount;
private double oldestPendingAgeSeconds;
private long deadCount;
public String getTargetRegion() {
return targetRegion;
}
public void setTargetRegion(String targetRegion) {
this.targetRegion = targetRegion;
}
public String getEventType() {
return eventType;
}
public void setEventType(String eventType) {
this.eventType = eventType;
}
public long getBacklogCount() {
return backlogCount;
}
public void setBacklogCount(long backlogCount) {
this.backlogCount = backlogCount;
}
public double getOldestPendingAgeSeconds() {
return oldestPendingAgeSeconds;
}
public void setOldestPendingAgeSeconds(double oldestPendingAgeSeconds) {
this.oldestPendingAgeSeconds = oldestPendingAgeSeconds;
}
public long getDeadCount() {
return deadCount;
}
public void setDeadCount(long deadCount) {
this.deadCount = deadCount;
}
}
@@ -5,6 +5,7 @@ import com.baiyue.common.event.EventPayloadCanonicalizer;
import com.baiyue.common.event.EventPayloadHash;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.dto.EventReceiveResult;
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.beans.factory.annotation.Value;
@@ -23,6 +24,7 @@ import java.util.Objects;
public class EventInboxService {
@Resource private InboxStore inboxStore;
@Resource private ObjectMapper objectMapper;
@Resource private EventBridgeMetrics metrics;
@Value("${baiyue.region:CN}")
private String localRegion;
@@ -44,6 +46,7 @@ public class EventInboxService {
BridgeInbox inbox = toInbox(envelope);
try {
inboxStore.insert(inbox);
metrics.recordReceive(envelope, EventBridgeMetrics.ReceiveResult.RECEIVED);
return new EventReceiveResult(envelope.eventId(), "已接收", inbox.getReceivedAt());
} catch (DuplicateKeyException exception) {
BridgeInbox concurrent = inboxStore.findByEventId(envelope.eventId());
@@ -56,8 +59,10 @@ public class EventInboxService {
private EventReceiveResult duplicateOrConflict(BridgeInbox existing, EventEnvelope envelope) {
if (!sameEnvelope(existing, envelope)) {
metrics.recordReceive(envelope, EventBridgeMetrics.ReceiveResult.CONFLICT);
throw new IllegalArgumentException("相同 eventId 对应的事件信封或 payloadHash 不一致,拒绝覆盖既有事件");
}
metrics.recordReceive(envelope, EventBridgeMetrics.ReceiveResult.DUPLICATE);
return new EventReceiveResult(envelope.eventId(), "重复", existing.getReceivedAt());
}
@@ -1,13 +1,23 @@
package com.baiyue.eventbridge.service;
import com.baiyue.eventbridge.domain.BridgeDeliveryAttempt;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
import jakarta.annotation.Resource;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
@Service
public class EventReplayService {
@Resource private BridgeInboxMapper inboxMapper;
@Resource private BridgeDeliveryAttemptMapper attemptMapper;
@Resource private EventBridgeMetrics metrics;
@Transactional
public void replayDead(String eventId, String operatorId, String reason) {
if (operatorId == null || operatorId.isBlank()) {
throw new IllegalArgumentException("人工补投必须提供操作人");
@@ -19,6 +29,22 @@ public class EventReplayService {
if (inboxMapper.replayDead(eventId, auditReason) != 1) {
throw new IllegalStateException("事件不存在或未处于 DEAD 状态");
}
BridgeInbox event = inboxMapper.selectById(eventId);
if (event == null) {
throw new IllegalStateException("事件补投状态已更新但事件记录不存在");
}
LocalDateTime now = LocalDateTime.now();
BridgeDeliveryAttempt audit = new BridgeDeliveryAttempt();
audit.setEventId(eventId);
audit.setSourceRegion(event.getSourceRegion());
audit.setTargetRegion(event.getTargetRegion());
audit.setAttemptNo(inboxMapper.nextAttemptNo(eventId));
audit.setStatus("MANUAL_REPLAY");
audit.setErrorMessage(auditReason);
audit.setStartedAt(now);
audit.setFinishedAt(now);
attemptMapper.insert(audit);
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.MANUAL_REPLAY);
}
private static String abbreviate(String value, int maxLength) {
@@ -42,11 +42,31 @@ public class HttpFirstlegEventClient implements FirstlegEventClient {
.header("X-Internal-Service-Token", serviceToken)
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(envelope))).build();
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
JsonNode responseBody = objectMapper.readTree(response.body());
JsonNode responseBody;
try {
responseBody = objectMapper.readTree(response.body());
} catch (Exception ignored) {
responseBody = null;
}
if (response.statusCode() / 100 != 2 || responseBody == null
|| !"200".equals(responseBody.path("code").asText())) {
throw new IllegalStateException("头程消费失败: HTTP " + response.statusCode());
throw new IllegalStateException(failureMessage(response.statusCode(), responseBody));
}
return String.valueOf(response.statusCode());
}
static String failureMessage(int statusCode, JsonNode body) {
String code = body == null ? "missing" : body.path("code").asText("missing");
String message = body == null ? "non-json response" : body.path("message").asText("missing");
String requestId = body == null ? "missing" : body.path("requestId").asText("missing");
String traceId = body == null ? "missing" : body.path("traceId").asText("missing");
return "头程消费失败: HTTP " + statusCode + ", code=" + bounded(code)
+ ", message=" + bounded(message) + ", requestId=" + bounded(requestId)
+ ", traceId=" + bounded(traceId);
}
private static String bounded(String value) {
String safe = value == null || value.isBlank() ? "missing" : value.replaceAll("[\\r\\n]+", " ");
return safe.length() <= 240 ? safe : safe.substring(0, 240);
}
}
@@ -0,0 +1,56 @@
package com.baiyue.eventbridge.service;
import com.baiyue.common.event.EventEnvelope;
import com.baiyue.common.event.RegionCode;
import com.baiyue.eventbridge.config.EventDispatcherProperties;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
@Component
public class HttpSecurityProjectionEventClient implements SecurityProjectionEventClient {
private final ObjectMapper objectMapper;
private final EventDispatcherProperties properties;
@Value("${baiyue.event-dispatcher.security-consumer-url}") private URI consumerUrl;
@Value("${baiyue.security.internal-service-token:}") private String serviceToken;
private HttpClient httpClient;
public HttpSecurityProjectionEventClient(ObjectMapper objectMapper, EventDispatcherProperties properties) {
this.objectMapper = objectMapper;
this.properties = properties;
}
@PostConstruct void initialize() {
httpClient = HttpClient.newBuilder().connectTimeout(properties.connectTimeout()).build();
}
@Override
public String deliver(BridgeInbox row) throws Exception {
EventEnvelope envelope = new EventEnvelope(row.getEventId(), row.getEventType(), row.getEventVersion(),
row.getOccurredAt(), RegionCode.valueOf(row.getSourceRegion()), RegionCode.valueOf(row.getTargetRegion()),
row.getCustomerId(), row.getAggregateType(), row.getAggregateId(), row.getAggregateVersion(),
row.getHandoverVersion(), row.getTraceId(), row.getCorrelationId(), row.getCausationId(),
objectMapper.readTree(row.getPayloadJson()), row.getPayloadHash());
HttpRequest request = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
.header("Content-Type", "application/json")
.header("X-Internal-Service-Token", serviceToken)
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(envelope))).build();
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
JsonNode body;
try { body = objectMapper.readTree(response.body()); }
catch (Exception exception) { body = null; }
if (response.statusCode() / 100 != 2 || body == null || !"200".equals(body.path("code").asText())
|| !row.getEventId().equals(body.path("data").path("eventId").asText())) {
throw new IllegalStateException("海外安全投影消费失败: HTTP " + response.statusCode());
}
return String.valueOf(response.statusCode());
}
}
@@ -1,10 +1,12 @@
package com.baiyue.eventbridge.service;
import com.baiyue.common.event.EventTypes;
import com.baiyue.eventbridge.config.EventDispatcherProperties;
import com.baiyue.eventbridge.domain.BridgeDeliveryAttempt;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
@@ -12,6 +14,7 @@ import org.springframework.stereotype.Component;
import java.lang.management.ManagementFactory;
import java.time.LocalDateTime;
import java.util.Set;
import java.util.UUID;
@Component
@@ -21,7 +24,9 @@ public class OverseasEventDispatcher {
@Resource private BridgeDeliveryAttemptMapper attemptMapper;
@Resource private LastlegEventClient eventClient;
@Resource private FirstlegEventClient firstlegEventClient;
@Resource private SecurityProjectionEventClient securityProjectionEventClient;
@Resource private EventDispatcherProperties properties;
@Resource private EventBridgeMetrics metrics;
private final String leaseOwner = ManagementFactory.getRuntimeMXBean().getName() + ":" + UUID.randomUUID();
@Scheduled(fixedDelayString = "${baiyue.event-dispatcher.fixed-delay:3000}")
@@ -40,26 +45,52 @@ public class OverseasEventDispatcher {
int retryAttempt = event.getRetryCount() == null ? 1 : event.getRetryCount() + 1;
int deliveryAttemptNo = inboxMapper.nextAttemptNo(eventId);
BridgeDeliveryAttempt attempt = startAttempt(event, deliveryAttemptNo);
String responseCode;
try {
String responseCode = deliver(event);
if (inboxMapper.markProcessed(eventId, leaseOwner) != 1) {
throw new IllegalStateException("事件租约已失效,无法确认投递结果");
}
finishAttempt(attempt, "DELIVERED", responseCode, null);
responseCode = deliver(event);
} catch (Exception exception) {
String error = abbreviate(exception.getMessage(), 1000);
inboxMapper.markFailed(eventId, leaseOwner, properties.maxAttempts(), backoffSeconds(retryAttempt), error);
finishAttempt(attempt, retryAttempt >= properties.maxAttempts() ? "DEAD" : "RETRY", null, error);
int marked = inboxMapper.markFailed(eventId, leaseOwner, properties.maxAttempts(),
backoffSeconds(retryAttempt), error);
String attemptStatus = marked == 1
? (retryAttempt >= properties.maxAttempts() ? "DEAD" : "RETRY")
: "FAILED_LEASE_LOST";
finishAttempt(attempt, attemptStatus, null, error);
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.valueOf(attemptStatus));
log.warn("跨区事件投递失败 eventId={}, retryAttempt={}, deliveryAttempt={}, reason={}",
eventId, retryAttempt, deliveryAttemptNo, error);
return;
}
if (inboxMapper.markProcessed(eventId, leaseOwner) != 1) {
String error = "下游已确认成功,但事件租约已失效;保留当前 Inbox 状态并由新租约持有者幂等续投";
finishAttempt(attempt, "ACKNOWLEDGED_LEASE_LOST", responseCode, error);
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.ACKNOWLEDGED_LEASE_LOST);
log.warn("跨区事件成功确认时租约已失效 eventId={}, deliveryAttempt={}", eventId, deliveryAttemptNo);
return;
}
finishAttempt(attempt, "DELIVERED", responseCode, null);
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.DELIVERED);
}
private String deliver(BridgeInbox event) throws Exception {
return "CN".equals(event.getTargetRegion())
? firstlegEventClient.deliver(event) : eventClient.deliver(event);
String eventType = event.getEventType();
if (eventType == null || eventType.isBlank()) {
throw new IllegalArgumentException("事件类型不能为空,无法选择下游消费者");
}
return switch (event.getTargetRegion()) {
case "CN" -> firstlegEventClient.deliver(event);
case "OVERSEAS" -> SECURITY_EVENT_TYPES.contains(eventType)
? securityProjectionEventClient.deliver(event) : eventClient.deliver(event);
default -> throw new IllegalArgumentException("不支持的事件目标区域: " + event.getTargetRegion());
};
}
private static final Set<String> SECURITY_EVENT_TYPES = Set.of(
EventTypes.IAM_USER_STATUS_CHANGED,
EventTypes.IAM_TENANT_STATUS_CHANGED,
EventTypes.IAM_PERMISSION_VERSION_CHANGED);
private BridgeDeliveryAttempt startAttempt(BridgeInbox event, int attemptNo) {
BridgeDeliveryAttempt attempt = new BridgeDeliveryAttempt();
attempt.setEventId(event.getEventId());
@@ -0,0 +1,7 @@
package com.baiyue.eventbridge.service;
import com.baiyue.eventbridge.domain.BridgeInbox;
public interface SecurityProjectionEventClient {
String deliver(BridgeInbox event) throws Exception;
}
@@ -33,6 +33,7 @@ baiyue:
event-dispatcher:
consumer-url: ${EVENT_CONSUMER_URL:${LASTLEG_CONSUMER_URL:http://127.0.0.1:9303/internal/lastleg/events}}
firstleg-consumer-url: ${FIRSTLEG_CONTAINER_EVENT_CONSUMER_URL:http://127.0.0.1:9202/internal/firstleg/v1/container-events}
security-consumer-url: ${OVERSEAS_SECURITY_EVENT_CONSUMER_URL:http://127.0.0.1:9307/internal/overseas-security/v1/events}
fixed-delay: ${EVENT_DISPATCH_FIXED_DELAY:3000}
batch-size: ${EVENT_DISPATCH_BATCH_SIZE:50}
max-attempts: ${EVENT_DISPATCH_MAX_ATTEMPTS:10}
@@ -0,0 +1,54 @@
package com.baiyue.eventbridge.metrics;
import com.baiyue.common.event.EventTypes;
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
class EventBridgeInboxMetricsSamplerTest {
@Test
void samplesDatabaseOnceAndServesCachedLowCardinalityGauges() {
BridgeInboxMapper mapper = mock(BridgeInboxMapper.class);
SimpleMeterRegistry registry = new SimpleMeterRegistry();
InboxMetricsSnapshot known = snapshot("OVERSEAS", EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED,
7, 121.5, 2);
InboxMetricsSnapshot unknown = snapshot("region-123", "dynamic-event-123", 3, 900.0, 1);
when(mapper.observabilitySnapshots()).thenReturn(List.of(known, unknown));
EventBridgeInboxMetricsSampler sampler = new EventBridgeInboxMetricsSampler(mapper, registry);
sampler.sample();
verify(mapper).observabilitySnapshots();
assertThat(gauge(registry, EventBridgeInboxMetricsSampler.BACKLOG_METRIC,
"OVERSEAS", EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED)).isEqualTo(7.0);
assertThat(gauge(registry, EventBridgeInboxMetricsSampler.OLDEST_AGE_METRIC,
"OVERSEAS", EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED)).isEqualTo(121.5);
assertThat(gauge(registry, EventBridgeInboxMetricsSampler.DEAD_METRIC,
"OTHER", "OTHER")).isEqualTo(1.0);
verifyNoMoreInteractions(mapper);
}
private InboxMetricsSnapshot snapshot(String region, String eventType, long backlog,
double oldestAge, long dead) {
InboxMetricsSnapshot snapshot = new InboxMetricsSnapshot();
snapshot.setTargetRegion(region);
snapshot.setEventType(eventType);
snapshot.setBacklogCount(backlog);
snapshot.setOldestPendingAgeSeconds(oldestAge);
snapshot.setDeadCount(dead);
return snapshot;
}
private double gauge(SimpleMeterRegistry registry, String name, String region, String eventType) {
return registry.get(name).tags("target_region", region, "event_type", eventType).gauge().value();
}
}
@@ -0,0 +1,58 @@
package com.baiyue.eventbridge.metrics;
import com.baiyue.eventbridge.domain.BridgeInbox;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import io.micrometer.core.instrument.MeterRegistry;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Modifier;
import java.util.Arrays;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.mockito.Mockito.mock;
class EventBridgeMetricsTest {
@Test
void collapsesUnboundedRegionAndEventTypeLabelsToOther() {
SimpleMeterRegistry registry = new SimpleMeterRegistry();
EventBridgeMetrics metrics = new EventBridgeMetrics(registry);
BridgeInbox event = new BridgeInbox();
event.setTargetRegion("customer-controlled-region");
event.setEventType("dynamic-event-123456");
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.RETRY);
assertThat(registry.get(EventBridgeMetrics.TRANSPORT_METRIC)
.tags("target_region", "OTHER", "event_type", "OTHER", "result", "RETRY")
.counter().count()).isEqualTo(1.0);
assertThat(registry.getMeters()).allSatisfy(meter -> assertThat(meter.getId().getTags())
.noneMatch(tag -> tag.getValue().contains("dynamic-event")));
assertThat(EventBridgeMetrics.eventType(null)).isEqualTo("OTHER");
}
@Test
void everyFrozenEventTypeHasABoundedTagValue() {
assertThat(Arrays.stream(com.baiyue.common.event.EventTypes.class.getFields())
.filter(field -> Modifier.isStatic(field.getModifiers()) && field.getType() == String.class)
.map(field -> {
try {
return (String) field.get(null);
} catch (IllegalAccessException exception) {
throw new AssertionError(exception);
}
})).allMatch(value -> EventBridgeMetrics.eventType(value).equals(value));
}
@Test
void metricsFailureDoesNotChangeTransportControlFlow() {
EventBridgeMetrics metrics = new EventBridgeMetrics(mock(MeterRegistry.class));
BridgeInbox event = new BridgeInbox();
event.setTargetRegion("CN");
event.setEventType(com.baiyue.common.event.EventTypes.OVERSEAS_CONTAINER_UNLOADED);
assertThatCode(() -> metrics.recordTransport(event, EventBridgeMetrics.TransportResult.RETRY))
.doesNotThrowAnyException();
}
}
@@ -0,0 +1,41 @@
package com.baiyue.eventbridge.metrics;
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
import org.apache.ibatis.annotations.Select;
import org.junit.jupiter.api.Test;
import java.nio.file.Files;
import java.nio.file.Path;
import static org.assertj.core.api.Assertions.assertThat;
class EventBridgeObservabilityContractTest {
@Test
void aggregationIsScheduledCacheInputAndUsesExistingStatusIndexBasis() throws Exception {
Select annotation = BridgeInboxMapper.class.getMethod("observabilitySnapshots")
.getAnnotation(Select.class);
String query = String.join(" ", annotation.value()).replaceAll("\\s+", " ");
String schema = Files.readString(Path.of("src/main/resources/db/event-bridge.sql"));
assertThat(query)
.contains("WHERE status IN ('RECEIVED','RETRY','PROCESSING','DEAD')")
.contains("GROUP BY target_region, event_type")
.doesNotContain("event_id AS", "customer_id AS", "last_error AS");
assertThat(schema).contains(
"KEY idx_bridge_inbox_consume (status, next_retry_at, lease_until)");
}
@Test
void developmentSloAndAlertThresholdsAreFrozenWithoutDeploymentClaim() throws Exception {
String contract = Files.readString(Path.of("docs/development-observability-slo.md"));
assertThat(contract)
.contains("99% 的新接收跨区事件在 60 秒内")
.contains("99.9% 的新接收跨区事件在 15 分钟内")
.contains("result=\"DEAD\"", "> 120", "> 900")
.contains("不表示任何环境已经部署")
.contains("idx_bridge_inbox_consume(status, next_retry_at, lease_until)")
.doesNotContain("customer_id\"", "event_id\"");
}
}
@@ -5,7 +5,9 @@ import com.baiyue.common.event.EventPayloadCanonicalizer;
import com.baiyue.common.event.EventPayloadHash;
import com.baiyue.common.event.RegionCode;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.jupiter.api.Test;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.test.util.ReflectionTestUtils;
@@ -24,6 +26,8 @@ import static org.mockito.Mockito.when;
class EventInboxServiceTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final InboxStore store = mock(InboxStore.class);
private final SimpleMeterRegistry registry = new SimpleMeterRegistry();
private final EventBridgeMetrics metrics = new EventBridgeMetrics(registry);
private final EventInboxService service = service("OVERSEAS");
@Test
@@ -33,6 +37,7 @@ class EventInboxServiceTest {
assertThat(service.receive(envelope).result()).isEqualTo("已接收");
verify(store).insert(any(BridgeInbox.class));
assertThat(receiveCount("RECEIVED")).isEqualTo(1.0);
}
@Test
@@ -57,6 +62,7 @@ class EventInboxServiceTest {
when(store.findByEventId(envelope.eventId())).thenReturn(existing);
assertThat(service.receive(envelope).result()).isEqualTo("重复");
assertThat(receiveCount("DUPLICATE")).isEqualTo(1.0);
}
@Test
@@ -67,6 +73,7 @@ class EventInboxServiceTest {
assertThatThrownBy(() -> service.receive(envelope))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("拒绝覆盖");
assertThat(receiveCount("CONFLICT")).isEqualTo(1.0);
}
@Test
@@ -113,10 +120,17 @@ class EventInboxServiceTest {
EventInboxService service = new EventInboxService();
ReflectionTestUtils.setField(service, "inboxStore", store);
ReflectionTestUtils.setField(service, "objectMapper", objectMapper);
ReflectionTestUtils.setField(service, "metrics", metrics);
ReflectionTestUtils.setField(service, "localRegion", region);
return service;
}
private double receiveCount(String result) {
return registry.get(EventBridgeMetrics.RECEIVE_METRIC)
.tags("target_region", "OVERSEAS", "event_type", "OrderApproved", "result", result)
.counter().count();
}
private EventEnvelope withHash(EventEnvelope source) {
String hash = EventPayloadHash.sha256(EventPayloadCanonicalizer.canonicalize(source.payload()));
return new EventEnvelope(source.eventId(), source.eventType(), source.eventVersion(), source.occurredAt(),
@@ -1,6 +1,11 @@
package com.baiyue.eventbridge.service;
import com.baiyue.eventbridge.domain.BridgeDeliveryAttempt;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import org.apache.ibatis.annotations.Update;
@@ -11,15 +16,31 @@ import static org.mockito.Mockito.*;
class EventReplayServiceTest {
private final BridgeInboxMapper mapper = mock(BridgeInboxMapper.class);
private final BridgeDeliveryAttemptMapper attemptMapper = mock(BridgeDeliveryAttemptMapper.class);
private final SimpleMeterRegistry registry = new SimpleMeterRegistry();
private final EventBridgeMetrics metrics = new EventBridgeMetrics(registry);
private final EventReplayService service = service();
@Test
void requeuesDeadEventWithAuditReason() {
when(mapper.replayDead(eq("event-1"), contains("incident-42"))).thenReturn(1);
BridgeInbox event = new BridgeInbox();
event.setEventId("event-1"); event.setSourceRegion("CN"); event.setTargetRegion("OVERSEAS");
event.setEventType(com.baiyue.common.event.EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED);
when(mapper.selectById("event-1")).thenReturn(event);
when(mapper.nextAttemptNo("event-1")).thenReturn(4);
service.replayDead("event-1", "operator-7", "incident-42 approved");
verify(mapper).replayDead(eq("event-1"), contains("operator-7"));
var captor = org.mockito.ArgumentCaptor.forClass(BridgeDeliveryAttempt.class);
verify(attemptMapper).insert(captor.capture());
assertThat(captor.getValue().getAttemptNo()).isEqualTo(4);
assertThat(captor.getValue().getStatus()).isEqualTo("MANUAL_REPLAY");
assertThat(captor.getValue().getErrorMessage()).contains("operator-7", "incident-42");
assertThat(registry.get(EventBridgeMetrics.TRANSPORT_METRIC)
.tags("target_region", "OVERSEAS", "event_type", event.getEventType(),
"result", "MANUAL_REPLAY").counter().count()).isEqualTo(1.0);
}
@Test
@@ -43,6 +64,8 @@ class EventReplayServiceTest {
private EventReplayService service() {
EventReplayService service = new EventReplayService();
ReflectionTestUtils.setField(service, "inboxMapper", mapper);
ReflectionTestUtils.setField(service, "attemptMapper", attemptMapper);
ReflectionTestUtils.setField(service, "metrics", metrics);
return service;
}
}
@@ -0,0 +1,26 @@
package com.baiyue.eventbridge.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class HttpFirstlegEventClientTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void businessFailureIsDiagnosableAndMustNotBeTreatedAsAcknowledged() throws Exception {
var body = objectMapper.readTree("""
{"code":"A0409","message":"事件摘要冲突","requestId":"request-cn-1","traceId":"trace-cn-1"}
""");
assertThat(HttpFirstlegEventClient.failureMessage(200, body))
.isEqualTo("头程消费失败: HTTP 200, code=A0409, message=事件摘要冲突, requestId=request-cn-1, traceId=trace-cn-1");
}
@Test
void nonJsonFailureIsStillDiagnosableWithoutRawBody() {
assertThat(HttpFirstlegEventClient.failureMessage(503, null))
.isEqualTo("头程消费失败: HTTP 503, code=missing, message=non-json response, requestId=missing, traceId=missing");
}
}
@@ -1,10 +1,13 @@
package com.baiyue.eventbridge.service;
import com.baiyue.common.event.EventTypes;
import com.baiyue.eventbridge.config.EventDispatcherProperties;
import com.baiyue.eventbridge.domain.BridgeDeliveryAttempt;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
@@ -12,6 +15,7 @@ import org.springframework.test.util.ReflectionTestUtils;
import java.net.URI;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
@@ -21,10 +25,14 @@ import static org.mockito.Mockito.*;
class OverseasEventDispatcherTest {
private final BridgeInboxMapper inboxMapper = mock(BridgeInboxMapper.class);
private final BridgeDeliveryAttemptMapper attemptMapper = mock(BridgeDeliveryAttemptMapper.class);
private final LastlegEventClient eventClient = mock(LastlegEventClient.class);
private final FirstlegEventClient firstlegEventClient = mock(FirstlegEventClient.class);
private final com.baiyue.eventbridge.service.LastlegEventClient eventClient =
mock(com.baiyue.eventbridge.service.LastlegEventClient.class);
private final com.baiyue.eventbridge.service.FirstlegEventClient firstlegEventClient =
mock(com.baiyue.eventbridge.service.FirstlegEventClient.class);
private final SecurityProjectionEventClient securityProjectionEventClient = mock(SecurityProjectionEventClient.class);
private OverseasEventDispatcher dispatcher;
private BridgeInbox event;
private SimpleMeterRegistry registry;
@BeforeEach
void setUp() {
@@ -33,16 +41,21 @@ class OverseasEventDispatcherTest {
Duration.ofSeconds(30), Duration.ofSeconds(5), Duration.ofMinutes(1),
Duration.ofSeconds(3), Duration.ofSeconds(10));
dispatcher = new OverseasEventDispatcher();
registry = new SimpleMeterRegistry();
ReflectionTestUtils.setField(dispatcher, "inboxMapper", inboxMapper);
ReflectionTestUtils.setField(dispatcher, "attemptMapper", attemptMapper);
ReflectionTestUtils.setField(dispatcher, "eventClient", eventClient);
ReflectionTestUtils.setField(dispatcher, "firstlegEventClient", firstlegEventClient);
ReflectionTestUtils.setField(dispatcher, "securityProjectionEventClient", securityProjectionEventClient);
ReflectionTestUtils.setField(dispatcher, "properties", properties);
ReflectionTestUtils.setField(dispatcher, "metrics", new EventBridgeMetrics(registry));
event = new BridgeInbox();
event.setEventId("event-1");
event.setEventType(EventTypes.FIRST_LEG_LOADING_PLAN_COMPLETED);
event.setSourceRegion("CN");
event.setTargetRegion("OVERSEAS");
event.setRetryCount(0);
event.setReceivedAt(LocalDateTime.now().minusSeconds(5));
when(inboxMapper.claim(eq("event-1"), anyString(), anyLong())).thenReturn(1);
when(inboxMapper.selectById("event-1")).thenReturn(event);
when(inboxMapper.nextAttemptNo("event-1")).thenReturn(1);
@@ -56,27 +69,50 @@ class OverseasEventDispatcherTest {
dispatcher.dispatchOne("event-1");
verify(inboxMapper).markProcessed(eq("event-1"), anyString());
verify(eventClient).deliver(event);
verifyNoInteractions(securityProjectionEventClient);
assertThat(updatedAttempt().getStatus()).isEqualTo("DELIVERED");
assertThat(transportCount("DELIVERED")).isEqualTo(1.0);
assertThat(registry.get(EventBridgeMetrics.DELIVERY_LATENCY_METRIC).timer().count()).isEqualTo(1);
}
@Test
void failedDeliveryIsScheduledForRetry() throws Exception {
when(eventClient.deliver(event)).thenThrow(new IllegalStateException("network down"));
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("network down")))
.thenReturn(1);
dispatcher.dispatchOne("event-1");
verify(inboxMapper).markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("network down"));
verify(inboxMapper).markFailed(eq("event-1"), anyString(), eq(3), eq(5L), eq("network down"));
assertThat(updatedAttempt().getStatus()).isEqualTo("RETRY");
assertThat(transportCount("RETRY")).isEqualTo(1.0);
}
@Test
void finalFailureIsMovedToDeadState() throws Exception {
event.setRetryCount(2);
when(eventClient.deliver(event)).thenThrow(new IllegalStateException("still down"));
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("still down")))
.thenReturn(1);
dispatcher.dispatchOne("event-1");
verify(inboxMapper).markFailed(eq("event-1"), anyString(), eq(3), eq(20L), eq("still down"));
assertThat(updatedAttempt().getStatus()).isEqualTo("DEAD");
assertThat(transportCount("DEAD")).isEqualTo(1.0);
}
@Test
void exponentialBackoffIsCappedAtConfiguredMaximum() throws Exception {
event.setRetryCount(4);
when(eventClient.deliver(event)).thenThrow(new IllegalStateException("still down"));
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("still down")))
.thenReturn(1);
dispatcher.dispatchOne("event-1");
verify(inboxMapper).markFailed(eq("event-1"), anyString(), eq(3), eq(60L), eq("still down"));
}
@Test
@@ -92,6 +128,7 @@ class OverseasEventDispatcherTest {
@Test
void cnTargetIsDeliveredToFirstleg() throws Exception {
event.setEventType(EventTypes.OVERSEAS_CONTAINER_UNLOADED);
event.setSourceRegion("OVERSEAS");
event.setTargetRegion("CN");
when(firstlegEventClient.deliver(event)).thenReturn("200");
@@ -101,9 +138,87 @@ class OverseasEventDispatcherTest {
verify(firstlegEventClient).deliver(event);
verifyNoInteractions(eventClient);
verifyNoInteractions(securityProjectionEventClient);
assertThat(updatedAttempt().getStatus()).isEqualTo("DELIVERED");
}
@Test
void iamSecurityEventsAreDeliveredToSecurityProjectionAndNeverLastleg() throws Exception {
for (String eventType : List.of(EventTypes.IAM_USER_STATUS_CHANGED,
EventTypes.IAM_TENANT_STATUS_CHANGED, EventTypes.IAM_PERMISSION_VERSION_CHANGED)) {
reset(inboxMapper, attemptMapper, eventClient, firstlegEventClient, securityProjectionEventClient);
event.setEventType(eventType);
when(inboxMapper.claim(eq("event-1"), anyString(), anyLong())).thenReturn(1);
when(inboxMapper.selectById("event-1")).thenReturn(event);
when(inboxMapper.nextAttemptNo("event-1")).thenReturn(1);
when(inboxMapper.markProcessed(eq("event-1"), anyString())).thenReturn(1);
when(securityProjectionEventClient.deliver(event)).thenReturn("200");
dispatcher.dispatchOne("event-1");
verify(securityProjectionEventClient).deliver(event);
verifyNoInteractions(eventClient);
verifyNoInteractions(firstlegEventClient);
}
}
@Test
void unsupportedTargetRegionIsNeverMisroutedToLastleg() {
event.setTargetRegion("UNKNOWN");
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), contains("不支持")))
.thenReturn(1);
dispatcher.dispatchOne("event-1");
verifyNoInteractions(eventClient);
verifyNoInteractions(firstlegEventClient);
verifyNoInteractions(securityProjectionEventClient);
assertThat(updatedAttempt().getStatus()).isEqualTo("RETRY");
}
@Test
void missingEventTypeFailsWithExplicitDiagnosticAndIsNeverDelivered() {
event.setEventType(null);
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(),
eq("事件类型不能为空,无法选择下游消费者"))).thenReturn(1);
dispatcher.dispatchOne("event-1");
verifyNoInteractions(eventClient);
verifyNoInteractions(firstlegEventClient);
verifyNoInteractions(securityProjectionEventClient);
BridgeDeliveryAttempt attempt = updatedAttempt();
assertThat(attempt.getStatus()).isEqualTo("RETRY");
assertThat(attempt.getErrorMessage()).isEqualTo("事件类型不能为空,无法选择下游消费者");
}
@Test
void acknowledgedDeliveryWithLostLeaseIsNotRecordedAsRetryOrDead() throws Exception {
when(eventClient.deliver(event)).thenReturn("200");
when(inboxMapper.markProcessed(eq("event-1"), anyString())).thenReturn(0);
dispatcher.dispatchOne("event-1");
verify(inboxMapper, never()).markFailed(anyString(), anyString(), anyInt(), anyLong(), anyString());
BridgeDeliveryAttempt attempt = updatedAttempt();
assertThat(attempt.getStatus()).isEqualTo("ACKNOWLEDGED_LEASE_LOST");
assertThat(attempt.getResponseCode()).isEqualTo("200");
assertThat(attempt.getErrorMessage()).contains("租约已失效");
assertThat(transportCount("ACKNOWLEDGED_LEASE_LOST")).isEqualTo(1.0);
}
@Test
void failedDeliveryAfterLeaseTakeoverDoesNotOverwriteNewOwnerState() throws Exception {
when(eventClient.deliver(event)).thenThrow(new IllegalStateException("network down"));
when(inboxMapper.markFailed(eq("event-1"), anyString(), eq(3), anyLong(), eq("network down")))
.thenReturn(0);
dispatcher.dispatchOne("event-1");
assertThat(updatedAttempt().getStatus()).isEqualTo("FAILED_LEASE_LOST");
assertThat(transportCount("FAILED_LEASE_LOST")).isEqualTo(1.0);
}
@Test
void downstream503RetriesThenDeliversWithDiagnosticAttemptsInBothDirections() throws Exception {
for (String target : List.of("OVERSEAS", "CN")) {
@@ -162,4 +277,11 @@ class OverseasEventDispatcherTest {
verify(attemptMapper).updateById(captor.capture());
return captor.getValue();
}
private double transportCount(String result) {
return registry.get(EventBridgeMetrics.TRANSPORT_METRIC)
.tags("target_region", event.getTargetRegion(),
"event_type", event.getEventType(), "result", result)
.counter().count();
}
}