This commit is contained in:
2026-08-25 19:07:34 +08:00
parent 65c01f8b3d
commit 7d36759f3f
28 changed files with 613 additions and 16 deletions
+2
View File
@@ -18,6 +18,7 @@
| 人工补投 | `service/EventReplayService.java`;只允许 `DEAD` 转回 `RETRY`,记录操作人与原因,并保留原 `eventId`、payload 与 `payloadHash` |
| Inbox 与投递记录 | `domain/BridgeInbox.java`、`BridgeDeliveryAttempt.java` 及对应 Mapper |
| 传输指标与 Inbox 采样 | `metrics/EventBridgeMetrics.java`、`EventBridgeInboxMetricsSampler.java`;SLO 与告警合同 `docs/development-observability-slo.md` |
| 事件 Trace 与原始 HTTP 传播 | `trace/EventBridgeTraceScope.java`、`EventBridgeHttpTracePropagation.java`;契约与已知缺口 `docs/event-trace-propagation.md` |
| 内部服务令牌 | `security/InternalServiceTokenGuard.java` |
| 数据库 | `src/main/resources/db/event-bridge.sql` |
| 测试 | `src/test/java/com/baiyue/eventbridge/` |
@@ -29,6 +30,7 @@
- 领取任务必须有数据库租约/并发保护,避免多实例重复发送;尾程幂等仍是最终保障。
- 内部接口使用 `X-Internal-Service-Token`,生产还应置于受信网络或 mTLS 后。
- 指标标签只允许有限 `target_region`、`event_type`、`result`,未知区域/类型折叠为 `OTHER`,禁止 eventId、customerId、错误消息等高基数标签。Inbox 聚合按现有状态索引定时采样,Prometheus scrape 不得查询数据库;开发阶段 SLO/告警只代表代码合同,不代表监控平台已部署。
- 投递线程只从 Inbox 已冻结 `trace_id` 建立 MDC;合法 32 位 trace ID 或规范 UUID 可贯通,非法历史值安全重建。每次 JDK HttpClient 投递生成 child span,并覆盖三类标准 Trace 头;不得把 correlationId、eventId、customerId、聚合 ID、JWT 或内部令牌写入 Trace 字段。
## 修改与验证
@@ -0,0 +1,16 @@
# Event Bridge Trace 传播基线
- Event Bridge 领取 Inbox 后,仅从事件信封已冻结的 `traceId` 建立投递线程 MDC。
- 32 位非零十六进制 trace ID 直接沿用;历史规范 UUID 去除连字符后沿用。
- 其他旧值不满足 W3C Trace ID 约束,投递时安全重建新 trace,但不覆盖或迁移原事件信封。
- 每次 HTTP 投递生成新的 `X-Request-Id` 和 child span,并统一覆盖 `X-Trace-Id`、`traceparent`。
- `correlationId`、`eventId`、customer ID、聚合 ID、JWT 和内部服务令牌不得写入 Trace 标准字段。
- 投递日志在 MDC 和显式 `traceId` 字段中记录本次标准 trace;投递 attempt 仍通过 `event_id`
关联 `bridge_inbox.trace_id`,本轮不增加数据库字段。
## 共享契约缺口
当前 `EventEnvelope.traceId` 仍是通用字符串契约,没有强制 32 位 W3C Trace ID;因此历史任意字符串不能
无损映射为 W3C trace。若后续要求所有事件跨重试、跨语言绝对保持同一标准 trace,必须由
`common-event-contract` 生产者、消费者和 Event Bridge 共同冻结字段校验后再收敛。本轮没有修改共享
Payload、信封字段、哈希规则或可靠性状态机。
@@ -4,6 +4,7 @@ import com.baiyue.common.event.EventEnvelope;
import com.baiyue.common.event.RegionCode;
import com.baiyue.eventbridge.config.EventDispatcherProperties;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.trace.EventBridgeHttpTracePropagation;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.annotation.PostConstruct;
@@ -20,6 +21,7 @@ import java.net.http.HttpResponse;
public class HttpFirstlegEventClient implements FirstlegEventClient {
@Resource private ObjectMapper objectMapper;
@Resource private EventDispatcherProperties properties;
@Resource private EventBridgeHttpTracePropagation tracePropagation;
@Value("${baiyue.event-dispatcher.firstleg-consumer-url:http://127.0.0.1:9203/internal/firstleg/v1/container-events}")
private URI consumerUrl;
@Value("${baiyue.security.internal-service-token:}") private String serviceToken;
@@ -37,9 +39,10 @@ public class HttpFirstlegEventClient implements FirstlegEventClient {
row.getCustomerId(), row.getAggregateType(), row.getAggregateId(), row.getAggregateVersion(),
row.getHandoverVersion(), row.getTraceId(), row.getCorrelationId(), row.getCausationId(),
objectMapper.readTree(row.getPayloadJson()), row.getPayloadHash());
HttpRequest request = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
.header("Content-Type", "application/json")
.header("X-Internal-Service-Token", serviceToken)
.header("X-Internal-Service-Token", serviceToken);
HttpRequest request = tracePropagation.apply(requestBuilder)
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(envelope))).build();
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
JsonNode responseBody;
@@ -4,6 +4,7 @@ import com.baiyue.common.event.EventEnvelope;
import com.baiyue.common.event.RegionCode;
import com.baiyue.eventbridge.config.EventDispatcherProperties;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.trace.EventBridgeHttpTracePropagation;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.beans.factory.annotation.Value;
@@ -19,6 +20,7 @@ import java.net.http.HttpResponse;
public class HttpLastlegEventClient implements LastlegEventClient {
@Resource private ObjectMapper objectMapper;
@Resource private EventDispatcherProperties properties;
@Resource private EventBridgeHttpTracePropagation tracePropagation;
@Value("${baiyue.security.internal-service-token:}")
private String serviceToken;
@@ -38,10 +40,11 @@ public class HttpLastlegEventClient implements LastlegEventClient {
row.getHandoverVersion(), row.getTraceId(), row.getCorrelationId(), row.getCausationId(),
objectMapper.readTree(row.getPayloadJson()), row.getPayloadHash());
String body = objectMapper.writeValueAsString(envelope);
HttpRequest request = HttpRequest.newBuilder(properties.consumerUrl())
HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(properties.consumerUrl())
.timeout(properties.requestTimeout())
.header("Content-Type", "application/json")
.header("X-Internal-Service-Token", serviceToken)
.header("X-Internal-Service-Token", serviceToken);
HttpRequest request = tracePropagation.apply(requestBuilder)
.POST(HttpRequest.BodyPublishers.ofString(body))
.build();
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
@@ -4,9 +4,11 @@ import com.baiyue.common.event.EventEnvelope;
import com.baiyue.common.event.RegionCode;
import com.baiyue.eventbridge.config.EventDispatcherProperties;
import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.trace.EventBridgeHttpTracePropagation;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.Resource;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -19,6 +21,7 @@ import java.net.http.HttpResponse;
public class HttpSecurityProjectionEventClient implements SecurityProjectionEventClient {
private final ObjectMapper objectMapper;
private final EventDispatcherProperties properties;
@Resource private EventBridgeHttpTracePropagation tracePropagation;
@Value("${baiyue.event-dispatcher.security-consumer-url}") private URI consumerUrl;
@Value("${baiyue.security.internal-service-token:}") private String serviceToken;
private HttpClient httpClient;
@@ -39,9 +42,10 @@ public class HttpSecurityProjectionEventClient implements SecurityProjectionEven
row.getCustomerId(), row.getAggregateType(), row.getAggregateId(), row.getAggregateVersion(),
row.getHandoverVersion(), row.getTraceId(), row.getCorrelationId(), row.getCausationId(),
objectMapper.readTree(row.getPayloadJson()), row.getPayloadHash());
HttpRequest request = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
.header("Content-Type", "application/json")
.header("X-Internal-Service-Token", serviceToken)
.header("X-Internal-Service-Token", serviceToken);
HttpRequest request = tracePropagation.apply(requestBuilder)
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(envelope))).build();
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
JsonNode body;
@@ -7,10 +7,13 @@ import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
import com.baiyue.eventbridge.trace.EventBridgeTraceScope;
import com.baiyue.common.web.CorrelationContext;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.slf4j.MDC;
import java.lang.management.ManagementFactory;
import java.time.LocalDateTime;
@@ -42,6 +45,12 @@ public class OverseasEventDispatcher {
}
BridgeInbox event = inboxMapper.selectById(eventId);
try (EventBridgeTraceScope ignored = EventBridgeTraceScope.open(event)) {
dispatchClaimed(eventId, event);
}
}
private void dispatchClaimed(String eventId, BridgeInbox event) {
int retryAttempt = event.getRetryCount() == null ? 1 : event.getRetryCount() + 1;
int deliveryAttemptNo = inboxMapper.nextAttemptNo(eventId);
BridgeDeliveryAttempt attempt = startAttempt(event, deliveryAttemptNo);
@@ -57,8 +66,8 @@ public class OverseasEventDispatcher {
: "FAILED_LEASE_LOST";
finishAttempt(attempt, attemptStatus, null, error);
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.valueOf(attemptStatus));
log.warn("跨区事件投递失败 eventId={}, retryAttempt={}, deliveryAttempt={}, reason={}",
eventId, retryAttempt, deliveryAttemptNo, error);
log.warn("跨区事件投递失败 eventId={}, retryAttempt={}, deliveryAttempt={}, traceId={}, reason={}",
eventId, retryAttempt, deliveryAttemptNo, MDC.get(CorrelationContext.TRACE_ID), error);
return;
}
@@ -66,7 +75,8 @@ public class OverseasEventDispatcher {
String error = "下游已确认成功,但事件租约已失效;保留当前 Inbox 状态并由新租约持有者幂等续投";
finishAttempt(attempt, "ACKNOWLEDGED_LEASE_LOST", responseCode, error);
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.ACKNOWLEDGED_LEASE_LOST);
log.warn("跨区事件成功确认时租约已失效 eventId={}, deliveryAttempt={}", eventId, deliveryAttemptNo);
log.warn("跨区事件成功确认时租约已失效 eventId={}, deliveryAttempt={}, traceId={}",
eventId, deliveryAttemptNo, MDC.get(CorrelationContext.TRACE_ID));
return;
}
finishAttempt(attempt, "DELIVERED", responseCode, null);
@@ -0,0 +1,19 @@
package com.baiyue.eventbridge.trace;
import com.baiyue.common.web.CorrelationContext;
import com.baiyue.common.web.OutboundTraceHeaders;
import org.springframework.stereotype.Component;
import java.net.http.HttpRequest;
/** Applies a fresh child span to one raw JDK HttpClient request. */
@Component
public class EventBridgeHttpTracePropagation {
public HttpRequest.Builder apply(HttpRequest.Builder builder) {
OutboundTraceHeaders outbound = OutboundTraceHeaders.fromCurrentOrNew();
return builder
.setHeader(CorrelationContext.REQUEST_ID_HEADER, outbound.requestId())
.setHeader(CorrelationContext.TRACE_ID_HEADER, outbound.traceId())
.setHeader(CorrelationContext.TRACE_PARENT_HEADER, outbound.traceParent());
}
}
@@ -0,0 +1,35 @@
package com.baiyue.eventbridge.trace;
import com.baiyue.common.web.CorrelationContext;
import com.baiyue.common.web.OutboundTraceHeaders;
import com.baiyue.eventbridge.domain.BridgeInbox;
import org.slf4j.MDC;
import java.util.Map;
/** Establishes one trusted MDC context for an Event Bridge delivery attempt. */
public final class EventBridgeTraceScope implements AutoCloseable {
private final Map<String, String> previous;
private EventBridgeTraceScope(Map<String, String> previous) {
this.previous = previous;
}
public static EventBridgeTraceScope open(BridgeInbox event) {
Map<String, String> previous = MDC.getCopyOfContextMap();
OutboundTraceHeaders local = OutboundTraceHeaders.fromTrustedTraceOrNew(event.getTraceId());
MDC.put(CorrelationContext.REQUEST_ID, local.requestId());
MDC.put(CorrelationContext.TRACE_ID, local.traceId());
MDC.put(CorrelationContext.SPAN_ID, local.spanId());
return new EventBridgeTraceScope(previous);
}
@Override
public void close() {
if (previous == null) {
MDC.clear();
} else {
MDC.setContextMap(previous);
}
}
}
@@ -7,11 +7,13 @@ import com.baiyue.eventbridge.domain.BridgeInbox;
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
import com.baiyue.common.web.CorrelationContext;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.test.util.ReflectionTestUtils;
import org.slf4j.MDC;
import java.net.URI;
import java.time.Duration;
@@ -63,7 +65,12 @@ class OverseasEventDispatcherTest {
@Test
void successfulDeliveryIsConfirmedAndAudited() throws Exception {
when(eventClient.deliver(event)).thenReturn("200");
event.setTraceId("550e8400-e29b-41d4-a716-446655440000");
when(eventClient.deliver(event)).thenAnswer(invocation -> {
assertThat(MDC.get(CorrelationContext.TRACE_ID))
.isEqualTo("550e8400e29b41d4a716446655440000");
return "200";
});
when(inboxMapper.markProcessed(eq("event-1"), anyString())).thenReturn(1);
dispatcher.dispatchOne("event-1");
@@ -74,6 +81,7 @@ class OverseasEventDispatcherTest {
assertThat(updatedAttempt().getStatus()).isEqualTo("DELIVERED");
assertThat(transportCount("DELIVERED")).isEqualTo(1.0);
assertThat(registry.get(EventBridgeMetrics.DELIVERY_LATENCY_METRIC).timer().count()).isEqualTo(1);
assertThat(MDC.get(CorrelationContext.TRACE_ID)).isNull();
}
@Test
@@ -0,0 +1,65 @@
package com.baiyue.eventbridge.trace;
import com.baiyue.common.web.CorrelationContext;
import com.baiyue.eventbridge.domain.BridgeInbox;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.slf4j.MDC;
import java.net.URI;
import java.net.http.HttpRequest;
import static org.assertj.core.api.Assertions.assertThat;
class EventBridgeTracePropagationTest {
@AfterEach
void clearMdc() {
MDC.clear();
}
@Test
void envelopeUuidFlowsThroughMdcAndRawHttpAsAChildSpan() {
BridgeInbox event = new BridgeInbox();
event.setTraceId("550e8400-e29b-41d4-a716-446655440000");
event.setCorrelationId("ORDER:customer-99:booking-123");
event.setEventId("event-business-id");
try (EventBridgeTraceScope ignored = EventBridgeTraceScope.open(event)) {
String localSpan = MDC.get(CorrelationContext.SPAN_ID);
HttpRequest.Builder builder = HttpRequest.newBuilder(URI.create("http://127.0.0.1/internal"))
.header(CorrelationContext.REQUEST_ID_HEADER, event.getEventId())
.header(CorrelationContext.TRACE_ID_HEADER, event.getCorrelationId())
.header(CorrelationContext.TRACE_PARENT_HEADER,
"00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-00");
HttpRequest request = new EventBridgeHttpTracePropagation().apply(builder).GET().build();
assertThat(MDC.get(CorrelationContext.TRACE_ID))
.isEqualTo("550e8400e29b41d4a716446655440000");
assertThat(request.headers().firstValue(CorrelationContext.TRACE_ID_HEADER))
.contains("550e8400e29b41d4a716446655440000");
assertThat(request.headers().firstValue(CorrelationContext.REQUEST_ID_HEADER).orElseThrow())
.matches("[0-9a-f-]{36}").isNotEqualTo(event.getEventId());
assertThat(request.headers().firstValue(CorrelationContext.TRACE_PARENT_HEADER).orElseThrow())
.matches("00-550e8400e29b41d4a716446655440000-[0-9a-f]{16}-01")
.doesNotContain(localSpan);
}
assertThat(MDC.get(CorrelationContext.TRACE_ID)).isNull();
}
@Test
void invalidLegacyEnvelopeTraceIsSafelyRebuiltAndPreviousMdcIsRestored() {
MDC.put("existing", "preserved");
BridgeInbox event = new BridgeInbox();
event.setTraceId("JWT-or-business-id-value");
try (EventBridgeTraceScope ignored = EventBridgeTraceScope.open(event)) {
assertThat(MDC.get(CorrelationContext.TRACE_ID)).matches("[0-9a-f]{32}")
.isNotEqualTo(event.getTraceId());
}
assertThat(MDC.get("existing")).isEqualTo("preserved");
assertThat(MDC.get(CorrelationContext.TRACE_ID)).isNull();
}
}