优化
This commit is contained in:
@@ -18,6 +18,7 @@
|
||||
| 人工补投 | `service/EventReplayService.java`;只允许 `DEAD` 转回 `RETRY`,记录操作人与原因,并保留原 `eventId`、payload 与 `payloadHash` |
|
||||
| Inbox 与投递记录 | `domain/BridgeInbox.java`、`BridgeDeliveryAttempt.java` 及对应 Mapper |
|
||||
| 传输指标与 Inbox 采样 | `metrics/EventBridgeMetrics.java`、`EventBridgeInboxMetricsSampler.java`;SLO 与告警合同 `docs/development-observability-slo.md` |
|
||||
| 事件 Trace 与原始 HTTP 传播 | `trace/EventBridgeTraceScope.java`、`EventBridgeHttpTracePropagation.java`;契约与已知缺口 `docs/event-trace-propagation.md` |
|
||||
| 内部服务令牌 | `security/InternalServiceTokenGuard.java` |
|
||||
| 数据库 | `src/main/resources/db/event-bridge.sql` |
|
||||
| 测试 | `src/test/java/com/baiyue/eventbridge/` |
|
||||
@@ -29,6 +30,7 @@
|
||||
- 领取任务必须有数据库租约/并发保护,避免多实例重复发送;尾程幂等仍是最终保障。
|
||||
- 内部接口使用 `X-Internal-Service-Token`,生产还应置于受信网络或 mTLS 后。
|
||||
- 指标标签只允许有限 `target_region`、`event_type`、`result`,未知区域/类型折叠为 `OTHER`,禁止 eventId、customerId、错误消息等高基数标签。Inbox 聚合按现有状态索引定时采样,Prometheus scrape 不得查询数据库;开发阶段 SLO/告警只代表代码合同,不代表监控平台已部署。
|
||||
- 投递线程只从 Inbox 已冻结 `trace_id` 建立 MDC;合法 32 位 trace ID 或规范 UUID 可贯通,非法历史值安全重建。每次 JDK HttpClient 投递生成 child span,并覆盖三类标准 Trace 头;不得把 correlationId、eventId、customerId、聚合 ID、JWT 或内部令牌写入 Trace 字段。
|
||||
|
||||
## 修改与验证
|
||||
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# Event Bridge Trace 传播基线
|
||||
|
||||
- Event Bridge 领取 Inbox 后,仅从事件信封已冻结的 `traceId` 建立投递线程 MDC。
|
||||
- 32 位非零十六进制 trace ID 直接沿用;历史规范 UUID 去除连字符后沿用。
|
||||
- 其他旧值不满足 W3C Trace ID 约束,投递时安全重建新 trace,但不覆盖或迁移原事件信封。
|
||||
- 每次 HTTP 投递生成新的 `X-Request-Id` 和 child span,并统一覆盖 `X-Trace-Id`、`traceparent`。
|
||||
- `correlationId`、`eventId`、customer ID、聚合 ID、JWT 和内部服务令牌不得写入 Trace 标准字段。
|
||||
- 投递日志在 MDC 和显式 `traceId` 字段中记录本次标准 trace;投递 attempt 仍通过 `event_id`
|
||||
关联 `bridge_inbox.trace_id`,本轮不增加数据库字段。
|
||||
|
||||
## 共享契约缺口
|
||||
|
||||
当前 `EventEnvelope.traceId` 仍是通用字符串契约,没有强制 32 位 W3C Trace ID;因此历史任意字符串不能
|
||||
无损映射为 W3C trace。若后续要求所有事件跨重试、跨语言绝对保持同一标准 trace,必须由
|
||||
`common-event-contract` 生产者、消费者和 Event Bridge 共同冻结字段校验后再收敛。本轮没有修改共享
|
||||
Payload、信封字段、哈希规则或可靠性状态机。
|
||||
+5
-2
@@ -4,6 +4,7 @@ import com.baiyue.common.event.EventEnvelope;
|
||||
import com.baiyue.common.event.RegionCode;
|
||||
import com.baiyue.eventbridge.config.EventDispatcherProperties;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.trace.EventBridgeHttpTracePropagation;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
@@ -20,6 +21,7 @@ import java.net.http.HttpResponse;
|
||||
public class HttpFirstlegEventClient implements FirstlegEventClient {
|
||||
@Resource private ObjectMapper objectMapper;
|
||||
@Resource private EventDispatcherProperties properties;
|
||||
@Resource private EventBridgeHttpTracePropagation tracePropagation;
|
||||
@Value("${baiyue.event-dispatcher.firstleg-consumer-url:http://127.0.0.1:9203/internal/firstleg/v1/container-events}")
|
||||
private URI consumerUrl;
|
||||
@Value("${baiyue.security.internal-service-token:}") private String serviceToken;
|
||||
@@ -37,9 +39,10 @@ public class HttpFirstlegEventClient implements FirstlegEventClient {
|
||||
row.getCustomerId(), row.getAggregateType(), row.getAggregateId(), row.getAggregateVersion(),
|
||||
row.getHandoverVersion(), row.getTraceId(), row.getCorrelationId(), row.getCausationId(),
|
||||
objectMapper.readTree(row.getPayloadJson()), row.getPayloadHash());
|
||||
HttpRequest request = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
|
||||
HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-Internal-Service-Token", serviceToken)
|
||||
.header("X-Internal-Service-Token", serviceToken);
|
||||
HttpRequest request = tracePropagation.apply(requestBuilder)
|
||||
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(envelope))).build();
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
JsonNode responseBody;
|
||||
|
||||
+5
-2
@@ -4,6 +4,7 @@ import com.baiyue.common.event.EventEnvelope;
|
||||
import com.baiyue.common.event.RegionCode;
|
||||
import com.baiyue.eventbridge.config.EventDispatcherProperties;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.trace.EventBridgeHttpTracePropagation;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -19,6 +20,7 @@ import java.net.http.HttpResponse;
|
||||
public class HttpLastlegEventClient implements LastlegEventClient {
|
||||
@Resource private ObjectMapper objectMapper;
|
||||
@Resource private EventDispatcherProperties properties;
|
||||
@Resource private EventBridgeHttpTracePropagation tracePropagation;
|
||||
|
||||
@Value("${baiyue.security.internal-service-token:}")
|
||||
private String serviceToken;
|
||||
@@ -38,10 +40,11 @@ public class HttpLastlegEventClient implements LastlegEventClient {
|
||||
row.getHandoverVersion(), row.getTraceId(), row.getCorrelationId(), row.getCausationId(),
|
||||
objectMapper.readTree(row.getPayloadJson()), row.getPayloadHash());
|
||||
String body = objectMapper.writeValueAsString(envelope);
|
||||
HttpRequest request = HttpRequest.newBuilder(properties.consumerUrl())
|
||||
HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(properties.consumerUrl())
|
||||
.timeout(properties.requestTimeout())
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-Internal-Service-Token", serviceToken)
|
||||
.header("X-Internal-Service-Token", serviceToken);
|
||||
HttpRequest request = tracePropagation.apply(requestBuilder)
|
||||
.POST(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build();
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
|
||||
+6
-2
@@ -4,9 +4,11 @@ import com.baiyue.common.event.EventEnvelope;
|
||||
import com.baiyue.common.event.RegionCode;
|
||||
import com.baiyue.eventbridge.config.EventDispatcherProperties;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.trace.EventBridgeHttpTracePropagation;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import jakarta.annotation.Resource;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -19,6 +21,7 @@ import java.net.http.HttpResponse;
|
||||
public class HttpSecurityProjectionEventClient implements SecurityProjectionEventClient {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final EventDispatcherProperties properties;
|
||||
@Resource private EventBridgeHttpTracePropagation tracePropagation;
|
||||
@Value("${baiyue.event-dispatcher.security-consumer-url}") private URI consumerUrl;
|
||||
@Value("${baiyue.security.internal-service-token:}") private String serviceToken;
|
||||
private HttpClient httpClient;
|
||||
@@ -39,9 +42,10 @@ public class HttpSecurityProjectionEventClient implements SecurityProjectionEven
|
||||
row.getCustomerId(), row.getAggregateType(), row.getAggregateId(), row.getAggregateVersion(),
|
||||
row.getHandoverVersion(), row.getTraceId(), row.getCorrelationId(), row.getCausationId(),
|
||||
objectMapper.readTree(row.getPayloadJson()), row.getPayloadHash());
|
||||
HttpRequest request = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
|
||||
HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(consumerUrl).timeout(properties.requestTimeout())
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-Internal-Service-Token", serviceToken)
|
||||
.header("X-Internal-Service-Token", serviceToken);
|
||||
HttpRequest request = tracePropagation.apply(requestBuilder)
|
||||
.POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(envelope))).build();
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
JsonNode body;
|
||||
|
||||
+13
-3
@@ -7,10 +7,13 @@ import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
|
||||
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
|
||||
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
|
||||
import com.baiyue.eventbridge.trace.EventBridgeTraceScope;
|
||||
import com.baiyue.common.web.CorrelationContext;
|
||||
import jakarta.annotation.Resource;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.slf4j.MDC;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.time.LocalDateTime;
|
||||
@@ -42,6 +45,12 @@ public class OverseasEventDispatcher {
|
||||
}
|
||||
|
||||
BridgeInbox event = inboxMapper.selectById(eventId);
|
||||
try (EventBridgeTraceScope ignored = EventBridgeTraceScope.open(event)) {
|
||||
dispatchClaimed(eventId, event);
|
||||
}
|
||||
}
|
||||
|
||||
private void dispatchClaimed(String eventId, BridgeInbox event) {
|
||||
int retryAttempt = event.getRetryCount() == null ? 1 : event.getRetryCount() + 1;
|
||||
int deliveryAttemptNo = inboxMapper.nextAttemptNo(eventId);
|
||||
BridgeDeliveryAttempt attempt = startAttempt(event, deliveryAttemptNo);
|
||||
@@ -57,8 +66,8 @@ public class OverseasEventDispatcher {
|
||||
: "FAILED_LEASE_LOST";
|
||||
finishAttempt(attempt, attemptStatus, null, error);
|
||||
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.valueOf(attemptStatus));
|
||||
log.warn("跨区事件投递失败 eventId={}, retryAttempt={}, deliveryAttempt={}, reason={}",
|
||||
eventId, retryAttempt, deliveryAttemptNo, error);
|
||||
log.warn("跨区事件投递失败 eventId={}, retryAttempt={}, deliveryAttempt={}, traceId={}, reason={}",
|
||||
eventId, retryAttempt, deliveryAttemptNo, MDC.get(CorrelationContext.TRACE_ID), error);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -66,7 +75,8 @@ public class OverseasEventDispatcher {
|
||||
String error = "下游已确认成功,但事件租约已失效;保留当前 Inbox 状态并由新租约持有者幂等续投";
|
||||
finishAttempt(attempt, "ACKNOWLEDGED_LEASE_LOST", responseCode, error);
|
||||
metrics.recordTransport(event, EventBridgeMetrics.TransportResult.ACKNOWLEDGED_LEASE_LOST);
|
||||
log.warn("跨区事件成功确认时租约已失效 eventId={}, deliveryAttempt={}", eventId, deliveryAttemptNo);
|
||||
log.warn("跨区事件成功确认时租约已失效 eventId={}, deliveryAttempt={}, traceId={}",
|
||||
eventId, deliveryAttemptNo, MDC.get(CorrelationContext.TRACE_ID));
|
||||
return;
|
||||
}
|
||||
finishAttempt(attempt, "DELIVERED", responseCode, null);
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package com.baiyue.eventbridge.trace;
|
||||
|
||||
import com.baiyue.common.web.CorrelationContext;
|
||||
import com.baiyue.common.web.OutboundTraceHeaders;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.net.http.HttpRequest;
|
||||
|
||||
/** Applies a fresh child span to one raw JDK HttpClient request. */
|
||||
@Component
|
||||
public class EventBridgeHttpTracePropagation {
|
||||
public HttpRequest.Builder apply(HttpRequest.Builder builder) {
|
||||
OutboundTraceHeaders outbound = OutboundTraceHeaders.fromCurrentOrNew();
|
||||
return builder
|
||||
.setHeader(CorrelationContext.REQUEST_ID_HEADER, outbound.requestId())
|
||||
.setHeader(CorrelationContext.TRACE_ID_HEADER, outbound.traceId())
|
||||
.setHeader(CorrelationContext.TRACE_PARENT_HEADER, outbound.traceParent());
|
||||
}
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.baiyue.eventbridge.trace;
|
||||
|
||||
import com.baiyue.common.web.CorrelationContext;
|
||||
import com.baiyue.common.web.OutboundTraceHeaders;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import org.slf4j.MDC;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/** Establishes one trusted MDC context for an Event Bridge delivery attempt. */
|
||||
public final class EventBridgeTraceScope implements AutoCloseable {
|
||||
private final Map<String, String> previous;
|
||||
|
||||
private EventBridgeTraceScope(Map<String, String> previous) {
|
||||
this.previous = previous;
|
||||
}
|
||||
|
||||
public static EventBridgeTraceScope open(BridgeInbox event) {
|
||||
Map<String, String> previous = MDC.getCopyOfContextMap();
|
||||
OutboundTraceHeaders local = OutboundTraceHeaders.fromTrustedTraceOrNew(event.getTraceId());
|
||||
MDC.put(CorrelationContext.REQUEST_ID, local.requestId());
|
||||
MDC.put(CorrelationContext.TRACE_ID, local.traceId());
|
||||
MDC.put(CorrelationContext.SPAN_ID, local.spanId());
|
||||
return new EventBridgeTraceScope(previous);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (previous == null) {
|
||||
MDC.clear();
|
||||
} else {
|
||||
MDC.setContextMap(previous);
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
-1
@@ -7,11 +7,13 @@ import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import com.baiyue.eventbridge.mapper.BridgeDeliveryAttemptMapper;
|
||||
import com.baiyue.eventbridge.mapper.BridgeInboxMapper;
|
||||
import com.baiyue.eventbridge.metrics.EventBridgeMetrics;
|
||||
import com.baiyue.common.web.CorrelationContext;
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.slf4j.MDC;
|
||||
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
@@ -63,7 +65,12 @@ class OverseasEventDispatcherTest {
|
||||
|
||||
@Test
|
||||
void successfulDeliveryIsConfirmedAndAudited() throws Exception {
|
||||
when(eventClient.deliver(event)).thenReturn("200");
|
||||
event.setTraceId("550e8400-e29b-41d4-a716-446655440000");
|
||||
when(eventClient.deliver(event)).thenAnswer(invocation -> {
|
||||
assertThat(MDC.get(CorrelationContext.TRACE_ID))
|
||||
.isEqualTo("550e8400e29b41d4a716446655440000");
|
||||
return "200";
|
||||
});
|
||||
when(inboxMapper.markProcessed(eq("event-1"), anyString())).thenReturn(1);
|
||||
|
||||
dispatcher.dispatchOne("event-1");
|
||||
@@ -74,6 +81,7 @@ class OverseasEventDispatcherTest {
|
||||
assertThat(updatedAttempt().getStatus()).isEqualTo("DELIVERED");
|
||||
assertThat(transportCount("DELIVERED")).isEqualTo(1.0);
|
||||
assertThat(registry.get(EventBridgeMetrics.DELIVERY_LATENCY_METRIC).timer().count()).isEqualTo(1);
|
||||
assertThat(MDC.get(CorrelationContext.TRACE_ID)).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.baiyue.eventbridge.trace;
|
||||
|
||||
import com.baiyue.common.web.CorrelationContext;
|
||||
import com.baiyue.eventbridge.domain.BridgeInbox;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.slf4j.MDC;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpRequest;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class EventBridgeTracePropagationTest {
|
||||
|
||||
@AfterEach
|
||||
void clearMdc() {
|
||||
MDC.clear();
|
||||
}
|
||||
|
||||
@Test
|
||||
void envelopeUuidFlowsThroughMdcAndRawHttpAsAChildSpan() {
|
||||
BridgeInbox event = new BridgeInbox();
|
||||
event.setTraceId("550e8400-e29b-41d4-a716-446655440000");
|
||||
event.setCorrelationId("ORDER:customer-99:booking-123");
|
||||
event.setEventId("event-business-id");
|
||||
|
||||
try (EventBridgeTraceScope ignored = EventBridgeTraceScope.open(event)) {
|
||||
String localSpan = MDC.get(CorrelationContext.SPAN_ID);
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder(URI.create("http://127.0.0.1/internal"))
|
||||
.header(CorrelationContext.REQUEST_ID_HEADER, event.getEventId())
|
||||
.header(CorrelationContext.TRACE_ID_HEADER, event.getCorrelationId())
|
||||
.header(CorrelationContext.TRACE_PARENT_HEADER,
|
||||
"00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-00");
|
||||
HttpRequest request = new EventBridgeHttpTracePropagation().apply(builder).GET().build();
|
||||
|
||||
assertThat(MDC.get(CorrelationContext.TRACE_ID))
|
||||
.isEqualTo("550e8400e29b41d4a716446655440000");
|
||||
assertThat(request.headers().firstValue(CorrelationContext.TRACE_ID_HEADER))
|
||||
.contains("550e8400e29b41d4a716446655440000");
|
||||
assertThat(request.headers().firstValue(CorrelationContext.REQUEST_ID_HEADER).orElseThrow())
|
||||
.matches("[0-9a-f-]{36}").isNotEqualTo(event.getEventId());
|
||||
assertThat(request.headers().firstValue(CorrelationContext.TRACE_PARENT_HEADER).orElseThrow())
|
||||
.matches("00-550e8400e29b41d4a716446655440000-[0-9a-f]{16}-01")
|
||||
.doesNotContain(localSpan);
|
||||
}
|
||||
|
||||
assertThat(MDC.get(CorrelationContext.TRACE_ID)).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidLegacyEnvelopeTraceIsSafelyRebuiltAndPreviousMdcIsRestored() {
|
||||
MDC.put("existing", "preserved");
|
||||
BridgeInbox event = new BridgeInbox();
|
||||
event.setTraceId("JWT-or-business-id-value");
|
||||
|
||||
try (EventBridgeTraceScope ignored = EventBridgeTraceScope.open(event)) {
|
||||
assertThat(MDC.get(CorrelationContext.TRACE_ID)).matches("[0-9a-f]{32}")
|
||||
.isNotEqualTo(event.getTraceId());
|
||||
}
|
||||
|
||||
assertThat(MDC.get("existing")).isEqualTo("preserved");
|
||||
assertThat(MDC.get(CorrelationContext.TRACE_ID)).isNull();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user