feat: expose reliable collection activity and stage estimates

This commit is contained in:
2026-10-13 16:50:53 +08:00
parent 03601ab0aa
commit 146d468ff4
15 changed files with 557 additions and 46 deletions
@@ -14,16 +14,29 @@ public final class SelfopProgressProjection {
"VALIDATION", "MERGE", "BUILD", "PUBLISH", "WAITING", "SAMPLE_DONE", "FAILED"); "VALIDATION", "MERGE", "BUILD", "PUBLISH", "WAITING", "SAMPLE_DONE", "FAILED");
/** Pure projection for callers already holding the scope/task/run locks and fence. */ /** Pure projection for callers already holding the scope/task/run locks and fence. */
public static void applyCheckpoint(SelfopSyncTask task,SelfopSyncRun run,String phase,String unit,long completed,Long total,int scan) { public static void applyCheckpoint(SelfopSyncTask task,SelfopSyncRun run,String phase,String unit,long completed,Long total,int scan) {
applyCheckpoint(task,run,phase,unit,completed,total,scan,true);
}
public static void applyCheckpoint(SelfopSyncTask task,SelfopSyncRun run,String phase,String unit,long completed,Long total,int scan,boolean realWork) {
require(PHASES.contains(phase),"SELFOP_PROGRESS_PHASE_INVALID"); require(PHASES.contains(phase),"SELFOP_PROGRESS_PHASE_INVALID");
require(Set.of("PAGE","ORDER","INVENTORY_ROW","BUSINESS_KEY","BATCH","NONE").contains(unit),"SELFOP_PROGRESS_UNIT_INVALID"); require(Set.of("PAGE","ORDER","INVENTORY_ROW","BUSINESS_KEY","BATCH","NONE").contains(unit),"SELFOP_PROGRESS_UNIT_INVALID");
require(completed>=0 && scan>=0 && (total==null || total>=completed),"SELFOP_PROGRESS_COUNTS_INVALID"); require(completed>=0 && scan>=0 && (total==null || total>=completed),"SELFOP_PROGRESS_COUNTS_INVALID");
Map<String,Object> p=progress(task); Map<String,Object> p=progress(task);
if(!Objects.equals(number(p.get("attemptNo")),run.getAttemptNo().longValue()))p=new LinkedHashMap<>(); if(!Objects.equals(number(p.get("attemptNo")),run.getAttemptNo().longValue())){
var prior=p;p=new LinkedHashMap<>();
if(prior.get("localWorkReady") instanceof Boolean
&&Objects.equals(prior.get("localWorkReadyScopeHash"),task.getScopeHash())
&&Objects.equals(prior.get("localWorkReadyUpstreamTaskId"),task.getUpstreamTaskId()==null?null:task.getUpstreamTaskId().toString()))
for(String key:List.of("localWorkReady","localWorkReadyScopeHash","localWorkReadyUpstreamTaskId"))p.put(key,prior.get(key));
}
String priorIdentity=SelfopStageEstimate.identity(p);
String countScope=scope(task); Object started=p.get("stageStartedAt"); String countScope=scope(task); Object started=p.get("stageStartedAt");
if(!phase.equals(p.get("phase")) || started==null)started=instant(utcNow()); if(!phase.equals(p.get("phase")) || started==null)started=instant(utcNow());
p.put("taskNo",task.getTaskNo());
p.put("attemptNo",run.getAttemptNo()); p.put("phase",phase); p.put("progressScope",countScope); p.put("attemptNo",run.getAttemptNo()); p.put("phase",phase); p.put("progressScope",countScope);
p.put("stageStartedAt",started); p.put("unit",unit); p.put("completed",completed); p.put("stageStartedAt",started); p.put("unit",unit); p.put("completed",completed);
p.put("total",total); p.put("totalKnown",total!=null); p.put("scanRound",scan); p.put("countBasis",Set.of("INVENTORY","BUILD").contains(phase)?"ATTEMPT_ONLY":"TASK_CUMULATIVE"); p.put("sliceFrom",instant(task.getQueryFromUtc())); p.put("sliceUntil",instant(task.getQueryUntilUtc())); p.put("stageFinishedAt",null); p.put("total",total); p.put("totalKnown",total!=null); p.put("scanRound",scan); p.put("countBasis",Set.of("INVENTORY","BUILD").contains(phase)?"ATTEMPT_ONLY":"TASK_CUMULATIVE"); p.put("sliceFrom",instant(task.getQueryFromUtc())); p.put("sliceUntil",instant(task.getQueryUntilUtc())); p.put("stageFinishedAt",null);
if(!priorIdentity.equals(SelfopStageEstimate.identity(p)))p.put("stageStartedAt",instant(utcNow()));
SelfopStageEstimate.checkpoint(p,utcNow().toInstant(ZoneOffset.UTC),realWork);
// A zero denominator is meaningful evidence, but cannot produce a percentage. // A zero denominator is meaningful evidence, but cannot produce a percentage.
p.put("phasePercent",total==null||total==0?null:BigDecimal.valueOf(completed).multiply(BigDecimal.valueOf(100)) p.put("phasePercent",total==null||total==0?null:BigDecimal.valueOf(completed).multiply(BigDecimal.valueOf(100))
.divide(BigDecimal.valueOf(total),1,RoundingMode.DOWN)); .divide(BigDecimal.valueOf(total),1,RoundingMode.DOWN));
@@ -42,6 +55,13 @@ public final class SelfopProgressProjection {
/** Pure terminal snapshot; historical attempt evidence is immutable. */ /** Pure terminal snapshot; historical attempt evidence is immutable. */
public static void freezeRun(SelfopSyncTask task,SelfopSyncRun run) { public static void freezeRun(SelfopSyncTask task,SelfopSyncRun run) {
// A yielded inventory attempt resumes with the same attempt number. Its live
// sampling baseline must not survive a budget pause or execution interruption.
if(task.getProgressJson()!=null){
var live=progress(task);
if(Objects.equals(number(live.get("attemptNo")),run.getAttemptNo().longValue())&&live.remove("etaSamples")!=null)
task.setProgressJson(bounded(live));
}
if(run.getProgressSnapshotJson()!=null)return; if(run.getProgressSnapshotJson()!=null)return;
if(task.getProgressJson()==null) { if(task.getProgressJson()==null) {
var fallback=new LinkedHashMap<String,Object>();fallback.put("attemptNo",run.getAttemptNo()); var fallback=new LinkedHashMap<String,Object>();fallback.put("attemptNo",run.getAttemptNo());
@@ -50,16 +70,50 @@ public final class SelfopProgressProjection {
fallback.put("unit","NONE");fallback.put("completed",0);fallback.put("total",null);fallback.put("totalKnown",false); fallback.put("unit","NONE");fallback.put("completed",0);fallback.put("total",null);fallback.put("totalKnown",false);
fallback.put("sliceFrom",instant(task.getQueryFromUtc()));fallback.put("sliceUntil",instant(task.getQueryUntilUtc())); fallback.put("sliceFrom",instant(task.getQueryFromUtc()));fallback.put("sliceUntil",instant(task.getQueryUntilUtc()));
fallback.put("stageStartedAt",instant(run.getFetchStartedAtUtc()));fallback.put("stageFinishedAt",null);fallback.put("stages",List.of()); fallback.put("stageStartedAt",instant(run.getFetchStartedAtUtc()));fallback.put("stageFinishedAt",null);fallback.put("stages",List.of());
fallback.put("taskNo",task.getTaskNo());fallback.put("summaryAsOf",instant(utcNow()));
run.setProgressSnapshotJson(bounded(fallback)); run.setProgressSnapshotJson(bounded(fallback));
return; return;
} }
var p=progress(task); var p=progress(task);
// A later attempt's projection must never be copied into an earlier historical run. // A later attempt's projection must never be copied into an earlier historical run.
require(Objects.equals(number(p.get("attemptNo")),run.getAttemptNo().longValue()),"SELFOP_PROGRESS_ATTEMPT_MISMATCH"); require(Objects.equals(number(p.get("attemptNo")),run.getAttemptNo().longValue()),"SELFOP_PROGRESS_ATTEMPT_MISMATCH");
p.remove("etaSamples");p.put("summaryAsOf",instant(utcNow()));
run.setProgressSnapshotJson(bounded(p));run.setUpdatedAt(utcNow()); run.setProgressSnapshotJson(bounded(p));run.setUpdatedAt(utcNow());
} }
private static String bounded(Map<String,Object> p) {String value=json(p);require(value.getBytes(StandardCharsets.UTF_8).length<=8192,"SELFOP_PROGRESS_TOO_LARGE");return value;} /** Route commits this proof together with its immutable effective-stage references. */
public static void localWorkReady(SelfopSyncTask task,boolean ready) {
var p=progress(task);p.put("localWorkReady",ready);
p.put("localWorkReadyScopeHash",task.getScopeHash());
p.put("localWorkReadyUpstreamTaskId",task.getUpstreamTaskId()==null?null:task.getUpstreamTaskId().toString());
task.setProgressJson(bounded(p));
}
private static String bounded(Map<String,Object> p) {
// Reserve the terminal timestamp before accepting a live checkpoint. Optional
// ETA and derived display fields never displace business/audit checkpoints.
int target=p.containsKey("summaryAsOf")?8192:8192-64;
String value=json(p);
if(bytes(value)>target){p.remove("etaSamples");value=json(p);}
if(bytes(value)>target){p.remove("phasePercent");value=json(p);}
if(bytes(value)>target&&p.get("stages") instanceof List<?> prior){
var stages=new ArrayList<Map<String,Object>>();
for(Object entry:prior){var stage=map(entry);stage.remove("phasePercent");stages.add(stage);}
p.put("stages",stages);value=json(p);
// The current top-level stage clock remains available. Duplicated stage
// display clocks can be omitted while retaining every stage count/identity.
if(bytes(value)>target){
for(var stage:stages){stage.remove("stageStartedAt");stage.remove("stageFinishedAt");}
value=json(p);
}
}
// A pre-existing legacy checkpoint may have used the whole 8 KiB with no
// optional display fields. Missing optional snapshot time is preferable to
// failing its business termination; never invent a replacement timestamp.
if(bytes(value)>8192&&p.containsKey("summaryAsOf")){p.remove("summaryAsOf");value=json(p);}
require(bytes(value)<=8192,"SELFOP_PROGRESS_TOO_LARGE");return value;
}
private static int bytes(String value){return value.getBytes(StandardCharsets.UTF_8).length;}
private static String scope(SelfopSyncTask task){return "ACCOUNT_DISCOVERY".equals(task.getTaskRole())?"ACCOUNT_SHARED":"WAREHOUSE";} private static String scope(SelfopSyncTask task){return "ACCOUNT_DISCOVERY".equals(task.getTaskRole())?"ACCOUNT_SHARED":"WAREHOUSE";}
@SuppressWarnings("unchecked") private static Map<String,Object> progress(SelfopSyncTask task){return task.getProgressJson()==null?new LinkedHashMap<>():read(task.getProgressJson(),LinkedHashMap.class);} @SuppressWarnings("unchecked") private static Map<String,Object> progress(SelfopSyncTask task){return task.getProgressJson()==null?new LinkedHashMap<>():read(task.getProgressJson(),LinkedHashMap.class);}
@SuppressWarnings("unchecked") private static Map<String,Object> map(Object value){return value instanceof Map<?,?>?new LinkedHashMap<>((Map<String,Object>)value):new LinkedHashMap<>();} @SuppressWarnings("unchecked") private static Map<String,Object> map(Object value){return value instanceof Map<?,?>?new LinkedHashMap<>((Map<String,Object>)value):new LinkedHashMap<>();}
@@ -0,0 +1,114 @@
package com.baiyue.lastleg.selfoperated.analytics.model;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.*;
import static com.baiyue.lastleg.selfoperated.analytics.model.SelfopSupport.*;
/** Bounded committed-checkpoint sampling; reads never manufacture progress. */
public final class SelfopStageEstimate {
private SelfopStageEstimate() { }
public static final int WINDOW_SECONDS=180, MIN_SECONDS=60, MIN_ADVANCES=3,
BUCKET_SECONDS=10, MAX_POINTS=20, STALE_SECONDS=60, VALID_SECONDS=10, MAX_BYTES=1024;
public record Estimate(String status,String progressIdentity,Long completed,Long total,
Instant sampleFrom,Instant sampleUntil,Long completedDelta,BigDecimal ratePerSecond,
Long remainingSeconds,Instant estimatedAt,Instant validUntil,String unavailableReason) { }
public static String identity(Map<String,Object> p) {
var values=new ArrayList<Object>();
for(String key:List.of("taskNo","attemptNo","phase","scanRound","progressScope","unit",
"countBasis","sliceFrom","sliceUntil","total"))values.add(p.get(key));
return hash(values);
}
/** A reuse/uncertain checkpoint resets the baseline, never contributes throughput. */
public static void checkpoint(Map<String,Object> p,Instant now,boolean realWork) {
try {
Long completed=number(p.get("completed")),total=number(p.get("total"));
if(total==null||total<=0||completed==null||completed<0||completed>=total||!Boolean.TRUE.equals(p.get("totalKnown"))||"NONE".equals(p.get("unit"))) {
p.remove("etaSamples");return;
}
String id=identity(p);long second=now.getEpochSecond();
var old=map(p.get("etaSamples"));var points=points(old.get("p"));
if(!id.equals(old.get("i"))||!realWork||points.isEmpty()
||second<points.get(points.size()-1).get(0)
||completed<points.get(points.size()-1).get(1)
||second-numberOr(old.get("a"),second)>=STALE_SECONDS)points.clear();
points.removeIf(point->second-point.get(0)>WINDOW_SECONDS);
long advances=points.isEmpty()?0:points.get(points.size()-1).get(2);
long lastAdvance=numberOr(old.get("a"),second);
if(points.isEmpty()) {points.add(List.of(second,completed,0L));lastAdvance=second;}
else {
var last=points.get(points.size()-1);
if(completed>last.get(1)) {advances++;lastAdvance=second;}
var point=List.of(second,completed,advances);
if(last.get(0)/BUCKET_SECONDS==second/BUCKET_SECONDS)points.set(points.size()-1,point);
else points.add(point);
}
while(points.size()>MAX_POINTS)points.remove(0);
var sample=new LinkedHashMap<String,Object>();sample.put("i",id);sample.put("a",lastAdvance);sample.put("p",points);
while(json(sample).getBytes(StandardCharsets.UTF_8).length>MAX_BYTES&&points.size()>1)points.remove(0);
if(json(sample).getBytes(StandardCharsets.UTF_8).length<=MAX_BYTES)p.put("etaSamples",sample);
else p.remove("etaSamples");
} catch(RuntimeException malformedOptionalSample) {p.remove("etaSamples");}
}
public static Estimate estimate(Map<String,Object> p,Instant now,boolean healthy,boolean paused,boolean terminal) {
String id=identity(p);Long completed=number(p.get("completed")),total=number(p.get("total"));
String reason=terminal?"TERMINAL":paused?"PAUSED":!healthy?"HEALTH_UNCONFIRMED":
total==null||!Boolean.TRUE.equals(p.get("totalKnown"))?"TOTAL_UNKNOWN":completed==null||completed<0||total<completed||"NONE".equals(p.get("unit"))?"INVALID_PROGRESS":
total.equals(completed)?"COUNT_COMPLETE":null;
Instant from=null;Long delta=null;BigDecimal rate=null;Long remaining=null;Instant until=null;
try {
var sample=map(p.get("etaSamples"));var points=points(sample.get("p"));
if(reason==null&&(!id.equals(sample.get("i"))||points.isEmpty()))reason="SAMPLING";
if(reason==null) {
long second=now.getEpochSecond(),lastAdvance=numberOr(sample.get("a"),0);
points.removeIf(point->second-point.get(0)>WINDOW_SECONDS);
if(points.isEmpty()||second-lastAdvance>=STALE_SECONDS)reason="NO_RECENT_PROGRESS";
else {
var first=points.get(0);var last=points.get(points.size()-1);
long elapsed=second-first.get(0),advances=last.get(2)-first.get(2);
if(second<last.get(0)||!completed.equals(last.get(1)))reason="INVALID_PROGRESS";
else if(elapsed<MIN_SECONDS||advances<MIN_ADVANCES)reason="SAMPLING";
else {
delta=last.get(1)-first.get(1);
if(delta<=0)reason="NO_RECENT_PROGRESS";
else {
from=Instant.ofEpochSecond(first.get(0));
rate=BigDecimal.valueOf(delta).divide(BigDecimal.valueOf(elapsed),6,RoundingMode.DOWN);
remaining=BigDecimal.valueOf(total-completed).multiply(BigDecimal.valueOf(elapsed))
.divide(BigDecimal.valueOf(delta),0,RoundingMode.CEILING).longValueExact();
until=Instant.ofEpochSecond(Math.min(second+VALID_SECONDS,Math.min(lastAdvance+STALE_SECONDS,first.get(0)+WINDOW_SECONDS)));
if(!until.isAfter(now))reason="SAMPLE_EXPIRED";
}
}
}
}
} catch(RuntimeException malformedOptionalSample) {reason="INVALID_SAMPLE";}
if(reason!=null)return new Estimate(paused&&!terminal?"PAUSED":"UNAVAILABLE",id,completed,total,null,null,null,null,null,now,null,reason);
return new Estimate("AVAILABLE",id,completed,total,from,now,delta,rate,remaining,now,until,null);
}
private static Long number(Object value){
if(!(value instanceof Number))return null;
try{return new BigDecimal(value.toString()).longValueExact();}catch(RuntimeException invalid){return null;}
}
private static long numberOr(Object value,long fallback){Long n=number(value);return n==null?fallback:n;}
@SuppressWarnings("unchecked") private static Map<String,Object> map(Object value){return value instanceof Map<?,?>?(Map<String,Object>)value:Map.of();}
private static List<List<Long>> points(Object value) {
var result=new ArrayList<List<Long>>();
if(value instanceof List<?> list)for(Object item:list) {
if(!(item instanceof List<?> point)||point.size()!=3||result.size()>=MAX_POINTS)throw new IllegalArgumentException("Invalid bounded sample");
var parsed=List.of(Objects.requireNonNull(number(point.get(0))),Objects.requireNonNull(number(point.get(1))),Objects.requireNonNull(number(point.get(2))));
if(parsed.stream().anyMatch(n->n<0))throw new IllegalArgumentException("Negative sample");
if(!result.isEmpty()){
var prior=result.get(result.size()-1);
if(parsed.get(0)<=prior.get(0)||parsed.get(1)<prior.get(1)||parsed.get(2)<prior.get(2))throw new IllegalArgumentException("Nonmonotonic sample");
}
result.add(parsed);
}
return result;
}
}
@@ -2,6 +2,7 @@ package com.baiyue.lastleg.selfoperated.analytics.query;
import com.baiyue.common.core.exception.BizException; import com.baiyue.common.core.exception.BizException;
import com.baiyue.lastleg.selfoperated.analytics.model.SelfopAnalyticsProperties; import com.baiyue.lastleg.selfoperated.analytics.model.SelfopAnalyticsProperties;
import com.baiyue.lastleg.selfoperated.analytics.model.SelfopStageEstimate;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@@ -23,6 +24,7 @@ public class CollectionQueryService {
private final Clock clock; private final Clock clock;
private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai"); private static final ZoneId BUSINESS_ZONE = ZoneId.of("Asia/Shanghai");
private static final Set<String> TERMINAL = Set.of("ROUTED","PUBLISHED","REUSED","YIELDED","FAILED","ABORTED","SAMPLE_DONE"); private static final Set<String> TERMINAL = Set.of("ROUTED","PUBLISHED","REUSED","YIELDED","FAILED","ABORTED","SAMPLE_DONE");
private static final Set<String> ACTIVE = Set.of("RUNNING","READY","QUEUED","WAITING","WAITING_SOURCE","RETRY_WAIT","YIELDED");
private static final Set<String> PROGRESS_FIELDS = Set.of("progressScope","countBasis","attemptNo","scanRound","phase", private static final Set<String> PROGRESS_FIELDS = Set.of("progressScope","countBasis","attemptNo","scanRound","phase",
"completed","total","totalKnown","unit","sliceFrom","sliceUntil","stageStartedAt","stageFinishedAt", "completed","total","totalKnown","unit","sliceFrom","sliceUntil","stageStartedAt","stageFinishedAt",
"validationState","invalidCount","unknownCount","pendingReviewCount","countUnit","summaryAsOf","directoryComplete"); "validationState","invalidCount","unknownCount","pendingReviewCount","countUnit","summaryAsOf","directoryComplete");
@@ -76,14 +78,20 @@ public class CollectionQueryService {
for(String family:List.of("OUTBOUND","INVENTORY")) { for(String family:List.of("OUTBOUND","INVENTORY")) {
if(query.dataFamily()!=null&&!family.equals(query.dataFamily()))continue; if(query.dataFamily()!=null&&!family.equals(query.dataFamily()))continue;
Scope scope=persisted.get(warehouse+":"+family); Scope scope=persisted.get(warehouse+":"+family);
Task current=currentTasks.stream().filter(t->warehouse.equals(t.warehouseCode())&&family.equals(t.dataFamily())).findFirst().orElse(null); List<Task> candidates=new ArrayList<>(currentTasks.stream().filter(t->warehouse.equals(t.warehouseCode())&&family.equals(t.dataFamily())).toList());
Task sampled=validationTasks.stream().filter(t->warehouse.equals(t.warehouseCode())).findFirst().orElse(null); Task sampled=validationTasks.stream().filter(t->warehouse.equals(t.warehouseCode())).findFirst().orElse(null);
if(sampled!=null&&(current==null||sampled.createdAt()!=null&&current.createdAt()!=null&&sampled.createdAt().isAfter(current.createdAt())))current=sampled; if(sampled!=null)candidates.add(sampled);
candidates.sort(Comparator.comparingInt((Task t)->"RUNNING".equals(t.status())?0:ACTIVE.contains(t.status())?1:2)
.thenComparing(Task::createdAt,Comparator.nullsLast(Comparator.reverseOrder()))
.thenComparing(Task::id,Comparator.nullsLast(Comparator.reverseOrder())));
Task current=candidates.isEmpty()?null:candidates.get(0);
List<CollectionResponses.Task> activity=candidates.stream().filter(t->ACTIVE.contains(t.status())).limit(8)
.map(t->taskView(t,scope,protection,context,family)).toList();
Task last=successes.stream().filter(t->warehouse.equals(t.warehouseCode())&&family.equals(t.dataFamily())).findFirst().orElse(null); Task last=successes.stream().filter(t->warehouse.equals(t.warehouseCode())&&family.equals(t.dataFamily())).findFirst().orElse(null);
Attempt published=scope==null?null:context.published().get(scope.publishedRunId()); Attempt published=scope==null?null:context.published().get(scope.publishedRunId());
if(published!=null&&(!warehouse.equals(published.warehouseCode())||!Objects.equals(scope.id(),published.scopeId()) if(published!=null&&(!warehouse.equals(published.warehouseCode())||!Objects.equals(scope.id(),published.scopeId())
||!Objects.equals(scope.scopeHash(),published.scopeHash())))published=null; ||!Objects.equals(scope.scopeHash(),published.scopeHash())))published=null;
Task shared="OUTBOUND".equals(family)&&sharedWindowRelated(active,scope)?active:null; Task shared=current==null?null:relatedUpstream(current,context);
String blocked=executionBlock(warehouse,protection); String blocked=executionBlock(warehouse,protection);
boolean enabled=blocked==null; boolean enabled=blocked==null;
boolean publication="BUSINESS".equals(config.getCollectionMode())&&config.isExecutionEnabled() boolean publication="BUSINESS".equals(config.getCollectionMode())&&config.isExecutionEnabled()
@@ -104,10 +112,11 @@ public class CollectionQueryService {
"INVENTORY".equals(family)&&published!=null?utc(published.fetchFinishedAtUtc()):null,freshness, "INVENTORY".equals(family)&&published!=null?utc(published.fetchFinishedAtUtc()):null,freshness,
"OUTBOUND".equals(family)&&scope!=null&&(Integer.valueOf(1).equals(scope.rebuildRequired())||barriers.contains(warehouse)),reviewStatus(scope), "OUTBOUND".equals(family)&&scope!=null&&(Integer.valueOf(1).equals(scope.rebuildRequired())||barriers.contains(warehouse)),reviewStatus(scope),
current==null?null:taskView(current,scope,protection,context,family),shared==null?null:taskView(shared,null,protection,context),protection, current==null?null:taskView(current,scope,protection,context,family),shared==null?null:taskView(shared,null,protection,context),protection,
"INVENTORY".equals(family)?inventorySamples.get(warehouse):null)); "INVENTORY".equals(family)?inventorySamples.get(warehouse):null,activity));
} }
} }
return new CollectionResponses.Overview(query.sourceAccount(),config.getEnvironment(),config.getCollectionMode(),clock.instant(),rows); return new CollectionResponses.Overview(query.sourceAccount(),config.getEnvironment(),config.getCollectionMode(),clock.instant(),rows,
active!=null&&ACTIVE.contains(active.status())&&"BUSINESS".equals(active.executionPurpose())?taskView(active,null,protection,context):null);
} }
private Map<String,CollectionResponses.InventoryDailySamples> inventorySamples(CollectionQuery query) { private Map<String,CollectionResponses.InventoryDailySamples> inventorySamples(CollectionQuery query) {
@@ -181,14 +190,6 @@ public class CollectionQueryService {
for(Scope scope:mapper.scopes(q.sourceAccount(),q.warehouseCode()))scopes.put(scope.dataFamily(),scope); for(Scope scope:mapper.scopes(q.sourceAccount(),q.warehouseCode()))scopes.put(scope.dataFamily(),scope);
return scopes; return scopes;
} }
private boolean sharedWindowRelated(Task active,Scope scope) {
if(active==null||scope==null||active.queryFromUtc()==null||active.queryUntilUtc()==null)return false;
return overlaps(active.queryFromUtc(),active.queryUntilUtc(),scope.initialFromUtc(),scope.initialUntilUtc())
||overlaps(active.queryFromUtc(),active.queryUntilUtc(),scope.discoveryFromUtc(),scope.discoveryUntilUtc());
}
private static boolean overlaps(LocalDateTime from,LocalDateTime until,LocalDateTime otherFrom,LocalDateTime otherUntil) {
return otherFrom!=null&&otherUntil!=null&&from.isBefore(otherUntil)&&until.isAfter(otherFrom);
}
private String reviewStatus(Scope scope) { private String reviewStatus(Scope scope) {
if(scope==null||scope.reviewTargetDays()==null||scope.reviewTargetDays()<1)return "UNKNOWN"; if(scope==null||scope.reviewTargetDays()==null||scope.reviewTargetDays()<1)return "UNKNOWN";
if(scope.knownOrderRoundFinishedAtUtc()==null||scope.historyRoundFinishedAtUtc()==null)return "PENDING"; if(scope.knownOrderRoundFinishedAtUtc()==null||scope.historyRoundFinishedAtUtc()==null)return "PENDING";
@@ -199,6 +200,7 @@ public class CollectionQueryService {
if(task.upstreamTaskId()==null)return null; if(task.upstreamTaskId()==null)return null;
Task parent=context.tasks().get(task.upstreamTaskId()); Task parent=context.tasks().get(task.upstreamTaskId());
return parent!=null&&"ACCOUNT_DISCOVERY".equals(parent.taskRole())&&Objects.equals(task.sourceAccount(),parent.sourceAccount()) return parent!=null&&"ACCOUNT_DISCOVERY".equals(parent.taskRole())&&Objects.equals(task.sourceAccount(),parent.sourceAccount())
&&Objects.equals(task.executionPurpose(),parent.executionPurpose())
&&(!"VALIDATION".equals(parent.executionPurpose())||validationAccountRoot(parent)&&Objects.equals(task.warehouseCode(),parent.warehouseCode()))?parent:null; &&(!"VALIDATION".equals(parent.executionPurpose())||validationAccountRoot(parent)&&Objects.equals(task.warehouseCode(),parent.warehouseCode()))?parent:null;
} }
private static boolean validationAccountRoot(Task task) { private static boolean validationAccountRoot(Task task) {
@@ -223,11 +225,55 @@ public class CollectionQueryService {
String resultState=t.resultPublishedRunId()==null?"NOT_PUBLISHED":result==null||scope==null||!Objects.equals(scope.id(),result.scopeId()) String resultState=t.resultPublishedRunId()==null?"NOT_PUBLISHED":result==null||scope==null||!Objects.equals(scope.id(),result.scopeId())
||!Objects.equals(t.warehouseCode(),result.warehouseCode())||!Objects.equals(t.scopeHash(),result.scopeHash())?"RESULT_UNAVAILABLE" ||!Objects.equals(t.warehouseCode(),result.warehouseCode())||!Objects.equals(t.scopeHash(),result.scopeHash())?"RESULT_UNAVAILABLE"
:latest!=null&&"REUSED".equals(latest.status())?"REUSED":latest!=null&&"PUBLISHED".equals(latest.status())?"PUBLISHED":"RESULT_AVAILABLE"; :latest!=null&&"REUSED".equals(latest.status())?"REUSED":latest!=null&&"PUBLISHED".equals(latest.status())?"PUBLISHED":"RESULT_AVAILABLE";
Task parent=relatedUpstream(t,context);
String dependency=dependencyState(t,parent);
String waiting=blockingReason(t,dependency,protection);
CollectionResponses.Attempt attempt=latest==null?null:attemptView(latest,t,blocked==null);
return new CollectionResponses.Task(t.taskNo(),t.taskType(),validationAccountRoot(t)&&family!=null?family:t.dataFamily(),t.warehouseCode(),t.taskRole(),t.executionPurpose(), return new CollectionResponses.Task(t.taskNo(),t.taskType(),validationAccountRoot(t)&&family!=null?family:t.dataFamily(),t.warehouseCode(),t.taskRole(),t.executionPurpose(),
utc(t.queryFromUtc()),utc(t.queryUntilUtc()),t.status(),t.applyState(),t.attemptCount(),latest==null?null:latest.attemptNo(), utc(t.queryFromUtc()),utc(t.queryUntilUtc()),t.status(),t.applyState(),t.attemptCount(),latest==null?null:latest.attemptNo(),
resultState,t.resultPublishedRunId(),utc(t.nextRetryAtUtc()), resultState,t.resultPublishedRunId(),utc(t.nextRetryAtUtc()),
blocked==null?List.of("RETRY"):List.of(),blocked,utc(t.progressUpdatedAtUtc()),progress(t.progressJson()), blocked==null?List.of("RETRY"):List.of(),blocked,utc(t.progressUpdatedAtUtc()),progress(t.progressJson()),
validation(root==null?t:root),latest==null?null:attemptView(latest,t,blocked==null)); validation(root==null?t:root),attempt,dependency,waiting,
"WAITING_CONFIRMED".equals(dependency)?parent.taskNo():t.taskNo(),stageEstimate(t,latest,attempt,waiting));
}
private String dependencyState(Task task,Task parent) {
if(task.upstreamTaskId()==null)return "WAITING_SOURCE".equals(task.status())?"WAITING_UNCONFIRMED":"NONE";
if(parent==null)return "WAITING_UNCONFIRMED";
if("SUCCEEDED".equals(parent.status()))return "SATISFIED";
if(Set.of("DEAD","FAILED","ABORTED","CANCELLED").contains(parent.status()))return "FAILED";
return "WAITING_CONFIRMED";
}
private String blockingReason(Task task,String dependency,CollectionResponses.Protection protection) {
if(!ACTIVE.contains(task.status()))return null;
String disabled=executionBlock(task.warehouseCode(),protection);
if(disabled!=null)return disabled;
if("test".equals(config.getEnvironment())&&"BUSINESS".equals(task.executionPurpose())) {
String budget=businessBudgetBlock(task);if(budget!=null)return budget;
}
if("WAITING_UNCONFIRMED".equals(dependency))return "UPSTREAM_UNCONFIRMED";
if("WAITING_CONFIRMED".equals(dependency))return "UPSTREAM_NOT_READY";
if("FAILED".equals(dependency))return "UPSTREAM_FAILED";
if("RETRY_WAIT".equals(task.status()))return "RETRY_SCHEDULED";
if("YIELDED".equals(task.status()))return "EXECUTION_YIELDED";
if(Set.of("READY","QUEUED","WAITING").contains(task.status()))return "QUEUED";
return null;
}
private SelfopStageEstimate.Estimate stageEstimate(Task task,Attempt run,CollectionResponses.Attempt attempt,String blocked) {
boolean terminal=!ACTIVE.contains(task.status())||run!=null&&TERMINAL.contains(run.status());
String source=task.progressJson();
JsonNode bounded=boundedObject(source);
Map<String,Object> raw=bounded==null?new LinkedHashMap<>():json.convertValue(bounded,new com.fasterxml.jackson.core.type.TypeReference<LinkedHashMap<String,Object>>(){});
raw.putAll(progress(source));
raw.put("taskNo",task.taskNo());
boolean healthy="RUNNING".equals(task.status())&&attempt!=null&&"HEALTHY".equals(attempt.executionHealth())
&&Objects.equals(task.attemptCount(),attempt.attemptNo())&&Objects.equals(number(bounded,"attemptNo"),attempt.attemptNo()==null?null:attempt.attemptNo().longValue());
var estimate=SelfopStageEstimate.estimate(raw,clock.instant(),healthy,blocked!=null,terminal);
if("AVAILABLE".equals(estimate.status())&&attempt.heartbeatAt()!=null) {
Instant healthUntil=attempt.heartbeatAt().plusSeconds(config.getLeaseSeconds());
if(healthUntil.isBefore(estimate.validUntil()))return new SelfopStageEstimate.Estimate(estimate.status(),estimate.progressIdentity(),estimate.completed(),estimate.total(),
estimate.sampleFrom(),estimate.sampleUntil(),estimate.completedDelta(),estimate.ratePerSecond(),estimate.remainingSeconds(),estimate.estimatedAt(),healthUntil,null);
}
return estimate;
} }
private String retryBlock(Task task,Scope scope,CollectionResponses.Protection protection,Context context) { private String retryBlock(Task task,Scope scope,CollectionResponses.Protection protection,Context context) {
if(!"BUSINESS".equals(task.executionPurpose()))return "VALIDATION_NOT_RETRYABLE"; if(!"BUSINESS".equals(task.executionPurpose()))return "VALIDATION_NOT_RETRYABLE";
@@ -294,9 +340,9 @@ public class CollectionQueryService {
if(!Set.of("BUSINESS","VALIDATION").contains(config.getCollectionMode()))return "COLLECTION_MODE_INVALID"; if(!Set.of("BUSINESS","VALIDATION").contains(config.getCollectionMode()))return "COLLECTION_MODE_INVALID";
if("BUSINESS".equals(config.getCollectionMode())&&!com.baiyue.lastleg.selfoperated.analytics.model.SelfopCollectionPolicy.businessEnvironment(config))return "BUSINESS_ENVIRONMENT_REJECTED"; if("BUSINESS".equals(config.getCollectionMode())&&!com.baiyue.lastleg.selfoperated.analytics.model.SelfopCollectionPolicy.businessEnvironment(config))return "BUSINESS_ENVIRONMENT_REJECTED";
if(!config.isExecutionEnabled())return "EXECUTION_DISABLED"; if(!config.isExecutionEnabled())return "EXECUTION_DISABLED";
if(!config.getWarehouses().containsKey(warehouse))return "WAREHOUSE_NOT_CONFIGURED"; if(warehouse!=null&&!config.getWarehouses().containsKey(warehouse))return "WAREHOUSE_NOT_CONFIGURED";
var w=config.getWarehouses().get(warehouse); var w=warehouse==null?null:config.getWarehouses().get(warehouse);
if(!"Asia/Shanghai".equals(w.getBusinessTimezone())||!"Asia/Shanghai".equals(w.getSourceTimezone())||!"PCS".equals(config.getQuantityUnit()))return "TIMEZONE_OR_UNIT_NOT_CONFIRMED"; if(w!=null&&(!"Asia/Shanghai".equals(w.getBusinessTimezone())||!"Asia/Shanghai".equals(w.getSourceTimezone()))||!"PCS".equals(config.getQuantityUnit()))return "TIMEZONE_OR_UNIT_NOT_CONFIRMED";
if(protection.accountPauseReason()!=null&&!protection.accountPauseReason().isBlank())return "ACCOUNT_PAUSED"; if(protection.accountPauseReason()!=null&&!protection.accountPauseReason().isBlank())return "ACCOUNT_PAUSED";
if(protection.accountCooldownUntil()!=null&&clock.instant().isBefore(protection.accountCooldownUntil()))return "ACCOUNT_COOLDOWN"; if(protection.accountCooldownUntil()!=null&&clock.instant().isBefore(protection.accountCooldownUntil()))return "ACCOUNT_COOLDOWN";
if(!config.isExclusiveHandoffVerified()||!config.isPriorEgressClosed()||!config.isInflightDrained()||!config.isTransferredBudgetVerified() if(!config.isExclusiveHandoffVerified()||!config.isPriorEgressClosed()||!config.isInflightDrained()||!config.isTransferredBudgetVerified()
@@ -39,16 +39,14 @@ public interface CollectionReadMapper {
+ " AND JSON_TYPE(JSON_EXTRACT(t.query_scope_json,'$.warehouses[0]'))='STRING')"; + " AND JSON_TYPE(JSON_EXTRACT(t.query_scope_json,'$.warehouses[0]'))='STRING')";
String TASK_WAREHOUSE = "CASE WHEN " + VALIDATION_ROOT String TASK_WAREHOUSE = "CASE WHEN " + VALIDATION_ROOT
+ " THEN JSON_UNQUOTE(JSON_EXTRACT(t.query_scope_json,'$.warehouses[0]')) ELSE t.warehouse_code END"; + " THEN JSON_UNQUOTE(JSON_EXTRACT(t.query_scope_json,'$.warehouses[0]')) ELSE t.warehouse_code END";
String TASK_FAMILY = "CASE WHEN " + VALIDATION_ROOT + " THEN 'OUTBOUND' ELSE s.data_family END"; String TASK_FAMILY = "CASE WHEN t.task_role='ACCOUNT_DISCOVERY' THEN 'OUTBOUND' ELSE s.data_family END";
String TASK_MEMBERSHIP = "((t.task_role='WAREHOUSE_SYNC' AND s.id IS NOT NULL) OR " + VALIDATION_ROOT + ")"; String TASK_MEMBERSHIP = "((t.task_role='WAREHOUSE_SYNC' AND s.id IS NOT NULL) OR " + VALIDATION_ROOT + ")";
String BUSINESS_LOCAL_WORK_READY = "CASE WHEN t.task_role='WAREHOUSE_SYNC' AND s.data_family='OUTBOUND'" String BUSINESS_LOCAL_WORK_READY = "CASE WHEN t.task_role='WAREHOUSE_SYNC' AND s.data_family='OUTBOUND'"
+ " AND (t.task_type='REBUILD' OR t.apply_state='APPLIED' OR (" + " AND (t.task_type='REBUILD' OR t.apply_state='APPLIED' OR ("
+ "JSON_EXTRACT(t.routing_manifest_json,'$.directoryComplete')=CAST('true' AS JSON)" + "OCTET_LENGTH(t.progress_json) BETWEEN 0 AND 8192 AND JSON_EXTRACT(t.progress_json,'$.localWorkReady')=CAST('true' AS JSON)"
+ " AND JSON_TYPE(JSON_EXTRACT(t.routing_manifest_json,'$.objects'))='OBJECT'" + " AND JSON_UNQUOTE(JSON_EXTRACT(t.progress_json,'$.localWorkReadyScopeHash'))=t.scope_hash"
+ " AND NOT EXISTS (SELECT 1 FROM JSON_TABLE(t.routing_manifest_json,'$.objects.*'" + " AND ((t.upstream_task_id IS NULL AND JSON_TYPE(JSON_EXTRACT(t.progress_json,'$.localWorkReadyUpstreamTaskId'))='NULL')"
+ " COLUMNS(entry JSON PATH '$')) manifest_item" + " OR (t.upstream_task_id IS NOT NULL AND JSON_UNQUOTE(JSON_EXTRACT(t.progress_json,'$.localWorkReadyUpstreamTaskId'))=CAST(t.upstream_task_id AS CHAR)))))"
+ " WHERE COALESCE(JSON_TYPE(JSON_EXTRACT(manifest_item.entry,'$.effectiveStageId')),'NULL') NOT IN ('INTEGER')"
+ " OR (CAST(JSON_UNQUOTE(JSON_EXTRACT(manifest_item.entry,'$.effectiveStageId')) AS DECIMAL(65,0)) BETWEEN 1 AND 9223372036854775807) IS NOT TRUE)))"
+ " THEN TRUE ELSE FALSE END"; + " THEN TRUE ELSE FALSE END";
String TASK = "t.id,t.task_no,t.source_account," + TASK_WAREHOUSE + " AS warehouse_code," + TASK_FAMILY + " AS data_family,t.task_role,t.task_type," String TASK = "t.id,t.task_no,t.source_account," + TASK_WAREHOUSE + " AS warehouse_code," + TASK_FAMILY + " AS data_family,t.task_role,t.task_type,"
+ "t.execution_purpose,t.scope_hash,t.status,t.apply_state,t.attempt_count,t.query_from_utc,t.query_until_utc," + "t.execution_purpose,t.scope_hash,t.status,t.apply_state,t.attempt_count,t.query_from_utc,t.query_until_utc,"
@@ -123,14 +121,17 @@ public interface CollectionReadMapper {
+ " WHERE r.source_account=#{account} AND r.status='PUBLISHED' AND r.execution_purpose='BUSINESS' AND r.run_role='WAREHOUSE_SYNC'" + " WHERE r.source_account=#{account} AND r.status='PUBLISHED' AND r.execution_purpose='BUSINESS' AND r.run_role='WAREHOUSE_SYNC'"
+ " AND r.id IN <foreach collection='ids' item='id' open='(' close=')' separator=','>#{id}</foreach></script>") + " AND r.id IN <foreach collection='ids' item='id' open='(' close=')' separator=','>#{id}</foreach></script>")
List<Attempt> publishedRuns(@Param("account") String account,@Param("ids") List<Long> ids); List<Attempt> publishedRuns(@Param("account") String account,@Param("ids") List<Long> ids);
String ACTIVE_STATUS = "('RUNNING','READY','QUEUED','WAITING','WAITING_SOURCE','RETRY_WAIT','YIELDED')";
String ACTIVITY_ORDER = "CASE WHEN status='RUNNING' THEN 0 WHEN status IN " + ACTIVE_STATUS + " THEN 1 ELSE 2 END";
@Select("<script>SELECT " + TASK + TASK_FROM @Select("<script>SELECT " + TASK + TASK_FROM
+ " JOIN (SELECT id,ROW_NUMBER() OVER(PARTITION BY scope_id ORDER BY created_at DESC,id DESC) rn FROM ll_selfop_sync_task" + " JOIN (SELECT id,ROW_NUMBER() OVER(PARTITION BY scope_id ORDER BY " + ACTIVITY_ORDER + ",created_at DESC,id DESC) rn FROM ll_selfop_sync_task"
+ " WHERE scope_id IN <foreach collection='ids' item='id' open='(' close=')' separator=','>#{id}</foreach>) ranked" + " WHERE source_account=#{account} AND task_role='WAREHOUSE_SYNC' AND scope_id IN <foreach collection='ids' item='id' open='(' close=')' separator=','>#{id}</foreach>) ranked"
+ " ON ranked.id=t.id AND ranked.rn=1 WHERE t.source_account=#{account}</script>") + " ON ranked.id=t.id AND ranked.rn &lt;= 8 WHERE t.source_account=#{account} AND (t.status IN " + ACTIVE_STATUS + " OR ranked.rn=1)"
+ " ORDER BY t.scope_id,ranked.rn</script>")
List<Task> currentTasks(@Param("account") String account,@Param("ids") List<Long> ids); List<Task> currentTasks(@Param("account") String account,@Param("ids") List<Long> ids);
@Select("<script>SELECT " + TASK + TASK_FROM @Select("<script>SELECT " + TASK + TASK_FROM
+ " JOIN (SELECT t.id,ROW_NUMBER() OVER(PARTITION BY " + TASK_WAREHOUSE + " JOIN (SELECT t.id,ROW_NUMBER() OVER(PARTITION BY " + TASK_WAREHOUSE
+ " ORDER BY t.created_at DESC,t.id DESC) rn FROM ll_selfop_sync_task t" + " ORDER BY " + ACTIVITY_ORDER + ",t.created_at DESC,t.id DESC) rn FROM ll_selfop_sync_task t"
+ " WHERE t.source_account=#{account} AND " + VALIDATION_ROOT + " AND (" + TASK_WAREHOUSE + ") IN" + " WHERE t.source_account=#{account} AND " + VALIDATION_ROOT + " AND (" + TASK_WAREHOUSE + ") IN"
+ " <foreach collection='warehouses' item='w' open='(' close=')' separator=','>#{w}</foreach>) ranked" + " <foreach collection='warehouses' item='w' open='(' close=')' separator=','>#{w}</foreach>) ranked"
+ " ON ranked.id=t.id AND ranked.rn=1 WHERE t.source_account=#{account} AND " + VALIDATION_ROOT + "</script>") + " ON ranked.id=t.id AND ranked.rn=1 WHERE t.source_account=#{account} AND " + VALIDATION_ROOT + "</script>")
@@ -141,7 +142,7 @@ public interface CollectionReadMapper {
+ " <foreach collection='ids' item='id' open='(' close=')' separator=','>#{id}</foreach>) ranked" + " <foreach collection='ids' item='id' open='(' close=')' separator=','>#{id}</foreach>) ranked"
+ " ON ranked.id=t.id AND ranked.rn=1 WHERE t.source_account=#{account}</script>") + " ON ranked.id=t.id AND ranked.rn=1 WHERE t.source_account=#{account}</script>")
List<Task> lastSuccessfulTasks(@Param("account") String account,@Param("ids") List<Long> ids); List<Task> lastSuccessfulTasks(@Param("account") String account,@Param("ids") List<Long> ids);
// The one account active task is separately related to warehouses by actual persisted windows. // Account activity is independent; only upstream_task_id proves a warehouse dependency.
@Select("SELECT " + TASK + " FROM ll_selfop_outbound_discovery_scope d JOIN ll_selfop_sync_task t ON t.id=d.active_task_id" @Select("SELECT " + TASK + " FROM ll_selfop_outbound_discovery_scope d JOIN ll_selfop_sync_task t ON t.id=d.active_task_id"
+ " AND t.source_account=d.source_account LEFT JOIN ll_selfop_sync_scope s ON s.id=t.scope_id" + " AND t.source_account=d.source_account LEFT JOIN ll_selfop_sync_scope s ON s.id=t.scope_id"
+ BUSINESS_ROOT_JOIN + BUSINESS_ROOT_JOIN
@@ -5,13 +5,14 @@ import java.time.LocalDate;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import com.baiyue.lastleg.selfoperated.analytics.model.SelfopStageEstimate;
public final class CollectionResponses { public final class CollectionResponses {
private CollectionResponses() {} private CollectionResponses() {}
public record Page<T>(int current, int size, long total, Instant listUpperCreatedAt, public record Page<T>(int current, int size, long total, Instant listUpperCreatedAt,
Long listUpperId, Instant observedAt, List<T> records) {} Long listUpperId, Instant observedAt, List<T> records) {}
public record Overview(String sourceAccount, String environmentLabel, String collectionMode, public record Overview(String sourceAccount, String environmentLabel, String collectionMode,
Instant observedAt, List<Warehouse> warehouses) {} Instant observedAt, List<Warehouse> warehouses, Task accountActivity) {}
public record Warehouse(String warehouseCode, String warehouseName, String dataFamily, public record Warehouse(String warehouseCode, String warehouseName, String dataFamily,
boolean executionEnabled, boolean publicationEnabled, String disabledReason, boolean executionEnabled, boolean publicationEnabled, String disabledReason,
Instant nextScheduledAt, Instant observedAt, Instant initialFrom, Instant initialUntil, Instant nextScheduledAt, Instant observedAt, Instant initialFrom, Instant initialUntil,
@@ -19,7 +20,8 @@ public final class CollectionResponses {
String lastSuccessfulTaskNo, Instant lastSucceededAt, Instant lastPublishedAt, String lastSuccessfulTaskNo, Instant lastSucceededAt, Instant lastPublishedAt,
Instant inventoryObservedFrom, Instant inventoryObservedUntil, String dataFreshness, Instant inventoryObservedFrom, Instant inventoryObservedUntil, String dataFreshness,
boolean rebuildRequired, String revisionReviewStatus, Task currentTask, boolean rebuildRequired, String revisionReviewStatus, Task currentTask,
Task sharedUpstream, Protection protection, InventoryDailySamples inventoryDailySamples) {} Task sharedUpstream, Protection protection, InventoryDailySamples inventoryDailySamples,
List<Task> activeTasks) {}
public record InventoryDailySamples(LocalDate windowFrom, LocalDate windowUntil, int expectedDays, public record InventoryDailySamples(LocalDate windowFrom, LocalDate windowUntil, int expectedDays,
int sampleCount, List<LocalDate> missingDates, LocalDate accumulationStartedOn) {} int sampleCount, List<LocalDate> missingDates, LocalDate accumulationStartedOn) {}
public record Task(String taskNo, String taskType, String dataFamily, String warehouseCode, public record Task(String taskNo, String taskType, String dataFamily, String warehouseCode,
@@ -27,7 +29,9 @@ public final class CollectionResponses {
String taskStatus, String applyState, Integer attemptCount, Integer latestAttemptNo, String taskStatus, String applyState, Integer attemptCount, Integer latestAttemptNo,
String resultPublicationState, Long resultPublishedRunId, Instant nextRetryAt, String resultPublicationState, Long resultPublishedRunId, Instant nextRetryAt,
List<String> allowedActions, String actionBlockReason, Instant progressUpdatedAt, List<String> allowedActions, String actionBlockReason, Instant progressUpdatedAt,
Map<String,Object> progress, Validation validation, Attempt latestAttempt) {} Map<String,Object> progress, Validation validation, Attempt latestAttempt,
String dependencyState, String blockingReason, String progressSourceTaskNo,
SelfopStageEstimate.Estimate stageEstimate) {}
public record TaskDetail(Task task, Task sharedUpstream, Protection protection, Instant observedAt) {} public record TaskDetail(Task task, Task sharedUpstream, Protection protection, Instant observedAt) {}
public record Attempt(Long runId, String taskNo, Integer attemptNo, String status, String phase, public record Attempt(Long runId, String taskNo, Integer attemptNo, String status, String phase,
String executionPurpose, Instant startedAt, Instant finishedAt, Instant phaseStartedAt, String executionPurpose, Instant startedAt, Instant finishedAt, Instant phaseStartedAt,
@@ -11,6 +11,6 @@ public interface SelfopEgressMapper {
@Update("UPDATE ll_selfop_outbound_discovery_scope SET api_budget_json=#{apiBudgetJson},api_cooldown_until_utc=#{apiCooldownUntilUtc},api_pause_reason=#{apiPauseReason},updated_at=UTC_TIMESTAMP(3) WHERE id=#{id}") int saveAccount(SelfopOutboundDiscoveryScope row); @Update("UPDATE ll_selfop_outbound_discovery_scope SET api_budget_json=#{apiBudgetJson},api_cooldown_until_utc=#{apiCooldownUntilUtc},api_pause_reason=#{apiPauseReason},updated_at=UTC_TIMESTAMP(3) WHERE id=#{id}") int saveAccount(SelfopOutboundDiscoveryScope row);
@Update("UPDATE ll_selfop_sync_task SET validation_budget_json=#{validationBudgetJson},business_budget_json=#{businessBudgetJson},status=#{status},next_retry_at_utc=#{nextRetryAtUtc},updated_at=UTC_TIMESTAMP(3) WHERE id=#{id}") int saveRoot(SelfopSyncTask row); @Update("UPDATE ll_selfop_sync_task SET validation_budget_json=#{validationBudgetJson},business_budget_json=#{businessBudgetJson},status=#{status},next_retry_at_utc=#{nextRetryAtUtc},updated_at=UTC_TIMESTAMP(3) WHERE id=#{id}") int saveRoot(SelfopSyncTask row);
@Update("UPDATE ll_selfop_sync_task SET status='DEAD',next_retry_at_utc=NULL,updated_at=UTC_TIMESTAMP(3) WHERE business_root_task_id=#{root} AND source_account=#{account} AND execution_purpose='BUSINESS' AND status IN ('WAITING_SOURCE','READY','RUNNING','RETRY_WAIT')") int terminateBusinessTasks(@Param("root") Long root,@Param("account") String account); @Update("UPDATE ll_selfop_sync_task SET status='DEAD',next_retry_at_utc=NULL,updated_at=UTC_TIMESTAMP(3) WHERE business_root_task_id=#{root} AND source_account=#{account} AND execution_purpose='BUSINESS' AND status IN ('WAITING_SOURCE','READY','RUNNING','RETRY_WAIT')") int terminateBusinessTasks(@Param("root") Long root,@Param("account") String account);
@Update("UPDATE ll_selfop_sync_run SET status='FAILED',error_code=#{reason},finished_at_utc=#{now},progress_snapshot_json=(SELECT progress_json FROM ll_selfop_sync_task WHERE id=task_id) WHERE source_account=#{account} AND execution_purpose='BUSINESS' AND task_id IN (SELECT id FROM ll_selfop_sync_task WHERE business_root_task_id=#{root} AND source_account=#{account} AND execution_purpose='BUSINESS') AND status IN ('FETCHING','VALIDATED','YIELDED','READY','BUILDING')") int terminateBusinessRuns(@Param("root") Long root,@Param("account") String account,@Param("reason") String reason,@Param("now") LocalDateTime now); @Update("UPDATE ll_selfop_sync_run SET status='FAILED',error_code=#{reason},finished_at_utc=#{now},progress_snapshot_json=(SELECT JSON_SET(JSON_REMOVE(COALESCE(progress_json,JSON_OBJECT()),'$.etaSamples'),'$.summaryAsOf',CONCAT(DATE_FORMAT(#{now},'%Y-%m-%dT%H:%i:%s.%f'),'Z')) FROM ll_selfop_sync_task WHERE id=task_id AND (progress_json IS NULL OR JSON_EXTRACT(progress_json,'$.attemptNo')=attempt_no)) WHERE source_account=#{account} AND execution_purpose='BUSINESS' AND task_id IN (SELECT id FROM ll_selfop_sync_task WHERE business_root_task_id=#{root} AND source_account=#{account} AND execution_purpose='BUSINESS') AND status IN ('FETCHING','VALIDATED','YIELDED','READY','BUILDING')") int terminateBusinessRuns(@Param("root") Long root,@Param("account") String account,@Param("reason") String reason,@Param("now") LocalDateTime now);
@Update("UPDATE ll_selfop_sync_run SET status=#{status},error_code=#{reason},finished_at_utc=#{now},progress_snapshot_json=(SELECT progress_json FROM ll_selfop_sync_task WHERE id=task_id) WHERE task_id IN (SELECT id FROM ll_selfop_sync_task WHERE validation_root_task_id=#{root}) AND status IN ('FETCHING','VALIDATED','YIELDED')") int terminateValidationRuns(@Param("root") Long root,@Param("status") String status,@Param("reason") String reason,@Param("now") LocalDateTime now); @Update("UPDATE ll_selfop_sync_run SET status=#{status},error_code=#{reason},finished_at_utc=#{now},progress_snapshot_json=(SELECT JSON_SET(JSON_REMOVE(COALESCE(progress_json,JSON_OBJECT()),'$.etaSamples'),'$.summaryAsOf',CONCAT(DATE_FORMAT(#{now},'%Y-%m-%dT%H:%i:%s.%f'),'Z')) FROM ll_selfop_sync_task WHERE id=task_id AND (progress_json IS NULL OR JSON_EXTRACT(progress_json,'$.attemptNo')=attempt_no)) WHERE task_id IN (SELECT id FROM ll_selfop_sync_task WHERE validation_root_task_id=#{root}) AND status IN ('FETCHING','VALIDATED','YIELDED')") int terminateValidationRuns(@Param("root") Long root,@Param("status") String status,@Param("reason") String reason,@Param("now") LocalDateTime now);
} }
@@ -153,10 +153,11 @@ public class SelfopDiscoveryService {
full=stage(finalFetchTask,finalFetchRun,acceptedDetail,existing==null?1:existing.getFirstPageNo(),acceptedObservation,"VALID"); full=stage(finalFetchTask,finalFetchRun,acceptedDetail,existing==null?1:existing.getFirstPageNo(),acceptedObservation,"VALID");
if(existing!=null){full.setId(existing.getId());full.setRunId(existing.getRunId());full.setCreatedAt(existing.getCreatedAt());stages.updateById(full);}else stages.insert(full); if(existing!=null){full.setId(existing.getId());full.setRunId(existing.getRunId());full.setCreatedAt(existing.getCreatedAt());stages.updateById(full);}else stages.insert(full);
} }
ref.setEffectiveStageId(full.getId());ref.setWarehouseCode(acceptedDetail.warehouseCode());saveManifest(checkpoint.task(),checkpoint.run(),manifest); ref.setEffectiveStageId(full.getId());ref.setWarehouseCode(acceptedDetail.warehouseCode());saveManifest(checkpoint.task(),checkpoint.run(),manifest,existing==null||!"VALID".equals(existing.getValidationStatus()));
if(!currentFetchTask.getId().equals(task.getId())){var refreshManifest=new SelfopRoutingManifest();refreshManifest.setDirectoryComplete(true);refreshManifest.setPages(1);refreshManifest.setTotal(1L); if(!currentFetchTask.getId().equals(task.getId())){var refreshManifest=new SelfopRoutingManifest();refreshManifest.setDirectoryComplete(true);refreshManifest.setPages(1);refreshManifest.setTotal(1L);
var refreshRef=new SelfopRoutingManifest.Entry();refreshRef.setCustomerCode(ref.getCustomerCode());refreshRef.setDeliveryNo(ref.getDeliveryNo());refreshRef.setWarehouseCode(ref.getWarehouseCode());refreshRef.setEffectiveStageId(full.getId()); var refreshRef=new SelfopRoutingManifest.Entry();refreshRef.setCustomerCode(ref.getCustomerCode());refreshRef.setDeliveryNo(ref.getDeliveryNo());refreshRef.setWarehouseCode(ref.getWarehouseCode());refreshRef.setEffectiveStageId(full.getId());
refreshManifest.getObjects().put(entry.getKey(),refreshRef);currentFetchTask.setRoutingManifestJson(json(refreshManifest)); refreshManifest.getObjects().put(entry.getKey(),refreshRef);currentFetchTask.setRoutingManifestJson(json(refreshManifest));
SelfopProgressProjection.applyCheckpoint(currentFetchTask,currentFetchRun,"DETAIL","ORDER",1,1L,refreshManifest.getScan(),existing==null||!"VALID".equals(existing.getValidationStatus()));
SelfopProgressProjection.freezeRun(currentFetchTask,currentFetchRun);currentFetchRun.setStatus("ROUTED");currentFetchRun.setFetchFinishedAtUtc(utcNow());currentFetchRun.setFinishedAtUtc(utcNow());currentFetchRun.setUpdatedAt(utcNow());runs.updateById(currentFetchRun); SelfopProgressProjection.freezeRun(currentFetchTask,currentFetchRun);currentFetchRun.setStatus("ROUTED");currentFetchRun.setFetchFinishedAtUtc(utcNow());currentFetchRun.setFinishedAtUtc(utcNow());currentFetchRun.setUpdatedAt(utcNow());runs.updateById(currentFetchRun);
currentFetchTask.setStatus("SUCCEEDED");currentFetchTask.setUpdatedAt(utcNow());tasks.updateById(currentFetchTask);} currentFetchTask.setStatus("SUCCEEDED");currentFetchTask.setUpdatedAt(utcNow());tasks.updateById(currentFetchTask);}
}); });
@@ -204,10 +205,14 @@ public class SelfopDiscoveryService {
s.setContentHash(SelfopOutboundFingerprint.order(payload));s.setPayloadJson(json(payload));s.setValidationStatus(status);s.setCreatedAt(utcNow());s.setUpdatedAt(utcNow());return s; s.setContentHash(SelfopOutboundFingerprint.order(payload));s.setPayloadJson(json(payload));s.setValidationStatus(status);s.setCreatedAt(utcNow());s.setUpdatedAt(utcNow());return s;
} }
private void saveManifest(SelfopSyncTask task,SelfopSyncRun active,SelfopRoutingManifest manifest){ private void saveManifest(SelfopSyncTask task,SelfopSyncRun active,SelfopRoutingManifest manifest){
// Directory pages are freshly fetched. Detail reference reuse cannot establish throughput.
saveManifest(task,active,manifest,!manifest.isDirectoryComplete());
}
private void saveManifest(SelfopSyncTask task,SelfopSyncRun active,SelfopRoutingManifest manifest,boolean realWork){
task.setRoutingManifestJson(json(manifest));task.setUpdatedAt(utcNow()); task.setRoutingManifestJson(json(manifest));task.setUpdatedAt(utcNow());
SelfopProgressProjection.applyCheckpoint(task,active,manifest.isDirectoryComplete()?"DETAIL":"DIRECTORY",manifest.isDirectoryComplete()?"ORDER":"PAGE", SelfopProgressProjection.applyCheckpoint(task,active,manifest.isDirectoryComplete()?"DETAIL":"DIRECTORY",manifest.isDirectoryComplete()?"ORDER":"PAGE",
manifest.isDirectoryComplete()?manifest.getObjects().values().stream().filter(e->e.getEffectiveStageId()!=null).count():Math.max(0,manifest.getNextPage()-1), manifest.isDirectoryComplete()?manifest.getObjects().values().stream().filter(e->e.getEffectiveStageId()!=null).count():Math.max(0,manifest.getNextPage()-1),
manifest.isDirectoryComplete()?(long)manifest.getObjects().size():manifest.getPages()==null?null:manifest.getPages().longValue(),manifest.getScan()); manifest.isDirectoryComplete()?(long)manifest.getObjects().size():manifest.getPages()==null?null:manifest.getPages().longValue(),manifest.getScan(),realWork);
runs.updateById(active);tasks.updateById(task); runs.updateById(active);tasks.updateById(task);
} }
private void route(SelfopSyncTask parent,SelfopSyncRun parentRun,SelfopRoutingManifest manifest,SelfopLeaseService.Lease lease,boolean detailsResolved){ private void route(SelfopSyncTask parent,SelfopSyncRun parentRun,SelfopRoutingManifest manifest,SelfopLeaseService.Lease lease,boolean detailsResolved){
@@ -238,7 +243,8 @@ public class SelfopDiscoveryService {
child.setQueryScopeJson(scope.getScopeJson());child.setScopeHash(scope.getScopeHash());child.setRequestHash(hash(List.of(parent.getId(),scope.getScopeHash())));child.setStatus(ready?"READY":"WAITING_SOURCE");child.setApplyState("NOT_STARTED"); child.setQueryScopeJson(scope.getScopeJson());child.setScopeHash(scope.getScopeHash());child.setRequestHash(hash(List.of(parent.getId(),scope.getScopeHash())));child.setStatus(ready?"READY":"WAITING_SOURCE");child.setApplyState("NOT_STARTED");
var saved=taskService.create(child); var saved=taskService.create(child);
if(!Set.of("SUCCEEDED","DEAD","SAMPLE_DONE").contains(saved.getStatus())&& !Set.of("APPLYING","APPLIED","BLOCKED").contains(saved.getApplyState())){ if(!Set.of("SUCCEEDED","DEAD","SAMPLE_DONE").contains(saved.getStatus())&& !Set.of("APPLYING","APPLIED","BLOCKED").contains(saved.getApplyState())){
saved.setRoutingManifestJson(json(own));saved.setStatus(ready?"READY":"WAITING_SOURCE");saved.setUpdatedAt(utcNow());tasks.updateById(saved);} saved.setRoutingManifestJson(json(own));saved.setStatus(ready?"READY":"WAITING_SOURCE");
SelfopProgressProjection.localWorkReady(saved,ready);saved.setUpdatedAt(utcNow());tasks.updateById(saved);}
} }
}); });
} }
@@ -145,8 +145,8 @@ public class SelfopTaskService {
var now=utcNow();var active=runs.selectOne(new QueryWrapper<SelfopSyncRun>().eq("task_id",taskId).eq("status","FETCHING").orderByDesc("id").last("LIMIT 1")); var now=utcNow();var active=runs.selectOne(new QueryWrapper<SelfopSyncRun>().eq("task_id",taskId).eq("status","FETCHING").orderByDesc("id").last("LIMIT 1"));
if(task.getScopeId()!=null && "INVENTORY".equals(scopes.selectById(task.getScopeId()).getDataFamily())) { if(task.getScopeId()!=null && "INVENTORY".equals(scopes.selectById(task.getScopeId()).getDataFamily())) {
var yielded=runs.selectOne(new QueryWrapper<SelfopSyncRun>().eq("task_id",taskId).eq("status","YIELDED").orderByDesc("id").last("LIMIT 1")); var yielded=runs.selectOne(new QueryWrapper<SelfopSyncRun>().eq("task_id",taskId).eq("status","YIELDED").orderByDesc("id").last("LIMIT 1"));
if(yielded!=null){yielded.setStatus("FETCHING");yielded.setFinishedAtUtc(null);yielded.setFenceToken(lease.fence());yielded.setUpdatedAt(now); if(yielded!=null){yielded.setStatus("FETCHING");yielded.setFinishedAtUtc(null);yielded.setProgressSnapshotJson(null);yielded.setFenceToken(lease.fence());yielded.setUpdatedAt(now);
runs.update(null,new UpdateWrapper<SelfopSyncRun>().eq("id",yielded.getId()).set("status","FETCHING").set("fence_token",lease.fence()).set("finished_at_utc",null).set("updated_at",now)); runs.update(null,new UpdateWrapper<SelfopSyncRun>().eq("id",yielded.getId()).set("status","FETCHING").set("fence_token",lease.fence()).set("finished_at_utc",null).set("progress_snapshot_json",null).set("updated_at",now));
task.setStatus("RUNNING");task.setUpdatedAt(now);tasks.updateById(task);return yielded;} task.setStatus("RUNNING");task.setUpdatedAt(now);tasks.updateById(task);return yielded;}
} }
if(active!=null){require(!Objects.equals(active.getFenceToken(),lease.fence()),"SELFOP_TASK_ALREADY_RUNNING"); if(active!=null){require(!Objects.equals(active.getFenceToken(),lease.fence()),"SELFOP_TASK_ALREADY_RUNNING");
@@ -750,6 +750,32 @@ components:
progress: {$ref: '#/components/schemas/Progress'} progress: {$ref: '#/components/schemas/Progress'}
validation: {$ref: '#/components/schemas/Validation'} validation: {$ref: '#/components/schemas/Validation'}
latestAttempt: {$ref: '#/components/schemas/Attempt'} latestAttempt: {$ref: '#/components/schemas/Attempt'}
dependencyState:
type: string
enum: [NONE, WAITING_CONFIRMED, WAITING_UNCONFIRMED, SATISFIED, FAILED]
description: 仅直接upstreamTaskId且账号及用途一致证明依赖;时间窗口相交不构成依赖。
blockingReason: {type: string, nullable: true, description: 当前执行阻塞原因码,与恢复操作资格actionBlockReason分开}
progressSourceTaskNo: {type: string, nullable: true, description: 确认等待时指向直接上游,否则指向本任务;计数仍保留在各自Task.progress,不复制到本仓}
stageEstimate: {$ref: '#/components/schemas/StageEstimate'}
StageEstimate:
type: object
description: 本任务当前阶段估算,不能当作全任务ETA;进度与估算整体按progressIdentity和validUntil接收。计数包含复用时测速基线重建,采样仅来自已提交检查点。
properties:
status: {type: string, enum: [AVAILABLE, UNAVAILABLE, PAUSED]}
progressIdentity: {type: string}
completed: {type: string, nullable: true}
total: {type: string, nullable: true}
sampleFrom: {type: string, format: date-time, nullable: true}
sampleUntil: {type: string, format: date-time, nullable: true}
completedDelta: {type: string, nullable: true}
ratePerSecond: {type: number, nullable: true}
remainingSeconds: {type: string, nullable: true}
estimatedAt: {type: string, format: date-time}
validUntil: {type: string, format: date-time, nullable: true}
unavailableReason:
type: string
nullable: true
enum: [TERMINAL, PAUSED, HEALTH_UNCONFIRMED, TOTAL_UNKNOWN, INVALID_PROGRESS, STAGE_COMPLETE, SAMPLING, NO_RECENT_PROGRESS, SAMPLE_EXPIRED, INVALID_SAMPLE]
TaskDetail: TaskDetail:
type: object type: object
properties: properties:
@@ -795,6 +821,11 @@ components:
revisionReviewStatus: {type: string, nullable: true} revisionReviewStatus: {type: string, nullable: true}
inventoryDailySamples: {$ref: '#/components/schemas/InventoryDailySamples'} inventoryDailySamples: {$ref: '#/components/schemas/InventoryDailySamples'}
currentTask: {$ref: '#/components/schemas/Task'} currentTask: {$ref: '#/components/schemas/Task'}
activeTasks:
type: array
maxItems: 8
description: 本仓本类型活动或等待任务,RUNNING优先,再按创建时间倒序;currentTask取活动首项,无活动时取最近任务。
items: {$ref: '#/components/schemas/Task'}
sharedUpstream: {$ref: '#/components/schemas/Task'} sharedUpstream: {$ref: '#/components/schemas/Task'}
protection: {$ref: '#/components/schemas/Protection'} protection: {$ref: '#/components/schemas/Protection'}
Overview: Overview:
@@ -805,6 +836,10 @@ components:
collectionMode: {type: string, nullable: true} collectionMode: {type: string, nullable: true}
observedAt: {type: string, format: date-time, nullable: true} observedAt: {type: string, format: date-time, nullable: true}
warehouses: {type: array, items: {$ref: '#/components/schemas/Warehouse'}} warehouses: {type: array, items: {$ref: '#/components/schemas/Warehouse'}}
accountActivity:
allOf: [{$ref: '#/components/schemas/Task'}]
nullable: true
description: 来源账号明确活动指针指向的正式采集任务,独立展示,不代表任一仓库的依赖或数量。
Progress: Progress:
type: object type: object
properties: properties:
@@ -0,0 +1,90 @@
package com.baiyue.lastleg.selfoperated.analytics.model;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.*;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
import static com.baiyue.lastleg.selfoperated.analytics.model.SelfopSupport.*;
class SelfopStageEstimateTest {
private final Instant start=Instant.parse("2026-10-13T00:00:00Z");
private Map<String,Object> progress(){
var p=new LinkedHashMap<String,Object>();p.put("taskNo","LXCOL-ETA");p.put("attemptNo",1);
p.put("phase","DETAIL");p.put("scanRound",1);p.put("progressScope","ACCOUNT_SHARED");
p.put("unit","ORDER");p.put("countBasis","TASK_CUMULATIVE");p.put("total",1000L);p.put("totalKnown",true);
p.put("sliceFrom","2026-10-12T00:00:00Z");p.put("sliceUntil","2026-10-13T00:00:00Z");return p;
}
private void commit(Map<String,Object> p,long seconds,long completed,boolean real){
p.put("completed",completed);SelfopStageEstimate.checkpoint(p,start.plusSeconds(seconds),real);
}
private Map<String,Object> ready(){var p=progress();for(int i=0;i<=3;i++)commit(p,i*20,1+i*20,true);return p;}
private SelfopStageEstimate.Estimate estimate(Map<String,Object> p,long second){return SelfopStageEstimate.estimate(p,start.plusSeconds(second),true,false,false);}
@Test void sufficientRealAdvancesEstimateCurrentStageAndIncludeIdleTime(){
var p=ready();var e=estimate(p,60);assertEquals("AVAILABLE",e.status());assertEquals(60L,e.completedDelta());
assertEquals(0,new BigDecimal("1").compareTo(e.ratePerSecond()));assertEquals(939L,e.remainingSeconds());
assertEquals(start,e.sampleFrom());assertEquals(start.plusSeconds(60),e.sampleUntil());assertEquals(start.plusSeconds(70),e.validUntil());
var idle=estimate(p,90);assertEquals("AVAILABLE",idle.status());assertEquals(new BigDecimal("0.666666"),idle.ratePerSecond());
assertEquals(1409L,idle.remainingSeconds());assertEquals("NO_RECENT_PROGRESS",estimate(p,120).unavailableReason());
}
@Test void shortOrInsufficientAdvanceWindowsNeverEstimate(){
var p=progress();commit(p,0,1,true);commit(p,20,2,true);commit(p,40,3,true);
assertEquals("SAMPLING",estimate(p,59).unavailableReason());assertEquals("SAMPLING",estimate(p,60).unavailableReason());
commit(p,60,4,true);assertEquals("AVAILABLE",estimate(p,60).status());
}
@Test void reuseBeforeInsideAndAfterWindowRestartsBaselineInsteadOfInflatingRate(){
var p=progress();commit(p,0,500,false);commit(p,20,501,true);commit(p,40,502,true);commit(p,60,503,true);
assertEquals(3L,estimate(p,60).completedDelta());
commit(p,70,900,false);assertEquals("SAMPLING",estimate(p,70).unavailableReason());
commit(p,90,901,true);commit(p,110,902,true);commit(p,130,903,true);assertEquals(3L,estimate(p,130).completedDelta());
commit(p,400,950,false);assertEquals("SAMPLING",estimate(p,400).unavailableReason());
}
@Test void eachIdentityDimensionInvalidatesEarlierEstimateAndCheckpointRebuildsIt(){
for(String key:List.of("taskNo","attemptNo","phase","scanRound","progressScope","unit","countBasis","sliceFrom","sliceUntil","total")){
var p=ready();String old=SelfopStageEstimate.identity(p);
Object value=p.get(key);p.put(key,value instanceof Number?((Number)value).longValue()+1:"changed");
assertNotEquals(old,SelfopStageEstimate.identity(p));assertEquals("SAMPLING",estimate(p,61).unavailableReason());
commit(p,61,62,true);assertEquals("SAMPLING",estimate(p,61).unavailableReason());
}
}
@Test void pausedUnhealthyTerminalUnknownAndCompleteNeverReturnActiveNumbers(){
var p=ready();
var paused=SelfopStageEstimate.estimate(p,start.plusSeconds(60),true,true,false);assertEquals("PAUSED",paused.status());assertNull(paused.ratePerSecond());
var terminal=SelfopStageEstimate.estimate(p,start.plusSeconds(60),true,true,true);assertEquals("UNAVAILABLE",terminal.status());assertEquals("TERMINAL",terminal.unavailableReason());
assertEquals("HEALTH_UNCONFIRMED",SelfopStageEstimate.estimate(p,start.plusSeconds(60),false,false,false).unavailableReason());
p.put("totalKnown",false);assertEquals("TOTAL_UNKNOWN",estimate(p,60).unavailableReason());
p.put("totalKnown",true);p.put("completed",1000L);assertEquals("COUNT_COMPLETE",estimate(p,60).unavailableReason());
commit(p,61,1000,true);assertFalse(p.containsKey("etaSamples"));
}
@Test void zeroAndFullCountsNeverInferStageCompletionOrPublication(){
for(long total:List.of(0L,1000L)){
var p=progress();p.put("total",total);p.put("completed",total);p.put("stageFinishedAt",null);
var result=estimate(p,60);
assertEquals("UNAVAILABLE",result.status());assertEquals("COUNT_COMPLETE",result.unavailableReason());
assertNull(result.remainingSeconds());assertNull(result.ratePerSecond());assertNull(result.validUntil());
assertNull(p.get("stageFinishedAt"));assertFalse(p.containsKey("publishedAt"));
assertEquals("DETAIL",p.get("phase"));
}
}
@Test void bucketBoundaryHighFrequencyAndLongHistoryStayBounded(){
var p=progress();p.put("total",100000L);
for(int second=0;second<=1000;second++)commit(p,second,second,true);
var sample=(Map<?,?>)p.get("etaSamples");var points=(List<?>)sample.get("p");assertTrue(points.size()<=20);
assertTrue(json(sample).getBytes(StandardCharsets.UTF_8).length<=1024);assertEquals("AVAILABLE",estimate(p,1000).status());
long first=((Number)((List<?>)points.get(0)).get(0)).longValue();assertTrue(start.plusSeconds(1000).getEpochSecond()-first<=180);
var boundary=progress();commit(boundary,9,1,true);commit(boundary,10,2,true);
assertEquals(2,((List<?>)((Map<?,?>)boundary.get("etaSamples")).get("p")).size());
}
@Test void sixtySecondGapAndCounterRegressionRestartSampling(){
var p=ready();commit(p,120,62,true);assertEquals("SAMPLING",estimate(p,120).unavailableReason());
commit(p,140,1,true);assertEquals("SAMPLING",estimate(p,140).unavailableReason());
}
@Test void corruptFractionalOutOfOrderOrOversizedSamplesFailClosed(){
var p=ready();p.put("completed",new BigDecimal("61.5"));assertEquals("INVALID_PROGRESS",estimate(p,60).unavailableReason());
p=ready();var sample=new LinkedHashMap<>((Map<String,Object>)p.get("etaSamples"));
sample.put("p",List.of(List.of(start.getEpochSecond()+40,1L,0L),List.of(start.getEpochSecond()+20,2L,1L)));p.put("etaSamples",sample);
assertEquals("INVALID_SAMPLE",estimate(p,60).unavailableReason());
p=ready();var future=estimate(p,50);assertEquals("INVALID_PROGRESS",future.unavailableReason());
}
}
@@ -25,6 +25,78 @@ class CollectionQueryServiceTest {
CollectionQuery q(Integer page,LocalDateTime upper,Long id) { CollectionQuery q(Integer page,LocalDateTime upper,Long id) {
return new CollectionQuery("primary","A",null,null,null,null,null,null,page,null,upper==null?null:upper.toInstant(ZoneOffset.UTC),id); return new CollectionQuery("primary","A",null,null,null,null,null,null,page,null,upper==null?null:upper.toInstant(ZoneOffset.UTC),id);
} }
private Task activity(long id,String status,Long upstream,boolean account,LocalDateTime created,String progress) {
return new Task(id,"LXCOL-"+id,"primary",account?null:"A","OUTBOUND",account?"ACCOUNT_DISCOVERY":"WAREHOUSE_SYNC","BACKFILL","BUSINESS",
"hash",status,"NOT_STARTED",1,now.minusDays(1),now,null,upstream,null,null,progress,now,null,null,false,created,created);
}
@Test void currentActivityWinsOverNewerHistoryAndIsBoundedToEight() {
var candidates=new ArrayList<Task>();
candidates.add(activity(99,"SUCCEEDED",null,false,now,null));
for(int i=0;i<10;i++)candidates.add(activity(10+i,"WAITING_SOURCE",null,false,now.minusMinutes(i+1),null));
candidates.add(activity(1,"RUNNING",null,false,now.minusDays(1),null));
when(mapper.currentTasks(eq("primary"),anyList())).thenReturn(candidates);
var row=service.overview(q(null,null,null)).warehouses().get(0);
assertEquals("LXCOL-1",row.currentTask().taskNo());assertEquals(8,row.activeTasks().size());
assertEquals("LXCOL-1",row.activeTasks().get(0).taskNo());
assertTrue(row.activeTasks().stream().noneMatch(t->"SUCCEEDED".equals(t.taskStatus())));
}
@Test void overlappingAccountActivityDoesNotProveWarehouseDependency() {
var child=activity(7,"WAITING_SOURCE",null,false,now,null);
var account=activity(8,"RUNNING",null,true,now,null);
when(mapper.currentTasks(eq("primary"),anyList())).thenReturn(List.of(child));
when(mapper.sharedActiveAccount("primary")).thenReturn(account);
when(mapper.task("primary","A",child.taskNo())).thenReturn(child);
var result=service.overview(q(null,null,null));var row=result.warehouses().get(0);
assertEquals(account.taskNo(),result.accountActivity().taskNo());assertNull(row.sharedUpstream());
assertEquals("WAITING_UNCONFIRMED",row.currentTask().dependencyState());
var detail=service.taskDetail(child.taskNo(),q(null,null,null));
assertNull(detail.sharedUpstream());assertEquals(row.currentTask().dependencyState(),detail.task().dependencyState());
}
@Test void directDependenciesAreConsistentInOverviewAndDetailIncludingFailureAndCompletion() {
enableBusiness("production");
var child=activity(7,"WAITING_SOURCE",8L,false,now,null);
when(mapper.currentTasks(eq("primary"),anyList())).thenReturn(List.of(child));
when(mapper.task("primary","A",child.taskNo())).thenReturn(child);
for(var pair:Map.of("RUNNING","WAITING_CONFIRMED","SUCCEEDED","SATISFIED","DEAD","FAILED").entrySet()) {
var parent=activity(8,pair.getKey(),null,true,now.minusHours(1),null);
when(mapper.tasksByIds(eq("primary"),anyList())).thenReturn(List.of(parent));
var row=service.overview(q(null,null,null)).warehouses().get(0);
var detail=service.taskDetail(child.taskNo(),q(null,null,null));
assertEquals(parent.taskNo(),row.sharedUpstream().taskNo());
assertEquals(parent.taskNo(),detail.sharedUpstream().taskNo());
assertEquals(pair.getValue(),row.currentTask().dependencyState());
assertEquals(pair.getValue(),detail.task().dependencyState());
if("RUNNING".equals(pair.getKey())) {
assertEquals(parent.taskNo(),detail.task().progressSourceTaskNo());
assertEquals("UPSTREAM_NOT_READY",detail.task().blockingReason());
assertEquals("TASK_NOT_RETRYABLE",detail.task().actionBlockReason());
}
}
when(mapper.tasksByIds(eq("primary"),anyList())).thenReturn(List.of());
assertEquals("WAITING_UNCONFIRMED",service.taskDetail(child.taskNo(),q(null,null,null)).task().dependencyState());
}
@Test void stageEstimateUsesCommittedSamplesWithoutExposingThemAndRejectsAttemptMismatch() throws Exception {
enableBusiness("production");
var p=new LinkedHashMap<String,Object>();
p.put("taskNo","LXCOL-7");p.put("attemptNo",1);p.put("phase","DETAIL");p.put("scanRound",1);
p.put("progressScope","WAREHOUSE");p.put("unit","ORDER");p.put("countBasis","TASK_CUMULATIVE");p.put("total",100);p.put("totalKnown",true);
Instant at=now.toInstant(ZoneOffset.UTC);
for(int i=0;i<4;i++) {
p.put("completed",10+i*10);
com.baiyue.lastleg.selfoperated.analytics.model.SelfopStageEstimate.checkpoint(p,at.minusSeconds(70-i*20),true);
}
var t=activity(7,"RUNNING",null,false,now.minusMinutes(2),new ObjectMapper().writeValueAsString(p));
when(mapper.task("primary","A",t.taskNo())).thenReturn(t);
when(mapper.latestAttempts(eq("primary"),anyList())).thenReturn(List.of(run(1,"FETCHING",null)));
var result=service.taskDetail(t.taskNo(),q(null,null,null)).task();
assertEquals("AVAILABLE",result.stageEstimate().status());
assertEquals(result.progress().get("completed"),result.stageEstimate().completed());
assertEquals(result.progress().get("total"),result.stageEstimate().total());
assertFalse(result.progress().containsKey("etaSamples"));
when(mapper.latestAttempts(eq("primary"),anyList())).thenReturn(List.of(run(2,"FETCHING",null)));
result=service.taskDetail(t.taskNo(),q(null,null,null)).task();
assertEquals("UNAVAILABLE",result.stageEstimate().status());assertNull(result.stageEstimate().remainingSeconds());
}
@Test void controlledTestBusinessProjectsEnabledInventoryPublicationWithoutAutomaticSchedule() { @Test void controlledTestBusinessProjectsEnabledInventoryPublicationWithoutAutomaticSchedule() {
config.setCollectionMode("BUSINESS");config.setEnvironment("test");config.setQuantityUnit("PCS"); config.setCollectionMode("BUSINESS");config.setEnvironment("test");config.setQuantityUnit("PCS");
config.setExclusiveHandoffVerified(true);config.setPriorEgressClosed(true);config.setInflightDrained(true);config.setTransferredBudgetVerified(true); config.setExclusiveHandoffVerified(true);config.setPriorEgressClosed(true);config.setInflightDrained(true);config.setTransferredBudgetVerified(true);
@@ -279,7 +351,7 @@ class CollectionQueryServiceTest {
"publishedRuns","currentTasks","latestValidationTasks","lastSuccessfulTasks","sharedActiveAccount","barrierWarehouses", "publishedRuns","currentTasks","latestValidationTasks","lastSuccessfulTasks","sharedActiveAccount","barrierWarehouses",
"inventoryDailySamples","inventoryAccumulationStarts")) { "inventoryDailySamples","inventoryAccumulationStarts")) {
String sql=configuration.getMappedStatement(CollectionReadMapper.class.getName()+"."+method).getBoundSql(parameters).getSql(); String sql=configuration.getMappedStatement(CollectionReadMapper.class.getName()+"."+method).getBoundSql(parameters).getSql();
assertFalse(sql.contains("AS routing_manifest_json"));assertFalse(sql.contains("ll_selfop_sku_daily_sales"));assertFalse(sql.contains("ll_selfop_inventory_snapshot")); assertFalse(sql.contains("routing_manifest_json"));assertFalse(sql.contains("JSON_TABLE"));assertFalse(sql.contains("ll_selfop_sku_daily_sales"));assertFalse(sql.contains("ll_selfop_inventory_snapshot"));
assertFalse(sql.contains("ll_selfop_sync_stage")); assertFalse(sql.contains("ll_selfop_sync_stage"));
} }
String runSql=configuration.getMappedStatement(CollectionReadMapper.class.getName()+".runs").getBoundSql(parameters).getSql(); String runSql=configuration.getMappedStatement(CollectionReadMapper.class.getName()+".runs").getBoundSql(parameters).getSql();
@@ -289,11 +361,15 @@ class CollectionQueryServiceTest {
assertTrue(sql.contains("b.source_account=t.source_account")); assertTrue(sql.contains("b.source_account=t.source_account"));
assertTrue(sql.contains("b.execution_purpose='BUSINESS' AND b.business_root_task_id=b.id")); assertTrue(sql.contains("b.execution_purpose='BUSINESS' AND b.business_root_task_id=b.id"));
assertTrue(sql.contains("AS business_local_work_ready")); assertTrue(sql.contains("AS business_local_work_ready"));
assertTrue(sql.contains("'$.objects'))='OBJECT'")); assertTrue(sql.contains("OCTET_LENGTH(t.progress_json) BETWEEN 0 AND 8192"));
assertTrue(sql.contains("'$.objects.*'")); assertTrue(sql.contains("'$.localWorkReady')=CAST('true' AS JSON)"));
assertTrue(sql.contains("COALESCE(JSON_TYPE(JSON_EXTRACT(manifest_item.entry,'$.effectiveStageId')),'NULL')")); assertTrue(sql.contains("'$.localWorkReadyScopeHash'))=t.scope_hash"));
assertTrue(sql.contains("BETWEEN 1 AND 9223372036854775807) IS NOT TRUE")); assertTrue(sql.contains("'$.localWorkReadyUpstreamTaskId'))=CAST(t.upstream_task_id AS CHAR)"));
} }
String currentSql=configuration.getMappedStatement(CollectionReadMapper.class.getName()+".currentTasks").getBoundSql(parameters).getSql();
assertTrue(currentSql.contains("ranked.rn <= 8"));
assertTrue(currentSql.contains("CASE WHEN status='RUNNING' THEN 0"));
assertTrue(currentSql.contains("source_account=? AND task_role='WAREHOUSE_SYNC'"));
assertTrue(runSql.contains("r.created_at <"));assertTrue(runSql.contains("r.id <="));assertTrue(runSql.contains("LIMIT ? OFFSET ?")); assertTrue(runSql.contains("r.created_at <"));assertTrue(runSql.contains("r.id <="));assertTrue(runSql.contains("LIMIT ? OFFSET ?"));
String samplesSql=configuration.getMappedStatement(CollectionReadMapper.class.getName()+".inventoryDailySamples").getBoundSql(parameters).getSql(); String samplesSql=configuration.getMappedStatement(CollectionReadMapper.class.getName()+".inventoryDailySamples").getBoundSql(parameters).getSql();
assertTrue(samplesSql.contains("r.inventory_daily_sample_date >= ?")); assertTrue(samplesSql.contains("r.inventory_daily_sample_date >= ?"));
@@ -225,6 +225,10 @@ class SelfopEgressServiceTest {
assertThat(tasks).contains("business_root_task_id=? AND source_account=? AND execution_purpose='BUSINESS'", "next_retry_at_utc=NULL", "status IN ('WAITING_SOURCE','READY','RUNNING','RETRY_WAIT')"); assertThat(tasks).contains("business_root_task_id=? AND source_account=? AND execution_purpose='BUSINESS'", "next_retry_at_utc=NULL", "status IN ('WAITING_SOURCE','READY','RUNNING','RETRY_WAIT')");
assertThat(runs).contains("business_root_task_id=? AND source_account=? AND execution_purpose='BUSINESS'", "error_code=?", "status IN ('FETCHING','VALIDATED','YIELDED','READY','BUILDING')"); assertThat(runs).contains("business_root_task_id=? AND source_account=? AND execution_purpose='BUSINESS'", "error_code=?", "status IN ('FETCHING','VALIDATED','YIELDED','READY','BUILDING')");
assertThat(tasks).doesNotContain("SUCCEEDED");assertThat(runs).doesNotContain("PUBLISHED","REUSED"); assertThat(tasks).doesNotContain("SUCCEEDED");assertThat(runs).doesNotContain("PUBLISHED","REUSED");
assertThat(runs).contains("JSON_SET(JSON_REMOVE", "'$.etaSamples'", "'$.summaryAsOf'", "DATE_FORMAT(?,", "JSON_EXTRACT(progress_json,'$.attemptNo')=attempt_no");
args.put("status","FAILED");
String validation=configuration.getMappedStatement(prefix+"terminateValidationRuns").getBoundSql(args).getSql();
assertThat(validation).contains("'$.etaSamples'", "'$.summaryAsOf'", "DATE_FORMAT(?,", "validation_root_task_id=?");
} }
@Test void committedCompletionWithLostReplyStaysPausedAcrossIntegrationRestart(){ @Test void committedCompletionWithLostReplyStaysPausedAcrossIntegrationRestart(){
business();var c=reserve(Operation.WAREHOUSES,1,null,null);service.authorize(new AuthorizationRequest(c,Operation.WAREHOUSES,PROVIDER)); business();var c=reserve(Operation.WAREHOUSES,1,null,null);service.authorize(new AuthorizationRequest(c,Operation.WAREHOUSES,PROVIDER));
@@ -167,6 +167,8 @@ class SelfopDiscoveryServiceTest {
service.collect(task,run,lease); service.collect(task,run,lease);
var warehouseA=children.get("route:1:A");var warehouseB=children.get("route:1:B"); var warehouseA=children.get("route:1:A");var warehouseB=children.get("route:1:B");
assertEquals("WAITING_SOURCE",warehouseA.getStatus());assertEquals("WAITING_SOURCE",warehouseB.getStatus()); assertEquals("WAITING_SOURCE",warehouseA.getStatus());assertEquals("WAITING_SOURCE",warehouseB.getStatus());
assertEquals(false,read(warehouseA.getProgressJson(),Map.class).get("localWorkReady"));
assertEquals(false,read(warehouseB.getProgressJson(),Map.class).get("localWorkReady"));
assertTrue(read(warehouseB.getRoutingManifestJson(),SelfopRoutingManifest.class).getObjects().isEmpty()); assertTrue(read(warehouseB.getRoutingManifestJson(),SelfopRoutingManifest.class).getObjects().isEmpty());
var partial=read(task.getRoutingManifestJson(),SelfopRoutingManifest.class); var partial=read(task.getRoutingManifestJson(),SelfopRoutingManifest.class);
assertTrue(partial.isDirectoryComplete());assertEquals(1,partial.getObjects().values().stream().filter(e->e.getEffectiveStageId()!=null).count()); assertTrue(partial.isDirectoryComplete());assertEquals(1,partial.getObjects().values().stream().filter(e->e.getEffectiveStageId()!=null).count());
@@ -175,6 +177,11 @@ class SelfopDiscoveryServiceTest {
account.setRequestBudget(10);service.collect(task,run,lease); account.setRequestBudget(10);service.collect(task,run,lease);
assertEquals("READY",warehouseA.getStatus());assertEquals("READY",warehouseB.getStatus()); assertEquals("READY",warehouseA.getStatus());assertEquals("READY",warehouseB.getStatus());
for(var child:List.of(warehouseA,warehouseB)){
var proof=read(child.getProgressJson(),Map.class);assertEquals(true,proof.get("localWorkReady"));
assertEquals(child.getScopeHash(),proof.get("localWorkReadyScopeHash"));
assertEquals(task.getId().toString(),proof.get("localWorkReadyUpstreamTaskId"));
}
var destinationManifest=read(warehouseB.getRoutingManifestJson(),SelfopRoutingManifest.class); var destinationManifest=read(warehouseB.getRoutingManifestJson(),SelfopRoutingManifest.class);
String movedKey=hash(List.of("demo","customer","two"));assertEquals(Set.of(movedKey),destinationManifest.getObjects().keySet()); String movedKey=hash(List.of("demo","customer","two"));assertEquals(Set.of(movedKey),destinationManifest.getObjects().keySet());
var moved=destinationManifest.getObjects().get(movedKey);assertNotNull(moved.getEffectiveStageId());assertEquals("B",moved.getWarehouseCode()); var moved=destinationManifest.getObjects().get(movedKey);assertNotNull(moved.getEffectiveStageId());assertEquals("B",moved.getWarehouseCode());
@@ -69,6 +69,27 @@ class SelfopInventoryRecoveryTest {
assertEquals(7L, resumed.getFenceToken()); assertEquals("FETCHING", resumed.getStatus()); assertNull(resumed.getFinishedAtUtc()); assertEquals(7L, resumed.getFenceToken()); assertEquals("FETCHING", resumed.getStatus()); assertNull(resumed.getFinishedAtUtc());
verify(runs, never()).insert(any(SelfopSyncRun.class)); verify(runs, never()).insert(any(SelfopSyncRun.class));
} }
@Test void resumedInventoryReplacesYieldSnapshotWithFinalProgressWithoutChangingHistoricalAttempt(){
original.setFenceToken(lease.fence());
SelfopProgressProjection.applyCheckpoint(task,original,"INVENTORY","PAGE",1,4L,1);
service.yield(1L,20L,lease);
var yielded=read(original.getProgressSnapshotJson(),java.util.Map.class);
assertEquals(1,yielded.get("completed"));assertNotNull(yielded.get("summaryAsOf"));
var history=new SelfopSyncRun();history.setId(19L);history.setAttemptNo(0);
String historical="{\"attemptNo\":0,\"completed\":9,\"summaryAsOf\":\"2026-10-12T00:00:00Z\"}";
history.setProgressSnapshotJson(historical);
when(runs.selectOne(any())).thenReturn(null,original);
var resumed=service.begin(1L,lease);assertSame(original,resumed);assertNull(resumed.getProgressSnapshotJson());
var update=ArgumentCaptor.forClass(com.baomidou.mybatisplus.core.conditions.Wrapper.class);
verify(runs).update(isNull(),update.capture());
assertTrue(((com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper<?>)update.getValue()).getSqlSet().contains("progress_snapshot_json"));
SelfopProgressProjection.applyCheckpoint(task,resumed,"INVENTORY","PAGE",4,4L,1);
java.time.Instant before=java.time.Instant.now();SelfopProgressProjection.freezeRun(task,resumed);
java.time.Instant after=java.time.Instant.now();var finished=read(resumed.getProgressSnapshotJson(),java.util.Map.class);
assertEquals(4,finished.get("completed"));var asOf=java.time.Instant.parse((String)finished.get("summaryAsOf"));
assertFalse(asOf.isBefore(before));assertFalse(asOf.isAfter(after));assertNotEquals(yielded,finished);
SelfopProgressProjection.freezeRun(task,history);assertEquals(historical,history.getProgressSnapshotJson());
}
@Test void genuineFailureClearsCursorAndNewAttemptStartsAtPageOneWithItsOwnStageBatch() { @Test void genuineFailureClearsCursorAndNewAttemptStartsAtPageOneWithItsOwnStageBatch() {
original.setFenceToken(lease.fence()); original.setFenceToken(lease.fence());
@@ -62,4 +62,57 @@ class SelfopProgressServiceTest {
verify(tasks,never()).updateById(any(SelfopSyncTask.class));verify(runs,never()).updateById(any(SelfopSyncRun.class)); verify(tasks,never()).updateById(any(SelfopSyncTask.class));verify(runs,never()).updateById(any(SelfopSyncRun.class));
var order=inOrder(leases,locks);order.verify(leases).verify(lease);order.verify(locks).lockTask(1);order.verify(locks).lockRun(2); var order=inOrder(leases,locks);order.verify(leases).verify(lease);order.verify(locks).lockTask(1);order.verify(locks).lockRun(2);
} }
@Test void terminalSnapshotHasActualFreezeTimeAndResumeCannotKeepPausedEtaBaseline(){
var t=task();var r=run();SelfopProgressProjection.applyCheckpoint(t,r,"DETAIL","ORDER",4,10L,1);
assertTrue(read(t.getProgressJson(),Map.class).containsKey("etaSamples"));
Instant before=Instant.now();SelfopProgressProjection.freezeRun(t,r);Instant after=Instant.now();
var snapshot=read(r.getProgressSnapshotJson(),Map.class);Instant asOf=Instant.parse((String)snapshot.get("summaryAsOf"));
assertFalse(asOf.isBefore(before));assertFalse(asOf.isAfter(after));assertFalse(snapshot.containsKey("etaSamples"));
assertFalse(read(t.getProgressJson(),Map.class).containsKey("etaSamples"));
SelfopProgressProjection.applyCheckpoint(t,r,"DETAIL","ORDER",5,10L,1);
var points=(List<?>)((Map<?,?>)read(t.getProgressJson(),Map.class).get("etaSamples")).get("p");assertEquals(1,points.size());
var fallback=run();SelfopProgressProjection.freezeRun(task(),fallback);
assertNotNull(read(fallback.getProgressSnapshotJson(),Map.class).get("summaryAsOf"));
}
@Test void etaIsFirstToBeDroppedAtEightKiBWhileCandidateAndAuditEvidenceRemain(){
var t=task();var r=run();SelfopProgressProjection.applyCheckpoint(t,r,"DETAIL","ORDER",1,100L,1);
var p=new java.util.LinkedHashMap<String,Object>(read(t.getProgressJson(),Map.class));p.remove("etaSamples");
p.put("candidateBuild",Map.of("committedBatches",7));
p.put("audit", "x".repeat(Math.max(0,8192-json(p).getBytes(StandardCharsets.UTF_8).length-80)));
t.setProgressJson(json(p));SelfopProgressProjection.applyCheckpoint(t,r,"DETAIL","ORDER",2,100L,1);
var result=read(t.getProgressJson(),Map.class);assertFalse(result.containsKey("etaSamples"));
assertEquals(Map.of("committedBatches",7),result.get("candidateBuild"));assertEquals(p.get("audit"),result.get("audit"));
assertTrue(t.getProgressJson().getBytes(StandardCharsets.UTF_8).length<=8192);
}
@Test void localWorkProofIsBoundToTheCommittedScopeAndUpstream(){
var t=task();t.setScopeHash("scope-1");t.setUpstreamTaskId(99L);
SelfopProgressProjection.localWorkReady(t,true);var p=read(t.getProgressJson(),Map.class);
assertEquals(true,p.get("localWorkReady"));assertEquals("scope-1",p.get("localWorkReadyScopeHash"));assertEquals("99",p.get("localWorkReadyUpstreamTaskId"));
SelfopProgressProjection.applyCheckpoint(t,run(),"MERGE","ORDER",0,4L,0);
assertEquals(true,read(t.getProgressJson(),Map.class).get("localWorkReady"));
var next=run();next.setAttemptNo(2);SelfopProgressProjection.applyCheckpoint(t,next,"MERGE","ORDER",1,4L,0);
assertEquals(true,read(t.getProgressJson(),Map.class).get("localWorkReady"));
t.setScopeHash("scope-2");next.setAttemptNo(3);SelfopProgressProjection.applyCheckpoint(t,next,"MERGE","ORDER",1,4L,0);
assertFalse(read(t.getProgressJson(),Map.class).containsKey("localWorkReady"));
SelfopProgressProjection.localWorkReady(t,false);assertEquals(false,read(t.getProgressJson(),Map.class).get("localWorkReady"));
}
@Test void exactLegacyEightKiBCheckpointFreezesWithoutLosingBusinessOrAuditCounts(){
var t=task();var r=run();SelfopProgressProjection.applyCheckpoint(t,r,"DETAIL","ORDER",4,10L,1);
var p=new java.util.LinkedHashMap<String,Object>(read(t.getProgressJson(),Map.class));p.remove("etaSamples");
p.put("candidateBuild",Map.of("committedBatches",7));p.put("audit","");
p.put("audit","x".repeat(8192-json(p).getBytes(StandardCharsets.UTF_8).length));t.setProgressJson(json(p));
assertEquals(8192,t.getProgressJson().getBytes(StandardCharsets.UTF_8).length);
assertDoesNotThrow(()->SelfopProgressProjection.freezeRun(t,r));var frozen=read(r.getProgressSnapshotJson(),Map.class);
assertNotNull(frozen.get("summaryAsOf"));assertEquals(4,frozen.get("completed"));assertEquals(10,frozen.get("total"));
assertEquals(p.get("candidateBuild"),frozen.get("candidateBuild"));assertEquals(p.get("audit"),frozen.get("audit"));
assertTrue(r.getProgressSnapshotJson().getBytes(StandardCharsets.UTF_8).length<=8192);
var stage=(Map<?,?>)((List<?>)frozen.get("stages")).get(0);assertEquals(4,stage.get("completed"));assertEquals(10,stage.get("total"));
}
@Test void minimalFullSizeLegacyCheckpointNeverFailsBusinessEndForOptionalTimestamp(){
var t=task();var r=run();var p=new java.util.LinkedHashMap<String,Object>();
p.put("attemptNo",1);p.put("completed",4);p.put("total",10);p.put("audit","");
p.put("audit","x".repeat(8192-json(p).getBytes(StandardCharsets.UTF_8).length));t.setProgressJson(json(p));
assertDoesNotThrow(()->SelfopProgressProjection.freezeRun(t,r));var frozen=read(r.getProgressSnapshotJson(),Map.class);
assertEquals(p,frozen);assertEquals(8192,r.getProgressSnapshotJson().getBytes(StandardCharsets.UTF_8).length);
}
} }