java-project/baiyue-firstleg
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dispatch): normalize candidate UNION text collations

Browse Source
This commit is contained in:
ouyangxq
2026-10-21 01:45:30 +08:00
parent eeb50d5f60
commit 42d548194d
3 changed files with 146 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lastleg-service/src/main/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidatePageMapper.java
+8 -3
View File
@@ -19,6 +19,7 @@ public interface DispatchCandidatePageMapper {
/** 复用既有资格SQL，只统一各来源投影的列序，不复制资格或客户隔离规则。 */
final class Sql {
private static final String FIELDS="planId,planPalletId,warehousePalletId,orderId,customerId,customerName,planNo,orderType,volumeWeightKg,grossWeightKg,containerNo,appointmentAt,expectedLoadingAt,overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,orderNo,orderRefs,inboundCode,palletNo,destinationWarehouseId,destinationWarehouseCode,destinationWarehouseName,fbaWarehouseCode,packageCount,volumeCbm,sourceStage,cartonMarks,relabelCartonMarks,destinationType,deliveryAddress,postalCode,contactName,contactPhone,returnReceiptId,returnReceiptNo,returnCartonSource";
private static final Set<String> TEXT_FIELDS=Set.of(("customerId,customerName,planNo,orderType,containerNo,overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,orderNo,orderRefs,inboundCode,palletNo,destinationWarehouseId,destinationWarehouseCode,destinationWarehouseName,fbaWarehouseCode,sourceStage,cartonMarks,relabelCartonMarks,destinationType,deliveryAddress,postalCode,contactName,contactPhone,returnReceiptNo").split(","));
private static final Set<String> PLAN_MISSING=Set.of("returnReceiptId","returnReceiptNo","returnCartonSource");
private static final Set<String> RETURN_FIELDS=Set.of(("returnReceiptId,returnReceiptNo,orderId,customerId,customerName,orderType,orderNo,orderRefs,inboundCode,warehousePalletId,palletNo,overseasWarehouseId,overseasWarehouseName,overseasWarehouseCode,destinationType,destinationWarehouseCode,fbaWarehouseCode,destinationWarehouseName,deliveryAddress,postalCode,contactName,contactPhone,sourceStage,packageCount").split(","));
private static final Set<String> NATIVE_FIELDS=Set.of(("customerId,orderId,overseasWarehouseId,overseasWarehouseName,orderType,orderNo,orderRefs,inboundCode,palletNo,packageCount,sourceStage,returnCartonSource,relabelCartonMarks,fbaWarehouseCode,grossWeightKg,volumeCbm,destinationWarehouseCode").split(","));
@@ -28,16 +29,20 @@ public interface DispatchCandidatePageMapper {
+"<bind name='sourceStage' value='query.sourceStage'/><bind name='draftToken' value='query.draftToken'/>";
/** 来源字段缺省仅表示该来源没有此字段，不伪造业务编号或实托身份。 */
private static String project(String body,Set<String> fields,boolean count) {
return "SELECT "+Arrays.stream((count?"overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,fbaWarehouseCode":FIELDS).split(",")).map(field->fields.contains(field)?"s."+field:
("returnCartonSource".equals(field)?"0":"NULL")+" AS "+field).collect(Collectors.joining(","))
return "SELECT "+Arrays.stream((count?"overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,fbaWarehouseCode":FIELDS).split(",")).map(field->column(field,fields.contains(field)?"s."+field:
("returnCartonSource".equals(field)?"0":"NULL"))).collect(Collectors.joining(","))
+" FROM ("+body+") s";
}
/** 在UNION前统一文本列，避免表字段、CAST及聚合结果采用不同collation而报1271；数值与时间保持原类型。 */
private static String column(String field,String expression) {
return (TEXT_FIELDS.contains(field)?"CONVERT("+expression+" USING utf8mb4) COLLATE utf8mb4_unicode_ci":expression)+" AS "+field;
}
/** 不同来源用同一组筛选和稳定行标识组合，排序前排除未选来源类型。 */
private static String rows(boolean count) {
var planFields=new HashSet<>(Arrays.asList(FIELDS.split(",")));planFields.removeAll(PLAN_MISSING);
var fields=count?"overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,fbaWarehouseCode":FIELDS;
return "<choose><when test=\"sourceStage != 'RELABEL'\">"+project(DispatchCandidateMapper.AVAILABLE_PLAN_BODY,planFields,count)
+"</when><otherwise>SELECT "+Arrays.stream(fields.split(",")).map(f->"NULL AS "+f).collect(Collectors.joining(","))+" WHERE 1=0</otherwise></choose>"
+"</when><otherwise>SELECT "+Arrays.stream(fields.split(",")).map(f->column(f,"NULL")).collect(Collectors.joining(","))+" WHERE 1=0</otherwise></choose>"
+"<if test='planIds == null or planIds.isEmpty()'>"
+"<if test=\"sourceStage == null or sourceStage == 'RECEIVED'\"> UNION ALL "+project(ReturnDispatchCandidateMapper.AVAILABLE_BODY,RETURN_FIELDS,count)+"</if>"
+"<if test=\"sourceStage == null or sourceStage == 'RECEIVED' or sourceStage == 'RELABEL'\"> UNION ALL "
lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidateMySqlReadOnlyTest.java
+127
View File
@@ -0,0 +1,127 @@
package com.baiyue.lastleg.dispatch.mapper;
import com.baiyue.lastleg.dispatch.model.DispatchPalletCandidate;
import com.baiyue.lastleg.dispatch.port.DispatchPlanningModels.CandidateQuery;
import org.apache.ibatis.scripting.defaults.DefaultParameterHandler;
import org.apache.ibatis.session.Configuration;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.*;
import java.util.*;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.*;
/** 显式指定TEST配置才运行；只读真实MySQL，不建表、不造数、不输出客户记录或连接凭据。 */
@EnabledIfSystemProperty(named="dispatch.mysql.profile",matches=".+")
class DispatchCandidateMySqlReadOnlyTest {
private Connection connect() throws Exception {
var profile=new HashMap<String,String>();
for(String line:Files.readAllLines(Path.of(System.getProperty("dispatch.mysql.profile")))) {
var match=java.util.regex.Pattern.compile("^\\s*(?:export\\s+)?([A-Z][A-Z0-9_]*)\\s*=\\s*(.*)$").matcher(line);
if(!match.matches())continue;
String value=match.group(2).trim();
if(value.length()>1&&(value.startsWith("\"")&&value.endsWith("\"")||value.startsWith("'")&&value.endsWith("'")))
value=value.substring(1,value.length()-1);
profile.put(match.group(1),value);
}
assertEquals("test",profile.get("CHONGCHONG_ENV"),"只允许显式TEST只读检查");
String url="jdbc:mysql://"+profile.get("CHONGCHONG_DB_HOST")+":"+profile.get("CHONGCHONG_DB_PORT")
+"/baiyue_lastleg?characterEncoding=UTF-8&connectTimeout=5000&socketTimeout=15000";
try {
var connection=DriverManager.getConnection(url,profile.get("CHONGCHONG_DB_USERNAME"),profile.get("CHONGCHONG_DB_PASSWORD"));
connection.setReadOnly(true);
return connection;
} catch(SQLException error) {
throw new AssertionError("MySQL connection failed; SQLState="+error.getSQLState()+", code="+error.getErrorCode());
}
}
/** 用纯SELECT构造冲突，确认原始UNION拒绝混合collation且实际生产投影保留中文/字符身份，无需修改测试库。 */
@Test void mixedCollationsAreNormalizedBeforeUnionForCountAndPage() throws Exception {
var fields=Arrays.stream(DispatchPalletCandidate.class.getDeclaredFields()).filter(f->f.getType()==String.class)
.map(java.lang.reflect.Field::getName).collect(Collectors.toSet());
var project=DispatchCandidatePageMapper.Sql.class.getDeclaredMethod("project",String.class,Set.class,boolean.class);
project.setAccessible(true);
String general="SELECT "+fields.stream().sorted().map(f->"_utf8mb4'洛杉矶-001' COLLATE utf8mb4_general_ci AS "+f).collect(Collectors.joining(","));
String unicode=general.replace("utf8mb4_general_ci","utf8mb4_unicode_ci");
try(var connection=connect();var statement=connection.createStatement()) {
statement.setQueryTimeout(10);
var failure=assertThrows(SQLException.class,()->statement.executeQuery(general+" UNION ALL "+unicode));
assertTrue(Set.of(1267,1271).contains(failure.getErrorCode()),"MySQL应报告collation冲突");
for(boolean count:List.of(true,false)) {
String sql=project.invoke(null,general,fields,count)+" UNION ALL "+project.invoke(null,unicode,fields,count);
try(var rows=statement.executeQuery(sql)) {
int total=0;
while(rows.next()) {
total++;
assertEquals("洛杉矶-001",rows.getString("overseasWarehouseId"));
assertEquals("洛杉矶-001",rows.getString("fbaWarehouseCode"));
if(!count) {
for(String field:fields)assertEquals("洛杉矶-001",rows.getString(field),field);
assertNull(rows.getObject("planPalletId"));
assertNull(rows.getObject("appointmentAt"));
assertNull(rows.getObject("grossWeightKg"));
}
}
assertEquals(2,total);
}
}
}
}
/** 执行MyBatis正式SQL和绑定参数，分别检查真实四来源count/page以及筛选分支；仅输出行数和SQL耗时。 */
@Test void actualCandidateQueriesExecuteOnMySql() throws Exception {
String warehouse=System.getProperty("dispatch.mysql.warehouse");
assertNotNull(warehouse,"指定只读验收仓");
var configuration=new Configuration();
configuration.addMapper(DispatchCandidatePageMapper.class);
List<CandidateQuery> queries=List.of(
new CandidateQuery(warehouse,null,null,null,null,null,1,20,"ALL"),
new CandidateQuery(warehouse,null,null,null,null,null,2,20,"ALL"),
new CandidateQuery(warehouse,null,null,null,"RELABEL",null,1,20,"ALL"),
new CandidateQuery(warehouse,null,null,null,"RECEIVED",null,1,20,"LAX"),
new CandidateQuery(warehouse,null,null,null,"PLAN",null,1,20,"NYC"),
new CandidateQuery(warehouse,null,null,"ONT8",null,null,1,20,"ALL"),
new CandidateQuery(warehouse,List.of(-1L),null,null,null,null,1,20,"ALL"));
try(var connection=connect()) {
try(var statement=connection.createStatement();var rows=statement.executeQuery("SELECT @@collation_connection,COLLATION(CAST(1 AS CHAR(128)))")) {
assertTrue(rows.next());
System.out.printf("MYSQL_COLLATION connection=%s castAsChar=%s%n",rows.getString(1),rows.getString(2));
}
try(var statement=connection.createStatement();var rows=statement.executeQuery(
"SELECT TABLE_NAME,COLUMN_NAME,COLLATION_NAME FROM information_schema.COLUMNS "
+"WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('ll_order','ll_inbound_plan','ll_return_receipt','ll_relabel_application') "
+"AND COLUMN_NAME IN ('customer_id','overseas_warehouse_id','overseas_warehouse_name','overseas_warehouse_code') ORDER BY TABLE_NAME,COLUMN_NAME")) {
while(rows.next())System.out.printf("MYSQL_COLLATION %s.%s=%s%n",rows.getString(1),rows.getString(2),rows.getString(3));
}
for(var query:queries) {
var parameters=new HashMap<String,Object>();
parameters.put("query",query);parameters.put("offset",(long)(query.page()-1)*query.size());
long total=0;int size=0;long started=System.nanoTime();
for(String method:List.of("count","page")) {
var mapped=configuration.getMappedStatement(DispatchCandidatePageMapper.class.getName()+"."+method);
var bound=mapped.getBoundSql(parameters);
assertTrue(bound.getSql().stripLeading().startsWith("SELECT"));
try(var statement=connection.prepareStatement(bound.getSql())) {
statement.setQueryTimeout(10);
new DefaultParameterHandler(mapped,parameters,bound).setParameters(statement);
try(var rows=statement.executeQuery()) {
if(method.equals("count")) { assertTrue(rows.next());total=rows.getLong(1); }
else while(rows.next()) {
size++;assertEquals(warehouse,rows.getString("overseasWarehouseId"));
if(query.destinationWarehouseCode()!=null)assertEquals(query.destinationWarehouseCode(),rows.getString("fbaWarehouseCode"));
}
}
}
}
assertTrue(size<=query.size());assertTrue(size<=total);
System.out.printf(Locale.ROOT,"MYSQL_READ_ONLY page=%d stage=%s tab=%s total=%d rows=%d countAndPageMs=%.3f%n",
query.page(),query.sourceStage(),query.warehouseTab(),total,size,(System.nanoTime()-started)/1_000_000d);
}
}
}
}
lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/service/DispatchMergedRelabelPersistenceTest.java
+11 -1
View File
@@ -242,7 +242,10 @@ class DispatchMergedRelabelPersistenceTest {
}
var provider=new RelabelDispatchSourceService(new RelabelDispatchSourceRepository(f.session.getMapper(RelabelDispatchSourceMapper.class)),f.json);
var origins=new DispatchItemSourceRepository(f.session.getMapper(DispatchItemSourceMapper.class));var snapshots=new DispatchItemSourceService(origins,provider,f.json);
f.session.getConfiguration().addMapper(DispatchCandidatePageMapper.class);
// H2不实现MySQL的字符集转换；这里只验证分页/完整组，collation由真实MySQL只读测试覆盖。
configuration.setDefaultScriptingLanguage(H2CandidateLanguageDriver.class);
configuration.addMapper(DispatchCandidatePageMapper.class);
configuration.setDefaultScriptingLanguage(org.apache.ibatis.scripting.xmltags.XMLLanguageDriver.class);
var sourceRepository=new DispatchSourceRepository(f.session.getMapper(DispatchCandidateMapper.class),f.session.getMapper(ReturnDispatchCandidateMapper.class),f.session.getMapper(DispatchSourceMapper.class));
ReflectionTestUtils.setField(sourceRepository,"pages",f.session.getMapper(DispatchCandidatePageMapper.class));
ReflectionTestUtils.setField(rules,"wholeSources",sourceRepository);ReflectionTestUtils.setField(rules,"wholeHashes",new DispatchCommandHasher(f.json));
@@ -289,4 +292,11 @@ class DispatchMergedRelabelPersistenceTest {
/** 历史由真实提供者解析，固定发生次序不依赖当前系统时钟。 */
void history(long id,long app,String customer,String action,Object detail,int day) throws Exception {insert("ll_relabel_pda_history","id",id,"task_id",1000+app,"application_id",app,"customer_id",customer,"action",action,"to_status","WAIT_REVIEW","operator_key","worker","operator_name","测试","detail_json",f.json.writeValueAsString(detail),"created_at",LocalDateTime.of(2026,1,day,0,0));}
}
public static class H2CandidateLanguageDriver extends org.apache.ibatis.scripting.xmltags.XMLLanguageDriver {
@Override public org.apache.ibatis.mapping.SqlSource createSqlSource(org.apache.ibatis.session.Configuration configuration,String script,Class<?> parameterType) {
return super.createSqlSource(configuration,script.replaceAll(
"CONVERT\\((s\\.\\w+|NULL) USING utf8mb4\\) COLLATE utf8mb4_unicode_ci","$1"),parameterType);
}
}
}