From 42d548194deffc48663d770e21ff1c804daefae2 Mon Sep 17 00:00:00 2001 From: ouyangxq <775153912@qq.com> Date: Wed, 21 Oct 2026 01:45:30 +0800 Subject: [PATCH] fix(dispatch): normalize candidate UNION text collations --- .../mapper/DispatchCandidatePageMapper.java | 11 +- .../DispatchCandidateMySqlReadOnlyTest.java | 127 ++++++++++++++++++ .../DispatchMergedRelabelPersistenceTest.java | 12 +- 3 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidateMySqlReadOnlyTest.java diff --git a/lastleg-service/src/main/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidatePageMapper.java b/lastleg-service/src/main/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidatePageMapper.java index 0c861ffd1..3e92404a7 100644 --- a/lastleg-service/src/main/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidatePageMapper.java +++ b/lastleg-service/src/main/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidatePageMapper.java @@ -19,6 +19,7 @@ public interface DispatchCandidatePageMapper { /** 复用既有资格SQL，只统一各来源投影的列序，不复制资格或客户隔离规则。 */ final class Sql { private static final String FIELDS="planId,planPalletId,warehousePalletId,orderId,customerId,customerName,planNo,orderType,volumeWeightKg,grossWeightKg,containerNo,appointmentAt,expectedLoadingAt,overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,orderNo,orderRefs,inboundCode,palletNo,destinationWarehouseId,destinationWarehouseCode,destinationWarehouseName,fbaWarehouseCode,packageCount,volumeCbm,sourceStage,cartonMarks,relabelCartonMarks,destinationType,deliveryAddress,postalCode,contactName,contactPhone,returnReceiptId,returnReceiptNo,returnCartonSource"; + private static final Set TEXT_FIELDS=Set.of(("customerId,customerName,planNo,orderType,containerNo,overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,orderNo,orderRefs,inboundCode,palletNo,destinationWarehouseId,destinationWarehouseCode,destinationWarehouseName,fbaWarehouseCode,sourceStage,cartonMarks,relabelCartonMarks,destinationType,deliveryAddress,postalCode,contactName,contactPhone,returnReceiptNo").split(",")); private static final Set PLAN_MISSING=Set.of("returnReceiptId","returnReceiptNo","returnCartonSource"); private static final Set RETURN_FIELDS=Set.of(("returnReceiptId,returnReceiptNo,orderId,customerId,customerName,orderType,orderNo,orderRefs,inboundCode,warehousePalletId,palletNo,overseasWarehouseId,overseasWarehouseName,overseasWarehouseCode,destinationType,destinationWarehouseCode,fbaWarehouseCode,destinationWarehouseName,deliveryAddress,postalCode,contactName,contactPhone,sourceStage,packageCount").split(",")); private static final Set NATIVE_FIELDS=Set.of(("customerId,orderId,overseasWarehouseId,overseasWarehouseName,orderType,orderNo,orderRefs,inboundCode,palletNo,packageCount,sourceStage,returnCartonSource,relabelCartonMarks,fbaWarehouseCode,grossWeightKg,volumeCbm,destinationWarehouseCode").split(",")); @@ -28,16 +29,20 @@ public interface DispatchCandidatePageMapper { +""; /** 来源字段缺省仅表示该来源没有此字段，不伪造业务编号或实托身份。 */ private static String project(String body,Set fields,boolean count) { - return "SELECT "+Arrays.stream((count?"overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,fbaWarehouseCode":FIELDS).split(",")).map(field->fields.contains(field)?"s."+field: - ("returnCartonSource".equals(field)?"0":"NULL")+" AS "+field).collect(Collectors.joining(",")) + return "SELECT "+Arrays.stream((count?"overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,fbaWarehouseCode":FIELDS).split(",")).map(field->column(field,fields.contains(field)?"s."+field: + ("returnCartonSource".equals(field)?"0":"NULL"))).collect(Collectors.joining(",")) +" FROM ("+body+") s"; } + /** 在UNION前统一文本列，避免表字段、CAST及聚合结果采用不同collation而报1271；数值与时间保持原类型。 */ + private static String column(String field,String expression) { + return (TEXT_FIELDS.contains(field)?"CONVERT("+expression+" USING utf8mb4) COLLATE utf8mb4_unicode_ci":expression)+" AS "+field; + } /** 不同来源用同一组筛选和稳定行标识组合，排序前排除未选来源类型。 */ private static String rows(boolean count) { var planFields=new HashSet<>(Arrays.asList(FIELDS.split(",")));planFields.removeAll(PLAN_MISSING); var fields=count?"overseasWarehouseId,overseasWarehouseCode,overseasWarehouseName,fbaWarehouseCode":FIELDS; return ""+project(DispatchCandidateMapper.AVAILABLE_PLAN_BODY,planFields,count) - +"SELECT "+Arrays.stream(fields.split(",")).map(f->"NULL AS "+f).collect(Collectors.joining(","))+" WHERE 1=0" + +"SELECT "+Arrays.stream(fields.split(",")).map(f->column(f,"NULL")).collect(Collectors.joining(","))+" WHERE 1=0" +"" +" UNION ALL "+project(ReturnDispatchCandidateMapper.AVAILABLE_BODY,RETURN_FIELDS,count)+"" +" UNION ALL " diff --git a/lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidateMySqlReadOnlyTest.java b/lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidateMySqlReadOnlyTest.java new file mode 100644 index 000000000..b47dc0792 --- /dev/null +++ b/lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/mapper/DispatchCandidateMySqlReadOnlyTest.java @@ -0,0 +1,127 @@ +package com.baiyue.lastleg.dispatch.mapper; + +import com.baiyue.lastleg.dispatch.model.DispatchPalletCandidate; +import com.baiyue.lastleg.dispatch.port.DispatchPlanningModels.CandidateQuery; +import org.apache.ibatis.scripting.defaults.DefaultParameterHandler; +import org.apache.ibatis.session.Configuration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.*; +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +/** 显式指定TEST配置才运行；只读真实MySQL，不建表、不造数、不输出客户记录或连接凭据。 */ +@EnabledIfSystemProperty(named="dispatch.mysql.profile",matches=".+") +class DispatchCandidateMySqlReadOnlyTest { + private Connection connect() throws Exception { + var profile=new HashMap(); + for(String line:Files.readAllLines(Path.of(System.getProperty("dispatch.mysql.profile")))) { + var match=java.util.regex.Pattern.compile("^\\s*(?:export\\s+)?([A-Z][A-Z0-9_]*)\\s*=\\s*(.*)$").matcher(line); + if(!match.matches())continue; + String value=match.group(2).trim(); + if(value.length()>1&&(value.startsWith("\"")&&value.endsWith("\"")||value.startsWith("'")&&value.endsWith("'"))) + value=value.substring(1,value.length()-1); + profile.put(match.group(1),value); + } + assertEquals("test",profile.get("CHONGCHONG_ENV"),"只允许显式TEST只读检查"); + String url="jdbc:mysql://"+profile.get("CHONGCHONG_DB_HOST")+":"+profile.get("CHONGCHONG_DB_PORT") + +"/baiyue_lastleg?characterEncoding=UTF-8&connectTimeout=5000&socketTimeout=15000"; + try { + var connection=DriverManager.getConnection(url,profile.get("CHONGCHONG_DB_USERNAME"),profile.get("CHONGCHONG_DB_PASSWORD")); + connection.setReadOnly(true); + return connection; + } catch(SQLException error) { + throw new AssertionError("MySQL connection failed; SQLState="+error.getSQLState()+", code="+error.getErrorCode()); + } + } + + /** 用纯SELECT构造冲突，确认原始UNION拒绝混合collation且实际生产投影保留中文/字符身份，无需修改测试库。 */ + @Test void mixedCollationsAreNormalizedBeforeUnionForCountAndPage() throws Exception { + var fields=Arrays.stream(DispatchPalletCandidate.class.getDeclaredFields()).filter(f->f.getType()==String.class) + .map(java.lang.reflect.Field::getName).collect(Collectors.toSet()); + var project=DispatchCandidatePageMapper.Sql.class.getDeclaredMethod("project",String.class,Set.class,boolean.class); + project.setAccessible(true); + String general="SELECT "+fields.stream().sorted().map(f->"_utf8mb4'洛杉矶-001' COLLATE utf8mb4_general_ci AS "+f).collect(Collectors.joining(",")); + String unicode=general.replace("utf8mb4_general_ci","utf8mb4_unicode_ci"); + try(var connection=connect();var statement=connection.createStatement()) { + statement.setQueryTimeout(10); + var failure=assertThrows(SQLException.class,()->statement.executeQuery(general+" UNION ALL "+unicode)); + assertTrue(Set.of(1267,1271).contains(failure.getErrorCode()),"MySQL应报告collation冲突"); + for(boolean count:List.of(true,false)) { + String sql=project.invoke(null,general,fields,count)+" UNION ALL "+project.invoke(null,unicode,fields,count); + try(var rows=statement.executeQuery(sql)) { + int total=0; + while(rows.next()) { + total++; + assertEquals("洛杉矶-001",rows.getString("overseasWarehouseId")); + assertEquals("洛杉矶-001",rows.getString("fbaWarehouseCode")); + if(!count) { + for(String field:fields)assertEquals("洛杉矶-001",rows.getString(field),field); + assertNull(rows.getObject("planPalletId")); + assertNull(rows.getObject("appointmentAt")); + assertNull(rows.getObject("grossWeightKg")); + } + } + assertEquals(2,total); + } + } + } + } + + /** 执行MyBatis正式SQL和绑定参数，分别检查真实四来源count/page以及筛选分支；仅输出行数和SQL耗时。 */ + @Test void actualCandidateQueriesExecuteOnMySql() throws Exception { + String warehouse=System.getProperty("dispatch.mysql.warehouse"); + assertNotNull(warehouse,"指定只读验收仓"); + var configuration=new Configuration(); + configuration.addMapper(DispatchCandidatePageMapper.class); + List queries=List.of( + new CandidateQuery(warehouse,null,null,null,null,null,1,20,"ALL"), + new CandidateQuery(warehouse,null,null,null,null,null,2,20,"ALL"), + new CandidateQuery(warehouse,null,null,null,"RELABEL",null,1,20,"ALL"), + new CandidateQuery(warehouse,null,null,null,"RECEIVED",null,1,20,"LAX"), + new CandidateQuery(warehouse,null,null,null,"PLAN",null,1,20,"NYC"), + new CandidateQuery(warehouse,null,null,"ONT8",null,null,1,20,"ALL"), + new CandidateQuery(warehouse,List.of(-1L),null,null,null,null,1,20,"ALL")); + try(var connection=connect()) { + try(var statement=connection.createStatement();var rows=statement.executeQuery("SELECT @@collation_connection,COLLATION(CAST(1 AS CHAR(128)))")) { + assertTrue(rows.next()); + System.out.printf("MYSQL_COLLATION connection=%s castAsChar=%s%n",rows.getString(1),rows.getString(2)); + } + try(var statement=connection.createStatement();var rows=statement.executeQuery( + "SELECT TABLE_NAME,COLUMN_NAME,COLLATION_NAME FROM information_schema.COLUMNS " + +"WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('ll_order','ll_inbound_plan','ll_return_receipt','ll_relabel_application') " + +"AND COLUMN_NAME IN ('customer_id','overseas_warehouse_id','overseas_warehouse_name','overseas_warehouse_code') ORDER BY TABLE_NAME,COLUMN_NAME")) { + while(rows.next())System.out.printf("MYSQL_COLLATION %s.%s=%s%n",rows.getString(1),rows.getString(2),rows.getString(3)); + } + for(var query:queries) { + var parameters=new HashMap(); + parameters.put("query",query);parameters.put("offset",(long)(query.page()-1)*query.size()); + long total=0;int size=0;long started=System.nanoTime(); + for(String method:List.of("count","page")) { + var mapped=configuration.getMappedStatement(DispatchCandidatePageMapper.class.getName()+"."+method); + var bound=mapped.getBoundSql(parameters); + assertTrue(bound.getSql().stripLeading().startsWith("SELECT")); + try(var statement=connection.prepareStatement(bound.getSql())) { + statement.setQueryTimeout(10); + new DefaultParameterHandler(mapped,parameters,bound).setParameters(statement); + try(var rows=statement.executeQuery()) { + if(method.equals("count")) { assertTrue(rows.next());total=rows.getLong(1); } + else while(rows.next()) { + size++;assertEquals(warehouse,rows.getString("overseasWarehouseId")); + if(query.destinationWarehouseCode()!=null)assertEquals(query.destinationWarehouseCode(),rows.getString("fbaWarehouseCode")); + } + } + } + } + assertTrue(size<=query.size());assertTrue(size<=total); + System.out.printf(Locale.ROOT,"MYSQL_READ_ONLY page=%d stage=%s tab=%s total=%d rows=%d countAndPageMs=%.3f%n", + query.page(),query.sourceStage(),query.warehouseTab(),total,size,(System.nanoTime()-started)/1_000_000d); + } + } + } +} diff --git a/lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/service/DispatchMergedRelabelPersistenceTest.java b/lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/service/DispatchMergedRelabelPersistenceTest.java index 185a1bf09..79fc0ee00 100644 --- a/lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/service/DispatchMergedRelabelPersistenceTest.java +++ b/lastleg-service/src/test/java/com/baiyue/lastleg/dispatch/service/DispatchMergedRelabelPersistenceTest.java @@ -242,7 +242,10 @@ class DispatchMergedRelabelPersistenceTest { } var provider=new RelabelDispatchSourceService(new RelabelDispatchSourceRepository(f.session.getMapper(RelabelDispatchSourceMapper.class)),f.json); var origins=new DispatchItemSourceRepository(f.session.getMapper(DispatchItemSourceMapper.class));var snapshots=new DispatchItemSourceService(origins,provider,f.json); - f.session.getConfiguration().addMapper(DispatchCandidatePageMapper.class); + // H2不实现MySQL的字符集转换；这里只验证分页/完整组，collation由真实MySQL只读测试覆盖。 + configuration.setDefaultScriptingLanguage(H2CandidateLanguageDriver.class); + configuration.addMapper(DispatchCandidatePageMapper.class); + configuration.setDefaultScriptingLanguage(org.apache.ibatis.scripting.xmltags.XMLLanguageDriver.class); var sourceRepository=new DispatchSourceRepository(f.session.getMapper(DispatchCandidateMapper.class),f.session.getMapper(ReturnDispatchCandidateMapper.class),f.session.getMapper(DispatchSourceMapper.class)); ReflectionTestUtils.setField(sourceRepository,"pages",f.session.getMapper(DispatchCandidatePageMapper.class)); ReflectionTestUtils.setField(rules,"wholeSources",sourceRepository);ReflectionTestUtils.setField(rules,"wholeHashes",new DispatchCommandHasher(f.json)); @@ -289,4 +292,11 @@ class DispatchMergedRelabelPersistenceTest { /** 历史由真实提供者解析，固定发生次序不依赖当前系统时钟。 */ void history(long id,long app,String customer,String action,Object detail,int day) throws Exception {insert("ll_relabel_pda_history","id",id,"task_id",1000+app,"application_id",app,"customer_id",customer,"action",action,"to_status","WAIT_REVIEW","operator_key","worker","operator_name","测试","detail_json",f.json.writeValueAsString(detail),"created_at",LocalDateTime.of(2026,1,day,0,0));} } + + public static class H2CandidateLanguageDriver extends org.apache.ibatis.scripting.xmltags.XMLLanguageDriver { + @Override public org.apache.ibatis.mapping.SqlSource createSqlSource(org.apache.ibatis.session.Configuration configuration,String script,Class parameterType) { + return super.createSqlSource(configuration,script.replaceAll( + "CONVERT\\((s\\.\\w+|NULL) USING utf8mb4\\) COLLATE utf8mb4_unicode_ci","$1"),parameterType); + } + } }