diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java index df529c0..02dbb19 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java @@ -11,6 +11,7 @@ import org.springframework.jdbc.datasource.DataSourceTransactionManager; public class SourceFilterIndex { public record BuildAvailable(String buildId) {} private static final int FORMAT = 1; + static final String CREATE_STAGE_SQL="CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB"; static final String MEMBER_INSERT_SQL="INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash"; static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)"; private final JdbcTemplate jdbc; @@ -46,11 +47,10 @@ public class SourceFilterIndex { jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build); jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build); // Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source. - jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB"); try { for(String stage:List.of("black","bright"))buildStage(build,stage); jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT); - } finally { jdbc.execute("DROP TEMPORARY TABLE source_filter_staging"); } + } finally { jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging"); } }); } @@ -68,7 +68,10 @@ public class SourceFilterIndex { jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end); String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')"; String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?"; - jdbc.update("DELETE FROM source_filter_staging"); + // DROP/CREATE TEMPORARY do not commit the owning build transaction. Reset the + // physical table too: DELETE retains undo/deleted records until that transaction ends. + jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging"); + jdbc.execute(CREATE_STAGE_SQL); jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end); jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash"); Integer collisions=jdbc.queryForObject(COLLISION_SQL,Integer.class); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 06d8b6e..4d1a681 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -469,7 +469,7 @@ class SourceColumnQueriesMySqlTest { installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black"); jdbc.execute((org.springframework.jdbc.core.ConnectionCallback)connection->{ try(var statement=connection.createStatement()) { - statement.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB"); + statement.execute(SourceFilterIndex.CREATE_STAGE_SQL); try { statement.execute("INSERT INTO source_filter_staging SELECT v.field_id,m.fact_id,v.value,v.value_hash FROM source_filter_value v JOIN source_filter_member m ON m.value_id=v.id LIMIT 4"); for(String sql:List.of(SourceFilterIndex.MEMBER_INSERT_SQL,SourceFilterIndex.COLLISION_SQL)) { @@ -487,14 +487,15 @@ class SourceColumnQueriesMySqlTest { @Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception { installFilterIndex(); - var fail=new java.util.concurrent.atomic.AtomicBoolean(true); + for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}"); + var memberWrites=new java.util.concurrent.atomic.AtomicInteger(); var failing=new JdbcTemplate(jdbc.getDataSource()) { @Override public int update(String sql) { - if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure"); + if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure"); return super.update(sql); } @Override public int update(String sql,Object... args) { - if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure"); + if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure"); return super.update(sql,args); } }; diff --git a/docs/当前进度.md b/docs/当前进度.md index 096b0d2..23cc969 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -9,7 +9,7 @@ 生命周期:启动仅在单线程后台补当前完整发布版本;导入和归类通过提交后事件进入同一有界去重队列。完整标记提交后才启用,故障保留原始查询,业务事务不等待索引。旧配置 `null`、来源字符串 `null`、空值、Unicode、同一事实行和当前权限分别保持;构建保留清理识别4张新表,未解除维护兼容保护。 -验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上2026.11.13.6已生效,后台补建期间发现每批全扫历史字典,实时计划约85万词条先行;改为500条事实窗口驱动及临时覆盖索引,专项47项(46通过、1跳过)及独立审查通过,准备2026.11.13.7补丁。已完成索引格式不变可复用。补建期间临时将数据库缓存从256MB调到1GB,验收前必须恢复原值;记录保存在本机私有发布目录。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 +验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上2026.11.13.6已生效,后台补建期间发现每批全扫历史字典,实时计划约85万词条先行;改为500条事实窗口驱动及临时覆盖索引,专项47项(46通过、1跳过)及独立审查通过,已发布2026.11.13.7(`5e02c6a`),完整构建986项(819通过、167跳过)。进一步将每批临时表物理重建,避免长事务中DELETE遗留窗口历史;第二窗口故障回滚及独立审查通过,准备2026.11.13.8。已完成索引格式不变可复用。补建期间临时将数据库缓存从256MB调到1GB,验收前必须恢复原值;记录保存在本机私有发布目录。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 ## 2026-11-13:来源字段筛选超时修复