优化:筛选索引按暂存窗口读取字典

This commit is contained in:
2026-11-13 18:20:32 +08:00
parent 7782e33bec
commit 5e02c6ad66
3 changed files with 26 additions and 4 deletions
@@ -11,6 +11,8 @@ import org.springframework.jdbc.datasource.DataSourceTransactionManager;
public class SourceFilterIndex { public class SourceFilterIndex {
public record BuildAvailable(String buildId) {} public record BuildAvailable(String buildId) {}
private static final int FORMAT = 1; private static final int FORMAT = 1;
static final String MEMBER_INSERT_SQL="INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash";
static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)";
private final JdbcTemplate jdbc; private final JdbcTemplate jdbc;
private volatile Boolean available; private volatile Boolean available;
public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; } public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; }
@@ -44,7 +46,7 @@ public class SourceFilterIndex {
jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build); jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build);
jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build); jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build);
// Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source. // Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source.
jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash)) ENGINE=InnoDB"); jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB");
try { try {
for(String stage:List.of("black","bright"))buildStage(build,stage); for(String stage:List.of("black","bright"))buildStage(build,stage);
jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT); jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT);
@@ -69,9 +71,9 @@ public class SourceFilterIndex {
jdbc.update("DELETE FROM source_filter_staging"); jdbc.update("DELETE FROM source_filter_staging");
jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end); jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end);
jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash"); jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash");
Integer collisions=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)",Integer.class); Integer collisions=jdbc.queryForObject(COLLISION_SQL,Integer.class);
if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布"); if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布");
jdbc.update("INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash"); jdbc.update(MEMBER_INSERT_SQL);
after=end; after=end;
} }
} }
@@ -465,6 +465,26 @@ class SourceColumnQueriesMySqlTest {
assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList()); assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
} }
@Test void memberBuildPlansStartFromTheBoundedStageWindowRatherThanTheGrowingDictionary() throws Exception {
installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black");
jdbc.execute((org.springframework.jdbc.core.ConnectionCallback<Void>)connection->{
try(var statement=connection.createStatement()) {
statement.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB");
try {
statement.execute("INSERT INTO source_filter_staging SELECT v.field_id,m.fact_id,v.value,v.value_hash FROM source_filter_value v JOIN source_filter_member m ON m.value_id=v.id LIMIT 4");
for(String sql:List.of(SourceFilterIndex.MEMBER_INSERT_SQL,SourceFilterIndex.COLLISION_SQL)) {
try(var rows=statement.executeQuery("EXPLAIN "+sql)) {
String first=null;
while(rows.next())if(List.of("s","v").contains(rows.getString("table"))){first=rows.getString("table");break;}
assertEquals("s",first,"每批构建只能从有界临时窗口开始,不能反复扫描历史字典");
}
}
} finally {statement.execute("DROP TEMPORARY TABLE source_filter_staging");}
}
return null;
});
}
@Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception { @Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception {
installFilterIndex(); installFilterIndex();
var fail=new java.util.concurrent.atomic.AtomicBoolean(true); var fail=new java.util.concurrent.atomic.AtomicBoolean(true);
+1 -1
View File
@@ -9,7 +9,7 @@
生命周期:启动仅在单线程后台补当前完整发布版本;导入和归类通过提交后事件进入同一有界去重队列。完整标记提交后才启用,故障保留原始查询,业务事务不等待索引。旧配置 `null`、来源字符串 `null`、空值、Unicode、同一事实行和当前权限分别保持;构建保留清理识别4张新表,未解除维护兼容保护。 生命周期:启动仅在单线程后台补当前完整发布版本;导入和归类通过提交后事件进入同一有界去重队列。完整标记提交后才启用,故障保留原始查询,业务事务不等待索引。旧配置 `null`、来源字符串 `null`、空值、Unicode、同一事实行和当前权限分别保持;构建保留清理识别4张新表,未解除维护兼容保护。
验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上2026.11.13.6已生效,后台补建期间发现每批全扫历史字典,实时计划约85万词条先行;改为500条事实窗口驱动及临时覆盖索引,专项47项(46通过、1跳过)及独立审查通过,准备2026.11.13.7补丁。已完成索引格式不变可复用。补建期间临时将数据库缓存从256MB调到1GB,验收前必须恢复原值;记录保存在本机私有发布目录。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。
## 2026-11-13:来源字段筛选超时修复 ## 2026-11-13:来源字段筛选超时修复