优化:按批次重置筛选索引临时窗口

This commit is contained in:
2026-11-13 18:37:11 +08:00
parent 5e02c6ad66
commit 7bd9e51004
3 changed files with 12 additions and 8 deletions
@@ -11,6 +11,7 @@ import org.springframework.jdbc.datasource.DataSourceTransactionManager;
public class SourceFilterIndex {
public record BuildAvailable(String buildId) {}
private static final int FORMAT = 1;
static final String CREATE_STAGE_SQL="CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB";
static final String MEMBER_INSERT_SQL="INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash";
static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)";
private final JdbcTemplate jdbc;
@@ -46,11 +47,10 @@ public class SourceFilterIndex {
jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build);
jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build);
// Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source.
jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB");
try {
for(String stage:List.of("black","bright"))buildStage(build,stage);
jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT);
} finally { jdbc.execute("DROP TEMPORARY TABLE source_filter_staging"); }
} finally { jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging"); }
});
}
@@ -68,7 +68,10 @@ public class SourceFilterIndex {
jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end);
String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')";
String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?";
jdbc.update("DELETE FROM source_filter_staging");
// DROP/CREATE TEMPORARY do not commit the owning build transaction. Reset the
// physical table too: DELETE retains undo/deleted records until that transaction ends.
jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging");
jdbc.execute(CREATE_STAGE_SQL);
jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end);
jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash");
Integer collisions=jdbc.queryForObject(COLLISION_SQL,Integer.class);
@@ -469,7 +469,7 @@ class SourceColumnQueriesMySqlTest {
installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black");
jdbc.execute((org.springframework.jdbc.core.ConnectionCallback<Void>)connection->{
try(var statement=connection.createStatement()) {
statement.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB");
statement.execute(SourceFilterIndex.CREATE_STAGE_SQL);
try {
statement.execute("INSERT INTO source_filter_staging SELECT v.field_id,m.fact_id,v.value,v.value_hash FROM source_filter_value v JOIN source_filter_member m ON m.value_id=v.id LIMIT 4");
for(String sql:List.of(SourceFilterIndex.MEMBER_INSERT_SQL,SourceFilterIndex.COLLISION_SQL)) {
@@ -487,14 +487,15 @@ class SourceColumnQueriesMySqlTest {
@Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception {
installFilterIndex();
var fail=new java.util.concurrent.atomic.AtomicBoolean(true);
for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}");
var memberWrites=new java.util.concurrent.atomic.AtomicInteger();
var failing=new JdbcTemplate(jdbc.getDataSource()) {
@Override public int update(String sql) {
if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure");
if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure");
return super.update(sql);
}
@Override public int update(String sql,Object... args) {
if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure");
if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure");
return super.update(sql,args);
}
};