Cover source candidate scope checks without wide fact row reads

This commit is contained in:
2026-11-13 19:19:18 +08:00
parent 7bd9e51004
commit fa70dc6504
3 changed files with 37 additions and 3 deletions
@@ -16,6 +16,7 @@ public class SourceFilterIndex {
static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)"; static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)";
private final JdbcTemplate jdbc; private final JdbcTemplate jdbc;
private volatile Boolean available; private volatile Boolean available;
private volatile Boolean candidateScopeAvailable;
public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; } public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; }
public boolean available() { public boolean available() {
@@ -26,6 +27,14 @@ public class SourceFilterIndex {
return available; return available;
} }
private String candidateFactHint(String stage) {
if (candidateScopeAvailable == null) {
Integer count=jdbc.queryForObject("SELECT COUNT(DISTINCT table_name) FROM information_schema.statistics WHERE table_schema=DATABASE() AND ((table_name='fact_black' AND index_name='ix_black_candidate_scope') OR (table_name='fact_bright' AND index_name='ix_bright_candidate_scope'))",Integer.class);
candidateScopeAvailable=count!=null&&count==2;
}
return candidateScopeAvailable ? " FORCE INDEX (ix_"+stage+"_candidate_scope)" : "";
}
public boolean ready(String release) { public boolean ready(String release) {
if(release==null||!available())return false; if(release==null||!available())return false;
// Only successful immutable builds are retained; a pending backfill is checked again later. // Only successful immutable builds are retained; a pending backfill is checked again later.
@@ -121,7 +130,7 @@ public class SourceFilterIndex {
public String candidateSelect(String stage,String column,String scope,List<Object> scopeArgs,List<Object> args) { public String candidateSelect(String stage,String column,String scope,List<Object> scopeArgs,List<Object> args) {
args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs); args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs);
return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id " return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id "
+"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id " +"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f"+candidateFactHint(stage)+" ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id "
+scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)"; +scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)";
} }
} }
@@ -0,0 +1,6 @@
-- Keep common candidate authorization/date/product checks off the wide JSON clustered rows.
-- The fact table remains authoritative. Changes to its dimensions update this index automatically.
ALTER TABLE fact_black ADD INDEX ix_black_candidate_scope
(id,build_id,source_revision_id,month_key,department,group_name,product);
ALTER TABLE fact_bright ADD INDEX ix_bright_candidate_scope
(id,build_id,source_revision_id,month_key,department,group_name,product);
@@ -485,6 +485,23 @@ class SourceColumnQueriesMySqlTest {
}); });
} }
@Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception {
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
var scopeArgs=new java.util.ArrayList<Object>(List.of(RELEASE,"2026-01","2026-12","部门A","组A","P-1"));
var args=new java.util.ArrayList<Object>();
String sql=index.candidateSelect("black","委托单"," WHERE rm.release_id=? AND f.month_key BETWEEN ? AND ? AND f.department=? AND f.group_name=? AND f.product=?",scopeArgs,args);
assertEquals(Set.of("0007","0008"),new java.util.HashSet<>(jdbc.queryForList(sql,String.class,args.toArray())));
boolean covering=false;
for(var row:jdbc.queryForList("EXPLAIN "+sql,args.toArray())) {
if("f".equals(row.get("table"))) {
assertEquals("ix_black_candidate_scope",row.get("key"));
covering=String.valueOf(row.get("Extra")).contains("Using index");
}
}
assertTrue(covering,"候选值的日期、权限和产品校验不应读取宽 JSON 原始行");
}
@Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception { @Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception {
installFilterIndex(); installFilterIndex();
for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}"); for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}");
@@ -563,9 +580,11 @@ class SourceColumnQueriesMySqlTest {
private void installFilterIndex() throws Exception { private void installFilterIndex() throws Exception {
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)"); jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id); for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id);
String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/V43__source_filter_indexes.sql")); for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql")) {
String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration));
for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement); for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement);
} }
}
private void schema() { private void schema() {
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))"); jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))");