From fa70dc65049989d3d8a74327b42987ac370aa1bf Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 19:19:18 +0800 Subject: [PATCH] Cover source candidate scope checks without wide fact row reads --- .../quality/filters/SourceFilterIndex.java | 11 ++++++++- .../migration/V44__source_candidate_scope.sql | 6 +++++ .../filters/SourceColumnQueriesMySqlTest.java | 23 +++++++++++++++++-- 3 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 backend/src/main/resources/db/migration/V44__source_candidate_scope.sql diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java index 02dbb19..3c56947 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java @@ -16,6 +16,7 @@ public class SourceFilterIndex { static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)"; private final JdbcTemplate jdbc; private volatile Boolean available; + private volatile Boolean candidateScopeAvailable; public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; } public boolean available() { @@ -26,6 +27,14 @@ public class SourceFilterIndex { return available; } + private String candidateFactHint(String stage) { + if (candidateScopeAvailable == null) { + Integer count=jdbc.queryForObject("SELECT COUNT(DISTINCT table_name) FROM information_schema.statistics WHERE table_schema=DATABASE() AND ((table_name='fact_black' AND index_name='ix_black_candidate_scope') OR (table_name='fact_bright' AND index_name='ix_bright_candidate_scope'))",Integer.class); + candidateScopeAvailable=count!=null&&count==2; + } + return candidateScopeAvailable ? " FORCE INDEX (ix_"+stage+"_candidate_scope)" : ""; + } + public boolean ready(String release) { if(release==null||!available())return false; // Only successful immutable builds are retained; a pending backfill is checked again later. @@ -121,7 +130,7 @@ public class SourceFilterIndex { public String candidateSelect(String stage,String column,String scope,List scopeArgs,List args) { args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs); return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id " - +"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id " + +"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f"+candidateFactHint(stage)+" ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id " +scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)"; } } diff --git a/backend/src/main/resources/db/migration/V44__source_candidate_scope.sql b/backend/src/main/resources/db/migration/V44__source_candidate_scope.sql new file mode 100644 index 0000000..a928672 --- /dev/null +++ b/backend/src/main/resources/db/migration/V44__source_candidate_scope.sql @@ -0,0 +1,6 @@ +-- Keep common candidate authorization/date/product checks off the wide JSON clustered rows. +-- The fact table remains authoritative. Changes to its dimensions update this index automatically. +ALTER TABLE fact_black ADD INDEX ix_black_candidate_scope + (id,build_id,source_revision_id,month_key,department,group_name,product); +ALTER TABLE fact_bright ADD INDEX ix_bright_candidate_scope + (id,build_id,source_revision_id,month_key,department,group_name,product); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 4d1a681..82b0239 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -485,6 +485,23 @@ class SourceColumnQueriesMySqlTest { }); } + @Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception { + installFilterIndex();var index=new SourceFilterIndex(jdbc); + for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build); + var scopeArgs=new java.util.ArrayList(List.of(RELEASE,"2026-01","2026-12","部门A","组A","P-1")); + var args=new java.util.ArrayList(); + String sql=index.candidateSelect("black","委托单"," WHERE rm.release_id=? AND f.month_key BETWEEN ? AND ? AND f.department=? AND f.group_name=? AND f.product=?",scopeArgs,args); + assertEquals(Set.of("0007","0008"),new java.util.HashSet<>(jdbc.queryForList(sql,String.class,args.toArray()))); + boolean covering=false; + for(var row:jdbc.queryForList("EXPLAIN "+sql,args.toArray())) { + if("f".equals(row.get("table"))) { + assertEquals("ix_black_candidate_scope",row.get("key")); + covering=String.valueOf(row.get("Extra")).contains("Using index"); + } + } + assertTrue(covering,"候选值的日期、权限和产品校验不应读取宽 JSON 原始行"); + } + @Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception { installFilterIndex(); for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}"); @@ -563,8 +580,10 @@ class SourceColumnQueriesMySqlTest { private void installFilterIndex() throws Exception { jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)"); for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id); - String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/V43__source_filter_indexes.sql")); - for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement); + for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql")) { + String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration)); + for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement); + } } private void schema() {