From 20f1c7d0e2dc5f0d366411c2807b4e549180492e Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 20:14:51 +0800 Subject: [PATCH] Seek the first visible source row without scanning all fact ids --- .../quality/filters/SourceColumnQueries.java | 12 ++++---- .../V45__source_catalog_first_visible.sql | 6 ++++ .../filters/SourceColumnQueriesMySqlTest.java | 28 +++++++++++++++++-- 3 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 backend/src/main/resources/db/migration/V45__source_catalog_first_visible.sql diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java index cde99bc..18fe501 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java @@ -110,16 +110,16 @@ public class SourceColumnQueries { Map stagePresent = new LinkedHashMap<>(); for (String stage : STAGES) { Set headers = new LinkedHashSet<>(); - StringBuilder representativeScope = new StringBuilder(" WHERE rm.release_id=?"); + StringBuilder representativeScope = new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"); List args = new ArrayList<>(List.of(releaseId)); DataScope.append(representativeScope, args, actor, "f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs"); + // Stop at the first authorized fact of each build instead of aggregating every fact id. String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) " - + "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id " - + "FROM release_member rm JOIN fact_" + stage + " f2" + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage)) + " " - + "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id" - + representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative " - + "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id"; + + "FROM (SELECT DISTINCT transform_build_id,source_revision_id FROM release_member WHERE release_id=?) rm " + + "JOIN fact_" + stage + " f ON f.id=(SELECT f2.id FROM fact_" + stage + " f2" + + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage)) + + representativeScope + " ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1) ORDER BY rm.transform_build_id"; List keyLists = rawHeaders.get(new QueryKey(sql,args,"raw-headers"), () -> Collections.unmodifiableList(new ArrayList<>(jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray())))); stagePresent.put(stage, !keyLists.isEmpty()); diff --git a/backend/src/main/resources/db/migration/V45__source_catalog_first_visible.sql b/backend/src/main/resources/db/migration/V45__source_catalog_first_visible.sql new file mode 100644 index 0000000..5e78c2f --- /dev/null +++ b/backend/src/main/resources/db/migration/V45__source_catalog_first_visible.sql @@ -0,0 +1,6 @@ +-- Equality on the owning build/revision followed by fact id supports the first visible row. +-- Keep authorization dimensions covered, without reading wide JSON rows while seeking it. +ALTER TABLE fact_black DROP INDEX ix_black_source_catalog, + ADD INDEX ix_black_source_catalog (build_id,source_revision_id,id,department,group_name); +ALTER TABLE fact_bright DROP INDEX ix_bright_source_catalog, + ADD INDEX ix_bright_source_catalog (build_id,source_revision_id,id,department,group_name); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 82b0239..3926326 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -101,7 +101,7 @@ class SourceColumnQueriesMySqlTest { assertTrue((Boolean)page.get("pageComplete")); } - @Test void groupedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() { + @Test void orderedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() { record Read(String sql,Object[] args,List result) {} List reads=new java.util.ArrayList<>(); var capturing=new JdbcTemplate(jdbc.getDataSource()) { @@ -114,7 +114,7 @@ class SourceColumnQueriesMySqlTest { assertEquals(2,representatives.size()); for(Read read:representatives) { String stage=read.sql().contains("fact_black f2")?"black":"bright"; - assertTrue(read.sql().contains("GROUP BY f2.build_id,f2.source_revision_id")); + assertTrue(read.sql().contains("ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1")); assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表"); StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"); List args=new java.util.ArrayList<>(); @@ -485,6 +485,28 @@ class SourceColumnQueriesMySqlTest { }); } + @Test void catalogStopsAtTheFirstAuthorizedFactWithoutSortingEveryFactId() throws Exception { + installFilterIndex(); + var captured=new java.util.ArrayList();var parameters=new java.util.ArrayList(); + var tracking=new JdbcTemplate(jdbc.getDataSource()) { + @Override public List query(String sql,org.springframework.jdbc.core.RowMapper mapper,Object... args) { + if(sql.startsWith("SELECT CAST(JSON_KEYS")){captured.add(sql);parameters.add(args);} + return super.query(sql,mapper,args); + } + }; + var catalog=new SourceColumnQueries(tracking,governance,JsonMapper.builder().build()).catalog(RELEASE,actor); + assertNotNull(field(catalog,"委托单")); + assertNull(field(catalog,"组织外行列"),"首条全局记录不可见时,必须继续寻找当前用户的第一条可见记录"); + assertNull(field(catalog,"其他组织列")); + assertEquals(2,captured.size()); + for(int i=0;iline.contains("Sort:")&&line.contains("f2.")),plan); + assertTrue(plan.contains("Covering index lookup on f2 using ix_"),plan); + } + } + @Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception { installFilterIndex();var index=new SourceFilterIndex(jdbc); for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build); @@ -580,7 +602,7 @@ class SourceColumnQueriesMySqlTest { private void installFilterIndex() throws Exception { jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)"); for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id); - for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql")) { + for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql","V45__source_catalog_first_visible.sql")) { String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration)); for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement); }