Seek the first visible source row without scanning all fact ids
This commit is contained in:
@@ -110,16 +110,16 @@ public class SourceColumnQueries {
|
|||||||
Map<String, Boolean> stagePresent = new LinkedHashMap<>();
|
Map<String, Boolean> stagePresent = new LinkedHashMap<>();
|
||||||
for (String stage : STAGES) {
|
for (String stage : STAGES) {
|
||||||
Set<String> headers = new LinkedHashSet<>();
|
Set<String> headers = new LinkedHashSet<>();
|
||||||
StringBuilder representativeScope = new StringBuilder(" WHERE rm.release_id=?");
|
StringBuilder representativeScope = new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
|
||||||
List<Object> args = new ArrayList<>(List.of(releaseId));
|
List<Object> args = new ArrayList<>(List.of(releaseId));
|
||||||
DataScope.append(representativeScope, args, actor,
|
DataScope.append(representativeScope, args, actor,
|
||||||
"f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs");
|
"f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs");
|
||||||
|
// Stop at the first authorized fact of each build instead of aggregating every fact id.
|
||||||
String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) "
|
String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) "
|
||||||
+ "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id "
|
+ "FROM (SELECT DISTINCT transform_build_id,source_revision_id FROM release_member WHERE release_id=?) rm "
|
||||||
+ "FROM release_member rm JOIN fact_" + stage + " f2" + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage)) + " "
|
+ "JOIN fact_" + stage + " f ON f.id=(SELECT f2.id FROM fact_" + stage + " f2"
|
||||||
+ "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"
|
+ (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage))
|
||||||
+ representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative "
|
+ representativeScope + " ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1) ORDER BY rm.transform_build_id";
|
||||||
+ "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id";
|
|
||||||
List<String> keyLists = rawHeaders.get(new QueryKey(sql,args,"raw-headers"),
|
List<String> keyLists = rawHeaders.get(new QueryKey(sql,args,"raw-headers"),
|
||||||
() -> Collections.unmodifiableList(new ArrayList<>(jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray()))));
|
() -> Collections.unmodifiableList(new ArrayList<>(jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray()))));
|
||||||
stagePresent.put(stage, !keyLists.isEmpty());
|
stagePresent.put(stage, !keyLists.isEmpty());
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
-- Equality on the owning build/revision followed by fact id supports the first visible row.
|
||||||
|
-- Keep authorization dimensions covered, without reading wide JSON rows while seeking it.
|
||||||
|
ALTER TABLE fact_black DROP INDEX ix_black_source_catalog,
|
||||||
|
ADD INDEX ix_black_source_catalog (build_id,source_revision_id,id,department,group_name);
|
||||||
|
ALTER TABLE fact_bright DROP INDEX ix_bright_source_catalog,
|
||||||
|
ADD INDEX ix_bright_source_catalog (build_id,source_revision_id,id,department,group_name);
|
||||||
+25
-3
@@ -101,7 +101,7 @@ class SourceColumnQueriesMySqlTest {
|
|||||||
assertTrue((Boolean)page.get("pageComplete"));
|
assertTrue((Boolean)page.get("pageComplete"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test void groupedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() {
|
@Test void orderedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() {
|
||||||
record Read(String sql,Object[] args,List<?> result) {}
|
record Read(String sql,Object[] args,List<?> result) {}
|
||||||
List<Read> reads=new java.util.ArrayList<>();
|
List<Read> reads=new java.util.ArrayList<>();
|
||||||
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
|
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
|
||||||
@@ -114,7 +114,7 @@ class SourceColumnQueriesMySqlTest {
|
|||||||
assertEquals(2,representatives.size());
|
assertEquals(2,representatives.size());
|
||||||
for(Read read:representatives) {
|
for(Read read:representatives) {
|
||||||
String stage=read.sql().contains("fact_black f2")?"black":"bright";
|
String stage=read.sql().contains("fact_black f2")?"black":"bright";
|
||||||
assertTrue(read.sql().contains("GROUP BY f2.build_id,f2.source_revision_id"));
|
assertTrue(read.sql().contains("ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1"));
|
||||||
assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表");
|
assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表");
|
||||||
StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
|
StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
|
||||||
List<Object> args=new java.util.ArrayList<>();
|
List<Object> args=new java.util.ArrayList<>();
|
||||||
@@ -485,6 +485,28 @@ class SourceColumnQueriesMySqlTest {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test void catalogStopsAtTheFirstAuthorizedFactWithoutSortingEveryFactId() throws Exception {
|
||||||
|
installFilterIndex();
|
||||||
|
var captured=new java.util.ArrayList<String>();var parameters=new java.util.ArrayList<Object[]>();
|
||||||
|
var tracking=new JdbcTemplate(jdbc.getDataSource()) {
|
||||||
|
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
|
||||||
|
if(sql.startsWith("SELECT CAST(JSON_KEYS")){captured.add(sql);parameters.add(args);}
|
||||||
|
return super.query(sql,mapper,args);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
var catalog=new SourceColumnQueries(tracking,governance,JsonMapper.builder().build()).catalog(RELEASE,actor);
|
||||||
|
assertNotNull(field(catalog,"委托单"));
|
||||||
|
assertNull(field(catalog,"组织外行列"),"首条全局记录不可见时,必须继续寻找当前用户的第一条可见记录");
|
||||||
|
assertNull(field(catalog,"其他组织列"));
|
||||||
|
assertEquals(2,captured.size());
|
||||||
|
for(int i=0;i<captured.size();i++) {
|
||||||
|
String plan=jdbc.queryForObject("EXPLAIN ANALYZE "+captured.get(i),String.class,parameters.get(i));
|
||||||
|
assertTrue(plan.contains("Limit: 1 row(s)"),plan);
|
||||||
|
assertFalse(plan.lines().anyMatch(line->line.contains("Sort:")&&line.contains("f2.")),plan);
|
||||||
|
assertTrue(plan.contains("Covering index lookup on f2 using ix_"),plan);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception {
|
@Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception {
|
||||||
installFilterIndex();var index=new SourceFilterIndex(jdbc);
|
installFilterIndex();var index=new SourceFilterIndex(jdbc);
|
||||||
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
|
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
|
||||||
@@ -580,7 +602,7 @@ class SourceColumnQueriesMySqlTest {
|
|||||||
private void installFilterIndex() throws Exception {
|
private void installFilterIndex() throws Exception {
|
||||||
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
|
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
|
||||||
for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id);
|
for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id);
|
||||||
for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql")) {
|
for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql","V45__source_catalog_first_visible.sql")) {
|
||||||
String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration));
|
String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration));
|
||||||
for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement);
|
for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user