修复:来源字段筛选超时与重复扫描

This commit is contained in:
2026-11-13 16:52:27 +08:00
parent 5457f5c786
commit ea84e26b2f
7 changed files with 89 additions and 43 deletions
@@ -60,6 +60,16 @@ public class SourceColumnQueries {
}
return List.copyOf(result);
}
// 只提取一次,并区分 JSON null 与字符串“null”,供筛选、候选值和成员读取共用。
private static final String SOURCE_TEXT = "JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON)))";
private static final String SOURCE_VALUE = "NULLIF(TRIM("+SOURCE_TEXT+"), '') COLLATE utf8mb4_0900_as_cs";
/** 原始字段筛选按发布构建读取,避免优化器从整年月份范围扫描大 JSON 原表。 */
public static String factIndexHint(String stage) {
if(!List.of("black","bright").contains(stage))throw bad("数据阶段无效");
return " FORCE INDEX (uq_fact_"+stage+"_origin)";
}
private static final List<String> STAGES = List.of("black", "bright");
private static final List<String> OPERATORS = List.of("in", "not_in", "is_empty", "not_empty");
private static final Map<String, String> CANONICAL_COLUMNS = Map.of(
@@ -100,7 +110,7 @@ public class SourceColumnQueries {
"f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs");
String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) "
+ "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id "
+ "FROM release_member rm JOIN fact_" + stage + " f2 "
+ "FROM release_member rm JOIN fact_" + stage + " f2" + factIndexHint(stage) + " "
+ "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"
+ representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative "
+ "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id";
@@ -240,10 +250,7 @@ public class SourceColumnQueries {
String expression;
Object column = ((Map<?, ?>) field.get("columns")).get(stage);
if ("source_value".equals(field.get("type"))) {
expression = "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL "
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
+ "COLLATE utf8mb4_0900_as_cs";
expression = SOURCE_VALUE;
} else {
String factColumn = CANONICAL_COLUMNS.get(key);
if (factColumn == null) throw bad("规范维度无效");
@@ -255,19 +262,19 @@ public class SourceColumnQueries {
if (operator.equals("in")) {
where.append(expression).append(" IN (")
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append(')');
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column));
if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
} else {
where.append('(').append(expression).append(" IS NULL OR ").append(expression).append(" NOT IN (")
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append("))");
if ("source_value".equals(field.get("type"))) {
bindSourcePaths(args, String.valueOf(column));
bindSourcePaths(args, String.valueOf(column));
bindSourcePath(args, String.valueOf(column));
bindSourcePath(args, String.valueOf(column));
}
}
args.addAll(values);
} else {
where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL");
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column));
if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
}
where.append(')');
}
@@ -578,12 +585,7 @@ public class SourceColumnQueries {
}
String path = jsonPath(column);
args.add(path);
args.add(path);
args.add(path);
return "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL "
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
+ "COLLATE utf8mb4_0900_as_cs";
return SOURCE_VALUE;
}
private static List<Object> combine(List<Object> first, List<Object> second) {
@@ -601,9 +603,7 @@ public class SourceColumnQueries {
}
String path = jsonPath(column);
args.add(path);
args.add(path);
return "CASE WHEN JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?)) END COLLATE utf8mb4_0900_as_cs";
return SOURCE_TEXT+" COLLATE utf8mb4_0900_as_cs";
}
private static String tupleColumns(int count) {
@@ -814,11 +814,9 @@ public class SourceColumnQueries {
catch (Exception e) { throw new IllegalArgumentException("来源列名无法编码", e); }
}
private void bindSourcePaths(List<Object> args, String header) {
private void bindSourcePath(List<Object> args, String header) {
String path = jsonPath(header);
args.add(path);
args.add(path);
args.add(path);
}
private static ResponseStatusException bad(String message) {
@@ -39,6 +39,7 @@ public class MetricsService {
private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128;
private final JdbcTemplate jdbc;
private final GovernanceAccessService governance;
private final SourceColumnQueries sourceColumns;
private final TransactionTemplate readSnapshot;
private final TransactionTemplate governanceSnapshot;
@@ -46,6 +47,7 @@ public class MetricsService {
public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance, PlatformTransactionManager transactionManager) {
this.jdbc = jdbc;
this.governance = governance;
this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
this.readSnapshot = new TransactionTemplate(transactionManager);
this.readSnapshot.setIsolationLevel(TransactionDefinition.ISOLATION_REPEATABLE_READ);
this.readSnapshot.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
@@ -56,7 +58,9 @@ public class MetricsService {
/** Keeps package-level in-memory aggregation tests independent of a database transaction manager. */
public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance) {
this.jdbc = jdbc; this.governance = governance; this.readSnapshot = null; this.governanceSnapshot = null;
this.jdbc = jdbc; this.governance = governance;
this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
this.readSnapshot = null; this.governanceSnapshot = null;
}
public MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor) {
@@ -158,7 +162,7 @@ public class MetricsService {
if (factFallback) {
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor);
if (!query.columnFilters().isEmpty())
new SourceColumnQueries(jdbc, governance, JsonMapper.builder().build())
sourceColumns
.append(where, args, stage, query.columnFilters(), releaseId, actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
@@ -180,7 +184,7 @@ public class MetricsService {
%s AS bright_processing_scrap,
%s AS bright_diffusion_scrap,
%s AS row_count
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
%s
GROUP BY trend_period
ORDER BY trend_period
@@ -189,7 +193,7 @@ public class MetricsService {
componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"),
componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"),
componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"),
countExpression, table, where);
countExpression, table, factFilterHint(stage,factFallback,query), where);
jdbc.query(sql, (RowCallbackHandler) rs -> {
String period = rs.getString("trend_period");
AggregateStats stats = new AggregateStats().read(rs, stage);
@@ -393,7 +397,7 @@ public class MetricsService {
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor);
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
@@ -413,7 +417,7 @@ public class MetricsService {
%s AS bright_processing_scrap,
%s AS bright_diffusion_scrap,
%s AS row_count
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
%s
GROUP BY %s
ORDER BY %s
@@ -423,7 +427,7 @@ public class MetricsService {
componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"),
componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"),
componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"),
factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, where,
factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, factFilterHint(stage,factFallback,query), where,
aggregateKeys, aggregateKeys);
jdbc.query(sql, (RowCallbackHandler) rs -> {
String period = rs.getString("trend_period");
@@ -595,6 +599,15 @@ public class MetricsService {
"sourceMonths", sourceMonths);
}
private static String factFilterHint(String stage,boolean factFallback,MetricsQuery query) {
if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return "";
// 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。
if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null
||query.department()!=null||query.group()!=null||query.product()!=null||query.projectGroup()!=null
||query.productSearch()!=null)return "";
return SourceColumnQueries.factIndexHint(stage);
}
private void coverageStage(Map<String, CoverageMonth> months, String stage, String releaseId,
MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup) {
if(query.stage()!=null&&!query.stage().equals(stage))return;
@@ -610,7 +623,7 @@ public class MetricsService {
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor);
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
String countExpression = factFallback ? "COUNT(*)" : "SUM(f.source_row_count)";
@@ -625,11 +638,11 @@ public class MetricsService {
String sql = """
SELECT f.month_key, %s AS row_count,
%s AS excluded_blank_rows, %s AS unmapped_rows, %s AS project_unconfigured_rows
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
%s
GROUP BY f.month_key
ORDER BY f.month_key
""".formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, where);
""".formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, factFilterHint(stage,factFallback,query), where);
jdbc.query(sql, (RowCallbackHandler) rs -> {
CoverageMonth coverage = months.computeIfAbsent(rs.getString("month_key"), ignored -> new CoverageMonth());
long rowCount = rs.getLong("row_count");
@@ -42,7 +42,7 @@ class FineDefectRollupsMySqlTest {
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)");
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))");
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,sheet_name VARCHAR(80),physical_row_no INT,KEY ix_fact_" + stage + "_month_dims(build_id,month_key),UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))");
for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql", "V42__fine_defect_compact_rollups.sql")) {
try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) {
for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql);
@@ -115,6 +115,7 @@ class SourceColumnQueriesMySqlTest {
for(Read read:representatives) {
String stage=read.sql().contains("fact_black f2")?"black":"bright";
assertTrue(read.sql().contains("GROUP BY f2.build_id,f2.source_revision_id"));
assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表");
StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
List<Object> args=new java.util.ArrayList<>();
DataScope.append(scope,args,actor,"f2.department COLLATE utf8mb4_0900_as_cs","f2.group_name COLLATE utf8mb4_0900_as_cs");args.add(RELEASE);
@@ -157,6 +158,23 @@ class SourceColumnQueriesMySqlTest {
assertThrows(ResponseStatusException.class, () -> queries.validate(RELEASE, unauthorized, actor));
}
@Test void sourceNullNormalizationMatchesThePreviousExpressionForEveryOperator() {
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
for(String raw:List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[]}","{\"委托单\":{\"n\":1}}"))
insert("bright","build-bright",BRIGHT_REVISION,"部门A","组A",raw);
String old="CASE WHEN JSON_EXTRACT(f.raw_values,'$.\"委托单\"') IS NULL OR JSON_TYPE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))='NULL' THEN NULL ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))),'') END COLLATE utf8mb4_0900_as_cs";
for(String operator:List.of("in","not_in","is_empty","not_empty")) {
Map<String,Object> rule=new java.util.LinkedHashMap<>(Map.of("field",key,"stages",List.of("bright"),"operator",operator));
List<Object> args=new java.util.ArrayList<>();String condition;
if(operator.equals("in")||operator.equals("not_in")) {
rule.put("values",List.of("null","0007"));args.add("null");args.add("0007");
condition=operator.equals("in")?old+" IN (?,?)":"("+old+" IS NULL OR "+old+" NOT IN (?,?))";
} else condition=old+(operator.equals("is_empty")?" IS NULL":" IS NOT NULL");
Long expected=jdbc.queryForObject("SELECT COUNT(*) FROM fact_bright f WHERE f.build_id='build-bright' AND f.department='部门A' AND f.group_name='组A' AND ("+condition+")",Long.class,args.toArray());
assertEquals(expected,countFiltered("bright",SourceColumnQueries.parse(List.of(rule))),operator);
}
}
@Test void explicitMappingWorksAcrossStagesAndMemberPageCountsMergedTuplesBeforePaging() {
List<Map<String, Object>> catalog = queries.catalog(RELEASE, actor);
String mappedKey = (String) catalog.stream().filter(item -> item.get("label").equals("跨阶段委托号"))
@@ -252,12 +270,19 @@ class SourceColumnQueriesMySqlTest {
jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE);
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
for(String stage:List.of("black","bright")) {
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT");
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=?,scrap_weight=?,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id",stage.equals("black")?90:80,stage.equals("black")?10:20);
}
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
var service=new cn.yunsheng.quality.metrics.MetricsService(jdbc,governance);
var headerReads=new java.util.concurrent.atomic.AtomicInteger();
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
if(sql.contains("JSON_KEYS(f.raw_values)"))headerReads.incrementAndGet();
return super.query(sql,mapper,args);
}
};
var service=new cn.yunsheng.quality.metrics.MetricsService(capturing,governance);
var releases=mock(cn.yunsheng.quality.release.ReleaseService.class);
when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE);
var mapper=JsonMapper.builder().build();
@@ -272,6 +297,7 @@ class SourceColumnQueriesMySqlTest {
assertEquals(stage.equals("black")?legacy.summary().black():legacy.summary().bright(),stage.equals("black")?selected.summary().black():selected.summary().bright());
assertEquals(0L,selected.summary().coverage().get(stage.equals("black")?"brightRows":"blackRows"));
assertEquals(1,selected.trend().size());
assertEquals(1,service.calculateTrend(base.withStage(stage),actor).trend().size());
var detail=records.recordsFiltered(stage,2026,1,null,"部门A","组A",null,null,null,false,RELEASE,1,50,null,null,null,"asc",null,null,mapper.writeValueAsString(List.of(filter)),actor);
assertEquals(1L,((Number)detail.get("total")).longValue());
java.nio.file.Path file=exporter.exportFiltered(2026,1,null,"部门A","组A",null,null,false,RELEASE,actor,Map.of(),null,stage,List.of(filter));
@@ -282,6 +308,7 @@ class SourceColumnQueriesMySqlTest {
var other=base.withColumnFilters(List.of(filter)).withStage(stage);
assertThrows(ResponseStatusException.class,()->other.withStage(stage.equals("black")?"bright":"black"));
}
assertEquals(2,headerReads.get(),"汇总、趋势与覆盖检查应复用同一份授权字段目录");
}
@Test void continuousRangeDetailsAndBothExcelSheetsExcludeOutsideMonthsAndUnauthorizedRows() throws Exception {
@@ -289,7 +316,7 @@ class SourceColumnQueriesMySqlTest {
when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE);
var mapper=JsonMapper.builder().build();
for(String stage:List.of("black","bright")) {
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT");
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=90,scrap_weight=10,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id");
String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION;
for(String month:List.of("2026-03","2026-09","2026-10","2025-03")) {
@@ -336,7 +363,7 @@ class SourceColumnQueriesMySqlTest {
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
for(String stage:List.of("black","bright")) {
jdbc.execute("DELETE FROM fact_"+stage);
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT");
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
}
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
@@ -395,7 +422,7 @@ class SourceColumnQueriesMySqlTest {
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage
+ "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),"
+ "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL)");
+ "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL,sheet_name VARCHAR(80),physical_row_no INT,UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))");
jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64) PRIMARY KEY,label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)");
}
@@ -143,9 +143,9 @@ class SourceColumnQueriesTest {
queries.append(where, args, "black", filters, "release", actor);
assertTrue(where.toString().contains("JSON_EXTRACT(f.raw_values, ?) IS NULL"));
assertTrue(where.toString().contains("NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON))"));
assertTrue(where.toString().contains("COLLATE utf8mb4_0900_as_cs IN (?,?)"));
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "00007", "x' OR 1=1 --"), args);
assertEquals(List.of("$.\"委托单\"", "00007", "x' OR 1=1 --"), args);
assertFalse(where.toString().contains("委托单"));
assertFalse(where.toString().contains("x' OR 1=1"));
}
@@ -160,7 +160,7 @@ class SourceColumnQueriesTest {
StringBuilder where = new StringBuilder();
queries.append(where, args, "black", filters, "release", actor);
assertTrue(where.toString().contains(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL"));
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\""), args);
assertEquals(List.of("$.\"委托单\""), args);
}
List<Map<String, Object>> filters = SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("bright"), "operator", "is_empty")));
@@ -179,8 +179,7 @@ class SourceColumnQueriesTest {
assertTrue(where.toString().contains("IS NULL OR"));
assertTrue(where.toString().contains(" NOT IN (?)"));
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"",
"$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "000123"), args);
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "000123"), args);
}
@Test void rejectsInvalidStageAndMissingFieldsInTheActiveCatalog() {