Add literal contains filters across source queries and analysis context

This commit is contained in:
2026-11-13 21:23:12 +08:00
parent 7cae4cfbdf
commit 3c022e8614
20 changed files with 194 additions and 41 deletions
@@ -891,7 +891,7 @@ final class AnalysisProtocolHistoryFacts {
String label=text(columnLabels.get(field),historyFieldLabel(section,sources,field));
if(label.isBlank())label="授权列";
List<String> prefix=strings(rule.get("stages")).stream().map(AnalysisProtocolHistoryFacts::stageLabel).toList();
String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";};
String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "contains"->"包含文字";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";};
List<String> values=scopeLiterals(rule.get("values"));
StringBuilder part=new StringBuilder();if(!prefix.isEmpty())part.append(String.join("、",prefix)).append(' ');
part.append(label).append(' ').append(operator);if(!values.isEmpty())part.append(' ').append(String.join("、",values));parts.add(part.toString());
@@ -71,7 +71,7 @@ public class SourceColumnQueries {
}
private static final List<String> STAGES = List.of("black", "bright");
private static final List<String> OPERATORS = List.of("in", "not_in", "is_empty", "not_empty");
private static final List<String> OPERATORS = List.of("in", "contains", "not_in", "is_empty", "not_empty");
private static final Map<String, String> CANONICAL_COLUMNS = Map.of(
"product", "product", "department", "department", "group", "group_name", "projectGroup", "project_group");
private static final Map<String, String> CANONICAL_LABELS = Map.of(
@@ -208,12 +208,13 @@ public class SourceColumnQueries {
normalizedStages.add(value);
}
List<String> values = new ArrayList<>();
if (operator.equals("in") || operator.equals("not_in")) {
if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) {
if (!(rule.get("values") instanceof List<?> rawValues) || rawValues.isEmpty() || rawValues.size() > 50)
throw bad("in/not_in 筛选必须提供1到50个字符串值");
throw bad("文本筛选必须提供1到50个字符串值");
for (Object value : rawValues) {
if (!(value instanceof String text) || text.isEmpty() || text.length() > 256)
throw bad("来源列筛选值必须是1到256字符的完整字符串;空值请使用空值操作符");
if(operator.equals("contains")&&text.isBlank())throw bad("包含筛选的关键词不能全为空白");
if (!values.contains(text)) values.add(text);
}
} else if (rule.containsKey("values")) {
@@ -223,7 +224,7 @@ public class SourceColumnQueries {
normalized.put("field", field);
normalized.put("stages", List.copyOf(normalizedStages));
normalized.put("operator", operator);
if (operator.equals("in") || operator.equals("not_in")) normalized.put("values", List.copyOf(values));
if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) normalized.put("values", List.copyOf(values));
parsed.add(Collections.unmodifiableMap(normalized));
}
return List.copyOf(parsed);
@@ -280,6 +281,13 @@ public class SourceColumnQueries {
}
}
args.addAll(values);
} else if (operator.equals("contains")) {
for(int i=0;i<selected.size();i++) {
if(i>0)where.append(" OR ");
where.append("LOCATE(?,").append(expression).append(")>0");
args.add(selected.get(i));
if("source_value".equals(field.get("type")))bindSourcePath(args,String.valueOf(column));
}
} else {
where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL");
if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
@@ -96,7 +96,7 @@ public class SourceFilterIndex {
if(!ready(release))return false;
where.append(" AND ");
membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"),
operator.equals("in")||operator.equals("not_in")?values:null,false);
operator.equals("in")||operator.equals("not_in")||operator.equals("contains")?values:null,false,operator.equals("contains"));
return true;
}
@@ -106,20 +106,27 @@ public class SourceFilterIndex {
var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList();
where.append(" AND (");
if(values.isEmpty())where.append("FALSE");
else membership(where,args,stage,release,column,false,values,true);
else membership(where,args,stage,release,column,false,values,true,false);
if(selected.contains(QueryFilterFields.EMPTY)) {
where.append(" OR ");membership(where,args,stage,release,column,true,null,true);
where.append(" OR ");membership(where,args,stage,release,column,true,null,true,false);
}
where.append(')');return true;
}
private void membership(StringBuilder where,List<Object> args,String stage,String release,String column,
boolean negative,List<String> values,boolean legacy) {
boolean negative,List<String> values,boolean legacy,boolean contains) {
where.append("f.id ").append(negative?"NOT IN (":"IN (")
.append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin");
args.add(release);args.add(stage);args.add(column);
if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)");
if(values!=null) {
if(contains) {
where.append(" AND (");
for(int i=0;i<values.size();i++) {
if(i>0)where.append(" OR ");
where.append("LOCATE(?,sv.value COLLATE utf8mb4_0900_as_cs)>0");args.add(values.get(i));
}
where.append(')');
} else if(values!=null) {
where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')');
args.addAll(values);
}
@@ -266,8 +266,8 @@ class SourceColumnQueriesMySqlTest {
}
@org.junit.jupiter.params.ParameterizedTest
@org.junit.jupiter.params.provider.ValueSource(booleans={false,true})
void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed) throws Exception {
@org.junit.jupiter.params.provider.CsvSource({"false,in","true,in","false,contains","true,contains"})
void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed,String operator) throws Exception {
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))");
jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE);
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
@@ -293,7 +293,7 @@ class SourceColumnQueriesMySqlTest {
var records=new cn.yunsheng.quality.records.RecordController(jdbc,releases,mapper,governance,exporter);
String fieldKey=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
for(String stage:List.of("black","bright")) {
var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator","in","values",List.of("0007"));
var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator",operator,"values",List.of(operator.equals("contains")?"007":"0007"));
var base=new cn.yunsheng.quality.metrics.MetricsService.MetricsQuery(Integer.valueOf(2026),Integer.valueOf(1),null,"部门A","组A",null,null,"product",false,RELEASE,"month",Map.of()).withColumnFilters(List.of(filter));
var legacy=service.calculate(base,actor);var selected=service.calculate(base.withStage(stage),actor);
assertEquals(1,selected.rows().size());assertEquals(1L,((Number)selected.coverage().get("includedRows")).longValue());
@@ -429,7 +429,7 @@ class SourceColumnQueriesMySqlTest {
insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}");
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
List<Map<String,Object>> rules=new java.util.ArrayList<>();
for(String op:List.of("in","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}"))
for(String op:List.of("in","contains","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}"))
rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value)));
var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList();
var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor);
@@ -465,6 +465,33 @@ class SourceColumnQueriesMySqlTest {
assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
}
@Test void containsMatchesLiteralSubstringsWithOrSemanticsInBothStagesAndBothQueryPaths() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
var mapper=JsonMapper.builder().build();
for(String stage:List.of("black","bright")) {
String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION;
for(String value:List.of("xxAlpha%_\\tail","中文日立公司","prefix x' OR 1=1 -- suffix","000123","null"))
insert(stage,"build-"+stage,revision,"部门A","组A",mapper.writeValueAsString(Map.of("委托单",value)));
insert(stage,"build-"+stage,revision,"部门B","组B",mapper.writeValueAsString(Map.of("委托单","日立-Alpha-hidden")));
}
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
List<List<String>> needles=List.of(List.of("Alpha"),List.of("alpha"),List.of("%"),List.of("_"),List.of("\\"),List.of("日立"),List.of("x' OR 1=1 --"),List.of("001"),List.of("null"),List.of("日立","Alpha"));
List<Long> expected=List.of(1L,0L,1L,1L,1L,1L,1L,1L,1L,2L);
for(boolean indexed:List.of(false,true)) {
if(indexed) {
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
queries=new SourceColumnQueries(jdbc,governance,mapper);
}
for(String stage:List.of("black","bright"))for(int n=0;n<needles.size();n++) {
var rule=Map.<String,Object>of("field",key,"stages",List.of(stage),"operator","contains","values",needles.get(n));
assertEquals(expected.get(n),countFiltered(stage,List.of(rule)),stage+" indexed="+indexed+" keywords="+needles.get(n));
}
var canonical=Map.<String,Object>of("field","product","stages",List.of("black"),"operator","contains","values",List.of("P-"));
assertTrue(countFiltered("black",List.of(canonical))>0);
}
}
@Test void memberBuildPlansStartFromTheBoundedStageWindowRatherThanTheGrowingDictionary() throws Exception {
installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black");
jdbc.execute((org.springframework.jdbc.core.ConnectionCallback<Void>)connection->{
@@ -32,7 +32,7 @@ class SourceColumnQueriesTest {
@Test void rejectsUnknownShapeOperatorsEmptyValuesAndMoreThanFiftyValues() {
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(Map.of()));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("black"), "operator", "contains", "values", List.of("x")))));
"field", SOURCE, "stages", List.of("black"), "operator", "starts_with", "values", List.of("x")))));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("black"), "operator", "in", "values", List.of("")))));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
@@ -42,6 +42,22 @@ class SourceColumnQueriesTest {
"field", SOURCE, "stages", List.of("bright"), "operator", "not_in", "values", tooMany))));
}
@Test void containsKeepsLiteralKeywordsAndBindsEachJsonPathAfterItsNeedle() {
var rule=Map.<String,Object>of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",List.of("000","%_\\","x' OR 1=1 --","000"));
var parsed=SourceColumnQueries.parse(List.of(rule));
assertEquals(List.of("000","%_\\","x' OR 1=1 --"),parsed.getFirst().get("values"));
var source=spy(new SourceColumnQueries(mock(JdbcTemplate.class),governance,JsonMapper.builder().build()));
doReturn(List.of(sourceField())).when(source).scopedCatalog("release",actor);
var sql=new StringBuilder();var args=new ArrayList<Object>();
source.append(sql,args,"black",parsed,"release",actor);
assertEquals(List.of("000","$.\"委托单\"","%_\\","$.\"委托单\"","x' OR 1=1 --","$.\"委托单\""),args);
assertEquals(3,sql.toString().split("LOCATE",-1).length-1);
assertFalse(sql.toString().contains("x' OR"));
for(Object values:List.of(List.of(),List.of(""),List.of(" "),List.of("x".repeat(257)),List.of(7))) {
assertThrows(ResponseStatusException.class,()->SourceColumnQueries.parse(List.of(Map.of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",values))));
}
}
@Test void monthRangeUsesInclusiveYearBoundariesAndKeepsLegacyMonthPredicate() {
Map<String,Object> range=SourceColumnQueries.period(Map.of("period",Map.of("year",2026,"monthStart",3,"monthEnd",9)));
StringBuilder rangeSql=new StringBuilder();List<Object> rangeArgs=new ArrayList<>();
@@ -222,6 +238,6 @@ class SourceColumnQueriesTest {
columns.put("black", stage.equals("black") ? header : null);
columns.put("bright", stage.equals("bright") ? header : null);
return Map.of("key", key, "label", header, "columns", columns, "stages", List.of(stage),
"type", "source_value", "operators", List.of("in", "not_in", "is_empty", "not_empty"));
"type", "source_value", "operators", List.of("in", "contains", "not_in", "is_empty", "not_empty"));
}
}