Add literal contains filters across source queries and analysis context

This commit is contained in:
2026-11-13 21:23:12 +08:00
parent 7cae4cfbdf
commit 3c022e8614
20 changed files with 194 additions and 41 deletions
@@ -891,7 +891,7 @@ final class AnalysisProtocolHistoryFacts {
String label=text(columnLabels.get(field),historyFieldLabel(section,sources,field));
if(label.isBlank())label="授权列";
List<String> prefix=strings(rule.get("stages")).stream().map(AnalysisProtocolHistoryFacts::stageLabel).toList();
String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";};
String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "contains"->"包含文字";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";};
List<String> values=scopeLiterals(rule.get("values"));
StringBuilder part=new StringBuilder();if(!prefix.isEmpty())part.append(String.join("、",prefix)).append(' ');
part.append(label).append(' ').append(operator);if(!values.isEmpty())part.append(' ').append(String.join("、",values));parts.add(part.toString());
@@ -71,7 +71,7 @@ public class SourceColumnQueries {
}
private static final List<String> STAGES = List.of("black", "bright");
private static final List<String> OPERATORS = List.of("in", "not_in", "is_empty", "not_empty");
private static final List<String> OPERATORS = List.of("in", "contains", "not_in", "is_empty", "not_empty");
private static final Map<String, String> CANONICAL_COLUMNS = Map.of(
"product", "product", "department", "department", "group", "group_name", "projectGroup", "project_group");
private static final Map<String, String> CANONICAL_LABELS = Map.of(
@@ -208,12 +208,13 @@ public class SourceColumnQueries {
normalizedStages.add(value);
}
List<String> values = new ArrayList<>();
if (operator.equals("in") || operator.equals("not_in")) {
if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) {
if (!(rule.get("values") instanceof List<?> rawValues) || rawValues.isEmpty() || rawValues.size() > 50)
throw bad("in/not_in 筛选必须提供1到50个字符串值");
throw bad("文本筛选必须提供1到50个字符串值");
for (Object value : rawValues) {
if (!(value instanceof String text) || text.isEmpty() || text.length() > 256)
throw bad("来源列筛选值必须是1到256字符的完整字符串;空值请使用空值操作符");
if(operator.equals("contains")&&text.isBlank())throw bad("包含筛选的关键词不能全为空白");
if (!values.contains(text)) values.add(text);
}
} else if (rule.containsKey("values")) {
@@ -223,7 +224,7 @@ public class SourceColumnQueries {
normalized.put("field", field);
normalized.put("stages", List.copyOf(normalizedStages));
normalized.put("operator", operator);
if (operator.equals("in") || operator.equals("not_in")) normalized.put("values", List.copyOf(values));
if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) normalized.put("values", List.copyOf(values));
parsed.add(Collections.unmodifiableMap(normalized));
}
return List.copyOf(parsed);
@@ -280,6 +281,13 @@ public class SourceColumnQueries {
}
}
args.addAll(values);
} else if (operator.equals("contains")) {
for(int i=0;i<selected.size();i++) {
if(i>0)where.append(" OR ");
where.append("LOCATE(?,").append(expression).append(")>0");
args.add(selected.get(i));
if("source_value".equals(field.get("type")))bindSourcePath(args,String.valueOf(column));
}
} else {
where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL");
if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
@@ -96,7 +96,7 @@ public class SourceFilterIndex {
if(!ready(release))return false;
where.append(" AND ");
membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"),
operator.equals("in")||operator.equals("not_in")?values:null,false);
operator.equals("in")||operator.equals("not_in")||operator.equals("contains")?values:null,false,operator.equals("contains"));
return true;
}
@@ -106,20 +106,27 @@ public class SourceFilterIndex {
var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList();
where.append(" AND (");
if(values.isEmpty())where.append("FALSE");
else membership(where,args,stage,release,column,false,values,true);
else membership(where,args,stage,release,column,false,values,true,false);
if(selected.contains(QueryFilterFields.EMPTY)) {
where.append(" OR ");membership(where,args,stage,release,column,true,null,true);
where.append(" OR ");membership(where,args,stage,release,column,true,null,true,false);
}
where.append(')');return true;
}
private void membership(StringBuilder where,List<Object> args,String stage,String release,String column,
boolean negative,List<String> values,boolean legacy) {
boolean negative,List<String> values,boolean legacy,boolean contains) {
where.append("f.id ").append(negative?"NOT IN (":"IN (")
.append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin");
args.add(release);args.add(stage);args.add(column);
if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)");
if(values!=null) {
if(contains) {
where.append(" AND (");
for(int i=0;i<values.size();i++) {
if(i>0)where.append(" OR ");
where.append("LOCATE(?,sv.value COLLATE utf8mb4_0900_as_cs)>0");args.add(values.get(i));
}
where.append(')');
} else if(values!=null) {
where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')');
args.addAll(values);
}
@@ -266,8 +266,8 @@ class SourceColumnQueriesMySqlTest {
}
@org.junit.jupiter.params.ParameterizedTest
@org.junit.jupiter.params.provider.ValueSource(booleans={false,true})
void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed) throws Exception {
@org.junit.jupiter.params.provider.CsvSource({"false,in","true,in","false,contains","true,contains"})
void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed,String operator) throws Exception {
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))");
jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE);
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
@@ -293,7 +293,7 @@ class SourceColumnQueriesMySqlTest {
var records=new cn.yunsheng.quality.records.RecordController(jdbc,releases,mapper,governance,exporter);
String fieldKey=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
for(String stage:List.of("black","bright")) {
var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator","in","values",List.of("0007"));
var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator",operator,"values",List.of(operator.equals("contains")?"007":"0007"));
var base=new cn.yunsheng.quality.metrics.MetricsService.MetricsQuery(Integer.valueOf(2026),Integer.valueOf(1),null,"部门A","组A",null,null,"product",false,RELEASE,"month",Map.of()).withColumnFilters(List.of(filter));
var legacy=service.calculate(base,actor);var selected=service.calculate(base.withStage(stage),actor);
assertEquals(1,selected.rows().size());assertEquals(1L,((Number)selected.coverage().get("includedRows")).longValue());
@@ -429,7 +429,7 @@ class SourceColumnQueriesMySqlTest {
insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}");
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
List<Map<String,Object>> rules=new java.util.ArrayList<>();
for(String op:List.of("in","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}"))
for(String op:List.of("in","contains","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}"))
rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value)));
var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList();
var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor);
@@ -465,6 +465,33 @@ class SourceColumnQueriesMySqlTest {
assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
}
@Test void containsMatchesLiteralSubstringsWithOrSemanticsInBothStagesAndBothQueryPaths() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
var mapper=JsonMapper.builder().build();
for(String stage:List.of("black","bright")) {
String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION;
for(String value:List.of("xxAlpha%_\\tail","中文日立公司","prefix x' OR 1=1 -- suffix","000123","null"))
insert(stage,"build-"+stage,revision,"部门A","组A",mapper.writeValueAsString(Map.of("委托单",value)));
insert(stage,"build-"+stage,revision,"部门B","组B",mapper.writeValueAsString(Map.of("委托单","日立-Alpha-hidden")));
}
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
List<List<String>> needles=List.of(List.of("Alpha"),List.of("alpha"),List.of("%"),List.of("_"),List.of("\\"),List.of("日立"),List.of("x' OR 1=1 --"),List.of("001"),List.of("null"),List.of("日立","Alpha"));
List<Long> expected=List.of(1L,0L,1L,1L,1L,1L,1L,1L,1L,2L);
for(boolean indexed:List.of(false,true)) {
if(indexed) {
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
queries=new SourceColumnQueries(jdbc,governance,mapper);
}
for(String stage:List.of("black","bright"))for(int n=0;n<needles.size();n++) {
var rule=Map.<String,Object>of("field",key,"stages",List.of(stage),"operator","contains","values",needles.get(n));
assertEquals(expected.get(n),countFiltered(stage,List.of(rule)),stage+" indexed="+indexed+" keywords="+needles.get(n));
}
var canonical=Map.<String,Object>of("field","product","stages",List.of("black"),"operator","contains","values",List.of("P-"));
assertTrue(countFiltered("black",List.of(canonical))>0);
}
}
@Test void memberBuildPlansStartFromTheBoundedStageWindowRatherThanTheGrowingDictionary() throws Exception {
installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black");
jdbc.execute((org.springframework.jdbc.core.ConnectionCallback<Void>)connection->{
@@ -32,7 +32,7 @@ class SourceColumnQueriesTest {
@Test void rejectsUnknownShapeOperatorsEmptyValuesAndMoreThanFiftyValues() {
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(Map.of()));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("black"), "operator", "contains", "values", List.of("x")))));
"field", SOURCE, "stages", List.of("black"), "operator", "starts_with", "values", List.of("x")))));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("black"), "operator", "in", "values", List.of("")))));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
@@ -42,6 +42,22 @@ class SourceColumnQueriesTest {
"field", SOURCE, "stages", List.of("bright"), "operator", "not_in", "values", tooMany))));
}
@Test void containsKeepsLiteralKeywordsAndBindsEachJsonPathAfterItsNeedle() {
var rule=Map.<String,Object>of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",List.of("000","%_\\","x' OR 1=1 --","000"));
var parsed=SourceColumnQueries.parse(List.of(rule));
assertEquals(List.of("000","%_\\","x' OR 1=1 --"),parsed.getFirst().get("values"));
var source=spy(new SourceColumnQueries(mock(JdbcTemplate.class),governance,JsonMapper.builder().build()));
doReturn(List.of(sourceField())).when(source).scopedCatalog("release",actor);
var sql=new StringBuilder();var args=new ArrayList<Object>();
source.append(sql,args,"black",parsed,"release",actor);
assertEquals(List.of("000","$.\"委托单\"","%_\\","$.\"委托单\"","x' OR 1=1 --","$.\"委托单\""),args);
assertEquals(3,sql.toString().split("LOCATE",-1).length-1);
assertFalse(sql.toString().contains("x' OR"));
for(Object values:List.of(List.of(),List.of(""),List.of(" "),List.of("x".repeat(257)),List.of(7))) {
assertThrows(ResponseStatusException.class,()->SourceColumnQueries.parse(List.of(Map.of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",values))));
}
}
@Test void monthRangeUsesInclusiveYearBoundariesAndKeepsLegacyMonthPredicate() {
Map<String,Object> range=SourceColumnQueries.period(Map.of("period",Map.of("year",2026,"monthStart",3,"monthEnd",9)));
StringBuilder rangeSql=new StringBuilder();List<Object> rangeArgs=new ArrayList<>();
@@ -222,6 +238,6 @@ class SourceColumnQueriesTest {
columns.put("black", stage.equals("black") ? header : null);
columns.put("bright", stage.equals("bright") ? header : null);
return Map.of("key", key, "label", header, "columns", columns, "stages", List.of(stage),
"type", "source_value", "operators", List.of("in", "not_in", "is_empty", "not_empty"));
"type", "source_value", "operators", List.of("in", "contains", "not_in", "is_empty", "not_empty"));
}
}
+5 -3
View File
@@ -144,6 +144,8 @@ def _resolve_source_column_values(task, *, release_id, resolver):
if not isinstance(rules, list):
return None
for rule_index, rule in enumerate(rules):
# Candidate lookup confirms complete values only. A contains keyword is
# intentionally not required to equal a full candidate source value.
if rule.get('operator') not in ('in', 'not_in'):
continue
field_key = rule['field']
@@ -1212,7 +1214,7 @@ def plan_task(*, base_url, api_key, model, question, context, previous=None, can
if aliases and field.get('type')=='source_value':item['aliases']=aliases
compact_fields.append(item)
compact_catalog['sourceFields']=compact_fields
compact_catalog['sourceColumnOperators']=['in','not_in','is_empty','not_empty']
compact_catalog['sourceColumnOperators']=['in','not_in','contains','is_empty','not_empty']
payload['businessCatalog']=compact_catalog
system += ('\n综合以上生成报告应整合previousVerifiedTurns中多个已完成轮次,historyMode=all、freshQuery=false;不得把最新局部追问当作全部历史。'
'只有用户要求更新数据时才freshQuery=true。报告与复述不自动开展新调查。'
@@ -1292,9 +1294,9 @@ JSON字段:goal(完整当前目标),kind(analysis/metrics/members/explain/re
filterUpdates允许department/group/product/projectGroup/extraFilters/exclusions/includeUnassigned。null明确清除该条件,{}表示没有变化;单一维度用目录原名字符串,extraFilters和exclusions用字段到值数组。额外字段仅限businessCatalog.extraFields;客户代码使用extraFilters.customerCode,保留完整字符串。未知字段不能丢弃或扩大范围,应澄清能力限制。汽车交通对应department=新能源汽车磁钢,国内/海外才是其业务分组。
productMatch为ambiguous且未选择时,只需列出授权候选并needsClarification=true,不选择或合并后缀、不换成全部产品。unique使用完整原名;18×15×4不能匹配18×15×4.1。用户选择前轮候选时保留原年份、指标和趋势目标。newTask=true仅用于用户明确另开无关任务,并声明新期间和条件。
今年、上月等使用serverCalendar;最新数据月才用目录,不猜测缺失期间。真正缺少期间或对象有歧义才澄清。普通读取业务指标freshQuery=true。
原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/is_empty/not_empty","values":[完整原文]}。in/not_in要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。大小写、空格、前导零都是字面值,不做归一化。规则AND、values OR;每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。
原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/contains/is_empty/not_empty","values":[完整原文或包含关键字]}。in/not_in/contains要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。规则AND、values OR;文本大小写、空格、前导零均保留原文,不对contains关键字归一化;contains是原文的字面子串筛选,不是通配符或正则。每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。
只问报废率时fields=[scrapRate];同时要求分子分母、计算基数、报废总量或总重时,分别保留对应重量字段;要求分子分母时fields=[scrapRate,scrapWeight,totalWeight]。不能只在goal中提及而漏掉实际fields。分子为现有正式报废总量,分母为该阶段正式总重,黑片使用现有修正总重。利用料、本部报废、过账外废沿用现有分量逻辑,不使用1-合格率,也不平均原表百分比。未指定阶段的报废率返回black和bright两个阶段;没有统一综合报废率定义,用户明确要综合报废率时请求其定义,不转成综合合格率。
按用户原文提出in/not_in文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。
按用户原文提出in/not_in完整文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验in/not_in取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。contains关键字是子串而非完整候选值,不做等值候选核验,按用户原文交由正式查询执行字面子串筛选。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。
kind=members仅用于列出有哪些委托单/批号/规格等唯一成员,不表示原始行;memberFields使用当前目录key且最多5个,必须与用户明确要列出的对象语义相符,不能把筛选字段当输出成员。裸“规格”表示产品规格,要选择目录中真实产品规格列;不能用图号或客户代码代替。明确问图号时选择图号列,明确问客户规格时选择客户规格列。若用户明确给出“成品规格”等目录label,优先精确匹配该label;裸“规格”有多个候选或没有可用字段时,needsClarification=true、memberFields=[],保留原目标并澄清。并列、否定或复合字段意图由规划器按用户表达处理,不能补入已取消字段。stages必须明确为一个或两个阶段,期间与普通filters照常保留。需要澄清时stages可为空,本轮不查询并保留目标。members的fields=[]、verifySource=false、fullDetails=false、freshQuery=true,不计算KPI、不改成metrics;清单由平台直接渲染,无需撰写业务数值。
单期投入重量排名或明确单阶段报废排名:kind=analysis、dimensions为实际product/department/group、fields仅totalWeight/scrapWeight、dimensionStages明确各维度black/bright、trendRequired=false;未限定阶段的报废排名使用queryPlan ranking、fields=[rate,scrapWeight]、stages=[]、dimensionStages对应维度=[];全量正式分母不随Top展示数量改变。普通年度重量排名默认periodCoverage=available,注明已发布月份;只有用户明确要求完整全年齐全时用complete。排名与趋势同时要求时仍保留真实要求,不删除trendRequired。
能力边界必须忠实表达:排名/缺陷保留analysis及真实dimensions和dimensionStages;两个独立期间比较保留comparisonRequired=true和comparisonPeriod;源表核验为source_verify及verifySource=true;原始逐行记录为details及fullDetails=true;报告为report。不得为了适配已接入能力而改写成普通指标。不得把“有哪些”理解成所有原始字段。后端负责说明仍未接入的能力。
+2 -1
View File
@@ -154,7 +154,8 @@ def _scope_label(filters):
if key=='columnFilters':
for rule in value:
stage='、'.join({'black':'黑片','bright':'亮片'}.get(s,s) for s in rule.get('stages',[]))
operation={'in':'为','not_in':'排除','is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'')
operation={'in':'为','not_in':'排除','contains':'包含片段',
'is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'')
parts.append(stage+'来源字段'+operation+'、'.join(rule.get('values',[])))
else: parts.append(names[key]+'='+str(value))
return ';'.join(parts) if parts else '当前授权范围内总体(默认不含未分类)'
+2 -1
View File
@@ -850,7 +850,8 @@ def _history_scope_caption(snapshot, section):
parts.append('包含未分配项')
column_filters = filters.get('columnFilters')
if isinstance(column_filters, list):
operator_labels = {'in': '包含', 'not_in': '不含', 'is_empty': '为空', 'not_empty': '非空'}
operator_labels = {'in': '为', 'not_in': '排除', 'contains': '包含片段',
'is_empty': '为空', 'not_empty': '非空'}
for rule in column_filters:
if not isinstance(rule, dict):
continue
+2 -2
View File
@@ -115,11 +115,11 @@ QUALITY_DATA_SCHEMA: dict[str, Any] = {
"field": {"type": "string", "minLength": 1},
"stages": {"type": "array", "minItems": 1, "maxItems": 2,
"items": {"type": "string", "enum": ["black", "bright"]}},
"operator": {"type": "string", "enum": ["in", "not_in", "is_empty", "not_empty"]},
"operator": {"type": "string", "enum": ["in", "not_in", "contains", "is_empty", "not_empty"]},
"values": {"type": "array", "minItems": 1, "maxItems": 50,
"items": {"type": "string", "maxLength": 256}},
}, "required": ["field", "stages", "operator"]},
"description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR,文本大小写、空格及前导零按原文匹配。is_empty/not_empty不得带values。"},
"description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR;in/not_in匹配完整文本值,contains按原文作字面子串匹配(MySQL LOCATE),不做关键字归一化,大小写、空格及前导零保留原文。每条文本规则提供1到50个不超过256字符的值;is_empty/not_empty不得带values。"},
"view": {"type": "string", "enum": ["overall", "department", "group", "product", "projectGroup"], "description": "query 的正式分组视图;changes 仅支持非 overall 维度。"},
"stage": {"type": ["string", "null"], "enum": ["black", "bright", None], "description": "top/details/defects/investigate必须明确black/bright。rank单阶段传black/bright;rank的metric=scrapWeight且stage省略或null时,合计黑片+亮片报废总重全量排序,并保留正式综合合格率。投入重量rank仍必须指定阶段。changes可选,未传为综合范围。"},
"dimension": {"type": "string", "enum": ["category", "defect", "product", "department", "group", "projectGroup"], "description": "top模式仅category/product/department/group/projectGroup,不能用defect;具体缺陷TOP必须使用mode=defects、dimension=defect。product是规格。changes 使用 product/department/group/projectGroup。defects 使用 defect 或 product;未指定具体 defect 字段时,省略 dimension 默认按 defect 查询阶段具体缺陷。"},
+6 -4
View File
@@ -3,7 +3,7 @@ import copy
DEPARTMENT_ALIASES = {'汽车交通':'新能源汽车磁钢'}
SOURCE_STAGES = {'black', 'bright'}
COLUMN_OPERATORS = {'in', 'not_in', 'is_empty', 'not_empty'}
COLUMN_OPERATORS = {'in', 'not_in', 'contains', 'is_empty', 'not_empty'}
def source_field_labels(source_fields):
@@ -35,11 +35,13 @@ def parse_column_filters(value):
raise ValueError(f'{name}.stages必须是非空black/bright数组。')
if operator not in COLUMN_OPERATORS:raise ValueError(f'{name}.operator不在当前字段支持目录中。')
clean={'field':key,'stages':list(stages),'operator':operator}
if operator in ('in','not_in'):
if operator in ('in','not_in','contains'):
values=rule.get('values')
if (not isinstance(values,list) or not 1<=len(values)<=50
or any(not isinstance(item,str) or item=='' or len(item)>256 for item in values)):
raise ValueError(f'{name}.values必须是1到50个不超过256字符的完整文本值。')
or any(not isinstance(item,str) or item==''
or operator=='contains' and not item.strip()
or len(item)>256 for item in values)):
raise ValueError(f'{name}.values必须是1到50个不超过256字符的文本值或包含关键字。')
clean['values']=list(dict.fromkeys(values))
elif 'values' in rule:raise ValueError(f'{name}的空值操作不接受values。')
result.append(clean)
+9
View File
@@ -34,6 +34,15 @@ def coverage_comparison():
class BoundReportTest(unittest.TestCase):
def test_scope_label_distinguishes_full_value_from_substring_filters(self):
from nao_evidence_report import _scope_label
label = _scope_label({'columnFilters': [
{'field': 'orderNo', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7']},
{'field': 'orderNo', 'stages': ['bright'], 'operator': 'contains', 'values': ['07']},
]})
self.assertIn('黑片来源字段为WO-7', label)
self.assertIn('亮片来源字段包含片段07', label)
def test_priority_summary_keeps_three_signals_and_rate_denominators_separate(self):
signals={'scrapIncrease':[{'label':'放量规格','scrapDelta':120}],
'currentScrap':[{'label':'存量规格','current':{'scrapWeight':500}}],
+8 -4
View File
@@ -456,9 +456,11 @@ class HistoryReportExecutionTest(unittest.TestCase):
first_section = candidate['sections'][0]
first_section['filters']['columnFilters'] = [{
'field': 'src_work_order', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7'],
}, {
'field': 'src_contains_code', 'stages': ['bright'], 'operator': 'contains', 'values': ['07'],
}]
set_source_task_context(candidate, SOURCE_A,
{'sourceFieldLabels': {'src_work_order': '黑片委托单号'}})
{'sourceFieldLabels': {'src_work_order': '黑片委托单号', 'src_contains_code': '客户代码'}})
set_source_task_context(candidate, SOURCE_B,
{'sourceFieldLabels': {'src_work_order': '亮片委托单号'}})
fact = candidate['facts'][0]
@@ -476,12 +478,14 @@ class HistoryReportExecutionTest(unittest.TestCase):
projected = _history_model_fact_projection(candidate, [fact])[0]
self.assertIn('产业部 消费电子产业部', projected['modelScope']['displayScope'])
self.assertIn('业务分组 消费电子-3C', projected['modelScope']['displayScope'])
self.assertIn('黑片委托单号 包含 WO-7', projected['modelScope']['displayScope'])
self.assertIn('黑片委托单号 为 WO-7', projected['modelScope']['displayScope'])
self.assertIn('亮片 客户代码 包含片段 07', projected['modelScope']['displayScope'])
self.assertIn('消费电子产业部', projected['modelText'])
self.assertIn('消费电子-3C', projected['modelText'])
self.assertIn('黑片委托单号', projected['modelText'])
self.assertIn('WO-7', projected['modelText'])
for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:', 'src_work_order'):
for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:',
'src_work_order', 'src_contains_code'):
self.assertNotIn(machine_label, projected['modelText'])
self.assertNotIn(machine_label, json.dumps(projected['modelScope'], ensure_ascii=False))
self.assertEqual(projected['text'], source_text)
@@ -581,7 +585,7 @@ class HistoryReportExecutionTest(unittest.TestCase):
goals = payload['historySourceGoals']
self.assertTrue(any('产业部 消费电子产业部' in goal['displayScopeLabel']
and '业务分组 消费电子-3C' in goal['displayScopeLabel']
and '黑片委托单号 包含 WO-7' in goal['displayScopeLabel']
and '黑片委托单号 为 WO-7' in goal['displayScopeLabel']
for goal in goals))
self.assertTrue(all('filters' not in goal and '_sourceTurnId' not in goal for goal in goals))
self.assertTrue(all('filters' not in scope and 'sourceTurnId' not in scope
+6 -2
View File
@@ -284,6 +284,10 @@ class QualityToolsTests(unittest.TestCase):
self.assertEqual(definition["inputSchema"]["properties"]["mode"]["enum"],
["catalog", "query", "rank", "compare", "top", "changes", "defects", "details", "investigate"])
self.assertIn("defect", definition["inputSchema"]["properties"])
column_filter = definition["inputSchema"]["properties"]["columnFilters"]
self.assertEqual(column_filter["items"]["properties"]["operator"]["enum"],
["in", "not_in", "contains", "is_empty", "not_empty"])
self.assertIn("MySQL LOCATE", column_filter["description"])
self.assertFalse(definition["inputSchema"]["additionalProperties"])
self.assertEqual(definition["inputSchema"]["allOf"][0]["then"]["required"], ["releaseId"])
@@ -529,11 +533,11 @@ class QualityToolsTests(unittest.TestCase):
if path == "/api/analysis/tools/catalog":
response["sourceFields"] = [{"key": "customerCode", "label": "客户代码",
"type": "source_value", "stages": ["black"],
"columns": {"black": "customer_code"}, "operators": ["in"]}]
"columns": {"black": "customer_code"}, "operators": ["in", "contains"]}]
return response
api = AuthorizedColumnApi()
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
rule = {"field": "customerCode", "stages": ["black"], "operator": "in", "values": ["C-17"]}
rule = {"field": "customerCode", "stages": ["black"], "operator": "contains", "values": ["C-17"]}
result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "monthStart": 3, "monthEnd": 9},
"filters": {"exclusions": {"product": ["规格甲"]}}, "columnFilters": [rule],
+29 -2
View File
@@ -25,7 +25,7 @@ from test_nao_quality_tools import FakeApi, RELEASE
def source_field(key, label, black, bright, stages=('black', 'bright')):
return {'key': key, 'label': label, 'columns': {'black': black, 'bright': bright},
'stages': list(stages), 'type': 'source_value',
'operators': ['in', 'not_in', 'is_empty', 'not_empty']}
'operators': ['in', 'not_in', 'contains', 'is_empty', 'not_empty']}
SOURCE_FIELDS = [
@@ -349,6 +349,15 @@ class SourceColumnProtocolTests(unittest.TestCase):
with self.assertRaisesRegex(TaskUnavailable, '不超过256字符'):
protocol_task(columnFilters=[{**rule, 'values': ['X' * 257]}])
def test_contains_keywords_are_preserved_as_literal_or_values(self):
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains',
'values': [' 001', 'A|B', 'abC']}
task = protocol_task(columnFilters=[rule])
self.assertEqual(task['filters']['columnFilters'], [rule])
for invalid in ([], [''], [' '], ['x' * 257], [1], ['one'] * 51):
with self.subTest(invalid=invalid), self.assertRaises(TaskUnavailable):
protocol_task(columnFilters=[{**rule, 'values': invalid}])
def test_protocol_presentation_enums_default_consistently_when_omitted_or_null(self):
base = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3}, comparisonRequired=False,
@@ -396,6 +405,10 @@ class SourceColumnProtocolTests(unittest.TestCase):
self.assertIn('periodCoverage(available/complete,可省略或null', prompt)
self.assertIn('freshQuery(bool)', prompt)
self.assertIn('needsClarification(bool)', prompt)
self.assertIn('in/not_in/contains/is_empty/not_empty', prompt)
self.assertIn('contains关键字是子串而非完整候选值', prompt)
self.assertEqual(model_call.call_args_list[0].kwargs['payload']['businessCatalog']['sourceColumnOperators'],
['in', 'not_in', 'contains', 'is_empty', 'not_empty'])
def test_three_arbitrary_source_keys_keep_literal_values_and_stage_scopes(self):
task = protocol_task()
@@ -429,7 +442,7 @@ class SourceColumnProtocolTests(unittest.TestCase):
def test_unauthorized_unknown_and_missing_stage_columns_fail_closed(self):
for bad in ([{**RULES[0], 'field': 'notInDirectory'}],
[{**RULES[0], 'field': 'blackOnlyCode', 'stages': ['bright']}],
[{**RULES[0], 'operator': 'contains'}]):
[{**RULES[0], 'operator': 'starts_with'}]):
with self.subTest(bad=bad), self.assertRaises(TaskUnavailable):
normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026}, columnFilters=bad), '筛选原表字段',
@@ -529,6 +542,20 @@ class SourceColumnProtocolTests(unittest.TestCase):
self.assertEqual(customer_lookup['filters']['columnFilters'], [
{'field': 'dispatchOrder', 'stages': ['bright'], 'operator': 'not_empty'}])
def test_contains_keyword_skips_exact_candidate_value_validation(self):
from nao_agent_task import _resolve_source_column_values
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains',
'values': ['001']}
task = protocol_task(columnFilters=[rule])
calls = []
def resolver(**contract):
calls.append(contract)
raise AssertionError('contains is a substring, not a complete candidate value')
result = _resolve_source_column_values(task, release_id=RELEASE, resolver=resolver)
self.assertEqual(result, {'status': 'exact'})
self.assertEqual(calls, [])
self.assertEqual(task['filters']['columnFilters'], [rule])
def test_missing_literal_offers_same_scope_alternatives_without_substitution(self):
from nao_agent_task import _resolve_source_column_values
task = protocol_task(columnFilters=[RULES[0]])
+1 -1
View File
@@ -32,6 +32,6 @@ export function StageSourceFilters({stage,fields,query,value,onChange,onUnauthor
const selected=value.map(rule=>rule.field);const patch=(key:string,change:Partial<SourceColumnFilter>)=>onChange(value.map(rule=>rule.field===key?{...rule,...change}:rule));
return <div className="stage-source-filters">
{part!=='rows'&&<Select {...readableSelectPopup} mode="multiple" maxCount={20} maxTagCount="responsive" showSearch optionFilterProp="label" allowClear aria-label="选择筛选字段" placeholder="添加筛选字段" value={selected} options={fields.map(field=>({label:field.label,value:field.key}))} onChange={keys=>onChange(keys.map(key=>value.find(rule=>rule.field===key)||{field:key,stages:[stage],operator:'in',values:[]}))}/>}
{part!=='picker'&&value.map(rule=>{const field=fields.find(item=>item.key===rule.field);return field?<div className="stage-source-filter-row" key={rule.field}><strong>{field.label}</strong><Select {...readableSelectPopup} aria-label={`${field.label}筛选方式`} value={rule.operator} options={[{label:'等于任一值',value:'in'},{label:'不等于这些值',value:'not_in'},{label:'为空',value:'is_empty'},{label:'不为空',value:'not_empty'}]} onChange={operator=>patch(rule.field,{operator,values:[]})}/>{['in','not_in'].includes(rule.operator)?<SourceValues field={field} stage={stage} query={query} value={rule.values||[]} onChange={values=>patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:<span/>}<Button type="text" aria-label={`移除${field.label}筛选`} icon={<CloseOutlined/>} onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}/></div>:<Alert key={rule.field} type="warning" message="所选来源字段当前不可用,请移除后重新选择" action={<Button onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}>移除</Button>}/>;})}
{part!=='picker'&&value.map(rule=>{const field=fields.find(item=>item.key===rule.field);return field?<div className="stage-source-filter-row" key={rule.field}><strong>{field.label}</strong><Select {...readableSelectPopup} aria-label={`${field.label}筛选方式`} value={rule.operator} options={[{label:'等于任一值',value:'in'},{label:'包含',value:'contains'},{label:'不等于这些值',value:'not_in'},{label:'为空',value:'is_empty'},{label:'不为空',value:'not_empty'}]} onChange={operator=>patch(rule.field,{operator,values:[]})}/>{rule.operator==='contains'?<Select {...readableSelectPopup} mode="tags" maxCount={50} maxLength={256} maxTagCount="responsive" allowClear aria-label={`${field.label}包含文字`} placeholder="输入关键词后按回车" value={rule.values||[]} onChange={values=>patch(rule.field,{values:values.filter(value=>value.trim().length>0)})} options={[]}/>:['in','not_in'].includes(rule.operator)?<SourceValues field={field} stage={stage} query={query} value={rule.values||[]} onChange={values=>patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:<span/>}<Button type="text" aria-label={`移除${field.label}筛选`} icon={<CloseOutlined/>} onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}/></div>:<Alert key={rule.field} type="warning" message="所选来源字段当前不可用,请移除后重新选择" action={<Button onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}>移除</Button>}/>;})}
</div>;
}
+4 -4
View File
@@ -7,7 +7,7 @@ export type AnalysisVisualSectionKey = 'overview' | 'stage' | 'department' | 'gr
export type AnalysisVisualDimension = 'department' | 'group' | 'product' | 'category' | 'defect';
export type AnalysisExclusionDimension = 'department' | 'group' | 'product' | 'projectGroup';
export type AnalysisExclusions = Partial<Record<AnalysisExclusionDimension, string[]>>;
export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export type AnalysisVisualContextValue = string | number | boolean | Array<string | number | boolean> | Record<string, string[]> | Record<string, string> | SourceColumnFilter[] | AnalysisExclusions;
export type AnalysisVisualContext = Record<string, AnalysisVisualContextValue>;
@@ -253,8 +253,8 @@ function safeContext(value: unknown): AnalysisVisualContext | null {
|| typeof rule.field !== 'string' || !/^[A-Za-z0-9_]{1,64}$/.test(rule.field)
|| !Array.isArray(rule.stages) || !rule.stages.length || rule.stages.length > 2 || new Set(rule.stages).size !== rule.stages.length
|| rule.stages.some(stage => stage !== 'black' && stage !== 'bright')
|| !['in', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null;
const needsValues = rule.operator === 'in' || rule.operator === 'not_in';
|| !['in', 'contains', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null;
const needsValues = rule.operator === 'in' || rule.operator === 'not_in' || rule.operator === 'contains';
if (needsValues && (!Array.isArray(rule.values) || !rule.values.length || rule.values.length > 50 || rule.values.some(value => typeof value !== 'string' || !value.trim() || value.length > 256))) return null;
if (!needsValues && rule.values !== undefined) return null;
filters.push({ field: rule.field, stages: [...rule.stages] as SourceColumnFilter['stages'], operator: rule.operator as SourceColumnFilter['operator'], ...(needsValues ? { values: [...rule.values as string[]] } : {}) });
@@ -1397,7 +1397,7 @@ export function columnFilterSummary(value: unknown, labelValue: unknown): string
if (!isRecord(rule) || typeof rule.field !== 'string' || !Array.isArray(rule.stages)) return [];
const stage = rule.stages.map(item => item === 'black' ? '黑片' : item === 'bright' ? '亮片' : '').filter(Boolean).join('、');
const label = typeof labels[rule.field] === 'string' ? labels[rule.field] : '来源字段';
const operation = ({ in: '为', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record<string, string>)[String(rule.operator)];
const operation = ({ in: '为', contains: '包含', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record<string, string>)[String(rule.operator)];
if (!operation) return [];
const values = Array.isArray(rule.values) ? rule.values.filter(item => typeof item === 'string').join('、') : '';
return [`${stage} ${label}${operation}${values}`.trim()];
+1 -1
View File
@@ -3,7 +3,7 @@ export type ViewKind = 'overall' | 'department' | 'group' | 'product' | 'project
export type PeriodKind = 'year' | 'quarter' | 'month';
export interface SourceColumnField { key: string; label: string; type: string; stages: Stage[]; columns: Partial<Record<Stage,string>> }
export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export interface QueryParams {
year: number;
@@ -1042,3 +1042,12 @@ test('adaptive trends cover every series, preserve missing values, and keep non-
assert.equal(formatVisualValue(.9639601707533932, 'ratio'), '96.40%');
assert.equal(formatVisualValue(3, 'kg'), '3.00 kg');
});
test('contains keywords survive analysis scope and display as substring filters',()=>{
const rule={field:'src_any',stages:['bright'],operator:'contains',values:['日立','%_']};
const parsed=parseAnalysisVisuals([visual({scope:{columnFilters:[rule],columnLabels:{src_any:'客户'}}})]);
assert.equal(parsed.visuals.length,1);
assert.deepEqual(parsed.visuals[0].scope.columnFilters,[rule]);
assert.match(columnFilterSummary([rule],{src_any:'客户'})[0],/包含.*日立/);
assert.equal(parseAnalysisVisuals([visual({scope:{columnFilters:[{...rule,values:[]}]}})]).visuals.length,0);
});
+30
View File
@@ -0,0 +1,30 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
const source=fs.readFileSync(new URL('../src/StageSourceFilters.tsx',import.meta.url),'utf8');
const output=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX,target:ts.ScriptTarget.ES2022}}).outputText;
const jsx=(type,props)=>({type,props});const exports={};
new Function('require','exports',output)(name=>{
if(name==='react/jsx-runtime')return {jsx,jsxs:jsx};
if(name==='antd')return {Select:'Select',Button:'Button',Alert:'Alert',Spin:'Spin'};
if(name==='@ant-design/icons')return {CloseOutlined:'CloseOutlined'};
if(name==='./readableSelectPopup')return {readableSelectPopup:{}};
return {};
},exports);
const nodes=value=>!value||typeof value!=='object'?[]:[value,...[value.props?.children].flat(Infinity).flatMap(nodes)];
const field={key:'customer',label:'客户名称',type:'source_value',stages:['bright'],columns:{bright:'客户名称'}};
test('contains is selectable and accepts bounded free keywords without a value-candidate lookup',()=>{
let changed;
const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立']};
const root=exports.StageSourceFilters({stage:'bright',fields:[field],query:{year:2026},value:[rule],onChange:value=>{changed=value;},onUnauthorized:()=>false});
const all=nodes(root);const operation=all.find(x=>x.type==='Select'&&x.props['aria-label']==='客户名称筛选方式');
assert.ok(operation.props.options.some(x=>x.value==='contains'&&x.label==='包含'));
const keywords=all.find(x=>x.type==='Select'&&x.props['aria-label']==='客户名称包含文字');
assert.equal(keywords.props.mode,'tags');assert.equal(keywords.props.maxCount,50);assert.equal(keywords.props.maxLength,256);assert.deepEqual(keywords.props.options,[]);
keywords.props.onChange(['日立','%_',' ']);
assert.deepEqual(changed,[{...rule,values:['日立','%_']}]);
operation.props.onChange('in');assert.deepEqual(changed,[{...rule,operator:'in',values:[]}]);
});
@@ -30,3 +30,9 @@ test('stage query includes unclassified source rows even when an old link exclud
assert.equal(base.includeUnassigned,false);
}
});
test('contains keywords are retained while an unfilled contains rule is inactive',()=>{
const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立','%_']};
assert.deepEqual(activeStageFilters([rule],'bright'),[rule]);
assert.deepEqual(activeStageFilters([{...rule,values:[]}],'bright'),[]);
});