Compare commits
13
Commits
5457f5c786
...
381901184c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
381901184c | ||
|
|
6983dc32eb | ||
|
|
f47341f3f0 | ||
|
|
f8432b4e02 | ||
|
|
3c022e8614 | ||
|
|
7cae4cfbdf | ||
|
|
20f1c7d0e2 | ||
|
|
fa70dc6504 | ||
|
|
7bd9e51004 | ||
|
|
5e02c6ad66 | ||
|
|
7782e33bec | ||
|
|
5068b8fbc9 | ||
|
|
ea84e26b2f |
@@ -58,6 +58,14 @@ DSH CLI 固定为 `0.2.2-rc.1`,依赖与官方 Python SDK 校验记录在 `run
|
||||
|
||||
底部表格同步展开当前分类的细项,重量/废率切换与 Excel 导出跟随当前显示;产品和期间行维度保持不变。细项使用正式阶段同期总重量作分母,月、季、年均先汇总分量再计算;没有来源或分母时显示缺失,真实零显示零。细项与正式分类总量不一致时提示覆盖不足,不补造分类。
|
||||
|
||||
## 筛选索引与查询加载
|
||||
|
||||
来源字段支持等于、包含、排除和空值筛选;“包含”输入关键词后按回车生效,多个关键词任一命中,区分大小写,`%`、`_` 按普通文字匹配,每个关键词最多256字符、最多50个。各类筛选共用按构建冻结的值字典及记录索引;汇总、趋势、来源明细和导出沿用当前授权与正式指标计算。源字段值保留空值、大小写及前导零语义,旧配置筛选的字符串 `null` 约定不变。下拉框使用只读候选模式,只取一页及是否还有;需要精确总数的原接口继续保留。
|
||||
|
||||
启动时在单线程后台补建当前完整正式版本的索引,导入及归类在事务提交后进入同一去重队列。完整构建标记提交后才启用;未完成、失败或未补建的历史构建沿用原始查询,不阻止启动和正式数据发布。索引增加持久存储及后台构建成本,规模取决于非空字段数和不同值数量;高基数宽表应先测容量。保留清理随无引用构建回收索引;升级后原维护兼容保护继续生效。
|
||||
|
||||
黑片/亮片页的普通查询在数据版本已解析后独立加载,来源字段条件仍等待对应版本的字段目录;产品搜索不因无关元数据失败而禁用。
|
||||
|
||||
## 旧版本时间迁移
|
||||
|
||||
连接与 MySQL 会话现在统一使用 UTC,界面仍按本地时区显示。V25 只校正旧代码按 UTC 墙上时间写入的截止/有效期、报告调度及报告租约列(前者由 Java 写入,租约使用 SQL UTC_TIMESTAMP),不改 SQL CURRENT_TIMESTAMP 自动生成的创建时间,也不修改业务日期或指标。若旧库存在此类值,迁移要求先备份并核实旧会话相对 UTC 的偏移,然后设置 `MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS`(旧会话 UTC 为 `0`,旧会话中国标准时间为 `28800`)。本机启动可在私有 local-config 中记录 `legacyTimestampOffsetSeconds`;不能仅凭电脑当前时区猜测。全新空库不需要该参数;迁移标记和历史值修改在同一事务提交,防止提交确认丢失后重复平移。还原旧备份时应重新核实其原运行时区。
|
||||
|
||||
@@ -369,7 +369,7 @@ public class AnalysisExplorationTools {
|
||||
SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value);
|
||||
SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value);
|
||||
appendDetailExclusions(sql, args, query);
|
||||
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor);
|
||||
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor,releaseId);
|
||||
DataScope.append(sql, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
|
||||
"f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
boolean includeUnassigned = query.includeUnassigned()
|
||||
|
||||
@@ -276,7 +276,7 @@ public class AnalysisFineDefectsService {
|
||||
SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value);
|
||||
if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))
|
||||
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
|
||||
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor);
|
||||
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor,releaseId);
|
||||
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
|
||||
"f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
}
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ public class AnalysisInvestigationService {
|
||||
SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x);
|
||||
SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x);
|
||||
if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
|
||||
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor);
|
||||
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor,release);
|
||||
DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
}
|
||||
if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());}
|
||||
|
||||
+1
-1
@@ -891,7 +891,7 @@ final class AnalysisProtocolHistoryFacts {
|
||||
String label=text(columnLabels.get(field),historyFieldLabel(section,sources,field));
|
||||
if(label.isBlank())label="授权列";
|
||||
List<String> prefix=strings(rule.get("stages")).stream().map(AnalysisProtocolHistoryFacts::stageLabel).toList();
|
||||
String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";};
|
||||
String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "contains"->"包含文字";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";};
|
||||
List<String> values=scopeLiterals(rule.get("values"));
|
||||
StringBuilder part=new StringBuilder();if(!prefix.isEmpty())part.append(String.join("、",prefix)).append(' ');
|
||||
part.append(label).append(' ').append(operator);if(!values.isEmpty())part.append(' ').append(String.join("、",values));parts.add(part.toString());
|
||||
|
||||
+7
-3
@@ -43,12 +43,16 @@ public class AnalysisSourceQueryController {
|
||||
@PostMapping("/values")
|
||||
public Map<String,Object> values(@RequestBody Map<String,Object> input,@AuthenticationPrincipal AppUserPrincipal actor) {
|
||||
String scope=captureScope(actor);
|
||||
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
|
||||
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset","candidateOnly").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
|
||||
String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null));
|
||||
Map<String,Object> context=context(input);
|
||||
if(!(input.get("field") instanceof String field)||field.isBlank()||input.get("search")!=null&&!(input.get("search") instanceof String))throw bad("候选字段或搜索值无效");
|
||||
Map<String,Object> result=new LinkedHashMap<>(columns.values(release,context,field,strings(input.get("stages"),2),Objects.toString(input.get("search"),null),
|
||||
integer(input.getOrDefault("limit",50),1,200),integer(input.getOrDefault("offset",0),0,100000),actor));
|
||||
if(input.containsKey("candidateOnly")&&!(input.get("candidateOnly") instanceof Boolean))throw bad("候选分页参数无效");
|
||||
List<String> stages=strings(input.get("stages"),2);String search=Objects.toString(input.get("search"),null);
|
||||
int limit=integer(input.getOrDefault("limit",50),1,200),offset=integer(input.getOrDefault("offset",0),0,100000);
|
||||
Map<String,Object> result=new LinkedHashMap<>(Boolean.TRUE.equals(input.get("candidateOnly"))
|
||||
? columns.values(release,context,field,stages,search,limit,offset,true,actor)
|
||||
: columns.values(release,context,field,stages,search,limit,offset,actor));
|
||||
result.put("period",input.get("period"));result.put("filters",input.get("filters"));
|
||||
result.put("values",AnalysisProtocolService.list(result.get("rows")).stream().map(row->AnalysisProtocolService.map(row).get("value")).toList());
|
||||
result.put("candidateOnly",true);reauthorize(actor,scope,false);return result;
|
||||
|
||||
@@ -111,7 +111,7 @@ public class AnalyticsController {
|
||||
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前字段授权不包含所选报废类别");
|
||||
List<Object> args = new ArrayList<>(extra.isEmpty()?List.of(selected, selected):List.of(selected));
|
||||
StringBuilder where = baseWhere(args, year, month, quarter, department, group, product, projectGroup, includeUnassigned, actor);
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor);
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor,selected);
|
||||
List<AggregateTop> aggregated;
|
||||
if (dimension.equals("category")) {
|
||||
List<Component> selectedComponents = selectedCategory == null ? visibleComponents : List.of(selectedCategory);
|
||||
|
||||
@@ -36,9 +36,11 @@ public class FilterFieldsController {
|
||||
governance.requireUserActivePolicy(actor);
|
||||
Map<String,List<String>> result=new LinkedHashMap<>();
|
||||
String release=releases.resolvePublishedRelease(null);
|
||||
SourceFilterIndex index=new SourceFilterIndex(jdbc);
|
||||
for(String stage:List.of("black","bright")) {
|
||||
// Raw source headers are stable within each build. Read a representative row of every current build.
|
||||
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2 WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
|
||||
String hint=index.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":"";
|
||||
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2"+hint+" WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
|
||||
Set<String> columns=new TreeSet<>();
|
||||
var json=tools.jackson.databind.json.JsonMapper.builder().build();
|
||||
Set<String> allowed=governance.allowedFields(actor);
|
||||
|
||||
@@ -48,6 +48,9 @@ public class QueryFilterFields {
|
||||
return Collections.unmodifiableMap(result);
|
||||
}
|
||||
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor) {
|
||||
append(where,args,stage,filters,actor,null);
|
||||
}
|
||||
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor,String releaseId) {
|
||||
if(filters==null || filters.isEmpty())return;
|
||||
Map<String,Field> fields=new HashMap<>(); visible(actor).forEach(f->fields.put(f.key(),f));
|
||||
for(var entry:filters.entrySet()) {
|
||||
@@ -61,6 +64,7 @@ public class QueryFilterFields {
|
||||
boolean empty=entry.getValue().contains(EMPTY);
|
||||
if(empty && !f.allowEmpty())throw bad(f.label()+"不允许筛选空值");
|
||||
String column=stage.equals("black")?f.blackColumn():f.brightColumn();
|
||||
if(releaseId!=null&&new SourceFilterIndex(jdbc).appendLegacy(where,args,stage,releaseId,column,entry.getValue()))continue;
|
||||
// Paths and selected values are bound parameters; never interpolate configured identifiers.
|
||||
String expression="NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs";
|
||||
List<String> values=entry.getValue().stream().filter(v->!v.equals(EMPTY)).toList();
|
||||
@@ -90,10 +94,15 @@ public class QueryFilterFields {
|
||||
StringBuilder scope=new StringBuilder(" WHERE rm.release_id=?");
|
||||
List<Object> scoped=new ArrayList<>();scoped.add(releaseId);
|
||||
DataScope.append(scope,scoped,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
var index=new SourceFilterIndex(jdbc);
|
||||
if(index.ready(releaseId)) {
|
||||
selects.add(index.candidateSelect(stage,column,scope.toString(),scoped,args).replace("AS candidate_value","AS value"));
|
||||
continue;
|
||||
}
|
||||
selects.add("SELECT NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs AS value FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+scope);
|
||||
args.add(path(column));args.addAll(scoped);
|
||||
}
|
||||
String predicate=" WHERE value IS NOT NULL";
|
||||
String predicate=" WHERE value IS NOT NULL AND CAST(value AS BINARY)<>CAST('null' AS BINARY)";
|
||||
if(search!=null && !search.isBlank()) { predicate+=" AND LOCATE(LOWER(?),LOWER(value))>0";args.add(search); }
|
||||
List<String> found=jdbc.query("SELECT DISTINCT value FROM ("+String.join(" UNION ALL ",selects)+") choices"+predicate+" ORDER BY value LIMIT 201",(rs,n)->rs.getString(1),args.toArray());
|
||||
return Map.of("values",found.stream().limit(200).toList(),"hasMore",found.size()>200);
|
||||
|
||||
@@ -60,8 +60,18 @@ public class SourceColumnQueries {
|
||||
}
|
||||
return List.copyOf(result);
|
||||
}
|
||||
// 只提取一次,并区分 JSON null 与字符串“null”,供筛选、候选值和成员读取共用。
|
||||
private static final String SOURCE_TEXT = "JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON)))";
|
||||
private static final String SOURCE_VALUE = "NULLIF(TRIM("+SOURCE_TEXT+"), '') COLLATE utf8mb4_0900_as_cs";
|
||||
|
||||
/** 原始字段筛选按发布构建读取,避免优化器从整年月份范围扫描大 JSON 原表。 */
|
||||
public static String factIndexHint(String stage) {
|
||||
if(!List.of("black","bright").contains(stage))throw bad("数据阶段无效");
|
||||
return " FORCE INDEX (uq_fact_"+stage+"_origin)";
|
||||
}
|
||||
|
||||
private static final List<String> STAGES = List.of("black", "bright");
|
||||
private static final List<String> OPERATORS = List.of("in", "not_in", "is_empty", "not_empty");
|
||||
private static final List<String> OPERATORS = List.of("in", "contains", "not_in", "is_empty", "not_empty");
|
||||
private static final Map<String, String> CANONICAL_COLUMNS = Map.of(
|
||||
"product", "product", "department", "department", "group", "group_name", "projectGroup", "project_group");
|
||||
private static final Map<String, String> CANONICAL_LABELS = Map.of(
|
||||
@@ -70,11 +80,17 @@ public class SourceColumnQueries {
|
||||
private final JdbcTemplate jdbc;
|
||||
private final GovernanceAccessService governance;
|
||||
private final JsonMapper json;
|
||||
private SourceFilterIndex filterIndex;
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
public void setFilterIndex(SourceFilterIndex index) { this.filterIndex=index; }
|
||||
public boolean indexed(String release) { return filterIndex.ready(release); }
|
||||
|
||||
|
||||
public SourceColumnQueries(JdbcTemplate jdbc, GovernanceAccessService governance, JsonMapper json) {
|
||||
this.jdbc = jdbc;
|
||||
this.governance = governance;
|
||||
this.json = json;
|
||||
this.filterIndex = new SourceFilterIndex(jdbc);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -94,16 +110,16 @@ public class SourceColumnQueries {
|
||||
Map<String, Boolean> stagePresent = new LinkedHashMap<>();
|
||||
for (String stage : STAGES) {
|
||||
Set<String> headers = new LinkedHashSet<>();
|
||||
StringBuilder representativeScope = new StringBuilder(" WHERE rm.release_id=?");
|
||||
StringBuilder representativeScope = new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
|
||||
List<Object> args = new ArrayList<>(List.of(releaseId));
|
||||
DataScope.append(representativeScope, args, actor,
|
||||
"f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
// Stop at the first authorized fact of each build instead of aggregating every fact id.
|
||||
String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) "
|
||||
+ "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id "
|
||||
+ "FROM release_member rm JOIN fact_" + stage + " f2 "
|
||||
+ "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"
|
||||
+ representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative "
|
||||
+ "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id";
|
||||
+ "FROM (SELECT DISTINCT transform_build_id,source_revision_id FROM release_member WHERE release_id=?) rm "
|
||||
+ "JOIN fact_" + stage + " f ON f.id=(SELECT f2.id FROM fact_" + stage + " f2"
|
||||
+ (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage))
|
||||
+ representativeScope + " ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1) ORDER BY rm.transform_build_id";
|
||||
List<String> keyLists = rawHeaders.get(new QueryKey(sql,args,"raw-headers"),
|
||||
() -> Collections.unmodifiableList(new ArrayList<>(jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray()))));
|
||||
stagePresent.put(stage, !keyLists.isEmpty());
|
||||
@@ -192,12 +208,13 @@ public class SourceColumnQueries {
|
||||
normalizedStages.add(value);
|
||||
}
|
||||
List<String> values = new ArrayList<>();
|
||||
if (operator.equals("in") || operator.equals("not_in")) {
|
||||
if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) {
|
||||
if (!(rule.get("values") instanceof List<?> rawValues) || rawValues.isEmpty() || rawValues.size() > 50)
|
||||
throw bad("in/not_in 筛选必须提供1到50个字符串值");
|
||||
throw bad("文本筛选必须提供1到50个字符串值");
|
||||
for (Object value : rawValues) {
|
||||
if (!(value instanceof String text) || text.isEmpty() || text.length() > 256)
|
||||
throw bad("来源列筛选值必须是1到256字符的完整字符串;空值请使用空值操作符");
|
||||
if(operator.equals("contains")&&text.isBlank())throw bad("包含筛选的关键词不能全为空白");
|
||||
if (!values.contains(text)) values.add(text);
|
||||
}
|
||||
} else if (rule.containsKey("values")) {
|
||||
@@ -207,7 +224,7 @@ public class SourceColumnQueries {
|
||||
normalized.put("field", field);
|
||||
normalized.put("stages", List.copyOf(normalizedStages));
|
||||
normalized.put("operator", operator);
|
||||
if (operator.equals("in") || operator.equals("not_in")) normalized.put("values", List.copyOf(values));
|
||||
if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) normalized.put("values", List.copyOf(values));
|
||||
parsed.add(Collections.unmodifiableMap(normalized));
|
||||
}
|
||||
return List.copyOf(parsed);
|
||||
@@ -239,11 +256,10 @@ public class SourceColumnQueries {
|
||||
String operator = (String) rule.get("operator");
|
||||
String expression;
|
||||
Object column = ((Map<?, ?>) field.get("columns")).get(stage);
|
||||
@SuppressWarnings("unchecked") List<String> selected = (List<String>) rule.getOrDefault("values",List.of());
|
||||
if("source_value".equals(field.get("type")) && filterIndex.append(where,args,stage,releaseId,String.valueOf(column),operator,selected))continue;
|
||||
if ("source_value".equals(field.get("type"))) {
|
||||
expression = "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL "
|
||||
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
|
||||
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
|
||||
+ "COLLATE utf8mb4_0900_as_cs";
|
||||
expression = SOURCE_VALUE;
|
||||
} else {
|
||||
String factColumn = CANONICAL_COLUMNS.get(key);
|
||||
if (factColumn == null) throw bad("规范维度无效");
|
||||
@@ -255,19 +271,26 @@ public class SourceColumnQueries {
|
||||
if (operator.equals("in")) {
|
||||
where.append(expression).append(" IN (")
|
||||
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append(')');
|
||||
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column));
|
||||
if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
|
||||
} else {
|
||||
where.append('(').append(expression).append(" IS NULL OR ").append(expression).append(" NOT IN (")
|
||||
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append("))");
|
||||
if ("source_value".equals(field.get("type"))) {
|
||||
bindSourcePaths(args, String.valueOf(column));
|
||||
bindSourcePaths(args, String.valueOf(column));
|
||||
bindSourcePath(args, String.valueOf(column));
|
||||
bindSourcePath(args, String.valueOf(column));
|
||||
}
|
||||
}
|
||||
args.addAll(values);
|
||||
} else if (operator.equals("contains")) {
|
||||
for(int i=0;i<selected.size();i++) {
|
||||
if(i>0)where.append(" OR ");
|
||||
where.append("LOCATE(?,").append(expression).append(")>0");
|
||||
args.add(selected.get(i));
|
||||
if("source_value".equals(field.get("type")))bindSourcePath(args,String.valueOf(column));
|
||||
}
|
||||
} else {
|
||||
where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL");
|
||||
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column));
|
||||
if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
|
||||
}
|
||||
where.append(')');
|
||||
}
|
||||
@@ -376,12 +399,16 @@ public class SourceColumnQueries {
|
||||
public Map<String, Object> values(String releaseId, Map<String, Object> context, String fieldKey,
|
||||
List<String> stages, String search, int limit, int offset,
|
||||
AppUserPrincipal actor) {
|
||||
return values(releaseId,context,fieldKey,stages,search,limit,offset,false,actor);
|
||||
}
|
||||
public Map<String,Object> values(String releaseId,Map<String,Object> context,String fieldKey,List<String> stages,
|
||||
String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
|
||||
requireQueryAccess(actor);
|
||||
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,actor));
|
||||
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,candidateOnly,actor));
|
||||
}
|
||||
|
||||
private Map<String,Object> valuesSnapshot(String releaseId,Map<String,Object> context,String fieldKey,
|
||||
List<String> stages,String search,int limit,int offset,AppUserPrincipal actor) {
|
||||
List<String> stages,String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
|
||||
if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效");
|
||||
if (fieldKey == null || fieldKey.isBlank()) throw bad("来源列标识无效");
|
||||
if (stages == null || stages.isEmpty() || stages.size() > STAGES.size()
|
||||
@@ -412,6 +439,10 @@ public class SourceColumnQueries {
|
||||
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
|
||||
List<Object> whereArgs = new ArrayList<>(List.of(releaseId));
|
||||
appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor);
|
||||
if("source_value".equals(field.get("type")) && filterIndex.ready(releaseId)) {
|
||||
selects.add(filterIndex.candidateSelect(stage,String.valueOf(columns.get(stage)),where.toString(),whereArgs,unionArgs));
|
||||
continue;
|
||||
}
|
||||
String fact = stage.equals("black") ? "fact_black" : "fact_bright";
|
||||
selects.add("SELECT " + expression + " AS candidate_value "
|
||||
+ "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id "
|
||||
@@ -430,6 +461,13 @@ public class SourceColumnQueries {
|
||||
}
|
||||
String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition;
|
||||
String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value";
|
||||
if(candidateOnly) {
|
||||
String sql="SELECT candidate_value FROM ("+candidates+") matches GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?";
|
||||
List<Object> parameters=combine(unionArgs,searchArgs);parameters.add(limit+1);parameters.add(offset);
|
||||
List<Map<String,Object>> found=pages.get(new QueryKey(sql,parameters,"candidates:"+fieldKey),()->List.copyOf(jdbc.query(sql,(rs,row)->Map.of("value",rs.getString(1)),parameters.toArray())));
|
||||
boolean more=found.size()>limit;
|
||||
return Map.of("rows",List.copyOf(found.subList(0,Math.min(limit,found.size()))),"hasMore",more,"limit",limit,"offset",offset,"pageComplete",true,"complete",offset==0&&!more,"releaseId",releaseId,"field",fieldKey,"selectedStages",List.copyOf(stages));
|
||||
}
|
||||
long total = total("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", combine(unionArgs,searchArgs));
|
||||
List<Map<String, Object>> rows = page("SELECT candidate_value FROM (" + candidates + ") matches "
|
||||
+ "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?",
|
||||
@@ -556,7 +594,7 @@ public class SourceColumnQueries {
|
||||
List<String> groups = parseSelections(context.dimensions().get("group"), "group");
|
||||
if (!context.includeUnassigned() && !groups.contains("__UNASSIGNED__"))
|
||||
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
|
||||
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor);
|
||||
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor,releaseId);
|
||||
if (!context.columnFilters().isEmpty()) append(where, args, stage, context.columnFilters(), releaseId, actor);
|
||||
DataScope.append(where, args, actor,
|
||||
"f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
@@ -578,12 +616,7 @@ public class SourceColumnQueries {
|
||||
}
|
||||
String path = jsonPath(column);
|
||||
args.add(path);
|
||||
args.add(path);
|
||||
args.add(path);
|
||||
return "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL "
|
||||
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
|
||||
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
|
||||
+ "COLLATE utf8mb4_0900_as_cs";
|
||||
return SOURCE_VALUE;
|
||||
}
|
||||
|
||||
private static List<Object> combine(List<Object> first, List<Object> second) {
|
||||
@@ -601,9 +634,7 @@ public class SourceColumnQueries {
|
||||
}
|
||||
String path = jsonPath(column);
|
||||
args.add(path);
|
||||
args.add(path);
|
||||
return "CASE WHEN JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
|
||||
+ "ELSE JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?)) END COLLATE utf8mb4_0900_as_cs";
|
||||
return SOURCE_TEXT+" COLLATE utf8mb4_0900_as_cs";
|
||||
}
|
||||
|
||||
private static String tupleColumns(int count) {
|
||||
@@ -814,11 +845,9 @@ public class SourceColumnQueries {
|
||||
catch (Exception e) { throw new IllegalArgumentException("来源列名无法编码", e); }
|
||||
}
|
||||
|
||||
private void bindSourcePaths(List<Object> args, String header) {
|
||||
private void bindSourcePath(List<Object> args, String header) {
|
||||
String path = jsonPath(header);
|
||||
args.add(path);
|
||||
args.add(path);
|
||||
args.add(path);
|
||||
}
|
||||
|
||||
private static ResponseStatusException bad(String message) {
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
package cn.yunsheng.quality.filters;
|
||||
|
||||
import java.util.List;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
|
||||
|
||||
/** Sparse, build-owned source values. Every consumer still authorizes and scopes the actual facts. */
|
||||
@Service
|
||||
public class SourceFilterIndex {
|
||||
public record BuildAvailable(String buildId) {}
|
||||
private static final int FORMAT = 1;
|
||||
static final String CREATE_STAGE_SQL="CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB";
|
||||
static final String MEMBER_INSERT_SQL="INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash";
|
||||
static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)";
|
||||
private final JdbcTemplate jdbc;
|
||||
private volatile Boolean available;
|
||||
private volatile Boolean candidateScopeAvailable;
|
||||
public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; }
|
||||
|
||||
public boolean available() {
|
||||
if (available == null) {
|
||||
Integer count=jdbc.queryForObject("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('source_filter_build','source_filter_field','source_filter_value','source_filter_member')",Integer.class);
|
||||
available=count!=null&&count==4;
|
||||
}
|
||||
return available;
|
||||
}
|
||||
|
||||
private String candidateFactHint(String stage) {
|
||||
if (candidateScopeAvailable == null) {
|
||||
Integer count=jdbc.queryForObject("SELECT COUNT(DISTINCT table_name) FROM information_schema.statistics WHERE table_schema=DATABASE() AND ((table_name='fact_black' AND index_name='ix_black_candidate_scope') OR (table_name='fact_bright' AND index_name='ix_bright_candidate_scope'))",Integer.class);
|
||||
candidateScopeAvailable=count!=null&&count==2;
|
||||
}
|
||||
return candidateScopeAvailable ? " FORCE INDEX (ix_"+stage+"_candidate_scope)" : "";
|
||||
}
|
||||
|
||||
public boolean ready(String release) {
|
||||
if(release==null||!available())return false;
|
||||
// Only successful immutable builds are retained; a pending backfill is checked again later.
|
||||
Integer missing=jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm LEFT JOIN source_filter_build sb ON sb.build_id=rm.transform_build_id AND sb.format_version=? WHERE rm.release_id=? AND sb.build_id IS NULL",Integer.class,FORMAT,release);
|
||||
if(missing==null||missing!=0)return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Idempotent and atomic with fact construction, including callers which instantiate this helper. */
|
||||
public void ensureBuild(String build) {
|
||||
if(!available())return;
|
||||
var tx=new TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource()));
|
||||
tx.executeWithoutResult(status->{
|
||||
jdbc.queryForObject("SELECT id FROM transform_build WHERE id=? FOR UPDATE",String.class,build);
|
||||
Integer done=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build WHERE build_id=? AND format_version=?",Integer.class,build,FORMAT);
|
||||
if(done!=null&&done>0)return;
|
||||
jdbc.update("DELETE m FROM source_filter_member m JOIN source_filter_field sf ON sf.id=m.field_id WHERE sf.build_id=?",build);
|
||||
jdbc.update("DELETE v FROM source_filter_value v JOIN source_filter_field sf ON sf.id=v.field_id WHERE sf.build_id=?",build);
|
||||
jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build);
|
||||
jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build);
|
||||
// Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source.
|
||||
try {
|
||||
for(String stage:List.of("black","bright"))buildStage(build,stage);
|
||||
jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT);
|
||||
} finally { jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging"); }
|
||||
});
|
||||
}
|
||||
|
||||
private void buildStage(String build,String stage) {
|
||||
var bounds=jdbc.queryForMap("SELECT MIN(id) AS first_id,MAX(id) AS last_id FROM fact_"+stage+" FORCE INDEX(uq_fact_"+stage+"_origin) WHERE build_id=?",build);
|
||||
if(bounds.get("first_id")==null)return;
|
||||
long after=((Number)bounds.get("first_id")).longValue()-1;
|
||||
long last=((Number)bounds.get("last_id")).longValue();
|
||||
while(after<last) {
|
||||
if(Thread.currentThread().isInterrupted())throw new IllegalStateException("来源筛选索引构建已中断");
|
||||
List<Long> ids=jdbc.queryForList("SELECT id FROM fact_"+stage+" FORCE INDEX(PRIMARY) WHERE build_id=? AND id>? AND id<=? ORDER BY id LIMIT 500",Long.class,build,after,last);
|
||||
if(ids.isEmpty())return;
|
||||
long end=ids.getLast();
|
||||
String rows=" FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE WHERE f.build_id=? AND f.id>? AND f.id<=?";
|
||||
jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end);
|
||||
String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')";
|
||||
String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?";
|
||||
// DROP/CREATE TEMPORARY do not commit the owning build transaction. Reset the
|
||||
// physical table too: DELETE retains undo/deleted records until that transaction ends.
|
||||
jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging");
|
||||
jdbc.execute(CREATE_STAGE_SQL);
|
||||
jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end);
|
||||
jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash");
|
||||
Integer collisions=jdbc.queryForObject(COLLISION_SQL,Integer.class);
|
||||
if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布");
|
||||
jdbc.update(MEMBER_INSERT_SQL);
|
||||
after=end;
|
||||
}
|
||||
}
|
||||
|
||||
/** Same-row membership is shared by metrics, trends, details and exports; empty values are sparse. */
|
||||
public boolean append(StringBuilder where,List<Object> args,String stage,String release,String column,
|
||||
String operator,List<String> values) {
|
||||
if(!ready(release))return false;
|
||||
where.append(" AND ");
|
||||
membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"),
|
||||
operator.equals("in")||operator.equals("not_in")||operator.equals("contains")?values:null,false,operator.equals("contains"));
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Legacy configured filters classify only the byte-exact literal string "null" as empty. */
|
||||
public boolean appendLegacy(StringBuilder where,List<Object> args,String stage,String release,String column,List<String> selected) {
|
||||
if(!ready(release))return false;
|
||||
var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList();
|
||||
where.append(" AND (");
|
||||
if(values.isEmpty())where.append("FALSE");
|
||||
else membership(where,args,stage,release,column,false,values,true,false);
|
||||
if(selected.contains(QueryFilterFields.EMPTY)) {
|
||||
where.append(" OR ");membership(where,args,stage,release,column,true,null,true,false);
|
||||
}
|
||||
where.append(')');return true;
|
||||
}
|
||||
|
||||
private void membership(StringBuilder where,List<Object> args,String stage,String release,String column,
|
||||
boolean negative,List<String> values,boolean legacy,boolean contains) {
|
||||
where.append("f.id ").append(negative?"NOT IN (":"IN (")
|
||||
.append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin");
|
||||
args.add(release);args.add(stage);args.add(column);
|
||||
if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)");
|
||||
if(contains) {
|
||||
where.append(" AND (");
|
||||
for(int i=0;i<values.size();i++) {
|
||||
if(i>0)where.append(" OR ");
|
||||
where.append("LOCATE(?,sv.value COLLATE utf8mb4_0900_as_cs)>0");args.add(values.get(i));
|
||||
}
|
||||
where.append(')');
|
||||
} else if(values!=null) {
|
||||
where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')');
|
||||
args.addAll(values);
|
||||
}
|
||||
where.append(')');
|
||||
}
|
||||
|
||||
/** Dictionary-first candidates short-circuit after one authorized matching member per value. */
|
||||
public String candidateSelect(String stage,String column,String scope,List<Object> scopeArgs,List<Object> args) {
|
||||
args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs);
|
||||
return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id "
|
||||
+"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f"+candidateFactHint(stage)+" ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id "
|
||||
+scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package cn.yunsheng.quality.filters;
|
||||
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Asynchronously builds current and newly completed snapshots; unindexed queries retain their raw-value fallback. */
|
||||
@Component
|
||||
public class SourceFilterIndexBackfill implements ApplicationRunner {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SourceFilterIndexBackfill.class);
|
||||
private static final int MAX_PENDING_BUILDS = 1024;
|
||||
|
||||
private final JdbcTemplate jdbc;
|
||||
private final SourceFilterIndex index;
|
||||
private final ExecutorService worker;
|
||||
private final Object pendingLock = new Object();
|
||||
private final LinkedHashSet<String> pending = new LinkedHashSet<>();
|
||||
private boolean drainScheduled;
|
||||
private String inFlight;
|
||||
|
||||
@Autowired
|
||||
public SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index) {
|
||||
this(jdbc, index, createWorker());
|
||||
}
|
||||
|
||||
SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index, ExecutorService worker) {
|
||||
this.jdbc = jdbc;
|
||||
this.index = index;
|
||||
this.worker = worker;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
List<String> currentBuilds;
|
||||
try {
|
||||
if (!index.available()) return;
|
||||
currentBuilds = jdbc.query("""
|
||||
SELECT DISTINCT b.id
|
||||
FROM current_release c
|
||||
JOIN data_release r ON r.id=c.release_id
|
||||
JOIN release_member rm ON rm.release_id=r.id
|
||||
JOIN transform_build b ON b.id=rm.transform_build_id
|
||||
WHERE c.dataset_id='quality'
|
||||
AND r.status='PUBLISHED'
|
||||
AND r.complete=TRUE
|
||||
AND b.status='COMPLETED'
|
||||
ORDER BY b.id
|
||||
""", (rs, row) -> rs.getString(1));
|
||||
} catch (RuntimeException failure) {
|
||||
LOG.error("Source filter index backfill discovery failed; raw-value fallback remains available", failure);
|
||||
return;
|
||||
}
|
||||
if (currentBuilds.isEmpty()) return;
|
||||
int accepted = enqueue(currentBuilds);
|
||||
LOG.info("Source filter index backfill queued: discovered={}, accepted={}", currentBuilds.size(), accepted);
|
||||
}
|
||||
|
||||
@EventListener
|
||||
public void onBuildAvailable(SourceFilterIndex.BuildAvailable event) {
|
||||
if (event == null || event.buildId() == null || event.buildId().isBlank()) return;
|
||||
int accepted = enqueue(List.of(event.buildId()));
|
||||
if (accepted > 0) LOG.info("Source filter index build queued after commit: build={}", event.buildId());
|
||||
}
|
||||
|
||||
private int enqueue(List<String> builds) {
|
||||
boolean schedule = false;
|
||||
int accepted = 0;
|
||||
int deferred = 0;
|
||||
synchronized (pendingLock) {
|
||||
for (String build : builds) {
|
||||
if (build == null || build.isBlank() || build.equals(inFlight) || pending.contains(build)) continue;
|
||||
if (pending.size() >= MAX_PENDING_BUILDS) {
|
||||
deferred++;
|
||||
continue;
|
||||
}
|
||||
pending.add(build);
|
||||
accepted++;
|
||||
}
|
||||
if (!pending.isEmpty() && !drainScheduled) {
|
||||
drainScheduled = true;
|
||||
schedule = true;
|
||||
}
|
||||
}
|
||||
if (deferred > 0) {
|
||||
LOG.warn("Source filter index queue is full; {} builds keep raw-value fallback", deferred);
|
||||
}
|
||||
if (schedule) scheduleDrain();
|
||||
return accepted;
|
||||
}
|
||||
|
||||
private void scheduleDrain() {
|
||||
try {
|
||||
worker.execute(this::drainPending);
|
||||
} catch (RejectedExecutionException failure) {
|
||||
int queued;
|
||||
synchronized (pendingLock) {
|
||||
drainScheduled = false;
|
||||
queued = pending.size();
|
||||
}
|
||||
LOG.error("Source filter index drain could not be scheduled; {} builds keep raw-value fallback", queued, failure);
|
||||
}
|
||||
}
|
||||
|
||||
private void drainPending() {
|
||||
long started = System.nanoTime();
|
||||
int succeeded = 0;
|
||||
int failed = 0;
|
||||
while (true) {
|
||||
String buildId;
|
||||
synchronized (pendingLock) {
|
||||
if (pending.isEmpty()) {
|
||||
drainScheduled = false;
|
||||
break;
|
||||
}
|
||||
var iterator = pending.iterator();
|
||||
buildId = iterator.next();
|
||||
iterator.remove();
|
||||
inFlight = buildId;
|
||||
}
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
synchronized (pendingLock) {
|
||||
pending.add(buildId);
|
||||
inFlight = null;
|
||||
drainScheduled = false;
|
||||
}
|
||||
LOG.info("Source filter index drain interrupted; {} builds remain queued", pendingCount());
|
||||
return;
|
||||
}
|
||||
|
||||
long buildStarted = System.nanoTime();
|
||||
try {
|
||||
index.ensureBuild(buildId);
|
||||
succeeded++;
|
||||
LOG.info("Source filter index build completed: build={}, queuedRemaining={}, elapsedMs={}", buildId,
|
||||
pendingCount(), (System.nanoTime() - buildStarted) / 1_000_000L);
|
||||
} catch (RuntimeException failure) {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
synchronized (pendingLock) {
|
||||
pending.add(buildId);
|
||||
inFlight = null;
|
||||
drainScheduled = false;
|
||||
}
|
||||
LOG.info("Source filter index drain interrupted while building {}; {} builds remain queued", buildId,
|
||||
pendingCount());
|
||||
return;
|
||||
}
|
||||
failed++;
|
||||
LOG.error("Source filter index build failed: build={}; raw-value fallback remains available, continuing",
|
||||
buildId, failure);
|
||||
} finally {
|
||||
synchronized (pendingLock) { inFlight = null; }
|
||||
}
|
||||
}
|
||||
LOG.info("Source filter index drain finished: succeeded={}, failed={}, elapsedMs={}", succeeded, failed,
|
||||
(System.nanoTime() - started) / 1_000_000L);
|
||||
}
|
||||
|
||||
private int pendingCount() {
|
||||
synchronized (pendingLock) { return pending.size(); }
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void stopWorker() {
|
||||
worker.shutdownNow();
|
||||
try {
|
||||
if (!worker.awaitTermination(2, TimeUnit.SECONDS)) {
|
||||
LOG.warn("Source filter index backfill worker did not stop within two seconds");
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
private static ExecutorService createWorker() {
|
||||
return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1), runnable -> {
|
||||
Thread thread = new Thread(runnable, "source-filter-index-backfill");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
}, new ThreadPoolExecutor.AbortPolicy());
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,21 @@
|
||||
package cn.yunsheng.quality.metrics;
|
||||
|
||||
import cn.yunsheng.quality.filters.SourceFilterIndex;
|
||||
import java.util.List;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
/** Builds immutable, per-transform-build source-component rollups for read-only metric queries. */
|
||||
@Service
|
||||
public class MetricsRollupService {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(MetricsRollupService.class);
|
||||
static final String DIMENSION_COLLATION = "utf8mb4_0900_as_cs";
|
||||
private static final List<String> FACT_TABLES = List.of("fact_black", "fact_bright");
|
||||
private static final String COMPONENTS = "JSON_OBJECT(" +
|
||||
@@ -22,10 +30,18 @@ public class MetricsRollupService {
|
||||
|
||||
private final JdbcTemplate jdbc;
|
||||
private final cn.yunsheng.quality.analysis.FineDefectRollups fineDefects;
|
||||
private final SourceFilterIndex sourceFilterIndex;
|
||||
private ApplicationEventPublisher eventPublisher;
|
||||
|
||||
public MetricsRollupService(JdbcTemplate jdbc) {
|
||||
this.jdbc = jdbc;
|
||||
this.fineDefects = new cn.yunsheng.quality.analysis.FineDefectRollups(jdbc, tools.jackson.databind.json.JsonMapper.builder().build());
|
||||
this.sourceFilterIndex = new SourceFilterIndex(jdbc);
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher eventPublisher) {
|
||||
this.eventPublisher = eventPublisher;
|
||||
}
|
||||
|
||||
public record GroupKey(String stage,String month,String department,String group,String project,String product) {}
|
||||
@@ -36,6 +52,7 @@ public class MetricsRollupService {
|
||||
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent);
|
||||
if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总");
|
||||
fineDefects.copyBuild(parent, child);
|
||||
announceSourceFilterBuild(child);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -74,6 +91,30 @@ public class MetricsRollupService {
|
||||
""", buildId, buildId, buildId);
|
||||
fineDefects.refreshBuild(buildId);
|
||||
new SalesOrderMetricRollups(jdbc).refreshCurrentBuild(buildId);
|
||||
announceSourceFilterBuild(buildId);
|
||||
}
|
||||
|
||||
void announceSourceFilterBuild(String buildId) {
|
||||
if (eventPublisher == null) {
|
||||
// Directly constructed fixtures have no application event bus; retain
|
||||
// their explicit synchronous behavior (legacy schemas no-op safely).
|
||||
sourceFilterIndex.ensureBuild(buildId);
|
||||
return;
|
||||
}
|
||||
Runnable publish = () -> {
|
||||
try {
|
||||
eventPublisher.publishEvent(new SourceFilterIndex.BuildAvailable(buildId));
|
||||
} catch (RuntimeException failure) {
|
||||
LOG.error("Source filter index notification failed for {}; raw-value fallback remains available", buildId, failure);
|
||||
}
|
||||
};
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@Override public void afterCommit() { publish.run(); }
|
||||
});
|
||||
} else {
|
||||
publish.run();
|
||||
}
|
||||
}
|
||||
|
||||
static String rollupInsertSql(String table) {
|
||||
|
||||
@@ -39,6 +39,10 @@ public class MetricsService {
|
||||
private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128;
|
||||
private final JdbcTemplate jdbc;
|
||||
private final GovernanceAccessService governance;
|
||||
private SourceColumnQueries sourceColumns;
|
||||
@Autowired
|
||||
public void setSourceColumns(SourceColumnQueries sourceColumns) { this.sourceColumns = sourceColumns; }
|
||||
|
||||
private final TransactionTemplate readSnapshot;
|
||||
private final TransactionTemplate governanceSnapshot;
|
||||
|
||||
@@ -46,6 +50,7 @@ public class MetricsService {
|
||||
public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance, PlatformTransactionManager transactionManager) {
|
||||
this.jdbc = jdbc;
|
||||
this.governance = governance;
|
||||
this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
|
||||
this.readSnapshot = new TransactionTemplate(transactionManager);
|
||||
this.readSnapshot.setIsolationLevel(TransactionDefinition.ISOLATION_REPEATABLE_READ);
|
||||
this.readSnapshot.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
|
||||
@@ -56,7 +61,9 @@ public class MetricsService {
|
||||
|
||||
/** Keeps package-level in-memory aggregation tests independent of a database transaction manager. */
|
||||
public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance) {
|
||||
this.jdbc = jdbc; this.governance = governance; this.readSnapshot = null; this.governanceSnapshot = null;
|
||||
this.jdbc = jdbc; this.governance = governance;
|
||||
this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
|
||||
this.readSnapshot = null; this.governanceSnapshot = null;
|
||||
}
|
||||
|
||||
public MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor) {
|
||||
@@ -156,9 +163,9 @@ public class MetricsService {
|
||||
appendDimensions(where, args, query);
|
||||
appendExclusions(where, args, query);
|
||||
if (factFallback) {
|
||||
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor);
|
||||
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor,releaseId);
|
||||
if (!query.columnFilters().isEmpty())
|
||||
new SourceColumnQueries(jdbc, governance, JsonMapper.builder().build())
|
||||
sourceColumns
|
||||
.append(where, args, stage, query.columnFilters(), releaseId, actor);
|
||||
}
|
||||
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
|
||||
@@ -180,7 +187,7 @@ public class MetricsService {
|
||||
%s AS bright_processing_scrap,
|
||||
%s AS bright_diffusion_scrap,
|
||||
%s AS row_count
|
||||
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
|
||||
FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
|
||||
%s
|
||||
GROUP BY trend_period
|
||||
ORDER BY trend_period
|
||||
@@ -189,7 +196,7 @@ public class MetricsService {
|
||||
componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"),
|
||||
componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"),
|
||||
componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"),
|
||||
countExpression, table, where);
|
||||
countExpression, table, factFilterHint(stage,factFallback,query), where);
|
||||
jdbc.query(sql, (RowCallbackHandler) rs -> {
|
||||
String period = rs.getString("trend_period");
|
||||
AggregateStats stats = new AggregateStats().read(rs, stage);
|
||||
@@ -392,8 +399,8 @@ public class MetricsService {
|
||||
appendDimensions(where, args, query);
|
||||
appendExclusions(where, args, query);
|
||||
if(factFallback) {
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
|
||||
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor);
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
|
||||
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
|
||||
}
|
||||
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
if (!effectiveIncludeUnassigned(query))
|
||||
@@ -413,7 +420,7 @@ public class MetricsService {
|
||||
%s AS bright_processing_scrap,
|
||||
%s AS bright_diffusion_scrap,
|
||||
%s AS row_count
|
||||
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
|
||||
FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
|
||||
%s
|
||||
GROUP BY %s
|
||||
ORDER BY %s
|
||||
@@ -423,7 +430,7 @@ public class MetricsService {
|
||||
componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"),
|
||||
componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"),
|
||||
componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"),
|
||||
factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, where,
|
||||
factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, factFilterHint(stage,factFallback,query), where,
|
||||
aggregateKeys, aggregateKeys);
|
||||
jdbc.query(sql, (RowCallbackHandler) rs -> {
|
||||
String period = rs.getString("trend_period");
|
||||
@@ -595,6 +602,16 @@ public class MetricsService {
|
||||
"sourceMonths", sourceMonths);
|
||||
}
|
||||
|
||||
private String factFilterHint(String stage,boolean factFallback,MetricsQuery query) {
|
||||
if(sourceColumns.indexed(query.releaseId()))return "";
|
||||
if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return "";
|
||||
// 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。
|
||||
if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null
|
||||
||query.department()!=null||query.group()!=null||query.product()!=null||query.projectGroup()!=null
|
||||
||query.productSearch()!=null)return "";
|
||||
return SourceColumnQueries.factIndexHint(stage);
|
||||
}
|
||||
|
||||
private void coverageStage(Map<String, CoverageMonth> months, String stage, String releaseId,
|
||||
MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup) {
|
||||
if(query.stage()!=null&&!query.stage().equals(stage))return;
|
||||
@@ -609,8 +626,8 @@ public class MetricsService {
|
||||
appendDimensions(where, args, query);
|
||||
appendExclusions(where, args, query);
|
||||
if(factFallback) {
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
|
||||
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor);
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
|
||||
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
|
||||
}
|
||||
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
String countExpression = factFallback ? "COUNT(*)" : "SUM(f.source_row_count)";
|
||||
@@ -625,11 +642,11 @@ public class MetricsService {
|
||||
String sql = """
|
||||
SELECT f.month_key, %s AS row_count,
|
||||
%s AS excluded_blank_rows, %s AS unmapped_rows, %s AS project_unconfigured_rows
|
||||
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
|
||||
FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
|
||||
%s
|
||||
GROUP BY f.month_key
|
||||
ORDER BY f.month_key
|
||||
""".formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, where);
|
||||
""".formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, factFilterHint(stage,factFallback,query), where);
|
||||
jdbc.query(sql, (RowCallbackHandler) rs -> {
|
||||
CoverageMonth coverage = months.computeIfAbsent(rs.getString("month_key"), ignored -> new CoverageMonth());
|
||||
long rowCount = rs.getLong("row_count");
|
||||
|
||||
@@ -53,6 +53,10 @@ public class RecordController {
|
||||
private final JsonMapper mapper;
|
||||
private final GovernanceAccessService governance;
|
||||
private final RecordsWorkbookExportService workbookExport;
|
||||
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
|
||||
|
||||
|
||||
public RecordController(JdbcTemplate jdbc, ReleaseService releases, JsonMapper mapper,
|
||||
GovernanceAccessService governance, RecordsWorkbookExportService workbookExport) {
|
||||
@@ -61,6 +65,7 @@ public class RecordController {
|
||||
this.mapper = mapper;
|
||||
this.governance = governance;
|
||||
this.workbookExport = workbookExport;
|
||||
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
|
||||
}
|
||||
|
||||
|
||||
@@ -156,10 +161,10 @@ public class RecordController {
|
||||
args.add(search);
|
||||
args.add(jsonPath(selectedField));
|
||||
}
|
||||
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor);
|
||||
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor,selectedRelease);
|
||||
var sourceFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters);
|
||||
for(var rule:sourceFilters)if(!List.of(stage).containsAll((List<?>)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段");
|
||||
if(!sourceFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,stage,sourceFilters,selectedRelease,actor);
|
||||
if(!sourceFilters.isEmpty())sourceColumns.append(predicate,args,stage,sourceFilters,selectedRelease,actor);
|
||||
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
|
||||
String countSql = """
|
||||
|
||||
+12
-4
@@ -52,6 +52,10 @@ public class RecordsWorkbookExportService {
|
||||
private final ReleaseService releases;
|
||||
private final GovernanceAccessService governance;
|
||||
private final JsonMapper mapper;
|
||||
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
|
||||
|
||||
|
||||
public RecordsWorkbookExportService(JdbcTemplate jdbc, ReleaseService releases,
|
||||
GovernanceAccessService governance, JsonMapper mapper) {
|
||||
@@ -59,6 +63,7 @@ public class RecordsWorkbookExportService {
|
||||
this.releases = releases;
|
||||
this.governance = governance;
|
||||
this.mapper = mapper;
|
||||
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
|
||||
}
|
||||
|
||||
public Path export(Integer year, Integer month, Integer quarter, String department, String group,
|
||||
@@ -120,8 +125,11 @@ public class RecordsWorkbookExportService {
|
||||
LinkedHashSet<String> rawFields = new LinkedHashSet<>();
|
||||
if (releaseId != null) {
|
||||
QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product,
|
||||
projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd);
|
||||
scan(rawScope, rs -> rawFields.addAll(visibleRaw(readJson(rs.getString(1)), allowedFields).keySet()));
|
||||
projectGroup, includeUnassigned, "CAST(JSON_KEYS(f.raw_values) AS CHAR)", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd);
|
||||
scan(rawScope, rs -> {
|
||||
Object decoded=mapper.readValue(rs.getString(1),Object.class);
|
||||
if(decoded instanceof List<?> keys)for(Object key:keys)if(key instanceof String name&&(allowedFields.contains("*")||allowedFields.contains(name)))rawFields.add(name);
|
||||
});
|
||||
}
|
||||
boolean showOrderType=SalesOrderTypes.allowed(allowedFields);
|
||||
if (rawFields.size() + HEADER_COUNT + (showOrderType?1:0) > MAX_COLUMNS)
|
||||
@@ -167,8 +175,8 @@ public class RecordsWorkbookExportService {
|
||||
SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value);
|
||||
if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__"))
|
||||
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
|
||||
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor);
|
||||
if(!columnFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
|
||||
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor,releaseId);
|
||||
if(!columnFilters.isEmpty())sourceColumns.append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
|
||||
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
|
||||
"f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
String columns = select == null ? "f.raw_values" : select;
|
||||
|
||||
@@ -15,6 +15,7 @@ import cn.yunsheng.quality.ingestion.ImportWorker;
|
||||
import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild;
|
||||
import cn.yunsheng.quality.ingestion.MappingValueCodec;
|
||||
import cn.yunsheng.quality.metrics.MetricsService;
|
||||
import cn.yunsheng.quality.metrics.MetricsRollupService;
|
||||
import cn.yunsheng.quality.release.ReleaseService;
|
||||
import cn.yunsheng.quality.source.SourceCatalogService;
|
||||
import cn.yunsheng.quality.source.SourceDeletionService;
|
||||
@@ -66,6 +67,7 @@ public class DataWorkflowService {
|
||||
private final TaskExecutor executor;
|
||||
private final TransactionTemplate transaction;
|
||||
private final MetricsService metrics;
|
||||
private MetricsRollupService rollups;
|
||||
private final AtomicReference<String> runningOperation = new AtomicReference<>();
|
||||
private final AtomicReference<String> runningLease = new AtomicReference<>();
|
||||
|
||||
@@ -108,6 +110,12 @@ public class DataWorkflowService {
|
||||
this.executor = executor;
|
||||
this.transaction = new TransactionTemplate(manager);
|
||||
this.metrics=metrics;
|
||||
this.rollups=new MetricsRollupService(jdbc);
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public void setMetricsRollupService(MetricsRollupService rollups) {
|
||||
this.rollups = rollups;
|
||||
}
|
||||
|
||||
public Map<String, Object> status(AppUserPrincipal actor) {
|
||||
@@ -376,7 +384,7 @@ public class DataWorkflowService {
|
||||
sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor);
|
||||
updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED");
|
||||
var selectedRules=unclassifiedRules;
|
||||
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
|
||||
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,rollups).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
|
||||
jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token);
|
||||
publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false);
|
||||
jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token);
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
CREATE TABLE source_filter_build (
|
||||
build_id CHAR(36) NOT NULL PRIMARY KEY,
|
||||
format_version INT NOT NULL,
|
||||
CONSTRAINT fk_source_filter_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
CREATE TABLE source_filter_field (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
build_id CHAR(36) NOT NULL,
|
||||
stage VARCHAR(16) NOT NULL,
|
||||
column_name LONGTEXT COLLATE utf8mb4_0900_bin NOT NULL,
|
||||
name_hash BINARY(32) NOT NULL,
|
||||
UNIQUE KEY uq_source_filter_field (build_id,stage,name_hash),
|
||||
CONSTRAINT fk_source_filter_field_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
CREATE TABLE source_filter_value (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
field_id BIGINT NOT NULL,
|
||||
value_hash BINARY(32) NOT NULL,
|
||||
value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,
|
||||
UNIQUE KEY uq_source_filter_value_hash (field_id,value_hash),
|
||||
KEY ix_source_filter_value_text (field_id,value(128)),
|
||||
CONSTRAINT fk_source_filter_value_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
CREATE TABLE source_filter_member (
|
||||
field_id BIGINT NOT NULL,
|
||||
fact_id BIGINT NOT NULL,
|
||||
value_id BIGINT NOT NULL,
|
||||
PRIMARY KEY (field_id,fact_id),
|
||||
KEY ix_source_filter_member_value (value_id,fact_id),
|
||||
CONSTRAINT fk_source_filter_member_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id),
|
||||
CONSTRAINT fk_source_filter_member_value FOREIGN KEY (value_id) REFERENCES source_filter_value(id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
-- Cover representative-row selection without reading every row's large source JSON.
|
||||
ALTER TABLE fact_black ADD KEY ix_black_source_catalog (build_id,source_revision_id,department,group_name,id);
|
||||
ALTER TABLE fact_bright ADD KEY ix_bright_source_catalog (build_id,source_revision_id,department,group_name,id);
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Keep common candidate authorization/date/product checks off the wide JSON clustered rows.
|
||||
-- The fact table remains authoritative. Changes to its dimensions update this index automatically.
|
||||
ALTER TABLE fact_black ADD INDEX ix_black_candidate_scope
|
||||
(id,build_id,source_revision_id,month_key,department,group_name,product);
|
||||
ALTER TABLE fact_bright ADD INDEX ix_bright_candidate_scope
|
||||
(id,build_id,source_revision_id,month_key,department,group_name,product);
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Equality on the owning build/revision followed by fact id supports the first visible row.
|
||||
-- Keep authorization dimensions covered, without reading wide JSON rows while seeking it.
|
||||
ALTER TABLE fact_black DROP INDEX ix_black_source_catalog,
|
||||
ADD INDEX ix_black_source_catalog (build_id,source_revision_id,id,department,group_name);
|
||||
ALTER TABLE fact_bright DROP INDEX ix_bright_source_catalog,
|
||||
ADD INDEX ix_bright_source_catalog (build_id,source_revision_id,id,department,group_name);
|
||||
@@ -42,7 +42,7 @@ class FineDefectRollupsMySqlTest {
|
||||
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)");
|
||||
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))");
|
||||
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))");
|
||||
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))");
|
||||
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,sheet_name VARCHAR(80),physical_row_no INT,KEY ix_fact_" + stage + "_month_dims(build_id,month_key),UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))");
|
||||
for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql", "V42__fine_defect_compact_rollups.sql")) {
|
||||
try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) {
|
||||
for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql);
|
||||
|
||||
+254
-9
@@ -101,7 +101,7 @@ class SourceColumnQueriesMySqlTest {
|
||||
assertTrue((Boolean)page.get("pageComplete"));
|
||||
}
|
||||
|
||||
@Test void groupedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() {
|
||||
@Test void orderedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() {
|
||||
record Read(String sql,Object[] args,List<?> result) {}
|
||||
List<Read> reads=new java.util.ArrayList<>();
|
||||
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
|
||||
@@ -114,7 +114,8 @@ class SourceColumnQueriesMySqlTest {
|
||||
assertEquals(2,representatives.size());
|
||||
for(Read read:representatives) {
|
||||
String stage=read.sql().contains("fact_black f2")?"black":"bright";
|
||||
assertTrue(read.sql().contains("GROUP BY f2.build_id,f2.source_revision_id"));
|
||||
assertTrue(read.sql().contains("ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1"));
|
||||
assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表");
|
||||
StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
|
||||
List<Object> args=new java.util.ArrayList<>();
|
||||
DataScope.append(scope,args,actor,"f2.department COLLATE utf8mb4_0900_as_cs","f2.group_name COLLATE utf8mb4_0900_as_cs");args.add(RELEASE);
|
||||
@@ -157,6 +158,23 @@ class SourceColumnQueriesMySqlTest {
|
||||
assertThrows(ResponseStatusException.class, () -> queries.validate(RELEASE, unauthorized, actor));
|
||||
}
|
||||
|
||||
@Test void sourceNullNormalizationMatchesThePreviousExpressionForEveryOperator() {
|
||||
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
|
||||
for(String raw:List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[]}","{\"委托单\":{\"n\":1}}"))
|
||||
insert("bright","build-bright",BRIGHT_REVISION,"部门A","组A",raw);
|
||||
String old="CASE WHEN JSON_EXTRACT(f.raw_values,'$.\"委托单\"') IS NULL OR JSON_TYPE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))='NULL' THEN NULL ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))),'') END COLLATE utf8mb4_0900_as_cs";
|
||||
for(String operator:List.of("in","not_in","is_empty","not_empty")) {
|
||||
Map<String,Object> rule=new java.util.LinkedHashMap<>(Map.of("field",key,"stages",List.of("bright"),"operator",operator));
|
||||
List<Object> args=new java.util.ArrayList<>();String condition;
|
||||
if(operator.equals("in")||operator.equals("not_in")) {
|
||||
rule.put("values",List.of("null","0007"));args.add("null");args.add("0007");
|
||||
condition=operator.equals("in")?old+" IN (?,?)":"("+old+" IS NULL OR "+old+" NOT IN (?,?))";
|
||||
} else condition=old+(operator.equals("is_empty")?" IS NULL":" IS NOT NULL");
|
||||
Long expected=jdbc.queryForObject("SELECT COUNT(*) FROM fact_bright f WHERE f.build_id='build-bright' AND f.department='部门A' AND f.group_name='组A' AND ("+condition+")",Long.class,args.toArray());
|
||||
assertEquals(expected,countFiltered("bright",SourceColumnQueries.parse(List.of(rule))),operator);
|
||||
}
|
||||
}
|
||||
|
||||
@Test void explicitMappingWorksAcrossStagesAndMemberPageCountsMergedTuplesBeforePaging() {
|
||||
List<Map<String, Object>> catalog = queries.catalog(RELEASE, actor);
|
||||
String mappedKey = (String) catalog.stream().filter(item -> item.get("label").equals("跨阶段委托号"))
|
||||
@@ -247,17 +265,27 @@ class SourceColumnQueriesMySqlTest {
|
||||
return fields.stream().filter(item -> item.get("label").equals(label)).findFirst().orElse(null);
|
||||
}
|
||||
|
||||
@Test void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters() throws Exception {
|
||||
@org.junit.jupiter.params.ParameterizedTest
|
||||
@org.junit.jupiter.params.provider.CsvSource({"false,in","true,in","false,contains","true,contains"})
|
||||
void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed,String operator) throws Exception {
|
||||
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))");
|
||||
jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE);
|
||||
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
|
||||
for(String stage:List.of("black","bright")) {
|
||||
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT");
|
||||
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
|
||||
jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=?,scrap_weight=?,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id",stage.equals("black")?90:80,stage.equals("black")?10:20);
|
||||
}
|
||||
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
|
||||
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
|
||||
var service=new cn.yunsheng.quality.metrics.MetricsService(jdbc,governance);
|
||||
if(indexed){installFilterIndex();var index=new SourceFilterIndex(jdbc);for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);}
|
||||
var headerReads=new java.util.concurrent.atomic.AtomicInteger();
|
||||
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
|
||||
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
|
||||
if(sql.contains("JSON_KEYS(f.raw_values)"))headerReads.incrementAndGet();
|
||||
return super.query(sql,mapper,args);
|
||||
}
|
||||
};
|
||||
var service=new cn.yunsheng.quality.metrics.MetricsService(capturing,governance);
|
||||
var releases=mock(cn.yunsheng.quality.release.ReleaseService.class);
|
||||
when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE);
|
||||
var mapper=JsonMapper.builder().build();
|
||||
@@ -265,13 +293,14 @@ class SourceColumnQueriesMySqlTest {
|
||||
var records=new cn.yunsheng.quality.records.RecordController(jdbc,releases,mapper,governance,exporter);
|
||||
String fieldKey=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
|
||||
for(String stage:List.of("black","bright")) {
|
||||
var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator","in","values",List.of("0007"));
|
||||
var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator",operator,"values",List.of(operator.equals("contains")?"007":"0007"));
|
||||
var base=new cn.yunsheng.quality.metrics.MetricsService.MetricsQuery(Integer.valueOf(2026),Integer.valueOf(1),null,"部门A","组A",null,null,"product",false,RELEASE,"month",Map.of()).withColumnFilters(List.of(filter));
|
||||
var legacy=service.calculate(base,actor);var selected=service.calculate(base.withStage(stage),actor);
|
||||
assertEquals(1,selected.rows().size());assertEquals(1L,((Number)selected.coverage().get("includedRows")).longValue());
|
||||
assertEquals(stage.equals("black")?legacy.summary().black():legacy.summary().bright(),stage.equals("black")?selected.summary().black():selected.summary().bright());
|
||||
assertEquals(0L,selected.summary().coverage().get(stage.equals("black")?"brightRows":"blackRows"));
|
||||
assertEquals(1,selected.trend().size());
|
||||
assertEquals(1,service.calculateTrend(base.withStage(stage),actor).trend().size());
|
||||
var detail=records.recordsFiltered(stage,2026,1,null,"部门A","组A",null,null,null,false,RELEASE,1,50,null,null,null,"asc",null,null,mapper.writeValueAsString(List.of(filter)),actor);
|
||||
assertEquals(1L,((Number)detail.get("total")).longValue());
|
||||
java.nio.file.Path file=exporter.exportFiltered(2026,1,null,"部门A","组A",null,null,false,RELEASE,actor,Map.of(),null,stage,List.of(filter));
|
||||
@@ -282,6 +311,7 @@ class SourceColumnQueriesMySqlTest {
|
||||
var other=base.withColumnFilters(List.of(filter)).withStage(stage);
|
||||
assertThrows(ResponseStatusException.class,()->other.withStage(stage.equals("black")?"bright":"black"));
|
||||
}
|
||||
assertEquals(2,headerReads.get(),"汇总、趋势与覆盖检查应复用同一份授权字段目录");
|
||||
}
|
||||
|
||||
@Test void continuousRangeDetailsAndBothExcelSheetsExcludeOutsideMonthsAndUnauthorizedRows() throws Exception {
|
||||
@@ -289,7 +319,7 @@ class SourceColumnQueriesMySqlTest {
|
||||
when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE);
|
||||
var mapper=JsonMapper.builder().build();
|
||||
for(String stage:List.of("black","bright")) {
|
||||
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT");
|
||||
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
|
||||
jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=90,scrap_weight=10,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id");
|
||||
String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION;
|
||||
for(String month:List.of("2026-03","2026-09","2026-10","2025-03")) {
|
||||
@@ -336,7 +366,7 @@ class SourceColumnQueriesMySqlTest {
|
||||
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
|
||||
for(String stage:List.of("black","bright")) {
|
||||
jdbc.execute("DELETE FROM fact_"+stage);
|
||||
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT");
|
||||
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
|
||||
}
|
||||
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
|
||||
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
|
||||
@@ -390,12 +420,227 @@ class SourceColumnQueriesMySqlTest {
|
||||
assertThrows(ResponseStatusException.class,()->controller.query(input,actor));
|
||||
}
|
||||
|
||||
@Test void indexedFiltersCandidatesAndTuplesMatchRawSemanticsAndStayAuthorized() throws Exception {
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
|
||||
List<String> samples=List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[1,2]}","{\"委托单\":{\"a\":1}}","{\"委托单\":\"é\"}","{\"委托单\":\"é\"}","{\"委托单\":\"A\"}","{\"委托单\":\"a\"}","{\"委托单\":\" padded \"}","{\"委托单\":\"null\"}");
|
||||
for(String raw:samples)insert("black","build-black",BLACK_REVISION,"部门A","组A",raw);
|
||||
String longKey="来源".repeat(300);
|
||||
insert("black","build-black",BLACK_REVISION,"部门A","组A",JsonMapper.builder().build().writeValueAsString(Map.of(longKey,"long-key-value")));
|
||||
insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}");
|
||||
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
|
||||
List<Map<String,Object>> rules=new java.util.ArrayList<>();
|
||||
for(String op:List.of("in","contains","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}"))
|
||||
rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value)));
|
||||
var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList();
|
||||
var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor);
|
||||
var rawMembers=queries.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor);
|
||||
jdbc.update("INSERT INTO query_filter_field VALUES ('plain','原始委托单','委托单','委托单',TRUE,TRUE,1)");
|
||||
List<List<String>> legacySelections=List.of(List.of("null"),List.of("0007"),List.of(QueryFilterFields.EMPTY),List.of("0007",QueryFilterFields.EMPTY));
|
||||
var rawLegacy=legacySelections.stream().map(this::legacyCount).toList();
|
||||
installFilterIndex();
|
||||
var index=new SourceFilterIndex(jdbc);
|
||||
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
|
||||
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
|
||||
assertTrue(indexed.indexed(RELEASE));
|
||||
assertEquals(rawLegacy,legacySelections.stream().map(this::legacyCount).toList());
|
||||
assertEquals("long-key-value",jdbc.queryForObject("SELECT sv.value FROM source_filter_field sf JOIN source_filter_value sv ON sv.field_id=sf.id WHERE sf.column_name=? AND sf.build_id='build-black'",String.class,longKey));
|
||||
var combined=List.of(Map.<String,Object>of("field",key,"stages",List.of("black"),"operator","in","values",List.of("0007")),Map.<String,Object>of("field",field(indexed.catalog(RELEASE,actor),"加工单位").get("key"),"stages",List.of("black"),"operator","in","values",List.of("机二")));
|
||||
assertEquals(0L,indexed.memberPage(RELEASE,Map.of("year",2026,"columnFilters",combined),List.of(key),List.of("black"),50,0,actor).get("totalCount"));
|
||||
assertEquals(List.of(),indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),"hidden-only-value",50,0,true,actor).get("rows"));
|
||||
jdbc.update("INSERT INTO transform_build VALUES ('build-child')");
|
||||
jdbc.update("INSERT INTO fact_black(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT 'build-child',source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no FROM fact_black WHERE build_id='build-black'");
|
||||
index.ensureBuild("build-child");
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member sm JOIN source_filter_field sf ON sf.id=sm.field_id JOIN fact_black f ON f.id=sm.fact_id WHERE sf.build_id='build-child' AND f.build_id<>'build-child'",Long.class));
|
||||
assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
|
||||
assertEquals(rawValues,indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor));
|
||||
assertEquals(rawMembers,indexed.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor));
|
||||
var page=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,2,0,true,actor);
|
||||
assertEquals(2,((List<?>)page.get("rows")).size());assertEquals(true,page.get("hasMore"));assertFalse(page.containsKey("totalCount"));
|
||||
long members=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class);
|
||||
index.ensureBuild("build-black");assertEquals(members,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("加工单位"));
|
||||
assertThrows(ResponseStatusException.class,()->indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),null,50,0,true,actor));
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
|
||||
jdbc.update("DELETE FROM source_filter_build WHERE build_id='build-black'");
|
||||
assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
|
||||
}
|
||||
|
||||
@Test void containsMatchesLiteralSubstringsWithOrSemanticsInBothStagesAndBothQueryPaths() throws Exception {
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
|
||||
var mapper=JsonMapper.builder().build();
|
||||
for(String stage:List.of("black","bright")) {
|
||||
String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION;
|
||||
for(String value:List.of("xxAlpha%_\\tail","中文日立公司","prefix x' OR 1=1 -- suffix","000123","null"))
|
||||
insert(stage,"build-"+stage,revision,"部门A","组A",mapper.writeValueAsString(Map.of("委托单",value)));
|
||||
insert(stage,"build-"+stage,revision,"部门B","组B",mapper.writeValueAsString(Map.of("委托单","日立-Alpha-hidden")));
|
||||
}
|
||||
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
|
||||
List<List<String>> needles=List.of(List.of("Alpha"),List.of("alpha"),List.of("%"),List.of("_"),List.of("\\"),List.of("日立"),List.of("x' OR 1=1 --"),List.of("001"),List.of("null"),List.of("日立","Alpha"));
|
||||
List<Long> expected=List.of(1L,0L,1L,1L,1L,1L,1L,1L,1L,2L);
|
||||
for(boolean indexed:List.of(false,true)) {
|
||||
if(indexed) {
|
||||
installFilterIndex();var index=new SourceFilterIndex(jdbc);
|
||||
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
|
||||
queries=new SourceColumnQueries(jdbc,governance,mapper);
|
||||
}
|
||||
for(String stage:List.of("black","bright"))for(int n=0;n<needles.size();n++) {
|
||||
var rule=Map.<String,Object>of("field",key,"stages",List.of(stage),"operator","contains","values",needles.get(n));
|
||||
assertEquals(expected.get(n),countFiltered(stage,List.of(rule)),stage+" indexed="+indexed+" keywords="+needles.get(n));
|
||||
}
|
||||
var canonical=Map.<String,Object>of("field","product","stages",List.of("black"),"operator","contains","values",List.of("P-"));
|
||||
assertTrue(countFiltered("black",List.of(canonical))>0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test void memberBuildPlansStartFromTheBoundedStageWindowRatherThanTheGrowingDictionary() throws Exception {
|
||||
installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black");
|
||||
jdbc.execute((org.springframework.jdbc.core.ConnectionCallback<Void>)connection->{
|
||||
try(var statement=connection.createStatement()) {
|
||||
statement.execute(SourceFilterIndex.CREATE_STAGE_SQL);
|
||||
try {
|
||||
statement.execute("INSERT INTO source_filter_staging SELECT v.field_id,m.fact_id,v.value,v.value_hash FROM source_filter_value v JOIN source_filter_member m ON m.value_id=v.id LIMIT 4");
|
||||
for(String sql:List.of(SourceFilterIndex.MEMBER_INSERT_SQL,SourceFilterIndex.COLLISION_SQL)) {
|
||||
try(var rows=statement.executeQuery("EXPLAIN "+sql)) {
|
||||
String first=null;
|
||||
while(rows.next())if(List.of("s","v").contains(rows.getString("table"))){first=rows.getString("table");break;}
|
||||
assertEquals("s",first,"每批构建只能从有界临时窗口开始,不能反复扫描历史字典");
|
||||
}
|
||||
}
|
||||
} finally {statement.execute("DROP TEMPORARY TABLE source_filter_staging");}
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
@Test void catalogStopsAtTheFirstAuthorizedFactWithoutSortingEveryFactId() throws Exception {
|
||||
installFilterIndex();
|
||||
var captured=new java.util.ArrayList<String>();var parameters=new java.util.ArrayList<Object[]>();
|
||||
var tracking=new JdbcTemplate(jdbc.getDataSource()) {
|
||||
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
|
||||
if(sql.startsWith("SELECT CAST(JSON_KEYS")){captured.add(sql);parameters.add(args);}
|
||||
return super.query(sql,mapper,args);
|
||||
}
|
||||
};
|
||||
var catalog=new SourceColumnQueries(tracking,governance,JsonMapper.builder().build()).catalog(RELEASE,actor);
|
||||
assertNotNull(field(catalog,"委托单"));
|
||||
assertNull(field(catalog,"组织外行列"),"首条全局记录不可见时,必须继续寻找当前用户的第一条可见记录");
|
||||
assertNull(field(catalog,"其他组织列"));
|
||||
assertEquals(2,captured.size());
|
||||
for(int i=0;i<captured.size();i++) {
|
||||
String plan=jdbc.queryForObject("EXPLAIN ANALYZE "+captured.get(i),String.class,parameters.get(i));
|
||||
assertTrue(plan.contains("Limit: 1 row(s)"),plan);
|
||||
assertFalse(plan.lines().anyMatch(line->line.contains("Sort:")&&line.contains("f2.")),plan);
|
||||
assertTrue(plan.contains("Covering index lookup on f2 using ix_"),plan);
|
||||
}
|
||||
}
|
||||
|
||||
@Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception {
|
||||
installFilterIndex();var index=new SourceFilterIndex(jdbc);
|
||||
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
|
||||
var scopeArgs=new java.util.ArrayList<Object>(List.of(RELEASE,"2026-01","2026-12","部门A","组A","P-1"));
|
||||
var args=new java.util.ArrayList<Object>();
|
||||
String sql=index.candidateSelect("black","委托单"," WHERE rm.release_id=? AND f.month_key BETWEEN ? AND ? AND f.department=? AND f.group_name=? AND f.product=?",scopeArgs,args);
|
||||
assertEquals(Set.of("0007","0008"),new java.util.HashSet<>(jdbc.queryForList(sql,String.class,args.toArray())));
|
||||
boolean covering=false;
|
||||
for(var row:jdbc.queryForList("EXPLAIN "+sql,args.toArray())) {
|
||||
if("f".equals(row.get("table"))) {
|
||||
assertEquals("ix_black_candidate_scope",row.get("key"));
|
||||
covering=String.valueOf(row.get("Extra")).contains("Using index");
|
||||
}
|
||||
}
|
||||
assertTrue(covering,"候选值的日期、权限和产品校验不应读取宽 JSON 原始行");
|
||||
}
|
||||
|
||||
@Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception {
|
||||
installFilterIndex();
|
||||
for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}");
|
||||
var memberWrites=new java.util.concurrent.atomic.AtomicInteger();
|
||||
var failing=new JdbcTemplate(jdbc.getDataSource()) {
|
||||
@Override public int update(String sql) {
|
||||
if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure");
|
||||
return super.update(sql);
|
||||
}
|
||||
@Override public int update(String sql,Object... args) {
|
||||
if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure");
|
||||
return super.update(sql,args);
|
||||
}
|
||||
};
|
||||
var index=new SourceFilterIndex(failing);
|
||||
assertThrows(IllegalStateException.class,()->index.ensureBuild("build-black"));
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build",Long.class));
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_field",Long.class));
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_value",Long.class));
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
|
||||
assertFalse(index.ready(RELEASE));
|
||||
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
|
||||
assertTrue(index.ready(RELEASE));
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_FILTER_BENCHMARK",matches="true")
|
||||
void syntheticWideSourceIndexBenchmark() throws Exception {
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
|
||||
int size=Integer.getInteger("quality.benchmarkRows",10000);
|
||||
jdbc.execute("CREATE TABLE benchmark_digit(n INT PRIMARY KEY)");for(int n=0;n<10;n++)jdbc.update("INSERT INTO benchmark_digit VALUES (?)",n);
|
||||
for(String stage:List.of("black","bright")) {
|
||||
jdbc.execute("DELETE FROM fact_"+stage);
|
||||
StringBuilder raw=new StringBuilder("JSON_OBJECT('客户名称',CONCAT('客户',MOD(n,300))");
|
||||
for(int f=0;f<120;f++)raw.append(",'字段").append(f).append("',CONCAT('value-").append(f).append("-',").append(stage.equals("black")?"MOD(n,50)":"n").append(")");
|
||||
raw.append(')');
|
||||
String numbers="SELECT a.n+10*b.n+100*c.n+1000*d.n+10000*e.n AS n FROM benchmark_digit a CROSS JOIN benchmark_digit b CROSS JOIN benchmark_digit c CROSS JOIN benchmark_digit d CROSS JOIN benchmark_digit e LIMIT "+size;
|
||||
jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT ?,?,'2026-01','部门A','组A','P-1','项目A',"+raw+",'data',n+1 FROM ("+numbers+") numbers","build-"+stage,stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION);
|
||||
}
|
||||
var rawQuery=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
|
||||
String field=(String)field(rawQuery.catalog(RELEASE,actor),"客户名称").get("key");
|
||||
var old=new java.util.LinkedHashMap<String,Object>();
|
||||
for(String stage:List.of("black","bright")) {
|
||||
long start=System.nanoTime();var result=rawQuery.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
|
||||
old.put(stage,Map.of("ms",(System.nanoTime()-start)/1_000_000L,"result",result));
|
||||
}
|
||||
installFilterIndex();var index=new SourceFilterIndex(jdbc);
|
||||
for(String build:List.of("build-black","build-bright","build-hidden")) {
|
||||
long start=System.nanoTime();index.ensureBuild(build);System.out.println("FILTER_BENCH_BUILD "+build+" rows="+size+" ms="+(System.nanoTime()-start)/1_000_000L);
|
||||
}
|
||||
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
|
||||
for(String stage:List.of("black","bright")) {
|
||||
long start=System.nanoTime();var result=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
|
||||
long elapsed=(System.nanoTime()-start)/1_000_000L;
|
||||
assertEquals(((Map<?,?>)old.get(stage)).get("result"),result);
|
||||
System.out.println("FILTER_BENCH_QUERY "+stage+" rawMs="+((Map<?,?>)old.get(stage)).get("ms")+" indexMs="+elapsed);
|
||||
}
|
||||
for(String table:List.of("source_filter_field","source_filter_value","source_filter_member"))jdbc.execute("ANALYZE TABLE "+table);
|
||||
System.out.println("FILTER_BENCH_SPACE "+jdbc.queryForList("SELECT table_name,table_rows,data_length,index_length FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name LIKE 'source_filter_%'"));
|
||||
}
|
||||
|
||||
private long legacyCount(List<String> selected) {
|
||||
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,"black",Map.of("plain",selected),actor,RELEASE);
|
||||
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
|
||||
}
|
||||
|
||||
private long filteredCount(SourceColumnQueries source,Map<String,Object> rule) {
|
||||
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
|
||||
source.append(where,args,"black",List.of(rule),RELEASE,actor);
|
||||
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
|
||||
}
|
||||
|
||||
private void installFilterIndex() throws Exception {
|
||||
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
|
||||
for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id);
|
||||
for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql","V45__source_catalog_first_visible.sql")) {
|
||||
String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration));
|
||||
for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement);
|
||||
}
|
||||
}
|
||||
|
||||
private void schema() {
|
||||
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))");
|
||||
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))");
|
||||
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage
|
||||
+ "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),"
|
||||
+ "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL)");
|
||||
+ "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL,sheet_name VARCHAR(80),physical_row_no INT,UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))");
|
||||
jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64) PRIMARY KEY,label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)");
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class SourceColumnQueriesTest {
|
||||
@Test void rejectsUnknownShapeOperatorsEmptyValuesAndMoreThanFiftyValues() {
|
||||
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(Map.of()));
|
||||
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
|
||||
"field", SOURCE, "stages", List.of("black"), "operator", "contains", "values", List.of("x")))));
|
||||
"field", SOURCE, "stages", List.of("black"), "operator", "starts_with", "values", List.of("x")))));
|
||||
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
|
||||
"field", SOURCE, "stages", List.of("black"), "operator", "in", "values", List.of("")))));
|
||||
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
|
||||
@@ -42,6 +42,22 @@ class SourceColumnQueriesTest {
|
||||
"field", SOURCE, "stages", List.of("bright"), "operator", "not_in", "values", tooMany))));
|
||||
}
|
||||
|
||||
@Test void containsKeepsLiteralKeywordsAndBindsEachJsonPathAfterItsNeedle() {
|
||||
var rule=Map.<String,Object>of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",List.of("000","%_\\","x' OR 1=1 --","000"));
|
||||
var parsed=SourceColumnQueries.parse(List.of(rule));
|
||||
assertEquals(List.of("000","%_\\","x' OR 1=1 --"),parsed.getFirst().get("values"));
|
||||
var source=spy(new SourceColumnQueries(mock(JdbcTemplate.class),governance,JsonMapper.builder().build()));
|
||||
doReturn(List.of(sourceField())).when(source).scopedCatalog("release",actor);
|
||||
var sql=new StringBuilder();var args=new ArrayList<Object>();
|
||||
source.append(sql,args,"black",parsed,"release",actor);
|
||||
assertEquals(List.of("000","$.\"委托单\"","%_\\","$.\"委托单\"","x' OR 1=1 --","$.\"委托单\""),args);
|
||||
assertEquals(3,sql.toString().split("LOCATE",-1).length-1);
|
||||
assertFalse(sql.toString().contains("x' OR"));
|
||||
for(Object values:List.of(List.of(),List.of(""),List.of(" "),List.of("x".repeat(257)),List.of(7))) {
|
||||
assertThrows(ResponseStatusException.class,()->SourceColumnQueries.parse(List.of(Map.of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",values))));
|
||||
}
|
||||
}
|
||||
|
||||
@Test void monthRangeUsesInclusiveYearBoundariesAndKeepsLegacyMonthPredicate() {
|
||||
Map<String,Object> range=SourceColumnQueries.period(Map.of("period",Map.of("year",2026,"monthStart",3,"monthEnd",9)));
|
||||
StringBuilder rangeSql=new StringBuilder();List<Object> rangeArgs=new ArrayList<>();
|
||||
@@ -143,9 +159,9 @@ class SourceColumnQueriesTest {
|
||||
|
||||
queries.append(where, args, "black", filters, "release", actor);
|
||||
|
||||
assertTrue(where.toString().contains("JSON_EXTRACT(f.raw_values, ?) IS NULL"));
|
||||
assertTrue(where.toString().contains("NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON))"));
|
||||
assertTrue(where.toString().contains("COLLATE utf8mb4_0900_as_cs IN (?,?)"));
|
||||
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "00007", "x' OR 1=1 --"), args);
|
||||
assertEquals(List.of("$.\"委托单\"", "00007", "x' OR 1=1 --"), args);
|
||||
assertFalse(where.toString().contains("委托单"));
|
||||
assertFalse(where.toString().contains("x' OR 1=1"));
|
||||
}
|
||||
@@ -160,7 +176,7 @@ class SourceColumnQueriesTest {
|
||||
StringBuilder where = new StringBuilder();
|
||||
queries.append(where, args, "black", filters, "release", actor);
|
||||
assertTrue(where.toString().contains(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL"));
|
||||
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\""), args);
|
||||
assertEquals(List.of("$.\"委托单\""), args);
|
||||
}
|
||||
List<Map<String, Object>> filters = SourceColumnQueries.parse(List.of(Map.of(
|
||||
"field", SOURCE, "stages", List.of("bright"), "operator", "is_empty")));
|
||||
@@ -179,8 +195,7 @@ class SourceColumnQueriesTest {
|
||||
|
||||
assertTrue(where.toString().contains("IS NULL OR"));
|
||||
assertTrue(where.toString().contains(" NOT IN (?)"));
|
||||
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"",
|
||||
"$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "000123"), args);
|
||||
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "000123"), args);
|
||||
}
|
||||
|
||||
@Test void rejectsInvalidStageAndMissingFieldsInTheActiveCatalog() {
|
||||
@@ -223,6 +238,6 @@ class SourceColumnQueriesTest {
|
||||
columns.put("black", stage.equals("black") ? header : null);
|
||||
columns.put("bright", stage.equals("bright") ? header : null);
|
||||
return Map.of("key", key, "label", header, "columns", columns, "stages", List.of(stage),
|
||||
"type", "source_value", "operators", List.of("in", "not_in", "is_empty", "not_empty"));
|
||||
"type", "source_value", "operators", List.of("in", "contains", "not_in", "is_empty", "not_empty"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package cn.yunsheng.quality.filters;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.AbstractExecutorService;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
|
||||
class SourceFilterIndexBackfillTest {
|
||||
@Test
|
||||
void skipsWhenIndexTablesAreUnavailable() {
|
||||
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
|
||||
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
|
||||
ManualExecutor worker = new ManualExecutor();
|
||||
doReturn(false).when(index).available();
|
||||
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
|
||||
|
||||
runner.run(null);
|
||||
|
||||
verify(jdbc, never()).query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
|
||||
verify(index, never()).ensureBuild(anyString());
|
||||
assertEquals(0, worker.queued());
|
||||
runner.stopWorker();
|
||||
}
|
||||
|
||||
@Test
|
||||
void coalescesEventsDuringDrainAndContinuesAfterAnIndividualFailure() {
|
||||
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
|
||||
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
|
||||
ManualExecutor worker = new ManualExecutor();
|
||||
doReturn(true).when(index).available();
|
||||
doReturn(List.of("failed-build", "ready-build")).when(jdbc)
|
||||
.query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
|
||||
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
|
||||
doAnswer(invocation -> {
|
||||
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
|
||||
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
|
||||
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("failed-build"));
|
||||
throw new IllegalStateException("synthetic index failure");
|
||||
}).when(index).ensureBuild("failed-build");
|
||||
|
||||
runner.run(null);
|
||||
|
||||
assertEquals(1, worker.queued());
|
||||
verify(index, never()).ensureBuild(anyString());
|
||||
ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
|
||||
verify(jdbc).query(sql.capture(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
|
||||
assertTrue(sql.getValue().contains("c.dataset_id='quality'"));
|
||||
assertTrue(sql.getValue().contains("r.status='PUBLISHED'"));
|
||||
assertTrue(sql.getValue().contains("r.complete=TRUE"));
|
||||
assertTrue(sql.getValue().contains("b.status='COMPLETED'"));
|
||||
|
||||
worker.runNext();
|
||||
|
||||
verify(index).ensureBuild("failed-build");
|
||||
verify(index).ensureBuild("ready-build");
|
||||
verify(index).ensureBuild("event-during-drain");
|
||||
assertEquals(0, worker.queued());
|
||||
runner.stopWorker();
|
||||
assertTrue(worker.isShutdown());
|
||||
}
|
||||
|
||||
@Test
|
||||
void componentHasAnAutowiredProductionConstructorAndCleanShutdown() {
|
||||
try (var context = new AnnotationConfigApplicationContext()) {
|
||||
context.registerBean(JdbcTemplate.class, () -> org.mockito.Mockito.mock(JdbcTemplate.class));
|
||||
context.registerBean(SourceFilterIndex.class, () -> org.mockito.Mockito.mock(SourceFilterIndex.class));
|
||||
context.register(SourceFilterIndexBackfill.class);
|
||||
context.refresh();
|
||||
|
||||
assertNotNull(context.getBean(SourceFilterIndexBackfill.class));
|
||||
}
|
||||
}
|
||||
|
||||
private static final class ManualExecutor extends AbstractExecutorService {
|
||||
private final ArrayDeque<Runnable> tasks = new ArrayDeque<>();
|
||||
private boolean shutdown;
|
||||
|
||||
@Override public void shutdown() { shutdown = true; }
|
||||
|
||||
@Override public List<Runnable> shutdownNow() {
|
||||
shutdown = true;
|
||||
List<Runnable> pending = List.copyOf(tasks);
|
||||
tasks.clear();
|
||||
return pending;
|
||||
}
|
||||
|
||||
@Override public boolean isShutdown() { return shutdown; }
|
||||
|
||||
@Override public boolean isTerminated() { return shutdown && tasks.isEmpty(); }
|
||||
|
||||
@Override public boolean awaitTermination(long timeout, TimeUnit unit) { return isTerminated(); }
|
||||
|
||||
@Override public void execute(Runnable command) {
|
||||
if (shutdown) throw new RejectedExecutionException("executor is stopped");
|
||||
tasks.add(command);
|
||||
}
|
||||
|
||||
int queued() { return tasks.size(); }
|
||||
|
||||
void runNext() { tasks.removeFirst().run(); }
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,21 @@
|
||||
package cn.yunsheng.quality.metrics;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.ArgumentMatchers.contains;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
|
||||
import cn.yunsheng.quality.filters.SourceFilterIndex;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
class MetricsRollupServiceTest {
|
||||
@Test
|
||||
@@ -17,4 +30,65 @@ class MetricsRollupServiceTest {
|
||||
assertTrue(sql.contains("GROUP BY f.build_id, f.month_key " + collation));
|
||||
assertTrue(sql.contains("f.group_name " + collation + ", f.project_group " + collation));
|
||||
}
|
||||
|
||||
@Test
|
||||
void publishesFilterBuildAvailableOnlyAfterTheOuterTransactionCommits() {
|
||||
var service = new MetricsRollupService(mock(JdbcTemplate.class));
|
||||
var publisher = mock(ApplicationEventPublisher.class);
|
||||
service.setApplicationEventPublisher(publisher);
|
||||
String build = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
try {
|
||||
service.announceSourceFilterBuild(build);
|
||||
verifyNoInteractions(publisher);
|
||||
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
|
||||
assertEquals(1, synchronizations.size());
|
||||
for (TransactionSynchronization synchronization : synchronizations) synchronization.afterCommit();
|
||||
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
|
||||
} finally {
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void publishesImmediatelyWhenThereIsNoTransactionSynchronization() {
|
||||
var service = new MetricsRollupService(mock(JdbcTemplate.class));
|
||||
var publisher = mock(ApplicationEventPublisher.class);
|
||||
service.setApplicationEventPublisher(publisher);
|
||||
String build = "22222222-2222-4222-8222-222222222222";
|
||||
|
||||
service.announceSourceFilterBuild(build);
|
||||
|
||||
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
|
||||
}
|
||||
|
||||
@Test
|
||||
void notificationFailureAfterCommitDoesNotEscapeTheTransactionSynchronization() {
|
||||
var service = new MetricsRollupService(mock(JdbcTemplate.class));
|
||||
var publisher = mock(ApplicationEventPublisher.class);
|
||||
service.setApplicationEventPublisher(publisher);
|
||||
String build = "33333333-3333-4333-8333-333333333333";
|
||||
var event = new SourceFilterIndex.BuildAvailable(build);
|
||||
doThrow(new IllegalStateException("synthetic event delivery failure")).when(publisher).publishEvent(event);
|
||||
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
try {
|
||||
service.announceSourceFilterBuild(build);
|
||||
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
|
||||
assertDoesNotThrow(() -> synchronizations.get(0).afterCommit());
|
||||
} finally {
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void directFixtureConstructionKeepsTheSynchronousLegacySchemaFallback() {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
var service = new MetricsRollupService(jdbc);
|
||||
|
||||
service.announceSourceFilterBuild("44444444-4444-4444-8444-444444444444");
|
||||
|
||||
verify(jdbc).queryForObject(contains("information_schema.tables"), eq(Integer.class));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
|
||||
TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED')
|
||||
BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup',
|
||||
'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build',
|
||||
'source_filter_member', 'source_filter_value', 'source_filter_field', 'source_filter_build',
|
||||
'fact_black', 'fact_bright')
|
||||
OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion',
|
||||
'release_action', 'candidate_reconciliation')
|
||||
@@ -253,6 +254,23 @@ class Database:
|
||||
self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid))
|
||||
for t in BUILD_CHILDREN:
|
||||
if t in self.columns:
|
||||
if t == 'source_filter_member':
|
||||
# Members have no build_id; drain each owned field in
|
||||
# order without aliases because the exclusion uses LOCK TABLES.
|
||||
count = 0
|
||||
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
|
||||
count = self.execute('DELETE FROM source_filter_member WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
|
||||
if count:
|
||||
break
|
||||
elif t == 'source_filter_value':
|
||||
# Values are likewise owned through fields and are deleted
|
||||
# only after all field memberships have been drained.
|
||||
count = 0
|
||||
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
|
||||
count = self.execute('DELETE FROM source_filter_value WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
|
||||
if count:
|
||||
break
|
||||
else:
|
||||
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
|
||||
if count:
|
||||
return count, 0
|
||||
|
||||
@@ -16,6 +16,7 @@ from unittest.mock import patch
|
||||
ROOT=Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0,str(ROOT/'scripts'))
|
||||
import backup_retention as br
|
||||
import managed_test_resources
|
||||
spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py'))
|
||||
ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret)
|
||||
|
||||
@@ -91,14 +92,18 @@ class DatabaseRetentionTests(unittest.TestCase):
|
||||
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
|
||||
cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True)
|
||||
cls.name='quality_retention_test_'+uuid.uuid4().hex[:12]
|
||||
managed_test_resources.create(ROOT,cls.name,cls.execute_admin)
|
||||
with cls.admin.cursor() as c:
|
||||
c.execute('CREATE DATABASE '+ret.ident(cls.name))
|
||||
c.execute('USE '+ret.ident(cls.name))
|
||||
ddl=[
|
||||
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
|
||||
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
|
||||
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)',
|
||||
'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE source_filter_build(build_id varchar(36) PRIMARY KEY,format_version int,FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
|
||||
'CREATE TABLE source_filter_member(field_id bigint NOT NULL,fact_id bigint NOT NULL,value_id bigint NOT NULL,PRIMARY KEY(field_id,fact_id),KEY(value_id,fact_id),FOREIGN KEY(field_id) REFERENCES source_filter_field(id),FOREIGN KEY(value_id) REFERENCES source_filter_value(id))',
|
||||
'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)',
|
||||
@@ -115,14 +120,25 @@ class DatabaseRetentionTests(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name))
|
||||
managed_test_resources.drop(ROOT,cls.name,cls.execute_admin)
|
||||
cls.admin.close()
|
||||
|
||||
@classmethod
|
||||
def execute_admin(cls,sql,args=None):
|
||||
result=''
|
||||
statements=[part.strip() for part in sql.split(';') if part.strip()]
|
||||
with cls.admin.cursor() as cursor:
|
||||
for statement in statements:
|
||||
cursor.execute(statement,args if len(statements)==1 else None)
|
||||
result='\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
|
||||
return result
|
||||
|
||||
def setUp(self):
|
||||
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
|
||||
self.db=ret.Database(ROOT,self.name);self.db.root=self.root
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
|
||||
for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table))
|
||||
for table in self.db.tables:
|
||||
if table!='__quality_test_owner':self.db.execute('TRUNCATE TABLE '+ret.ident(table))
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
|
||||
self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')")
|
||||
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
|
||||
@@ -187,14 +203,24 @@ class DatabaseRetentionTests(unittest.TestCase):
|
||||
|
||||
def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self):
|
||||
self.retire_old();state={'builds':{'obsolete':1}}
|
||||
self.db.execute("INSERT INTO source_filter_build VALUES('obsolete',1)")
|
||||
self.db.execute("INSERT INTO source_filter_field VALUES(9,'obsolete','black','空字段',%s),(10,'obsolete','black','客户',%s),(11,'obsolete','black','工单',%s)", (b'0'*32,b'1'*32,b'2'*32))
|
||||
self.db.execute("INSERT INTO source_filter_value VALUES(100,10,%s,'客户A'),(101,10,%s,'客户B'),(102,11,%s,'工单A')", (b'1'*32,b'2'*32,b'3'*32))
|
||||
self.db.execute("INSERT INTO source_filter_member VALUES(10,1,100),(10,2,101),(11,3,102)")
|
||||
with self.db.frozen('current'):
|
||||
self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0))
|
||||
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_member')[0]['n'],2)
|
||||
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_value')[0]['n'],3)
|
||||
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_field')[0]['n'],3)
|
||||
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_build')[0]['n'],1)
|
||||
old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0]
|
||||
self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64)
|
||||
self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,))
|
||||
for _ in range(4):
|
||||
for _ in range(20):
|
||||
with self.db.frozen('current'):self.db.build_step(state,1800,4000,1)
|
||||
self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'"))
|
||||
for table in ('source_filter_member','source_filter_value','source_filter_field','source_filter_build'):
|
||||
self.assertFalse(self.db.rows('SELECT * FROM '+ret.ident(table)))
|
||||
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'"))
|
||||
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'"))
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from unittest.mock import patch
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
import backup_retention as br
|
||||
import managed_test_resources
|
||||
import revision_retention as revisions
|
||||
spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py'))
|
||||
ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret)
|
||||
@@ -26,12 +27,15 @@ class SourceRevisionTests(unittest.TestCase):
|
||||
cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root',
|
||||
password=cfg['rootPassword'], autocommit=True)
|
||||
cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
|
||||
managed_test_resources.create(ROOT, cls.name, cls.execute_admin)
|
||||
ddl = [
|
||||
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))',
|
||||
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))',
|
||||
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
|
||||
"CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)",
|
||||
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
|
||||
'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||
'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))',
|
||||
@@ -42,23 +46,34 @@ class SourceRevisionTests(unittest.TestCase):
|
||||
'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))']
|
||||
try:
|
||||
with cls.admin.cursor() as c:
|
||||
c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name))
|
||||
c.execute('USE ' + ret.ident(cls.name))
|
||||
for statement in ddl:
|
||||
c.execute(statement + ' ENGINE=InnoDB')
|
||||
except BaseException:
|
||||
with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name))
|
||||
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
|
||||
cls.admin.close(); raise
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name))
|
||||
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
|
||||
cls.admin.close()
|
||||
|
||||
@classmethod
|
||||
def execute_admin(cls, sql, args=None):
|
||||
result = ''
|
||||
statements = [part.strip() for part in sql.split(';') if part.strip()]
|
||||
with cls.admin.cursor() as cursor:
|
||||
for statement in statements:
|
||||
cursor.execute(statement, args if len(statements) == 1 else None)
|
||||
result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
|
||||
return result
|
||||
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve()
|
||||
self.db = ret.Database(ROOT, self.name); self.db.root = self.root
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
|
||||
for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
|
||||
for table in self.db.tables:
|
||||
if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
|
||||
self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')")
|
||||
self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')")
|
||||
@@ -121,6 +136,21 @@ class SourceRevisionTests(unittest.TestCase):
|
||||
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
||||
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
||||
|
||||
def test_indexed_business_values_do_not_count_as_revision_metadata(self):
|
||||
path = self.add_revision('indexed-only', b'indexed-only workbook')
|
||||
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,))
|
||||
self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,))
|
||||
self.collect(100); result = self.collect(2000)
|
||||
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||
self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists())
|
||||
|
||||
def test_indexed_field_names_do_not_count_as_revision_metadata(self):
|
||||
path = self.add_revision('indexed-field-only', b'field-only workbook')
|
||||
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,))
|
||||
self.collect(100); result = self.collect(2000)
|
||||
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||
self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists())
|
||||
|
||||
def test_new_reference_resets_continuous_grace(self):
|
||||
self.add_revision(); self.collect(100)
|
||||
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500)
|
||||
|
||||
+41
-1
@@ -1,8 +1,48 @@
|
||||
# 当前进度
|
||||
|
||||
核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。
|
||||
核实日期:2026年11月13日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。
|
||||
|
||||
|
||||
## 2026-11-13:Gitea交付(筛选提速与包含条件)
|
||||
|
||||
本轮仓库交付整合此前来源筛选超时修复、统一字典/记录索引、后台补建、目录与候选读取优化,以及“包含”条件和输入边界修正;保留11个实施与验收提交,整合提交为 `6983dc3`。目标为 `lyun/quality_analyse` 的 `main`,对应当前已上线2026.11.13.12的功能源码。
|
||||
|
||||
整合后功能代码与已验收分支逐文件一致,无冲突或新增业务修改,沿用既有构建、隔离数据库与线上验收结果。本次只同步源码和交付记录,不重复发布、重启服务或运行数据库迁移;私有配置、业务数据、运行包和验收附件不进入仓库。下方“未同步”保留各次实施时点,本节为后续接续的最新仓库交付记录。
|
||||
|
||||
## 2026-11-13:来源字段“包含”筛选(已发布)
|
||||
|
||||
来源字段操作菜单新增“包含”:自由输入关键词按回车,多个关键词按OR匹配;不同筛选条件之间仍为AND。后台用绑定参数的LOCATE执行字面子串匹配,大小写及正式来源值语义保持,`%`、`_`、反斜杠和引号不是通配或SQL内容。支持1–50个非空白关键词,每个最多256字符;前端控制搜索长度并避免截断代理对,重复关键词失焦后清空草稿。
|
||||
|
||||
汇总、趋势、来源明细与Excel统一走同一解析/授权/查询逻辑,兼容原始JSON回退与现有索引。DSH活跃共享契约、图表上下文和历史/证据显示同步接受contains;关键词不做完整候选值的等值核验。打包只替换5个相关DSH业务模块,保留当前已验收的流式助手和原固定发行版;模型配置及状态不变,未调用在线模型。不新增迁移、表或索引,不重建既有52个构建。
|
||||
|
||||
验证:前端61项及类型/生产构建通过;完整后端990项(820执行通过、170按条件跳过);专项隔离MySQL及服务45项(44通过、1基准跳过),覆盖两阶段、原表/索引、规范维度、大小写、特殊字符、零/字符串null/空值、组织权限,以及汇总/趋势/明细/Excel。DSH契约/显示Python135项(133通过、2跳过),应用构建16项通过,新打包应用的contains契约及固定运行时检查通过,独立审查无阻断。未执行私有原表全量回归或在线模型验收。
|
||||
|
||||
线上关键词命中2个完整客户值,contains与对应in集合的汇总业务数据、趋势及来源结果一致;代表查询约1965/721/465ms。来源29条、121个原始字段,导出29条且ZIP/XML有效;首次导出12907ms、复用后844ms。实际菜单、键盘提交、多关键词字面值、257字符粘贴限制及重复失焦清理已检查,页面无失败提示。前端2026.11.13.12(`f8432b4`),后端功能提交`3c022e8`,后续仅前端输入边界修正,数据库进程保持;DSH固定引擎、模型配置及状态保持。
|
||||
|
||||
代码已在独立分支 `codex/source-filter-contains-20261113` 本地提交,本轮未同步Gitea。私有请求、结果及效果截图位于 `output/20261113-source-contains/`,不进入Git。
|
||||
|
||||
## 2026-11-13:统一筛选提速(已发布验收)
|
||||
|
||||
已发布2026.11.13.10,功能提交 `20f1c7d`;当前正式版52/52个构建的筛选索引全部就绪。新增按构建保存的来源字段字典与稀疏记录索引,来源/配置筛选、候选、指标、趋势、明细及明细导出共用;正式算法、当前字段/组织权限和发布版绑定保持。候选模式只取上限加一,不再为下拉精确计数;常规页面查询不等待无关目录,本字段改选不重复读取自己的候选,明细导出先只读字段名。
|
||||
|
||||
生命周期:启动和构建提交后通过同一单线程有界去重队列补索引,完整提交后才启用;失败或尚未就绪保留原查询,导入事务不等待索引。构建过程按500条事实窗口驱动字典关联,并物理重建临时窗口,避免回扫增长字典和长事务临时表累积。保留清理已识别4张新增表,未解除维护兼容保护。
|
||||
|
||||
首次加载还存在两条慢路径:候选为校验年份/可见范围反复读取宽原始行,V44新增薄覆盖索引;字段目录扫描约110万事实只为找每构建首条可见记录,V45改索引列序及带完整权限的有序LIMIT1查询。实际计划回归要求覆盖读取、不排序全部事实、找到首条即停;与旧MIN查询直接对照原始字段目录,保留空/隐藏构建和重复成员语义。
|
||||
|
||||
验证:前端相关46项及生产构建通过;完整后端988项(819执行通过、169按条件跳过),专项隔离MySQL/服务49项(48通过、1基准跳过),保留清理34项通过,多次独立审查无阻断。隔离场景覆盖黑片/亮片、全部4类来源运算、旧配置null和字面字符串null、Unicode、同一事实行、多字段、当前权限、导出、跨窗口故障回滚与重试。未执行私有原表全量回归。
|
||||
|
||||
线上实测在恢复原256MiB数据库缓存、后端重新启动且无并行测试时完成:同一客户筛选首次汇总1366ms、趋势347ms、来源明细513ms、原候选条件381ms、去掉自身约束的完整候选首开3546ms;月度458ms、季度369ms。原始超时场景汇总/趋势约48.3秒;前一版索引已全就绪时首次汇总仍18843ms,此次首条目录优化后降到1366ms。汇总、趋势、29条来源明细的完整规范化结果与原基准一致,月/季与原路径一致。Excel约3479ms,1张表、29条明细、A1:EA30,ZIP/XML有效;页面查询完成,来源抽屉载入121字段、无失败提示,点击产品与来源范围一致。以上是代表场景的只读重放,不是数据库完整冷启动或所有字段组合的固定耗时承诺。
|
||||
|
||||
空间实测:新增字典/成员等4表文件共13,727,023,104字节,事实表4个覆盖索引页约372,064,256字节;本口径合计约14.10GB(13.13GiB),不含既有原表、redo/undo和临时文件,也不等同于数据库目录净增差值。此次覆盖全部原始字段,空间显著高于先前紧凑柏拉图汇总。补建期间临时1GiB缓存已恢复256MiB,核验16384页且调整完成。数据库原进程和DSH配置保留,无业务版本指针变更。
|
||||
|
||||
实现与验收记录已本地提交;本轮新优化尚未同步Gitea。私有请求、结果校验及页面截图保存在 `output/20261113-filter-speed/`,不进入仓库。主工作目录已有历史改动保持,当前交付在独立分支 `codex/unified-filter-speed-20261113`。
|
||||
|
||||
## 2026-11-13:来源字段筛选超时修复
|
||||
|
||||
定位:客户等原始字段筛选时,汇总、趋势和覆盖检查重复读取字段目录,数据库对年度查询选择了原表扫描,约48秒超过页面30秒期限。复用服务级字段目录缓存,目录及无窄范围限制的年度来源筛选使用既有构建索引;JSON值每次表达式只提取一次,保留空值、字符串和大小写语义。单月、季度、产品及扩展条件保留原查询计划。查询超时提示改为重试或调整范围,不再误报连接断开。权限和正式计算规则不变,无新增表或索引。
|
||||
|
||||
验证:后端完整构建975项(812执行通过、163按条件跳过);隔离MySQL及快照专项34项(33通过、1跳过),前端相关17项及生产构建通过。独立复核无实质问题;已验证主目录本轮6文件与交付目录一致,新旧运行包数据库迁移文件一致。已发布2026.11.13.5(功能提交 `ea84e26`)。线上同条件汇总/趋势从48282/48281ms降至17818/17816ms,完整结果规范化校验值均一致;页面正常加载时两接口约5528/5529ms,无失败提示。实际来源明细返回29条、121个字段,Excel下载有效。数据库迁移文件不变,MySQL原进程和DSH配置保留。只读同条件复测不等于私有原表全量回归。本轮独立分支已本地提交,尚未同步Gitea;私有验收材料位于 `output/20261113-source-filter-timeout/`。
|
||||
|
||||
## 2026-11-13:Gitea 同步交付
|
||||
|
||||
本次将已发布的来源明细与期间范围修复、换年保持产品层级、查询默认年度、不良率曲线细分、表格及 Excel 联动、趋势与表格独立加载、标题旁钻取路径,以及紧凑预汇总提速整合同步至 `lyun/quality_analyse` 的 `main`。保留各次发布的提交记录,线上版本为2026.11.13.4,本次仓库同步不重复部署。
|
||||
|
||||
@@ -144,6 +144,8 @@ def _resolve_source_column_values(task, *, release_id, resolver):
|
||||
if not isinstance(rules, list):
|
||||
return None
|
||||
for rule_index, rule in enumerate(rules):
|
||||
# Candidate lookup confirms complete values only. A contains keyword is
|
||||
# intentionally not required to equal a full candidate source value.
|
||||
if rule.get('operator') not in ('in', 'not_in'):
|
||||
continue
|
||||
field_key = rule['field']
|
||||
@@ -1212,7 +1214,7 @@ def plan_task(*, base_url, api_key, model, question, context, previous=None, can
|
||||
if aliases and field.get('type')=='source_value':item['aliases']=aliases
|
||||
compact_fields.append(item)
|
||||
compact_catalog['sourceFields']=compact_fields
|
||||
compact_catalog['sourceColumnOperators']=['in','not_in','is_empty','not_empty']
|
||||
compact_catalog['sourceColumnOperators']=['in','not_in','contains','is_empty','not_empty']
|
||||
payload['businessCatalog']=compact_catalog
|
||||
system += ('\n综合以上生成报告应整合previousVerifiedTurns中多个已完成轮次,historyMode=all、freshQuery=false;不得把最新局部追问当作全部历史。'
|
||||
'只有用户要求更新数据时才freshQuery=true。报告与复述不自动开展新调查。'
|
||||
@@ -1292,9 +1294,9 @@ JSON字段:goal(完整当前目标),kind(analysis/metrics/members/explain/re
|
||||
filterUpdates允许department/group/product/projectGroup/extraFilters/exclusions/includeUnassigned。null明确清除该条件,{}表示没有变化;单一维度用目录原名字符串,extraFilters和exclusions用字段到值数组。额外字段仅限businessCatalog.extraFields;客户代码使用extraFilters.customerCode,保留完整字符串。未知字段不能丢弃或扩大范围,应澄清能力限制。汽车交通对应department=新能源汽车磁钢,国内/海外才是其业务分组。
|
||||
productMatch为ambiguous且未选择时,只需列出授权候选并needsClarification=true,不选择或合并后缀、不换成全部产品。unique使用完整原名;18×15×4不能匹配18×15×4.1。用户选择前轮候选时保留原年份、指标和趋势目标。newTask=true仅用于用户明确另开无关任务,并声明新期间和条件。
|
||||
今年、上月等使用serverCalendar;最新数据月才用目录,不猜测缺失期间。真正缺少期间或对象有歧义才澄清。普通读取业务指标freshQuery=true。
|
||||
原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/is_empty/not_empty","values":[完整原文]}。in/not_in要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。大小写、空格、前导零都是字面值,不做归一化。规则AND、values OR;每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。
|
||||
原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/contains/is_empty/not_empty","values":[完整原文或包含关键字]}。in/not_in/contains要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。规则AND、values OR;文本大小写、空格、前导零均保留原文,不对contains关键字归一化;contains是原文的字面子串筛选,不是通配符或正则。每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。
|
||||
只问报废率时fields=[scrapRate];同时要求分子分母、计算基数、报废总量或总重时,分别保留对应重量字段;要求分子分母时fields=[scrapRate,scrapWeight,totalWeight]。不能只在goal中提及而漏掉实际fields。分子为现有正式报废总量,分母为该阶段正式总重,黑片使用现有修正总重。利用料、本部报废、过账外废沿用现有分量逻辑,不使用1-合格率,也不平均原表百分比。未指定阶段的报废率返回black和bright两个阶段;没有统一综合报废率定义,用户明确要综合报废率时请求其定义,不转成综合合格率。
|
||||
按用户原文提出in/not_in文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。
|
||||
按用户原文提出in/not_in完整文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验in/not_in取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。contains关键字是子串而非完整候选值,不做等值候选核验,按用户原文交由正式查询执行字面子串筛选。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。
|
||||
kind=members仅用于列出有哪些委托单/批号/规格等唯一成员,不表示原始行;memberFields使用当前目录key且最多5个,必须与用户明确要列出的对象语义相符,不能把筛选字段当输出成员。裸“规格”表示产品规格,要选择目录中真实产品规格列;不能用图号或客户代码代替。明确问图号时选择图号列,明确问客户规格时选择客户规格列。若用户明确给出“成品规格”等目录label,优先精确匹配该label;裸“规格”有多个候选或没有可用字段时,needsClarification=true、memberFields=[],保留原目标并澄清。并列、否定或复合字段意图由规划器按用户表达处理,不能补入已取消字段。stages必须明确为一个或两个阶段,期间与普通filters照常保留。需要澄清时stages可为空,本轮不查询并保留目标。members的fields=[]、verifySource=false、fullDetails=false、freshQuery=true,不计算KPI、不改成metrics;清单由平台直接渲染,无需撰写业务数值。
|
||||
单期投入重量排名或明确单阶段报废排名:kind=analysis、dimensions为实际product/department/group、fields仅totalWeight/scrapWeight、dimensionStages明确各维度black/bright、trendRequired=false;未限定阶段的报废排名使用queryPlan ranking、fields=[rate,scrapWeight]、stages=[]、dimensionStages对应维度=[];全量正式分母不随Top展示数量改变。普通年度重量排名默认periodCoverage=available,注明已发布月份;只有用户明确要求完整全年齐全时用complete。排名与趋势同时要求时仍保留真实要求,不删除trendRequired。
|
||||
能力边界必须忠实表达:排名/缺陷保留analysis及真实dimensions和dimensionStages;两个独立期间比较保留comparisonRequired=true和comparisonPeriod;源表核验为source_verify及verifySource=true;原始逐行记录为details及fullDetails=true;报告为report。不得为了适配已接入能力而改写成普通指标。不得把“有哪些”理解成所有原始字段。后端负责说明仍未接入的能力。
|
||||
|
||||
@@ -154,7 +154,8 @@ def _scope_label(filters):
|
||||
if key=='columnFilters':
|
||||
for rule in value:
|
||||
stage='、'.join({'black':'黑片','bright':'亮片'}.get(s,s) for s in rule.get('stages',[]))
|
||||
operation={'in':'为','not_in':'排除','is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'')
|
||||
operation={'in':'为','not_in':'排除','contains':'包含片段',
|
||||
'is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'')
|
||||
parts.append(stage+'来源字段'+operation+'、'.join(rule.get('values',[])))
|
||||
else: parts.append(names[key]+'='+str(value))
|
||||
return ';'.join(parts) if parts else '当前授权范围内总体(默认不含未分类)'
|
||||
|
||||
@@ -850,7 +850,8 @@ def _history_scope_caption(snapshot, section):
|
||||
parts.append('包含未分配项')
|
||||
column_filters = filters.get('columnFilters')
|
||||
if isinstance(column_filters, list):
|
||||
operator_labels = {'in': '包含', 'not_in': '不含', 'is_empty': '为空', 'not_empty': '非空'}
|
||||
operator_labels = {'in': '为', 'not_in': '排除', 'contains': '包含片段',
|
||||
'is_empty': '为空', 'not_empty': '非空'}
|
||||
for rule in column_filters:
|
||||
if not isinstance(rule, dict):
|
||||
continue
|
||||
|
||||
@@ -115,11 +115,11 @@ QUALITY_DATA_SCHEMA: dict[str, Any] = {
|
||||
"field": {"type": "string", "minLength": 1},
|
||||
"stages": {"type": "array", "minItems": 1, "maxItems": 2,
|
||||
"items": {"type": "string", "enum": ["black", "bright"]}},
|
||||
"operator": {"type": "string", "enum": ["in", "not_in", "is_empty", "not_empty"]},
|
||||
"operator": {"type": "string", "enum": ["in", "not_in", "contains", "is_empty", "not_empty"]},
|
||||
"values": {"type": "array", "minItems": 1, "maxItems": 50,
|
||||
"items": {"type": "string", "maxLength": 256}},
|
||||
}, "required": ["field", "stages", "operator"]},
|
||||
"description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR,文本大小写、空格及前导零按原文匹配。is_empty/not_empty不得带values。"},
|
||||
"description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR;in/not_in匹配完整文本值,contains按原文作字面子串匹配(MySQL LOCATE),不做关键字归一化,大小写、空格及前导零保留原文。每条文本规则提供1到50个不超过256字符的值;is_empty/not_empty不得带values。"},
|
||||
"view": {"type": "string", "enum": ["overall", "department", "group", "product", "projectGroup"], "description": "query 的正式分组视图;changes 仅支持非 overall 维度。"},
|
||||
"stage": {"type": ["string", "null"], "enum": ["black", "bright", None], "description": "top/details/defects/investigate必须明确black/bright。rank单阶段传black/bright;rank的metric=scrapWeight且stage省略或null时,合计黑片+亮片报废总重全量排序,并保留正式综合合格率。投入重量rank仍必须指定阶段。changes可选,未传为综合范围。"},
|
||||
"dimension": {"type": "string", "enum": ["category", "defect", "product", "department", "group", "projectGroup"], "description": "top模式仅category/product/department/group/projectGroup,不能用defect;具体缺陷TOP必须使用mode=defects、dimension=defect。product是规格。changes 使用 product/department/group/projectGroup。defects 使用 defect 或 product;未指定具体 defect 字段时,省略 dimension 默认按 defect 查询阶段具体缺陷。"},
|
||||
|
||||
@@ -3,7 +3,7 @@ import copy
|
||||
|
||||
DEPARTMENT_ALIASES = {'汽车交通':'新能源汽车磁钢'}
|
||||
SOURCE_STAGES = {'black', 'bright'}
|
||||
COLUMN_OPERATORS = {'in', 'not_in', 'is_empty', 'not_empty'}
|
||||
COLUMN_OPERATORS = {'in', 'not_in', 'contains', 'is_empty', 'not_empty'}
|
||||
|
||||
|
||||
def source_field_labels(source_fields):
|
||||
@@ -35,11 +35,13 @@ def parse_column_filters(value):
|
||||
raise ValueError(f'{name}.stages必须是非空black/bright数组。')
|
||||
if operator not in COLUMN_OPERATORS:raise ValueError(f'{name}.operator不在当前字段支持目录中。')
|
||||
clean={'field':key,'stages':list(stages),'operator':operator}
|
||||
if operator in ('in','not_in'):
|
||||
if operator in ('in','not_in','contains'):
|
||||
values=rule.get('values')
|
||||
if (not isinstance(values,list) or not 1<=len(values)<=50
|
||||
or any(not isinstance(item,str) or item=='' or len(item)>256 for item in values)):
|
||||
raise ValueError(f'{name}.values必须是1到50个不超过256字符的完整文本值。')
|
||||
or any(not isinstance(item,str) or item==''
|
||||
or operator=='contains' and not item.strip()
|
||||
or len(item)>256 for item in values)):
|
||||
raise ValueError(f'{name}.values必须是1到50个不超过256字符的文本值或包含关键字。')
|
||||
clean['values']=list(dict.fromkeys(values))
|
||||
elif 'values' in rule:raise ValueError(f'{name}的空值操作不接受values。')
|
||||
result.append(clean)
|
||||
|
||||
@@ -9,9 +9,11 @@ import uuid
|
||||
|
||||
|
||||
# These are measured business values, not documents that refer to source IDs.
|
||||
VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'),
|
||||
BUSINESS_VALUE_COLUMNS = {('fact_black', 'raw_values'), ('fact_black', 'components'),
|
||||
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
|
||||
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')}
|
||||
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components'),
|
||||
('source_filter_field', 'column_name'),
|
||||
('source_filter_value', 'value')}
|
||||
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
|
||||
|
||||
|
||||
@@ -54,7 +56,7 @@ class Cleaner:
|
||||
def metadata_mentions(self, needles, excluded=None):
|
||||
for table, columns in self.db.column_types.items():
|
||||
for column, kind in columns.items():
|
||||
if kind not in TEXT_TYPES or (table, column) in VALUE_JSON:
|
||||
if kind not in TEXT_TYPES or (table, column) in BUSINESS_VALUE_COLUMNS:
|
||||
continue
|
||||
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
|
||||
args = list(needles)
|
||||
|
||||
@@ -34,6 +34,15 @@ def coverage_comparison():
|
||||
|
||||
|
||||
class BoundReportTest(unittest.TestCase):
|
||||
def test_scope_label_distinguishes_full_value_from_substring_filters(self):
|
||||
from nao_evidence_report import _scope_label
|
||||
label = _scope_label({'columnFilters': [
|
||||
{'field': 'orderNo', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7']},
|
||||
{'field': 'orderNo', 'stages': ['bright'], 'operator': 'contains', 'values': ['07']},
|
||||
]})
|
||||
self.assertIn('黑片来源字段为WO-7', label)
|
||||
self.assertIn('亮片来源字段包含片段07', label)
|
||||
|
||||
def test_priority_summary_keeps_three_signals_and_rate_denominators_separate(self):
|
||||
signals={'scrapIncrease':[{'label':'放量规格','scrapDelta':120}],
|
||||
'currentScrap':[{'label':'存量规格','current':{'scrapWeight':500}}],
|
||||
|
||||
@@ -456,9 +456,11 @@ class HistoryReportExecutionTest(unittest.TestCase):
|
||||
first_section = candidate['sections'][0]
|
||||
first_section['filters']['columnFilters'] = [{
|
||||
'field': 'src_work_order', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7'],
|
||||
}, {
|
||||
'field': 'src_contains_code', 'stages': ['bright'], 'operator': 'contains', 'values': ['07'],
|
||||
}]
|
||||
set_source_task_context(candidate, SOURCE_A,
|
||||
{'sourceFieldLabels': {'src_work_order': '黑片委托单号'}})
|
||||
{'sourceFieldLabels': {'src_work_order': '黑片委托单号', 'src_contains_code': '客户代码'}})
|
||||
set_source_task_context(candidate, SOURCE_B,
|
||||
{'sourceFieldLabels': {'src_work_order': '亮片委托单号'}})
|
||||
fact = candidate['facts'][0]
|
||||
@@ -476,12 +478,14 @@ class HistoryReportExecutionTest(unittest.TestCase):
|
||||
projected = _history_model_fact_projection(candidate, [fact])[0]
|
||||
self.assertIn('产业部 消费电子产业部', projected['modelScope']['displayScope'])
|
||||
self.assertIn('业务分组 消费电子-3C', projected['modelScope']['displayScope'])
|
||||
self.assertIn('黑片委托单号 包含 WO-7', projected['modelScope']['displayScope'])
|
||||
self.assertIn('黑片委托单号 为 WO-7', projected['modelScope']['displayScope'])
|
||||
self.assertIn('亮片 客户代码 包含片段 07', projected['modelScope']['displayScope'])
|
||||
self.assertIn('消费电子产业部', projected['modelText'])
|
||||
self.assertIn('消费电子-3C', projected['modelText'])
|
||||
self.assertIn('黑片委托单号', projected['modelText'])
|
||||
self.assertIn('WO-7', projected['modelText'])
|
||||
for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:', 'src_work_order'):
|
||||
for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:',
|
||||
'src_work_order', 'src_contains_code'):
|
||||
self.assertNotIn(machine_label, projected['modelText'])
|
||||
self.assertNotIn(machine_label, json.dumps(projected['modelScope'], ensure_ascii=False))
|
||||
self.assertEqual(projected['text'], source_text)
|
||||
@@ -581,7 +585,7 @@ class HistoryReportExecutionTest(unittest.TestCase):
|
||||
goals = payload['historySourceGoals']
|
||||
self.assertTrue(any('产业部 消费电子产业部' in goal['displayScopeLabel']
|
||||
and '业务分组 消费电子-3C' in goal['displayScopeLabel']
|
||||
and '黑片委托单号 包含 WO-7' in goal['displayScopeLabel']
|
||||
and '黑片委托单号 为 WO-7' in goal['displayScopeLabel']
|
||||
for goal in goals))
|
||||
self.assertTrue(all('filters' not in goal and '_sourceTurnId' not in goal for goal in goals))
|
||||
self.assertTrue(all('filters' not in scope and 'sourceTurnId' not in scope
|
||||
|
||||
@@ -284,6 +284,10 @@ class QualityToolsTests(unittest.TestCase):
|
||||
self.assertEqual(definition["inputSchema"]["properties"]["mode"]["enum"],
|
||||
["catalog", "query", "rank", "compare", "top", "changes", "defects", "details", "investigate"])
|
||||
self.assertIn("defect", definition["inputSchema"]["properties"])
|
||||
column_filter = definition["inputSchema"]["properties"]["columnFilters"]
|
||||
self.assertEqual(column_filter["items"]["properties"]["operator"]["enum"],
|
||||
["in", "not_in", "contains", "is_empty", "not_empty"])
|
||||
self.assertIn("MySQL LOCATE", column_filter["description"])
|
||||
self.assertFalse(definition["inputSchema"]["additionalProperties"])
|
||||
self.assertEqual(definition["inputSchema"]["allOf"][0]["then"]["required"], ["releaseId"])
|
||||
|
||||
@@ -529,11 +533,11 @@ class QualityToolsTests(unittest.TestCase):
|
||||
if path == "/api/analysis/tools/catalog":
|
||||
response["sourceFields"] = [{"key": "customerCode", "label": "客户代码",
|
||||
"type": "source_value", "stages": ["black"],
|
||||
"columns": {"black": "customer_code"}, "operators": ["in"]}]
|
||||
"columns": {"black": "customer_code"}, "operators": ["in", "contains"]}]
|
||||
return response
|
||||
api = AuthorizedColumnApi()
|
||||
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
|
||||
rule = {"field": "customerCode", "stages": ["black"], "operator": "in", "values": ["C-17"]}
|
||||
rule = {"field": "customerCode", "stages": ["black"], "operator": "contains", "values": ["C-17"]}
|
||||
result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
|
||||
"period": {"year": 2026, "monthStart": 3, "monthEnd": 9},
|
||||
"filters": {"exclusions": {"product": ["规格甲"]}}, "columnFilters": [rule],
|
||||
|
||||
@@ -25,7 +25,7 @@ from test_nao_quality_tools import FakeApi, RELEASE
|
||||
def source_field(key, label, black, bright, stages=('black', 'bright')):
|
||||
return {'key': key, 'label': label, 'columns': {'black': black, 'bright': bright},
|
||||
'stages': list(stages), 'type': 'source_value',
|
||||
'operators': ['in', 'not_in', 'is_empty', 'not_empty']}
|
||||
'operators': ['in', 'not_in', 'contains', 'is_empty', 'not_empty']}
|
||||
|
||||
|
||||
SOURCE_FIELDS = [
|
||||
@@ -349,6 +349,15 @@ class SourceColumnProtocolTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(TaskUnavailable, '不超过256字符'):
|
||||
protocol_task(columnFilters=[{**rule, 'values': ['X' * 257]}])
|
||||
|
||||
def test_contains_keywords_are_preserved_as_literal_or_values(self):
|
||||
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains',
|
||||
'values': [' 001', 'A|B', 'abC']}
|
||||
task = protocol_task(columnFilters=[rule])
|
||||
self.assertEqual(task['filters']['columnFilters'], [rule])
|
||||
for invalid in ([], [''], [' '], ['x' * 257], [1], ['one'] * 51):
|
||||
with self.subTest(invalid=invalid), self.assertRaises(TaskUnavailable):
|
||||
protocol_task(columnFilters=[{**rule, 'values': invalid}])
|
||||
|
||||
def test_protocol_presentation_enums_default_consistently_when_omitted_or_null(self):
|
||||
base = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026, 'month': 3}, comparisonRequired=False,
|
||||
@@ -396,6 +405,10 @@ class SourceColumnProtocolTests(unittest.TestCase):
|
||||
self.assertIn('periodCoverage(available/complete,可省略或null', prompt)
|
||||
self.assertIn('freshQuery(bool)', prompt)
|
||||
self.assertIn('needsClarification(bool)', prompt)
|
||||
self.assertIn('in/not_in/contains/is_empty/not_empty', prompt)
|
||||
self.assertIn('contains关键字是子串而非完整候选值', prompt)
|
||||
self.assertEqual(model_call.call_args_list[0].kwargs['payload']['businessCatalog']['sourceColumnOperators'],
|
||||
['in', 'not_in', 'contains', 'is_empty', 'not_empty'])
|
||||
|
||||
def test_three_arbitrary_source_keys_keep_literal_values_and_stage_scopes(self):
|
||||
task = protocol_task()
|
||||
@@ -429,7 +442,7 @@ class SourceColumnProtocolTests(unittest.TestCase):
|
||||
def test_unauthorized_unknown_and_missing_stage_columns_fail_closed(self):
|
||||
for bad in ([{**RULES[0], 'field': 'notInDirectory'}],
|
||||
[{**RULES[0], 'field': 'blackOnlyCode', 'stages': ['bright']}],
|
||||
[{**RULES[0], 'operator': 'contains'}]):
|
||||
[{**RULES[0], 'operator': 'starts_with'}]):
|
||||
with self.subTest(bad=bad), self.assertRaises(TaskUnavailable):
|
||||
normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
|
||||
period={'year': 2026}, columnFilters=bad), '筛选原表字段',
|
||||
@@ -529,6 +542,20 @@ class SourceColumnProtocolTests(unittest.TestCase):
|
||||
self.assertEqual(customer_lookup['filters']['columnFilters'], [
|
||||
{'field': 'dispatchOrder', 'stages': ['bright'], 'operator': 'not_empty'}])
|
||||
|
||||
def test_contains_keyword_skips_exact_candidate_value_validation(self):
|
||||
from nao_agent_task import _resolve_source_column_values
|
||||
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains',
|
||||
'values': ['001']}
|
||||
task = protocol_task(columnFilters=[rule])
|
||||
calls = []
|
||||
def resolver(**contract):
|
||||
calls.append(contract)
|
||||
raise AssertionError('contains is a substring, not a complete candidate value')
|
||||
result = _resolve_source_column_values(task, release_id=RELEASE, resolver=resolver)
|
||||
self.assertEqual(result, {'status': 'exact'})
|
||||
self.assertEqual(calls, [])
|
||||
self.assertEqual(task['filters']['columnFilters'], [rule])
|
||||
|
||||
def test_missing_literal_offers_same_scope_alternatives_without_substitution(self):
|
||||
from nao_agent_task import _resolve_source_column_values
|
||||
task = protocol_task(columnFilters=[RULES[0]])
|
||||
|
||||
@@ -37,7 +37,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
const [moreFiltersOpen,setMoreFiltersOpen]=useState(false);
|
||||
const [extraFilters,setExtraFilters]=useState<Record<string,string[]>>(initial.extraFilters||{});
|
||||
const [extraFields,setExtraFields]=useState<FilterField[]>([]);
|
||||
const [options,setOptions]=useState<OptionsResponse|null>(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');const [supportLoading,setSupportLoading]=useState(true);
|
||||
const [options,setOptions]=useState<OptionsResponse|null>(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');
|
||||
const [applied,setApplied]=useState<QueryParams|null>(null);const [revision,setRevision]=useState(0);const [response,setResponse]=useState<{key:string;value:MetricsResponse}|null>(null);const [loading,setLoading]=useState(false);const [failure,setFailure]=useState<{key:string;message:string}|null>(null);
|
||||
const [defectDisplay,setDefectDisplay]=useState<'weight'|'rate'>('weight');
|
||||
const [fullFields,setFullFields]=useState(true);const [rateBasis,setRateBasis]=useState<'original'|'withoutHeadquartersScrap'>('original');const [basisOpen,setBasisOpen]=useState(false);const [trendMode,setTrendMode]=useState<'production'|'defects'|'data'>('production');
|
||||
@@ -50,29 +50,39 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
const supportGeneration=useRef(0);const supportController=useRef<AbortController|null>(null);
|
||||
const loadSupport=useCallback(async()=>{
|
||||
const generation=++supportGeneration.current;supportController.current?.abort();const controller=new AbortController();supportController.current=controller;
|
||||
setSupportLoading(true);setSupportError('');
|
||||
try{const [choices,source,filterFields]=await Promise.all([api.options(controller.signal),api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal),api.filterFields(false,controller.signal)]);if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);setCatalog(source);setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));}
|
||||
catch(error){if(generation===supportGeneration.current&&!onUnauthorized(error))setSupportError(error instanceof Error?error.message:'查询条件读取失败');}
|
||||
finally{if(generation===supportGeneration.current)setSupportLoading(false);}
|
||||
setSupportError('');
|
||||
const fail=(error:unknown)=>{if(generation===supportGeneration.current&&!controller.signal.aborted&&!onUnauthorized(error))setSupportError(previous=>previous||(error instanceof Error?error.message:'查询条件读取失败'));};
|
||||
const requests=[
|
||||
api.options(controller.signal).then(choices=>{if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);}).catch(fail),
|
||||
api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal).then(source=>{if(generation===supportGeneration.current)setCatalog(source);}).catch(fail),
|
||||
api.filterFields(false,controller.signal).then(filterFields=>{if(generation===supportGeneration.current)setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));}).catch(fail),
|
||||
];
|
||||
await Promise.all(requests);
|
||||
},[onUnauthorized,initial.releaseId,currentReleaseId]);
|
||||
useEffect(()=>{if(active)void loadSupport();return()=>{supportGeneration.current++;supportController.current?.abort();};},[active,refreshKey,loadSupport]);
|
||||
const fields=useMemo(()=>catalog?.sourceFields.filter(field=>field.type==='source_value'&&field.stages.includes(stage))||[],[catalog,stage]);
|
||||
const exactProductSearch=useMemo(()=>(options?.products||[]).find(name=>name.trim().toLocaleLowerCase()===appliedSearch.trim().toLocaleLowerCase())||'', [options,appliedSearch]);
|
||||
const queryProducts=useMemo(()=>drilledProduct?[drilledProduct]:exactProductSearch?product.length&&!product.includes(exactProductSearch)?['__codex_no_product_search_match__']:[exactProductSearch]:product,[drilledProduct,exactProductSearch,product]);
|
||||
useEffect(()=>{setDrilledProduct('');},[department,group,product,projectGroup,sourceFilters,extraFilters]);
|
||||
const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:initial.releaseId||currentReleaseId||catalog?.releaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,catalog,stage,sourceFilters,extraFilters,initial.releaseId,currentReleaseId]);
|
||||
const queryReleaseId=initial.releaseId||(currentReleaseId!==undefined?currentReleaseId||undefined:catalog?.releaseId);
|
||||
const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,queryReleaseId,stage,sourceFilters,extraFilters]);
|
||||
const catalogMatchesDraft=!!catalog&&!!draft.releaseId&&catalog.releaseId===draft.releaseId;
|
||||
const queryReady=(query:QueryParams)=>!!queryReleaseId&&query.releaseId===queryReleaseId&&(!query.productSearch||!!options)&&(!(query.columnFilters||[]).length||!!catalog&&catalog.releaseId===query.releaseId);
|
||||
const draftReady=queryReady(draft);
|
||||
const appliedReady=!!applied&&queryReady(applied);
|
||||
// Apply organizational/source filters directly, like the other query pages. Product text search keeps its explicit submit.
|
||||
useEffect(()=>{if(active&&catalog&&!supportLoading&&!supportError)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,catalog,draft,supportLoading,supportError]);
|
||||
useEffect(()=>{if(active&&draftReady)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,draft,draftReady]);
|
||||
const queryKey=JSON.stringify({query:applied,revision});const current=response?.key===queryKey?response.value:null;const error=failure?.key===queryKey?failure.message:'';
|
||||
useEffect(()=>{
|
||||
if(!active||!applied||supportError)return;const controller=new AbortController();setLoading(true);setFailure(null);
|
||||
if(!active||!applied||!appliedReady)return;const controller=new AbortController();setLoading(true);setFailure(null);
|
||||
void api.metrics(applied,controller.signal).then(value=>{if(!controller.signal.aborted)setResponse({key:queryKey,value});}).catch(e=>{if(!controller.signal.aborted&&!onUnauthorized(e))setFailure({key:queryKey,message:e instanceof Error?e.message:'查询失败'});}).finally(()=>{if(!controller.signal.aborted)setLoading(false);});
|
||||
return()=>controller.abort();
|
||||
},[active,queryKey,onUnauthorized,supportError]);
|
||||
},[active,queryKey,onUnauthorized,appliedReady]);
|
||||
useEffect(()=>{if(!active||!applied)return;const bytes=new TextEncoder().encode(JSON.stringify({query:queryForLocation(applied,initial.releaseId)}));const encoded=btoa(Array.from(bytes,b=>String.fromCharCode(b)).join('')).replace(/=/g,'').replace(/\+/g,'-').replace(/\//g,'_');const url=`?page=${stage==='black'?'blackProduct':'brightProduct'}&context=${encoded}`;window.history.replaceState({},'',`${location.pathname}${url}`);onLocationChange(url);},[active,applied,stage]);
|
||||
const trendQuery=useMemo(()=>applied?{...applied,month:undefined,quarter:undefined,...(trendGranularity==='year'?{year:undefined}:{}),trendGranularity}:null,[applied,trendGranularity]);
|
||||
const trendKey=JSON.stringify({query:trendQuery,revision});
|
||||
useEffect(()=>{if(!active||!trendQuery||supportError)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,supportError]);
|
||||
const trendReady=!!trendQuery&&queryReady(trendQuery);
|
||||
useEffect(()=>{if(!active||!trendQuery||!trendReady)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,trendReady]);
|
||||
const visibleTrend=useMemo(()=>fullYearTrend(trendResponse?.key===trendKey?trendResponse.value.trend:[],applied?.year||year,trendGranularity),[trendResponse,trendKey,applied?.year,year,trendGranularity]);
|
||||
const displayedTrend=useRetainedResult(trendResponse?.key===trendKey?visibleTrend:null)||visibleTrend;
|
||||
const selectedTrendPeriod=trendGranularity==='month'&&applied?.month
|
||||
@@ -80,12 +90,14 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
: trendGranularity==='quarter'&&applied?.quarter ? `${applied.year}-Q${applied.quarter}` : undefined;
|
||||
const selectTrendPeriod=(value:number|string)=>{automaticPeriod.current=false;const next=nextTrendPeriod(value,{period,year,month,quarter});setYear(next.year);setMonth(next.month);setQuarter(next.quarter);setPeriod(next.period);};
|
||||
const dirty=!!applied&&JSON.stringify(draft)!==JSON.stringify(applied);
|
||||
const queryBusy=supportLoading||loading||(!error&&!supportError&&(dirty||!current));
|
||||
const queryFailed=!!(error||supportError);
|
||||
const metadataPending=!draftReady;
|
||||
const metadataFailed=!draftReady&&!!supportError;
|
||||
const queryBusy=!metadataFailed&&(metadataPending||loading||(!error&&(dirty||!current)));
|
||||
const queryFailed=!!error||metadataFailed;
|
||||
const submit=()=>{automaticPeriod.current=false;setAppliedSearch(search.trim());setDrilledProduct('');setRevision(v=>v+1);};
|
||||
const setTrendObject=(value?:string)=>{const selected=value||'';setDrilledProduct('');setSearch(selected);setAppliedSearch(selected);};
|
||||
const returnToProducts=()=>{setDrilledProduct('');setSearch(drillParentSearch.current);setAppliedSearch(drillParentSearch.current);if(trendMode!=='defects')setTrendMode('production');};
|
||||
const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:catalog?.releaseId},stage,[]));setRevision(v=>v+1);};
|
||||
const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,[]));setRevision(v=>v+1);};
|
||||
const groups=(options?.groups||[]).filter(name=>!department.length||department.includes(options?.groupDepartments?.[name]||''));
|
||||
const paths=options?.organizationPaths||[];const products=department.length||group.length?[...new Set(paths.filter(path=>(!department.length||department.includes(path.department||''))&&(!group.length||group.includes(path.group||''))).map(path=>path.product).filter((v):v is string=>!!v))]:options?.products||[];
|
||||
const trendOptions=useMemo(()=>[...products.map(value=>({label:value,value})),...(drilledProduct==='__UNASSIGNED__'?[{label:'未分类',value:drilledProduct}]:[])],[products,drilledProduct]);
|
||||
@@ -124,9 +136,9 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
</div>
|
||||
<div className="query-toolbar-tail"><span className={`query-status ${queryBusy?'busy':queryFailed?'failed':''}`}><i/>{queryBusy?'正在查询':queryFailed?'查询失败':'查询完成'}</span><QuerySnapshotButton title={`${label}数据`} queryKey={JSON.stringify([draft,trendMode,trendObject,trendGranularity,rateBasis,fullFields,defectDisplay,revision])} disabled={queryBusy || !current} onUnauthorized={onUnauthorized} /><Button type="default" onClick={reset}>重置</Button></div>
|
||||
{moreFiltersOpen&&<div className="query-extra-fields" id={`stage-more-filters-${stage}`}>
|
||||
{extraFields.map(field=><SalesOrderTypeSelect key={field.key} field={field} filters={extraFilters} releaseId={draft.releaseId} stage={stage} onUnauthorized={onUnauthorized} onChange={setExtraFilters}/>)}
|
||||
{draft.releaseId&&extraFields.map(field=><SalesOrderTypeSelect key={field.key} field={field} filters={extraFilters} releaseId={draft.releaseId} stage={stage} onUnauthorized={onUnauthorized} onChange={setExtraFilters}/>)}
|
||||
{options?.projectGroupStatus==='READY'&&select('项目组',projectGroup,options.projectGroups||[],setProjectGroup,true)}
|
||||
{catalog&&<StageSourceFilters stage={stage} fields={fields} query={draft} value={sourceFilters} onChange={setSourceFilters} onUnauthorized={onUnauthorized}/>}
|
||||
{catalogMatchesDraft&&<StageSourceFilters stage={stage} fields={fields} query={draft} value={sourceFilters} onChange={setSourceFilters} onUnauthorized={onUnauthorized}/>}
|
||||
</div>}
|
||||
</section>
|
||||
{error&&<Alert showIcon type="error" message={response?"更新失败,当前显示上次结果":"查询失败"} description={error} action={<Button onClick={()=>setRevision(v=>v+1)}>重试</Button>}/>}
|
||||
@@ -150,7 +162,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
</div>
|
||||
</QueryTrendPanel>
|
||||
<QueryTablePanel heading={<SectionHeading title={table?.drilledProduct?`产品${table.granularity==='quarter'?'季度':table.granularity==='year'?'年度':'月度'}质量数据`:`${label}产品质量汇总`} subtitle={table?.drilledProduct?`${table.query.year} 年 · ${table.drilledProduct==='__UNASSIGNED__'?'未分类':table.drilledProduct} · 重量 / kg 合格率 / %`:'重量 / kg 合格率 / %'}/>}
|
||||
busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!catalog||!!supportError}
|
||||
busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!options}
|
||||
fullFields={fullFields} onFullFields={setFullFields} defectDisplay={defectDisplay} onDefectDisplay={setDefectDisplay}
|
||||
returnControls={drilledProduct&&<Button className="query-return-button" icon={<ArrowLeftOutlined/>} onClick={returnToProducts}>返回上一级</Button>}
|
||||
exporting={exporting} exportDisabled={!!defectDrill.category&&(!defectDrill.table||defectDrill.pending||!!defectDrill.error)||!current||loading||!tableReady||!table?.rows.length} onExport={()=>void runExport()}>
|
||||
|
||||
@@ -11,13 +11,15 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S
|
||||
const [search,setSearch]=useState('');const [choices,setChoices]=useState<string[]>([]);const [busy,setBusy]=useState(false);const [error,setError]=useState('');const [more,setMore]=useState(false);const [revision,setRevision]=useState(0);
|
||||
const {active}=usePageCache();
|
||||
const showBusy=useDelayedIndicator(busy);
|
||||
const contextKey=JSON.stringify(query);const current=useRef(0);
|
||||
const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query;
|
||||
const period={year,...(month?{month}:{}),...(quarter?{quarter}:{})};
|
||||
const filters={...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)};
|
||||
const contextKey=JSON.stringify({releaseId,period,filters,field:field.key,stage});const current=useRef(0);
|
||||
useEffect(()=>{
|
||||
if(!active)return;
|
||||
const generation=++current.current,controller=new AbortController();setBusy(true);setError('');setChoices([]);
|
||||
const timer=window.setTimeout(()=>{
|
||||
const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query;
|
||||
void api.sourceColumnValues({releaseId,period:{year,...(month?{month}:{}),...(quarter?{quarter}:{})},filters:{...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)},field:field.key,stages:[stage],search,limit:50,offset:0},controller.signal)
|
||||
void api.sourceColumnValues({releaseId,period,filters,field:field.key,stages:[stage],search,limit:50,offset:0,candidateOnly:true},controller.signal)
|
||||
.then(result=>{if(current.current!==generation)return;setChoices(result.values.filter((v):v is string=>typeof v==='string'));setMore(result.hasMore===true||(result.total??0)>50);})
|
||||
.catch(failure=>{if(controller.signal.aborted||current.current!==generation)return;if(!onUnauthorized(failure))setError(failure instanceof Error?failure.message:'选项读取失败');})
|
||||
.finally(()=>{if(current.current===generation)setBusy(false);});
|
||||
@@ -26,10 +28,16 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S
|
||||
},[active,field.key,stage,contextKey,search,revision,onUnauthorized]);
|
||||
return <Select {...readableSelectPopup} mode="multiple" allowClear showSearch aria-label={`${field.label}筛选值`} placeholder="选择筛选值" value={value} searchValue={search} filterOption={false} onSearch={setSearch} loading={showBusy} maxTagCount="responsive" onChange={values=>{onChange(values);setSearch('');}} options={[...new Set([...choices,...value])].map(v=>({label:v,value:v}))} notFoundContent={showBusy?<Spin size="small"/>:error?<Button size="small" onClick={()=>setRevision(v=>v+1)}>读取失败,重试</Button>:'暂无匹配值'} popupRender={menu=><>{menu}{more&&<div className="query-options-hint">输入关键词搜索更多值</div>}{error&&<div role="alert" className="query-options-hint">{error}</div>}</>}/>;
|
||||
}
|
||||
function ContainsValues({field,value,onChange}:{field:SourceColumnField;value:string[];onChange:(value:string[])=>void}) {
|
||||
const [search,setSearch]=useState('');
|
||||
return <Select {...readableSelectPopup} mode="tags" maxCount={50} maxTagCount="responsive" allowClear aria-label={`${field.label}包含文字`} placeholder="输入关键词后按回车" value={value} searchValue={search}
|
||||
onSearch={text=>setSearch(text.slice(0,256).replace(/[\uD800-\uDBFF]$/u,''))} onBlur={()=>setSearch('')}
|
||||
onChange={values=>{onChange(values.filter(value=>value.trim().length>0&&value.length<=256));setSearch('');}} options={[]}/>;
|
||||
}
|
||||
export function StageSourceFilters({stage,fields,query,value,onChange,onUnauthorized,part='both'}:{part?:'both'|'picker'|'rows';stage:Stage;fields:SourceColumnField[];query:QueryParams;value:SourceColumnFilter[];onChange:(value:SourceColumnFilter[])=>void;onUnauthorized:(error:unknown)=>boolean}) {
|
||||
const selected=value.map(rule=>rule.field);const patch=(key:string,change:Partial<SourceColumnFilter>)=>onChange(value.map(rule=>rule.field===key?{...rule,...change}:rule));
|
||||
return <div className="stage-source-filters">
|
||||
{part!=='rows'&&<Select {...readableSelectPopup} mode="multiple" maxCount={20} maxTagCount="responsive" showSearch optionFilterProp="label" allowClear aria-label="选择筛选字段" placeholder="添加筛选字段" value={selected} options={fields.map(field=>({label:field.label,value:field.key}))} onChange={keys=>onChange(keys.map(key=>value.find(rule=>rule.field===key)||{field:key,stages:[stage],operator:'in',values:[]}))}/>}
|
||||
{part!=='picker'&&value.map(rule=>{const field=fields.find(item=>item.key===rule.field);return field?<div className="stage-source-filter-row" key={rule.field}><strong>{field.label}</strong><Select {...readableSelectPopup} aria-label={`${field.label}筛选方式`} value={rule.operator} options={[{label:'等于任一值',value:'in'},{label:'不等于这些值',value:'not_in'},{label:'为空',value:'is_empty'},{label:'不为空',value:'not_empty'}]} onChange={operator=>patch(rule.field,{operator,values:[]})}/>{['in','not_in'].includes(rule.operator)?<SourceValues field={field} stage={stage} query={query} value={rule.values||[]} onChange={values=>patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:<span/>}<Button type="text" aria-label={`移除${field.label}筛选`} icon={<CloseOutlined/>} onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}/></div>:<Alert key={rule.field} type="warning" message="所选来源字段当前不可用,请移除后重新选择" action={<Button onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}>移除</Button>}/>;})}
|
||||
{part!=='picker'&&value.map(rule=>{const field=fields.find(item=>item.key===rule.field);return field?<div className="stage-source-filter-row" key={rule.field}><strong>{field.label}</strong><Select {...readableSelectPopup} aria-label={`${field.label}筛选方式`} value={rule.operator} options={[{label:'等于任一值',value:'in'},{label:'包含',value:'contains'},{label:'不等于这些值',value:'not_in'},{label:'为空',value:'is_empty'},{label:'不为空',value:'not_empty'}]} onChange={operator=>patch(rule.field,{operator,values:[]})}/>{rule.operator==='contains'?<ContainsValues field={field} value={rule.values||[]} onChange={values=>patch(rule.field,{values})}/>:['in','not_in'].includes(rule.operator)?<SourceValues field={field} stage={stage} query={query} value={rule.values||[]} onChange={values=>patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:<span/>}<Button type="text" aria-label={`移除${field.label}筛选`} icon={<CloseOutlined/>} onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}/></div>:<Alert key={rule.field} type="warning" message="所选来源字段当前不可用,请移除后重新选择" action={<Button onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}>移除</Button>}/>;})}
|
||||
</div>;
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ export type AnalysisVisualSectionKey = 'overview' | 'stage' | 'department' | 'gr
|
||||
export type AnalysisVisualDimension = 'department' | 'group' | 'product' | 'category' | 'defect';
|
||||
export type AnalysisExclusionDimension = 'department' | 'group' | 'product' | 'projectGroup';
|
||||
export type AnalysisExclusions = Partial<Record<AnalysisExclusionDimension, string[]>>;
|
||||
export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
|
||||
export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
|
||||
export type AnalysisVisualContextValue = string | number | boolean | Array<string | number | boolean> | Record<string, string[]> | Record<string, string> | SourceColumnFilter[] | AnalysisExclusions;
|
||||
export type AnalysisVisualContext = Record<string, AnalysisVisualContextValue>;
|
||||
|
||||
@@ -253,8 +253,8 @@ function safeContext(value: unknown): AnalysisVisualContext | null {
|
||||
|| typeof rule.field !== 'string' || !/^[A-Za-z0-9_]{1,64}$/.test(rule.field)
|
||||
|| !Array.isArray(rule.stages) || !rule.stages.length || rule.stages.length > 2 || new Set(rule.stages).size !== rule.stages.length
|
||||
|| rule.stages.some(stage => stage !== 'black' && stage !== 'bright')
|
||||
|| !['in', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null;
|
||||
const needsValues = rule.operator === 'in' || rule.operator === 'not_in';
|
||||
|| !['in', 'contains', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null;
|
||||
const needsValues = rule.operator === 'in' || rule.operator === 'not_in' || rule.operator === 'contains';
|
||||
if (needsValues && (!Array.isArray(rule.values) || !rule.values.length || rule.values.length > 50 || rule.values.some(value => typeof value !== 'string' || !value.trim() || value.length > 256))) return null;
|
||||
if (!needsValues && rule.values !== undefined) return null;
|
||||
filters.push({ field: rule.field, stages: [...rule.stages] as SourceColumnFilter['stages'], operator: rule.operator as SourceColumnFilter['operator'], ...(needsValues ? { values: [...rule.values as string[]] } : {}) });
|
||||
@@ -1397,7 +1397,7 @@ export function columnFilterSummary(value: unknown, labelValue: unknown): string
|
||||
if (!isRecord(rule) || typeof rule.field !== 'string' || !Array.isArray(rule.stages)) return [];
|
||||
const stage = rule.stages.map(item => item === 'black' ? '黑片' : item === 'bright' ? '亮片' : '').filter(Boolean).join('、');
|
||||
const label = typeof labels[rule.field] === 'string' ? labels[rule.field] : '来源字段';
|
||||
const operation = ({ in: '为', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record<string, string>)[String(rule.operator)];
|
||||
const operation = ({ in: '为', contains: '包含', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record<string, string>)[String(rule.operator)];
|
||||
if (!operation) return [];
|
||||
const values = Array.isArray(rule.values) ? rule.values.filter(item => typeof item === 'string').join('、') : '';
|
||||
return [`${stage} ${label}${operation}${values}`.trim()];
|
||||
|
||||
+5
-2
@@ -122,7 +122,10 @@ export async function request<T>(path: string, init: RequestInit = {}, timeoutMs
|
||||
}
|
||||
catch (error) {
|
||||
handleAccessFailure(error);
|
||||
if (error instanceof Error && error.name === 'RequestTimeoutError') throw new ApiError(408, '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。');
|
||||
if (error instanceof Error && error.name === 'RequestTimeoutError') {
|
||||
const query = /^\/api\/(?:metrics(?:\/|$)|records(?:\/|$)|analysis\/(?:source\/|defects(?:\/|$)))/.test(requestPath(path));
|
||||
throw new ApiError(408, query ? '查询处理超时,请重试或调整筛选范围。' : '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -236,7 +239,7 @@ export const api = {
|
||||
},
|
||||
activateSalesOrderTypes(versionId:string,expectedVersion:string) { return mutate<{versionId:string}>('/api/sales-order-types/activate',{versionId,expectedVersion}); },
|
||||
sourceColumnCatalog(releaseId?: string, signal?: AbortSignal) { return request<{releaseId:string;sourceFields:SourceColumnField[]}>(`/api/analysis/source/catalog?${toQuery({releaseId})}`, { signal }); },
|
||||
sourceColumnValues(body: Record<string,unknown>, signal?: AbortSignal) { return queryPost<{values:Array<string|null>;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); },
|
||||
sourceColumnValues(body: Record<string,unknown> & {candidateOnly?:true}, signal?: AbortSignal) { return queryPost<{values:Array<string|null>;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); },
|
||||
workspaceState() { return request<{ authRevision: string; dataRevision: string; currentReleaseId: string | null }>('/api/workspace/state', {}, 8000); },
|
||||
filterFields(manage = false, signal?: AbortSignal) { return request<FilterField[]>(`/api/filter-fields?manage=${manage}`, { signal }); },
|
||||
filterFieldOptions(key: string, search = '', signal?: AbortSignal,releaseId?:string,stage?:string) { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search,releaseId,stage})}`, { signal }); },
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ export type ViewKind = 'overall' | 'department' | 'group' | 'product' | 'project
|
||||
export type PeriodKind = 'year' | 'quarter' | 'month';
|
||||
|
||||
export interface SourceColumnField { key: string; label: string; type: string; stages: Stage[]; columns: Partial<Record<Stage,string>> }
|
||||
export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
|
||||
export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
|
||||
|
||||
export interface QueryParams {
|
||||
year: number;
|
||||
|
||||
@@ -1042,3 +1042,12 @@ test('adaptive trends cover every series, preserve missing values, and keep non-
|
||||
assert.equal(formatVisualValue(.9639601707533932, 'ratio'), '96.40%');
|
||||
assert.equal(formatVisualValue(3, 'kg'), '3.00 kg');
|
||||
});
|
||||
|
||||
test('contains keywords survive analysis scope and display as substring filters',()=>{
|
||||
const rule={field:'src_any',stages:['bright'],operator:'contains',values:['日立','%_']};
|
||||
const parsed=parseAnalysisVisuals([visual({scope:{columnFilters:[rule],columnLabels:{src_any:'客户'}}})]);
|
||||
assert.equal(parsed.visuals.length,1);
|
||||
assert.deepEqual(parsed.visuals[0].scope.columnFilters,[rule]);
|
||||
assert.match(columnFilterSummary([rule],{src_any:'客户'})[0],/包含.*日立/);
|
||||
assert.equal(parseAnalysisVisuals([visual({scope:{columnFilters:[{...rule,values:[]}]}})]).visuals.length,0);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import ts from 'typescript';
|
||||
|
||||
const source=fs.readFileSync(new URL('../src/StageSourceFilters.tsx',import.meta.url),'utf8');
|
||||
const output=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX,target:ts.ScriptTarget.ES2022}}).outputText;
|
||||
let lastSearch;
|
||||
const jsx=(type,props)=>({type,props});const exports={};
|
||||
new Function('require','exports',output)(name=>{
|
||||
if(name==='react/jsx-runtime')return {jsx,jsxs:jsx};
|
||||
if(name==='react')return {useState:initial=>[initial,value=>{lastSearch=value;} ]};
|
||||
if(name==='antd')return {Select:'Select',Button:'Button',Alert:'Alert',Spin:'Spin'};
|
||||
if(name==='@ant-design/icons')return {CloseOutlined:'CloseOutlined'};
|
||||
if(name==='./readableSelectPopup')return {readableSelectPopup:{}};
|
||||
return {};
|
||||
},exports);
|
||||
const nodes=value=>!value||typeof value!=='object'?[]:[value,...[value.props?.children].flat(Infinity).flatMap(nodes)];
|
||||
const field={key:'customer',label:'客户名称',type:'source_value',stages:['bright'],columns:{bright:'客户名称'}};
|
||||
|
||||
test('contains is selectable and accepts bounded free keywords without a value-candidate lookup',()=>{
|
||||
let changed;
|
||||
const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立']};
|
||||
const root=exports.StageSourceFilters({stage:'bright',fields:[field],query:{year:2026},value:[rule],onChange:value=>{changed=value;},onUnauthorized:()=>false});
|
||||
const all=nodes(root);const operation=all.find(x=>x.type==='Select'&&x.props['aria-label']==='客户名称筛选方式');
|
||||
assert.ok(operation.props.options.some(x=>x.value==='contains'&&x.label==='包含'));
|
||||
const free=all.find(x=>typeof x.type==='function'&&x.type.name==='ContainsValues');
|
||||
const keywords=free.type(free.props);
|
||||
assert.equal(keywords.props.mode,'tags');assert.equal(keywords.props.maxCount,50);assert.deepEqual(keywords.props.options,[]);
|
||||
keywords.props.onSearch('x'.repeat(257));assert.equal(lastSearch.length,256);
|
||||
keywords.props.onSearch('x'.repeat(255)+'😀');assert.equal(lastSearch,'x'.repeat(255));
|
||||
keywords.props.onSearch('日立');keywords.props.onBlur();assert.equal(lastSearch,'');
|
||||
keywords.props.onChange(['日立','%_',' ']);
|
||||
assert.deepEqual(changed,[{...rule,values:['日立','%_']}]);
|
||||
operation.props.onChange('in');assert.deepEqual(changed,[{...rule,operator:'in',values:[]}]);
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import ts from 'typescript';
|
||||
|
||||
const read=name=>fs.readFileSync(new URL(`../src/${name}`,import.meta.url),'utf8');
|
||||
|
||||
test('stage product common queries use a resolved release and do not wait for source catalog when unfiltered',()=>{
|
||||
const source=read('StageProductPage.tsx');
|
||||
const releaseExpression=source.match(/const queryReleaseId=([^;]+);/)[1];
|
||||
const resolvedRelease=new Function('initial','currentReleaseId','catalog',`return ${releaseExpression}`);
|
||||
assert.equal(resolvedRelease({releaseId:'historical'},'current',{releaseId:'catalog'}),'historical');
|
||||
assert.equal(resolvedRelease({},'current',{releaseId:'catalog'}),'current');
|
||||
assert.equal(resolvedRelease({},null,{releaseId:'stale'}),undefined);
|
||||
assert.equal(resolvedRelease({},undefined,{releaseId:'resolved'}),'resolved');
|
||||
const expression=source.match(/const queryReady=\(query:QueryParams\)=>((?:[^;]|\n)+);/)[1];
|
||||
const ready=new Function('query','queryReleaseId','catalog','options',`return ${expression}`);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[]},'r1',null,null),true);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[]},undefined,null,null),false);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',null,null),false);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',{releaseId:'r1'},null),true);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r2',{releaseId:'r1'},null),false);
|
||||
assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,null),false);
|
||||
assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,{}),true);
|
||||
assert.match(source,/if\(active&&draftReady\)setApplied/);
|
||||
assert.doesNotMatch(source,/if\(!active\|\|!applied\|\|supportError\)/);
|
||||
});
|
||||
|
||||
test('source option request context excludes its own selected values and tracks other filters',()=>{
|
||||
const source=read('StageSourceFilters.tsx');
|
||||
const declarations=source.match(/ const \{year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters\}=query;\n const period=.*;\n const filters=.*;\n const contextKey=[^;]+;/)[0];
|
||||
const run=new Function('query','field','stage',`${ts.transpileModule(declarations,{compilerOptions:{target:ts.ScriptTarget.ES2022}}).outputText};return {filters,contextKey};`);
|
||||
const base={year:2026,month:4,releaseId:'r1',department:'D1',columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['North']}]};
|
||||
const first=run(base,{key:'customer'},'black');
|
||||
const ownValueChanged=run({...base,columnFilters:[{field:'customer',values:['OTHER']},{field:'region',values:['North']}]},{key:'customer'},'black');
|
||||
const otherValueChanged=run({...base,columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['South']}]},{key:'customer'},'black');
|
||||
assert.deepEqual(first.filters.columnFilters,[{field:'region',values:['North']}]);
|
||||
assert.equal(first.contextKey,ownValueChanged.contextKey);
|
||||
assert.notEqual(first.contextKey,otherValueChanged.contextKey);
|
||||
});
|
||||
|
||||
test('source value dropdowns request candidate-only results while preserving search and debounce behavior',()=>{
|
||||
const source=read('StageSourceFilters.tsx');
|
||||
assert.match(source,/sourceColumnValues\(\{releaseId,period,filters,field:field\.key,stages:\[stage\],search,limit:50,offset:0,candidateOnly:true\}/);
|
||||
assert.match(source,/search\?300:0/);
|
||||
assert.match(source,/result\.values\.filter\(\(v\):v is string=>typeof v==='string'\)/);
|
||||
assert.match(read('api.ts'),/sourceColumnValues\(body: Record<string,unknown> & \{candidateOnly\?:true\}/);
|
||||
});
|
||||
|
||||
|
||||
test('ordinary product search only depends on product options', () => {
|
||||
assert.match(read('StageProductPage.tsx'), /searchUnavailable=\{!options\}/);
|
||||
});
|
||||
@@ -30,3 +30,9 @@ test('stage query includes unclassified source rows even when an old link exclud
|
||||
assert.equal(base.includeUnassigned,false);
|
||||
}
|
||||
});
|
||||
|
||||
test('contains keywords are retained while an unfilled contains rule is inactive',()=>{
|
||||
const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立','%_']};
|
||||
assert.deepEqual(activeStageFilters([rule],'bright'),[rule]);
|
||||
assert.deepEqual(activeStageFilters([{...rule,values:[]}],'bright'),[]);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user