18 Commits
Author SHA1 Message Date
lyun ebb722df36 文档:记录分页发布与全量测试验收 2026-11-14 14:44:52 +08:00
lyun 8f1b790b1d 优化:产品查询数据库分页并修复报告测试契约 2026-11-14 14:34:56 +08:00
lyun d948e1e2a4 文档:确认分步加载已交付主线 2026-11-14 12:46:19 +08:00
lyun bfb91efc2b 文档:记录分步加载发布与线上验收 2026-11-14 12:44:51 +08:00
lyun 374f08c526 优化:查询按范围分步加载并精简表格聚合 2026-11-14 12:34:50 +08:00
lyun 381901184c 文档:记录筛选提速与包含条件的仓库交付 2026-11-13 22:16:57 +08:00
lyun 6983dc32eb 同步:来源筛选提速与包含条件 2026-11-13 22:15:58 +08:00
lyun f47341f3f0 Document contains filter usage and live acceptance 2026-11-13 22:10:50 +08:00
lyun f8432b4e02 Bound contains keyword input and clear duplicate search drafts 2026-11-13 21:51:31 +08:00
lyun 3c022e8614 Add literal contains filters across source queries and analysis context 2026-11-13 21:23:12 +08:00
lyun 7cae4cfbdf Record accepted filter-speed release and measured index storage 2026-11-13 20:32:16 +08:00
lyun 20f1c7d0e2 Seek the first visible source row without scanning all fact ids 2026-11-13 20:14:51 +08:00
lyun fa70dc6504 Cover source candidate scope checks without wide fact row reads 2026-11-13 19:19:18 +08:00
lyun 7bd9e51004 优化:按批次重置筛选索引临时窗口 2026-11-13 18:37:11 +08:00
lyun 5e02c6ad66 优化:筛选索引按暂存窗口读取字典 2026-11-13 18:20:32 +08:00
lyun 7782e33bec 优化:统一来源筛选索引与后台构建 2026-11-13 17:56:18 +08:00
lyun 5068b8fbc9 文档:记录来源筛选修复的线上验收 2026-11-13 16:56:38 +08:00
lyun ea84e26b2f 修复:来源字段筛选超时与重复扫描 2026-11-13 16:52:27 +08:00
71 changed files with 2285 additions and 229 deletions
+10
View File
@@ -58,6 +58,16 @@ DSH CLI 固定为 `0.2.2-rc.1`,依赖与官方 Python SDK 校验记录在 `run
底部表格同步展开当前分类的细项,重量/废率切换与 Excel 导出跟随当前显示;产品和期间行维度保持不变。细项使用正式阶段同期总重量作分母,月、季、年均先汇总分量再计算;没有来源或分母时显示缺失,真实零显示零。细项与正式分类总量不一致时提示覆盖不足,不补造分类。
## 筛选索引与查询加载
来源字段支持等于、包含、排除和空值筛选;“包含”输入关键词后按回车生效,多个关键词任一命中,区分大小写,`%`、`_` 按普通文字匹配,每个关键词最多256字符、最多50个。各类筛选共用按构建冻结的值字典及记录索引;汇总、趋势、来源明细和导出沿用当前授权与正式指标计算。源字段值保留空值、大小写及前导零语义,旧配置筛选的字符串 `null` 约定不变。下拉框使用只读候选模式,只取一页及是否还有;需要精确总数的原接口继续保留。
启动时在单线程后台补建当前完整正式版本的索引,导入及归类在事务提交后进入同一去重队列。完整构建标记提交后才启用;未完成、失败或未补建的历史构建沿用原始查询,不阻止启动和正式数据发布。索引增加持久存储及后台构建成本,规模取决于非空字段数和不同值数量;高基数宽表应先测容量。保留清理随无引用构建回收索引;升级后原维护兼容保护继续生效。
年度全产品查询先显示正式指标和同范围趋势,再读取产品表格;月/季及产品搜索并行读取摘要和产品页,单产品及总体/产业部等范围复用完整结果。不同范围的趋势独立加载,底部柏拉图接近视口且主查询就绪后读取。刷新和筛选变化会使旧结果失去当前资格;表格失败时,已就绪的指标与趋势仍可使用。
普通查询在数据版本已解析后开始,来源字段条件仍等待对应版本的字段目录;产品文本搜索等待必要产品选项,不因无关元数据失败而禁用。产品汇总、黑片和亮片的宽产品表由数据库分页,默认每页15行,可选择10/15/20/50/100行;换年、筛选和修改每页数量时回到第一页。指标与趋势继续统计完整范围,细分废率随当前页读取。Excel 重新读取完整授权范围并全量导出,产品钻取后的期间表保持原有展示。
## 旧版本时间迁移
连接与 MySQL 会话现在统一使用 UTC,界面仍按本地时区显示。V25 只校正旧代码按 UTC 墙上时间写入的截止/有效期、报告调度及报告租约列(前者由 Java 写入,租约使用 SQL UTC_TIMESTAMP),不改 SQL CURRENT_TIMESTAMP 自动生成的创建时间,也不修改业务日期或指标。若旧库存在此类值,迁移要求先备份并核实旧会话相对 UTC 的偏移,然后设置 `MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS`(旧会话 UTC 为 `0`,旧会话中国标准时间为 `28800`)。本机启动可在私有 local-config 中记录 `legacyTimestampOffsetSeconds`;不能仅凭电脑当前时区猜测。全新空库不需要该参数;迁移标记和历史值修改在同一事务提交,防止提交确认丢失后重复平移。还原旧备份时应重新核实其原运行时区。
@@ -369,7 +369,7 @@ public class AnalysisExplorationTools {
SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value);
SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value);
appendDetailExclusions(sql, args, query);
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor);
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor,releaseId);
DataScope.append(sql, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
boolean includeUnassigned = query.includeUnassigned()
@@ -158,6 +158,12 @@ public class AnalysisFineDefectsService {
/** One authorized scan (or rollup query) for every period/object and all visible children. */
@Transactional(readOnly = true, isolation = Isolation.REPEATABLE_READ)
public Map<String, Object> grouped(Map<String, Object> input, AppUserPrincipal actor, String dimension, String granularity) {
return grouped(input,actor,dimension,granularity,null);
}
public Map<String,Object> grouped(Map<String,Object> input,AppUserPrincipal actor,String dimension,String granularity,List<String> productKeys) {
productKeys=cn.yunsheng.quality.metrics.ProductBuckets.parse(productKeys);
if(productKeys!=null&&!dimension.equals("product"))throw bad("产品桶只能用于产品细分表");
Set<String> allowed = governance.allowedFields(actor);
Request request = parse(input, allowed, true);
String bucket = switch (dimension) {
@@ -184,6 +190,7 @@ public class AnalysisFineDefectsService {
if (request.productSearch() != null) {
predicate.append(" AND LOCATE(LOWER(?),LOWER(COALESCE(f.product,'')))>0"); args.add(request.productSearch());
}
cn.yunsheng.quality.metrics.ProductBuckets.append(predicate,args,productKeys);
List<DefectField> fields = applicableFields(request,allowed);
List<Category> categories = categories(request.stage());
Set<String> components = new LinkedHashSet<>();
@@ -276,7 +283,7 @@ public class AnalysisFineDefectsService {
SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value);
if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor);
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor,releaseId);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
}
@@ -90,7 +90,7 @@ public class AnalysisInvestigationService {
SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x);
SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x);
if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor);
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor,release);
DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
}
if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());}
@@ -891,7 +891,7 @@ final class AnalysisProtocolHistoryFacts {
String label=text(columnLabels.get(field),historyFieldLabel(section,sources,field));
if(label.isBlank())label="授权列";
List<String> prefix=strings(rule.get("stages")).stream().map(AnalysisProtocolHistoryFacts::stageLabel).toList();
String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";};
String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "contains"->"包含文字";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";};
List<String> values=scopeLiterals(rule.get("values"));
StringBuilder part=new StringBuilder();if(!prefix.isEmpty())part.append(String.join("、",prefix)).append(' ');
part.append(label).append(' ').append(operator);if(!values.isEmpty())part.append(' ').append(String.join("、",values));parts.add(part.toString());
@@ -43,12 +43,16 @@ public class AnalysisSourceQueryController {
@PostMapping("/values")
public Map<String,Object> values(@RequestBody Map<String,Object> input,@AuthenticationPrincipal AppUserPrincipal actor) {
String scope=captureScope(actor);
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset","candidateOnly").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null));
Map<String,Object> context=context(input);
if(!(input.get("field") instanceof String field)||field.isBlank()||input.get("search")!=null&&!(input.get("search") instanceof String))throw bad("候选字段或搜索值无效");
Map<String,Object> result=new LinkedHashMap<>(columns.values(release,context,field,strings(input.get("stages"),2),Objects.toString(input.get("search"),null),
integer(input.getOrDefault("limit",50),1,200),integer(input.getOrDefault("offset",0),0,100000),actor));
if(input.containsKey("candidateOnly")&&!(input.get("candidateOnly") instanceof Boolean))throw bad("候选分页参数无效");
List<String> stages=strings(input.get("stages"),2);String search=Objects.toString(input.get("search"),null);
int limit=integer(input.getOrDefault("limit",50),1,200),offset=integer(input.getOrDefault("offset",0),0,100000);
Map<String,Object> result=new LinkedHashMap<>(Boolean.TRUE.equals(input.get("candidateOnly"))
? columns.values(release,context,field,stages,search,limit,offset,true,actor)
: columns.values(release,context,field,stages,search,limit,offset,actor));
result.put("period",input.get("period"));result.put("filters",input.get("filters"));
result.put("values",AnalysisProtocolService.list(result.get("rows")).stream().map(row->AnalysisProtocolService.map(row).get("value")).toList());
result.put("candidateOnly",true);reauthorize(actor,scope,false);return result;
@@ -32,13 +32,17 @@ public class DefectDrilldownController {
String granularity=String.valueOf(input.getOrDefault("trendGranularity","month"));
if(!Set.of("period","overall","department","group","product","projectGroup").contains(dimension))throw bad("细分维度无效");
if(!Set.of("month","quarter","year").contains(granularity))throw bad("趋势粒度无效");
Map<String,Object> scope=new LinkedHashMap<>(input);scope.remove("trendGranularity");scope.remove("view");scope.put("dimension","defect");scope.put("mode","top");scope.put("limit",100);
List<String> productKeys=cn.yunsheng.quality.metrics.ProductBuckets.parse(input.get("productKeys"));
if(productKeys!=null&&!dimension.equals("product"))throw bad("产品桶只能用于产品细分表");
Map<String,Object> scope=new LinkedHashMap<>(input);scope.remove("productKeys");scope.remove("trendGranularity");scope.remove("view");scope.put("dimension","defect");scope.put("mode","top");scope.put("limit",100);
// Reuse the source parser to reject unknown filters and validate the selected category/stage.
AnalysisFineDefectsService.Request request=AnalysisFineDefectsService.parse(scope,governance.allowedFields(actor),true);
if(request.category()==null)throw bad("必须指定不良分类");
var query=new MetricsService.MetricsQuery(number(scope,"year"),number(scope,"month"),number(scope,"quarter"),request.department(),request.group(),request.product(),request.projectGroup(),dimension.equals("period")?"overall":dimension,request.includeUnassigned(),request.releaseId(),granularity,AnalysisExplorationTools.exclusions(scope.get("exclusions")))
.withProductSearch(request.productSearch()).withExtraFilters(scope.get("extraFilters")).withColumnFilters(scope.get("columnFilters")).withMonthRange(number(scope,"monthStart"),number(scope,"monthEnd")).withStage(request.stage());
var paired=metrics.calculatePairedSnapshot(query,actor,dimension.equals("period"),() -> fine.grouped(scope,actor,dimension,granularity));
var paired=productKeys==null
? metrics.calculatePairedSnapshot(query,actor,dimension.equals("period"),() -> fine.grouped(scope,actor,dimension,granularity))
: metrics.calculatePairedProductSnapshot(query,actor,productKeys,() -> fine.grouped(scope,actor,dimension,granularity,productKeys));
// Refresh the visible fields after the snapshot; a concurrent policy revocation cannot release old fields.
governance.requireDetail(actor);governance.requireBoundAction("QUERY","query.detail");
Set<String> currentAllowed=governance.allowedFields(actor);
@@ -111,7 +111,7 @@ public class AnalyticsController {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前字段授权不包含所选报废类别");
List<Object> args = new ArrayList<>(extra.isEmpty()?List.of(selected, selected):List.of(selected));
StringBuilder where = baseWhere(args, year, month, quarter, department, group, product, projectGroup, includeUnassigned, actor);
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor);
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor,selected);
List<AggregateTop> aggregated;
if (dimension.equals("category")) {
List<Component> selectedComponents = selectedCategory == null ? visibleComponents : List.of(selectedCategory);
@@ -36,9 +36,11 @@ public class FilterFieldsController {
governance.requireUserActivePolicy(actor);
Map<String,List<String>> result=new LinkedHashMap<>();
String release=releases.resolvePublishedRelease(null);
SourceFilterIndex index=new SourceFilterIndex(jdbc);
for(String stage:List.of("black","bright")) {
// Raw source headers are stable within each build. Read a representative row of every current build.
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2 WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
String hint=index.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":"";
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2"+hint+" WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
Set<String> columns=new TreeSet<>();
var json=tools.jackson.databind.json.JsonMapper.builder().build();
Set<String> allowed=governance.allowedFields(actor);
@@ -48,6 +48,9 @@ public class QueryFilterFields {
return Collections.unmodifiableMap(result);
}
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor) {
append(where,args,stage,filters,actor,null);
}
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor,String releaseId) {
if(filters==null || filters.isEmpty())return;
Map<String,Field> fields=new HashMap<>(); visible(actor).forEach(f->fields.put(f.key(),f));
for(var entry:filters.entrySet()) {
@@ -61,6 +64,7 @@ public class QueryFilterFields {
boolean empty=entry.getValue().contains(EMPTY);
if(empty && !f.allowEmpty())throw bad(f.label()+"不允许筛选空值");
String column=stage.equals("black")?f.blackColumn():f.brightColumn();
if(releaseId!=null&&new SourceFilterIndex(jdbc).appendLegacy(where,args,stage,releaseId,column,entry.getValue()))continue;
// Paths and selected values are bound parameters; never interpolate configured identifiers.
String expression="NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs";
List<String> values=entry.getValue().stream().filter(v->!v.equals(EMPTY)).toList();
@@ -90,10 +94,15 @@ public class QueryFilterFields {
StringBuilder scope=new StringBuilder(" WHERE rm.release_id=?");
List<Object> scoped=new ArrayList<>();scoped.add(releaseId);
DataScope.append(scope,scoped,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
var index=new SourceFilterIndex(jdbc);
if(index.ready(releaseId)) {
selects.add(index.candidateSelect(stage,column,scope.toString(),scoped,args).replace("AS candidate_value","AS value"));
continue;
}
selects.add("SELECT NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs AS value FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+scope);
args.add(path(column));args.addAll(scoped);
}
String predicate=" WHERE value IS NOT NULL";
String predicate=" WHERE value IS NOT NULL AND CAST(value AS BINARY)<>CAST('null' AS BINARY)";
if(search!=null && !search.isBlank()) { predicate+=" AND LOCATE(LOWER(?),LOWER(value))>0";args.add(search); }
List<String> found=jdbc.query("SELECT DISTINCT value FROM ("+String.join(" UNION ALL ",selects)+") choices"+predicate+" ORDER BY value LIMIT 201",(rs,n)->rs.getString(1),args.toArray());
return Map.of("values",found.stream().limit(200).toList(),"hasMore",found.size()>200);
@@ -60,8 +60,18 @@ public class SourceColumnQueries {
}
return List.copyOf(result);
}
// 只提取一次,并区分 JSON null 与字符串“null”,供筛选、候选值和成员读取共用。
private static final String SOURCE_TEXT = "JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON)))";
private static final String SOURCE_VALUE = "NULLIF(TRIM("+SOURCE_TEXT+"), '') COLLATE utf8mb4_0900_as_cs";
/** 原始字段筛选按发布构建读取,避免优化器从整年月份范围扫描大 JSON 原表。 */
public static String factIndexHint(String stage) {
if(!List.of("black","bright").contains(stage))throw bad("数据阶段无效");
return " FORCE INDEX (uq_fact_"+stage+"_origin)";
}
private static final List<String> STAGES = List.of("black", "bright");
private static final List<String> OPERATORS = List.of("in", "not_in", "is_empty", "not_empty");
private static final List<String> OPERATORS = List.of("in", "contains", "not_in", "is_empty", "not_empty");
private static final Map<String, String> CANONICAL_COLUMNS = Map.of(
"product", "product", "department", "department", "group", "group_name", "projectGroup", "project_group");
private static final Map<String, String> CANONICAL_LABELS = Map.of(
@@ -70,11 +80,17 @@ public class SourceColumnQueries {
private final JdbcTemplate jdbc;
private final GovernanceAccessService governance;
private final JsonMapper json;
private SourceFilterIndex filterIndex;
@org.springframework.beans.factory.annotation.Autowired
public void setFilterIndex(SourceFilterIndex index) { this.filterIndex=index; }
public boolean indexed(String release) { return filterIndex.ready(release); }
public SourceColumnQueries(JdbcTemplate jdbc, GovernanceAccessService governance, JsonMapper json) {
this.jdbc = jdbc;
this.governance = governance;
this.json = json;
this.filterIndex = new SourceFilterIndex(jdbc);
}
/**
@@ -94,16 +110,16 @@ public class SourceColumnQueries {
Map<String, Boolean> stagePresent = new LinkedHashMap<>();
for (String stage : STAGES) {
Set<String> headers = new LinkedHashSet<>();
StringBuilder representativeScope = new StringBuilder(" WHERE rm.release_id=?");
StringBuilder representativeScope = new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
List<Object> args = new ArrayList<>(List.of(releaseId));
DataScope.append(representativeScope, args, actor,
"f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs");
// Stop at the first authorized fact of each build instead of aggregating every fact id.
String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) "
+ "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id "
+ "FROM release_member rm JOIN fact_" + stage + " f2 "
+ "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"
+ representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative "
+ "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id";
+ "FROM (SELECT DISTINCT transform_build_id,source_revision_id FROM release_member WHERE release_id=?) rm "
+ "JOIN fact_" + stage + " f ON f.id=(SELECT f2.id FROM fact_" + stage + " f2"
+ (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage))
+ representativeScope + " ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1) ORDER BY rm.transform_build_id";
List<String> keyLists = rawHeaders.get(new QueryKey(sql,args,"raw-headers"),
() -> Collections.unmodifiableList(new ArrayList<>(jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray()))));
stagePresent.put(stage, !keyLists.isEmpty());
@@ -192,12 +208,13 @@ public class SourceColumnQueries {
normalizedStages.add(value);
}
List<String> values = new ArrayList<>();
if (operator.equals("in") || operator.equals("not_in")) {
if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) {
if (!(rule.get("values") instanceof List<?> rawValues) || rawValues.isEmpty() || rawValues.size() > 50)
throw bad("in/not_in 筛选必须提供1到50个字符串值");
throw bad("文本筛选必须提供1到50个字符串值");
for (Object value : rawValues) {
if (!(value instanceof String text) || text.isEmpty() || text.length() > 256)
throw bad("来源列筛选值必须是1到256字符的完整字符串;空值请使用空值操作符");
if(operator.equals("contains")&&text.isBlank())throw bad("包含筛选的关键词不能全为空白");
if (!values.contains(text)) values.add(text);
}
} else if (rule.containsKey("values")) {
@@ -207,7 +224,7 @@ public class SourceColumnQueries {
normalized.put("field", field);
normalized.put("stages", List.copyOf(normalizedStages));
normalized.put("operator", operator);
if (operator.equals("in") || operator.equals("not_in")) normalized.put("values", List.copyOf(values));
if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) normalized.put("values", List.copyOf(values));
parsed.add(Collections.unmodifiableMap(normalized));
}
return List.copyOf(parsed);
@@ -239,11 +256,10 @@ public class SourceColumnQueries {
String operator = (String) rule.get("operator");
String expression;
Object column = ((Map<?, ?>) field.get("columns")).get(stage);
@SuppressWarnings("unchecked") List<String> selected = (List<String>) rule.getOrDefault("values",List.of());
if("source_value".equals(field.get("type")) && filterIndex.append(where,args,stage,releaseId,String.valueOf(column),operator,selected))continue;
if ("source_value".equals(field.get("type"))) {
expression = "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL "
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
+ "COLLATE utf8mb4_0900_as_cs";
expression = SOURCE_VALUE;
} else {
String factColumn = CANONICAL_COLUMNS.get(key);
if (factColumn == null) throw bad("规范维度无效");
@@ -255,19 +271,26 @@ public class SourceColumnQueries {
if (operator.equals("in")) {
where.append(expression).append(" IN (")
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append(')');
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column));
if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
} else {
where.append('(').append(expression).append(" IS NULL OR ").append(expression).append(" NOT IN (")
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append("))");
if ("source_value".equals(field.get("type"))) {
bindSourcePaths(args, String.valueOf(column));
bindSourcePaths(args, String.valueOf(column));
bindSourcePath(args, String.valueOf(column));
bindSourcePath(args, String.valueOf(column));
}
}
args.addAll(values);
} else if (operator.equals("contains")) {
for(int i=0;i<selected.size();i++) {
if(i>0)where.append(" OR ");
where.append("LOCATE(?,").append(expression).append(")>0");
args.add(selected.get(i));
if("source_value".equals(field.get("type")))bindSourcePath(args,String.valueOf(column));
}
} else {
where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL");
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column));
if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
}
where.append(')');
}
@@ -376,12 +399,16 @@ public class SourceColumnQueries {
public Map<String, Object> values(String releaseId, Map<String, Object> context, String fieldKey,
List<String> stages, String search, int limit, int offset,
AppUserPrincipal actor) {
return values(releaseId,context,fieldKey,stages,search,limit,offset,false,actor);
}
public Map<String,Object> values(String releaseId,Map<String,Object> context,String fieldKey,List<String> stages,
String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
requireQueryAccess(actor);
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,actor));
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,candidateOnly,actor));
}
private Map<String,Object> valuesSnapshot(String releaseId,Map<String,Object> context,String fieldKey,
List<String> stages,String search,int limit,int offset,AppUserPrincipal actor) {
List<String> stages,String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效");
if (fieldKey == null || fieldKey.isBlank()) throw bad("来源列标识无效");
if (stages == null || stages.isEmpty() || stages.size() > STAGES.size()
@@ -412,6 +439,10 @@ public class SourceColumnQueries {
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> whereArgs = new ArrayList<>(List.of(releaseId));
appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor);
if("source_value".equals(field.get("type")) && filterIndex.ready(releaseId)) {
selects.add(filterIndex.candidateSelect(stage,String.valueOf(columns.get(stage)),where.toString(),whereArgs,unionArgs));
continue;
}
String fact = stage.equals("black") ? "fact_black" : "fact_bright";
selects.add("SELECT " + expression + " AS candidate_value "
+ "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id "
@@ -430,6 +461,13 @@ public class SourceColumnQueries {
}
String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition;
String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value";
if(candidateOnly) {
String sql="SELECT candidate_value FROM ("+candidates+") matches GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?";
List<Object> parameters=combine(unionArgs,searchArgs);parameters.add(limit+1);parameters.add(offset);
List<Map<String,Object>> found=pages.get(new QueryKey(sql,parameters,"candidates:"+fieldKey),()->List.copyOf(jdbc.query(sql,(rs,row)->Map.of("value",rs.getString(1)),parameters.toArray())));
boolean more=found.size()>limit;
return Map.of("rows",List.copyOf(found.subList(0,Math.min(limit,found.size()))),"hasMore",more,"limit",limit,"offset",offset,"pageComplete",true,"complete",offset==0&&!more,"releaseId",releaseId,"field",fieldKey,"selectedStages",List.copyOf(stages));
}
long total = total("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", combine(unionArgs,searchArgs));
List<Map<String, Object>> rows = page("SELECT candidate_value FROM (" + candidates + ") matches "
+ "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?",
@@ -556,7 +594,7 @@ public class SourceColumnQueries {
List<String> groups = parseSelections(context.dimensions().get("group"), "group");
if (!context.includeUnassigned() && !groups.contains("__UNASSIGNED__"))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor);
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor,releaseId);
if (!context.columnFilters().isEmpty()) append(where, args, stage, context.columnFilters(), releaseId, actor);
DataScope.append(where, args, actor,
"f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
@@ -578,12 +616,7 @@ public class SourceColumnQueries {
}
String path = jsonPath(column);
args.add(path);
args.add(path);
args.add(path);
return "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL "
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
+ "COLLATE utf8mb4_0900_as_cs";
return SOURCE_VALUE;
}
private static List<Object> combine(List<Object> first, List<Object> second) {
@@ -601,9 +634,7 @@ public class SourceColumnQueries {
}
String path = jsonPath(column);
args.add(path);
args.add(path);
return "CASE WHEN JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?)) END COLLATE utf8mb4_0900_as_cs";
return SOURCE_TEXT+" COLLATE utf8mb4_0900_as_cs";
}
private static String tupleColumns(int count) {
@@ -814,11 +845,9 @@ public class SourceColumnQueries {
catch (Exception e) { throw new IllegalArgumentException("来源列名无法编码", e); }
}
private void bindSourcePaths(List<Object> args, String header) {
private void bindSourcePath(List<Object> args, String header) {
String path = jsonPath(header);
args.add(path);
args.add(path);
args.add(path);
}
private static ResponseStatusException bad(String message) {
@@ -0,0 +1,143 @@
package cn.yunsheng.quality.filters;
import java.util.List;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
/** Sparse, build-owned source values. Every consumer still authorizes and scopes the actual facts. */
@Service
public class SourceFilterIndex {
public record BuildAvailable(String buildId) {}
private static final int FORMAT = 1;
static final String CREATE_STAGE_SQL="CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB";
static final String MEMBER_INSERT_SQL="INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash";
static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)";
private final JdbcTemplate jdbc;
private volatile Boolean available;
private volatile Boolean candidateScopeAvailable;
public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; }
public boolean available() {
if (available == null) {
Integer count=jdbc.queryForObject("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('source_filter_build','source_filter_field','source_filter_value','source_filter_member')",Integer.class);
available=count!=null&&count==4;
}
return available;
}
private String candidateFactHint(String stage) {
if (candidateScopeAvailable == null) {
Integer count=jdbc.queryForObject("SELECT COUNT(DISTINCT table_name) FROM information_schema.statistics WHERE table_schema=DATABASE() AND ((table_name='fact_black' AND index_name='ix_black_candidate_scope') OR (table_name='fact_bright' AND index_name='ix_bright_candidate_scope'))",Integer.class);
candidateScopeAvailable=count!=null&&count==2;
}
return candidateScopeAvailable ? " FORCE INDEX (ix_"+stage+"_candidate_scope)" : "";
}
public boolean ready(String release) {
if(release==null||!available())return false;
// Only successful immutable builds are retained; a pending backfill is checked again later.
Integer missing=jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm LEFT JOIN source_filter_build sb ON sb.build_id=rm.transform_build_id AND sb.format_version=? WHERE rm.release_id=? AND sb.build_id IS NULL",Integer.class,FORMAT,release);
if(missing==null||missing!=0)return false;
return true;
}
/** Idempotent and atomic with fact construction, including callers which instantiate this helper. */
public void ensureBuild(String build) {
if(!available())return;
var tx=new TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource()));
tx.executeWithoutResult(status->{
jdbc.queryForObject("SELECT id FROM transform_build WHERE id=? FOR UPDATE",String.class,build);
Integer done=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build WHERE build_id=? AND format_version=?",Integer.class,build,FORMAT);
if(done!=null&&done>0)return;
jdbc.update("DELETE m FROM source_filter_member m JOIN source_filter_field sf ON sf.id=m.field_id WHERE sf.build_id=?",build);
jdbc.update("DELETE v FROM source_filter_value v JOIN source_filter_field sf ON sf.id=v.field_id WHERE sf.build_id=?",build);
jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build);
jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build);
// Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source.
try {
for(String stage:List.of("black","bright"))buildStage(build,stage);
jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT);
} finally { jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging"); }
});
}
private void buildStage(String build,String stage) {
var bounds=jdbc.queryForMap("SELECT MIN(id) AS first_id,MAX(id) AS last_id FROM fact_"+stage+" FORCE INDEX(uq_fact_"+stage+"_origin) WHERE build_id=?",build);
if(bounds.get("first_id")==null)return;
long after=((Number)bounds.get("first_id")).longValue()-1;
long last=((Number)bounds.get("last_id")).longValue();
while(after<last) {
if(Thread.currentThread().isInterrupted())throw new IllegalStateException("来源筛选索引构建已中断");
List<Long> ids=jdbc.queryForList("SELECT id FROM fact_"+stage+" FORCE INDEX(PRIMARY) WHERE build_id=? AND id>? AND id<=? ORDER BY id LIMIT 500",Long.class,build,after,last);
if(ids.isEmpty())return;
long end=ids.getLast();
String rows=" FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE WHERE f.build_id=? AND f.id>? AND f.id<=?";
jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end);
String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')";
String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?";
// DROP/CREATE TEMPORARY do not commit the owning build transaction. Reset the
// physical table too: DELETE retains undo/deleted records until that transaction ends.
jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging");
jdbc.execute(CREATE_STAGE_SQL);
jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end);
jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash");
Integer collisions=jdbc.queryForObject(COLLISION_SQL,Integer.class);
if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布");
jdbc.update(MEMBER_INSERT_SQL);
after=end;
}
}
/** Same-row membership is shared by metrics, trends, details and exports; empty values are sparse. */
public boolean append(StringBuilder where,List<Object> args,String stage,String release,String column,
String operator,List<String> values) {
if(!ready(release))return false;
where.append(" AND ");
membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"),
operator.equals("in")||operator.equals("not_in")||operator.equals("contains")?values:null,false,operator.equals("contains"));
return true;
}
/** Legacy configured filters classify only the byte-exact literal string "null" as empty. */
public boolean appendLegacy(StringBuilder where,List<Object> args,String stage,String release,String column,List<String> selected) {
if(!ready(release))return false;
var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList();
where.append(" AND (");
if(values.isEmpty())where.append("FALSE");
else membership(where,args,stage,release,column,false,values,true,false);
if(selected.contains(QueryFilterFields.EMPTY)) {
where.append(" OR ");membership(where,args,stage,release,column,true,null,true,false);
}
where.append(')');return true;
}
private void membership(StringBuilder where,List<Object> args,String stage,String release,String column,
boolean negative,List<String> values,boolean legacy,boolean contains) {
where.append("f.id ").append(negative?"NOT IN (":"IN (")
.append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin");
args.add(release);args.add(stage);args.add(column);
if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)");
if(contains) {
where.append(" AND (");
for(int i=0;i<values.size();i++) {
if(i>0)where.append(" OR ");
where.append("LOCATE(?,sv.value COLLATE utf8mb4_0900_as_cs)>0");args.add(values.get(i));
}
where.append(')');
} else if(values!=null) {
where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')');
args.addAll(values);
}
where.append(')');
}
/** Dictionary-first candidates short-circuit after one authorized matching member per value. */
public String candidateSelect(String stage,String column,String scope,List<Object> scopeArgs,List<Object> args) {
args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs);
return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id "
+"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f"+candidateFactHint(stage)+" ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id "
+scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)";
}
}
@@ -0,0 +1,194 @@
package cn.yunsheng.quality.filters;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.context.event.EventListener;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
/** Asynchronously builds current and newly completed snapshots; unindexed queries retain their raw-value fallback. */
@Component
public class SourceFilterIndexBackfill implements ApplicationRunner {
private static final Logger LOG = LoggerFactory.getLogger(SourceFilterIndexBackfill.class);
private static final int MAX_PENDING_BUILDS = 1024;
private final JdbcTemplate jdbc;
private final SourceFilterIndex index;
private final ExecutorService worker;
private final Object pendingLock = new Object();
private final LinkedHashSet<String> pending = new LinkedHashSet<>();
private boolean drainScheduled;
private String inFlight;
@Autowired
public SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index) {
this(jdbc, index, createWorker());
}
SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index, ExecutorService worker) {
this.jdbc = jdbc;
this.index = index;
this.worker = worker;
}
@Override
public void run(ApplicationArguments args) {
List<String> currentBuilds;
try {
if (!index.available()) return;
currentBuilds = jdbc.query("""
SELECT DISTINCT b.id
FROM current_release c
JOIN data_release r ON r.id=c.release_id
JOIN release_member rm ON rm.release_id=r.id
JOIN transform_build b ON b.id=rm.transform_build_id
WHERE c.dataset_id='quality'
AND r.status='PUBLISHED'
AND r.complete=TRUE
AND b.status='COMPLETED'
ORDER BY b.id
""", (rs, row) -> rs.getString(1));
} catch (RuntimeException failure) {
LOG.error("Source filter index backfill discovery failed; raw-value fallback remains available", failure);
return;
}
if (currentBuilds.isEmpty()) return;
int accepted = enqueue(currentBuilds);
LOG.info("Source filter index backfill queued: discovered={}, accepted={}", currentBuilds.size(), accepted);
}
@EventListener
public void onBuildAvailable(SourceFilterIndex.BuildAvailable event) {
if (event == null || event.buildId() == null || event.buildId().isBlank()) return;
int accepted = enqueue(List.of(event.buildId()));
if (accepted > 0) LOG.info("Source filter index build queued after commit: build={}", event.buildId());
}
private int enqueue(List<String> builds) {
boolean schedule = false;
int accepted = 0;
int deferred = 0;
synchronized (pendingLock) {
for (String build : builds) {
if (build == null || build.isBlank() || build.equals(inFlight) || pending.contains(build)) continue;
if (pending.size() >= MAX_PENDING_BUILDS) {
deferred++;
continue;
}
pending.add(build);
accepted++;
}
if (!pending.isEmpty() && !drainScheduled) {
drainScheduled = true;
schedule = true;
}
}
if (deferred > 0) {
LOG.warn("Source filter index queue is full; {} builds keep raw-value fallback", deferred);
}
if (schedule) scheduleDrain();
return accepted;
}
private void scheduleDrain() {
try {
worker.execute(this::drainPending);
} catch (RejectedExecutionException failure) {
int queued;
synchronized (pendingLock) {
drainScheduled = false;
queued = pending.size();
}
LOG.error("Source filter index drain could not be scheduled; {} builds keep raw-value fallback", queued, failure);
}
}
private void drainPending() {
long started = System.nanoTime();
int succeeded = 0;
int failed = 0;
while (true) {
String buildId;
synchronized (pendingLock) {
if (pending.isEmpty()) {
drainScheduled = false;
break;
}
var iterator = pending.iterator();
buildId = iterator.next();
iterator.remove();
inFlight = buildId;
}
if (Thread.currentThread().isInterrupted()) {
synchronized (pendingLock) {
pending.add(buildId);
inFlight = null;
drainScheduled = false;
}
LOG.info("Source filter index drain interrupted; {} builds remain queued", pendingCount());
return;
}
long buildStarted = System.nanoTime();
try {
index.ensureBuild(buildId);
succeeded++;
LOG.info("Source filter index build completed: build={}, queuedRemaining={}, elapsedMs={}", buildId,
pendingCount(), (System.nanoTime() - buildStarted) / 1_000_000L);
} catch (RuntimeException failure) {
if (Thread.currentThread().isInterrupted()) {
synchronized (pendingLock) {
pending.add(buildId);
inFlight = null;
drainScheduled = false;
}
LOG.info("Source filter index drain interrupted while building {}; {} builds remain queued", buildId,
pendingCount());
return;
}
failed++;
LOG.error("Source filter index build failed: build={}; raw-value fallback remains available, continuing",
buildId, failure);
} finally {
synchronized (pendingLock) { inFlight = null; }
}
}
LOG.info("Source filter index drain finished: succeeded={}, failed={}, elapsedMs={}", succeeded, failed,
(System.nanoTime() - started) / 1_000_000L);
}
private int pendingCount() {
synchronized (pendingLock) { return pending.size(); }
}
@PreDestroy
public void stopWorker() {
worker.shutdownNow();
try {
if (!worker.awaitTermination(2, TimeUnit.SECONDS)) {
LOG.warn("Source filter index backfill worker did not stop within two seconds");
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
}
}
private static ExecutorService createWorker() {
return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1), runnable -> {
Thread thread = new Thread(runnable, "source-filter-index-backfill");
thread.setDaemon(true);
return thread;
}, new ThreadPoolExecutor.AbortPolicy());
}
}
@@ -70,10 +70,12 @@ public class MetricsController {
@RequestParam(required=false) String exclusions,
@RequestParam(required=false) String columnFilters,
@RequestParam(required=false) String stage,
@RequestParam(defaultValue="full") String part,
@AuthenticationPrincipal AppUserPrincipal actor) {
if (!List.of("full", "table").contains(part)) throw new IllegalArgumentException("part 必须是 full 或 table");
return calculateRangeStage(year, month, monthStart, monthEnd, quarter, department, group, product, productSearch,
projectGroup, view, includeUnassigned, releaseId, trendGranularity, rowLimit, rowOffset, rowOrder,
extraFilters, exclusions, columnFilters, stage, actor);
extraFilters, exclusions, columnFilters, stage, actor, "table".equals(part));
}
/** 轻量趋势接口:查询参数与正式指标接口一致,但只返回版本、上下文、趋势和覆盖信息。 */
@@ -107,6 +109,40 @@ public class MetricsController {
return responseAccess.visibleTrend(metrics.calculateTrend(query, actor), actor);
}
/** 产品分页接口:范围、授权和正式规则与主查询一致。 */
@GetMapping("/products")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN', 'DATA_ADMIN', 'USER')")
public MetricsService.ProductPage productsHttp(@RequestParam(required = false) Integer year,
@RequestParam(required = false) Integer month,
@RequestParam(required = false) Integer monthStart,
@RequestParam(required = false) Integer monthEnd,
@RequestParam(required = false) String quarter,
@RequestParam(required = false) String department,
@RequestParam(required = false) String group,
@RequestParam(required = false) String product,
@RequestParam(required = false) String productSearch,
@RequestParam(required = false) String projectGroup,
@RequestParam(defaultValue = "product") String view,
@RequestParam(defaultValue = "true") boolean includeUnassigned,
@RequestParam(required = false) String releaseId,
@RequestParam(defaultValue = "month") String trendGranularity,
@RequestParam(required = false) String extraFilters,
@RequestParam(required = false) String exclusions,
@RequestParam(required = false) String columnFilters,
@RequestParam(required = false) String stage,
@RequestParam(defaultValue="1") int page,
@RequestParam(defaultValue="15") int pageSize,
@RequestParam(defaultValue="brightQualifiedDesc") String order,
@AuthenticationPrincipal AppUserPrincipal actor) {
Integer quarterNumber = parseQuarter(quarter);
MetricsService.MetricsQuery query = new MetricsService.MetricsQuery(year, month, quarterNumber, department,
group, product, projectGroup, view, includeUnassigned, releaseId, trendGranularity,
parseExclusions(exclusions));
query = query.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters)
.withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage);
return responseAccess.visibleProductPage(metrics.calculateProductPage(query,actor,page,pageSize,order),actor);
}
/** Existing direct callers retain the pre-range controller signature. */
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter,
String department, String group, String product, String productSearch,
@@ -131,19 +167,25 @@ public class MetricsController {
}
private MetricsService.MetricSnapshot calculateRangeStage(Integer year,Integer month,Integer monthStart,Integer monthEnd,String quarter,String department,String group,String product,String productSearch,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Integer rowLimit,int rowOffset,String rowOrder,String extraFilters,String exclusions,String columnFilters,String stage,AppUserPrincipal actor) {
return calculateRangeStage(year,month,monthStart,monthEnd,quarter,department,group,product,productSearch,projectGroup,view,includeUnassigned,releaseId,trendGranularity,rowLimit,rowOffset,rowOrder,extraFilters,exclusions,columnFilters,stage,actor,false);
}
private MetricsService.MetricSnapshot calculateRangeStage(Integer year,Integer month,Integer monthStart,Integer monthEnd,String quarter,String department,String group,String product,String productSearch,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Integer rowLimit,int rowOffset,String rowOrder,String extraFilters,String exclusions,String columnFilters,String stage,AppUserPrincipal actor,boolean tableOnly) {
Integer quarterNumber = parseQuarter(quarter);
MetricsService.MetricSnapshot visible = responseAccess.visible(metrics.calculate(new MetricsService.MetricsQuery(year, month, quarterNumber, department, group,
MetricsService.MetricsQuery query = new MetricsService.MetricsQuery(year, month, quarterNumber, department, group,
product, projectGroup, view, includeUnassigned, releaseId, trendGranularity, parseExclusions(exclusions))
.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters).withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage), actor), actor);
if (rowLimit == null) return visible;
if (rowLimit < 1 || rowLimit > 500 || rowOffset < 0) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
if (rowOrder != null && !rowOrder.equals("scrapDesc")) throw new IllegalArgumentException("rowOrder 无效");
.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters).withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage);
MetricsService.MetricSnapshot visible = responseAccess.visible(tableOnly ? metrics.calculateTable(query,actor) : metrics.calculate(query,actor), actor);
if (rowLimit == null && rowOrder == null) return visible;
if (rowLimit != null && (rowLimit < 1 || rowLimit > 500 || rowOffset < 0)) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
ProductTableOrder productOrder=rowOrder==null||rowOrder.equals("scrapDesc")?null:ProductTableOrder.parse(rowOrder);
List<MetricsService.MetricRow> ordered = new ArrayList<>(visible.rows());
if ("scrapDesc".equals(rowOrder)) ordered.sort(Comparator
.comparing(this::totalScrap).reversed()
.thenComparing(row -> row.key() == null ? "" : row.key()));
int start = Math.min(rowOffset, ordered.size());
int end = (int) Math.min((long) start + rowLimit, ordered.size());
if(productOrder!=null)ordered.sort(productOrder.comparator());
int start = rowLimit==null?0:Math.min(rowOffset, ordered.size());
int end = rowLimit==null?ordered.size():(int) Math.min((long) start + rowLimit, ordered.size());
Map<String, Object> context = new LinkedHashMap<>(visible.context());
context.put("rowCount", ordered.size());
return new MetricsService.MetricSnapshot(visible.release(), context, visible.summary(), ordered.subList(start, end),
@@ -31,6 +31,17 @@ public class MetricsResponseAccess {
return new MetricSnapshot(filtered.release(), filtered.context(), hideDefects(filtered.summary()), List.of(), List.of(), defects, filtered.coverage());
}
/** 分页结果返回前重新核验当前用户、字段和明细服务绑定。 */
public MetricsService.ProductPage visibleProductPage(MetricsService.ProductPage page,AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);
if(!Boolean.TRUE.equals(governance.policy(actor.getId().toString()).get("detailAllowed")))
return new MetricsService.ProductPage(page.release(),page.context(),List.of(),0,page.page(),page.pageSize(),page.order());
governance.requireBoundAction("QUERY","query.detail");
Set<String> allowed=governance.allowedFields(actor);
var rows=allowed.contains("*")?page.rows():page.rows().stream().map(row->restrictDefects(row,allowed)).toList();
return new MetricsService.ProductPage(page.release(),page.context(),rows,page.total(),page.page(),page.pageSize(),page.order());
}
/** 趋势查询沿用明细授权和指标字段可见性边界。 */
public MetricTrendSnapshot visibleTrend(MetricTrendSnapshot snapshot, AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);
@@ -1,13 +1,21 @@
package cn.yunsheng.quality.metrics;
import cn.yunsheng.quality.filters.SourceFilterIndex;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
/** Builds immutable, per-transform-build source-component rollups for read-only metric queries. */
@Service
public class MetricsRollupService {
private static final Logger LOG = LoggerFactory.getLogger(MetricsRollupService.class);
static final String DIMENSION_COLLATION = "utf8mb4_0900_as_cs";
private static final List<String> FACT_TABLES = List.of("fact_black", "fact_bright");
private static final String COMPONENTS = "JSON_OBJECT(" +
@@ -22,10 +30,18 @@ public class MetricsRollupService {
private final JdbcTemplate jdbc;
private final cn.yunsheng.quality.analysis.FineDefectRollups fineDefects;
private final SourceFilterIndex sourceFilterIndex;
private ApplicationEventPublisher eventPublisher;
public MetricsRollupService(JdbcTemplate jdbc) {
this.jdbc = jdbc;
this.fineDefects = new cn.yunsheng.quality.analysis.FineDefectRollups(jdbc, tools.jackson.databind.json.JsonMapper.builder().build());
this.sourceFilterIndex = new SourceFilterIndex(jdbc);
}
@Autowired
public void setApplicationEventPublisher(ApplicationEventPublisher eventPublisher) {
this.eventPublisher = eventPublisher;
}
public record GroupKey(String stage,String month,String department,String group,String project,String product) {}
@@ -36,6 +52,7 @@ public class MetricsRollupService {
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent);
if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总");
fineDefects.copyBuild(parent, child);
announceSourceFilterBuild(child);
}
@Transactional
@@ -74,6 +91,30 @@ public class MetricsRollupService {
""", buildId, buildId, buildId);
fineDefects.refreshBuild(buildId);
new SalesOrderMetricRollups(jdbc).refreshCurrentBuild(buildId);
announceSourceFilterBuild(buildId);
}
void announceSourceFilterBuild(String buildId) {
if (eventPublisher == null) {
// Directly constructed fixtures have no application event bus; retain
// their explicit synchronous behavior (legacy schemas no-op safely).
sourceFilterIndex.ensureBuild(buildId);
return;
}
Runnable publish = () -> {
try {
eventPublisher.publishEvent(new SourceFilterIndex.BuildAvailable(buildId));
} catch (RuntimeException failure) {
LOG.error("Source filter index notification failed for {}; raw-value fallback remains available", buildId, failure);
}
};
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override public void afterCommit() { publish.run(); }
});
} else {
publish.run();
}
}
static String rollupInsertSql(String table) {
@@ -39,6 +39,10 @@ public class MetricsService {
private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128;
private final JdbcTemplate jdbc;
private final GovernanceAccessService governance;
private SourceColumnQueries sourceColumns;
@Autowired
public void setSourceColumns(SourceColumnQueries sourceColumns) { this.sourceColumns = sourceColumns; }
private final TransactionTemplate readSnapshot;
private final TransactionTemplate governanceSnapshot;
@@ -46,6 +50,7 @@ public class MetricsService {
public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance, PlatformTransactionManager transactionManager) {
this.jdbc = jdbc;
this.governance = governance;
this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
this.readSnapshot = new TransactionTemplate(transactionManager);
this.readSnapshot.setIsolationLevel(TransactionDefinition.ISOLATION_REPEATABLE_READ);
this.readSnapshot.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
@@ -56,10 +61,21 @@ public class MetricsService {
/** Keeps package-level in-memory aggregation tests independent of a database transaction manager. */
public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance) {
this.jdbc = jdbc; this.governance = governance; this.readSnapshot = null; this.governanceSnapshot = null;
this.jdbc = jdbc; this.governance = governance;
this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
this.readSnapshot = null; this.governanceSnapshot = null;
}
public MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor) {
return calculate(query, actor, false);
}
/** 仅按对象汇总表格,省去时间与对象交叉分组的中间行。 */
public MetricSnapshot calculateTable(MetricsQuery query, AppUserPrincipal actor) {
return calculate(query, actor, true);
}
private MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor, boolean tableOnly) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor);
governance.requireBoundAction("QUERY", "query.aggregate");
@@ -68,7 +84,55 @@ public class MetricsService {
? currentReleaseId() : query.releaseId();
if (releaseId == null) return emptySnapshot(query);
ReleaseInfo release = release(releaseId, actor);
return calculateForRelease(query, actor, release);
return withConsistentSnapshot(() -> calculateForReleaseSnapshot(query, actor, release, tableOnly));
}
public record ProductPage(Map<String,Object> release,Map<String,Object> context,List<MetricRow> rows,long total,int page,int pageSize,String order) {}
/** 数据库选出当前页的产品桶,再按正式规则计算这些桶的完整分量。 */
public ProductPage calculateProductPage(MetricsQuery query,AppUserPrincipal actor,int page,int pageSize,String orderName) {
if(actor==null)throw new ResponseStatusException(HttpStatus.FORBIDDEN,"指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor);governance.requireBoundAction("QUERY","query.aggregate");validate(query);
if(!"product".equals(query.view()))throw new IllegalArgumentException("产品分页只支持product视图");
if(page<1||page>1_000_000||pageSize<1||pageSize>100)throw new IllegalArgumentException("页码或每页数量无效");
ProductTableOrder requestedOrder=ProductTableOrder.parse(orderName);
ProductTableOrder order=requestedOrder.visibleTo(governance.allowedFields(actor));
String releaseId=query.releaseId()==null||query.releaseId().isBlank()?currentReleaseId():query.releaseId();
if(releaseId==null){var empty=emptySnapshot(query);return new ProductPage(empty.release(),empty.context(),List.of(),0,page,pageSize,requestedOrder.wire());}
ReleaseInfo release=release(releaseId,actor);
return withConsistentSnapshot(()->{
boolean unrolled=hasUnrolledBuilds(releaseId),sales=salesOrderRollupReady(releaseId,query,actor);
boolean source=!query.extraFilters().isEmpty()||!query.columnFilters().isEmpty();
var branches=new ArrayList<String>();var args=new ArrayList<Object>();
for(String stage:List.of("black","bright")) {
if(query.stage()!=null&&!query.stage().equals(stage))continue;
if(!source||sales)addProductRankBranch(branches,args,stageScope(stage,releaseId,query,actor,false,sales));
if(!sales&&(source||unrolled))addProductRankBranch(branches,args,stageScope(stage,releaseId,query,actor,true,false));
}
String candidates="WITH candidates AS ("+String.join(" UNION ALL ",branches)+") ";
String ranked="SELECT ANY_VALUE(bucket_key) AS bucket_key,"+order.rankExpression()+" AS sort_weight FROM candidates GROUP BY CAST(bucket_key AS BINARY)";
String sql=candidates+", ranked AS ("+ranked+") SELECT bucket_key,COUNT(*) OVER() AS row_count FROM ranked ORDER BY "+order.sqlOrder()+" LIMIT ? OFFSET ?";
var pageArgs=new ArrayList<Object>(args);pageArgs.add(pageSize);pageArgs.add((long)(page-1)*pageSize);
var keys=new ArrayList<String>();long[] count={0};
jdbc.query(sql,(RowCallbackHandler)rs->{keys.add(rs.getString("bucket_key"));count[0]=rs.getLong("row_count");},pageArgs.toArray());
if(keys.isEmpty()) {
Long total=jdbc.queryForObject(candidates+"SELECT COUNT(*) FROM (SELECT CAST(bucket_key AS BINARY) FROM candidates GROUP BY CAST(bucket_key AS BINARY)) all_products",Long.class,args.toArray());
count[0]=total==null?0:total;
}
var snapshot=calculateForReleaseSnapshot(query,actor,release,true,keys);
var byKey=new LinkedHashMap<String,MetricRow>();snapshot.rows().forEach(row->byKey.put(row.key(),row));
var rows=keys.stream().map(byKey::get).filter(Objects::nonNull).toList();
return new ProductPage(snapshot.release(),snapshot.context(),rows,count[0],page,pageSize,requestedOrder.wire());
});
}
private void addProductRankBranch(List<String> branches,List<Object> args,StageScope scope) {
boolean black=scope.stage().equals("black");
String nil="CAST(NULL AS DECIMAL(38,8))";
branches.add("SELECT "+ProductBuckets.EXPRESSION+" AS bucket_key,"+(black?"SUM(f.total_weight)":nil)+" AS black_total,"+
(black?nil:"SUM(f.qualified_weight)")+" AS bright_good,"+(black?nil:"SUM(f.scrap_weight)")+" AS bright_scrap "+
"FROM release_member rm JOIN "+scope.table()+" f"+scope.hint()+" ON f.build_id=rm.transform_build_id"+scope.where()+" GROUP BY bucket_key");
args.addAll(scope.args());
}
/** 只按时间桶生成趋势,不构造总体摘要或产品/分组维度行。 */
@@ -89,6 +153,15 @@ public class MetricsService {
/** Authorize before opening the read snapshot, so service-binding locks do not span source scans. */
public <T> PairedSnapshot<T> calculatePairedSnapshot(MetricsQuery query, AppUserPrincipal actor,
boolean trendOnly, Supplier<T> source) {
return calculatePairedSnapshot(query,actor,trendOnly,source,null);
}
public <T> PairedSnapshot<T> calculatePairedProductSnapshot(MetricsQuery query,AppUserPrincipal actor,List<String> productKeys,Supplier<T> source) {
if(!"product".equals(query.view()))throw new IllegalArgumentException("产品桶范围无效");
return calculatePairedSnapshot(query,actor,false,source,ProductBuckets.parse(productKeys));
}
private <T> PairedSnapshot<T> calculatePairedSnapshot(MetricsQuery query,AppUserPrincipal actor,boolean trendOnly,Supplier<T> source,List<String> productKeys) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN,"指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor);
governance.requireBoundAction("QUERY","query.aggregate");
@@ -98,7 +171,7 @@ public class MetricsService {
ReleaseInfo release=release(releaseId,actor);
return withConsistentSnapshot(() -> trendOnly
? new PairedSnapshot<T>(null,calculateTrendForReleaseSnapshot(query,actor,release),source.get())
: new PairedSnapshot<T>(calculateForReleaseSnapshot(query,actor,release),null,source.get()));
: new PairedSnapshot<T>(calculateForReleaseSnapshot(query,actor,release,productKeys!=null,productKeys),null,source.get()));
}
private MetricTrendSnapshot calculateTrendForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor,
@@ -156,9 +229,9 @@ public class MetricsService {
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if (factFallback) {
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor);
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor,releaseId);
if (!query.columnFilters().isEmpty())
new SourceColumnQueries(jdbc, governance, JsonMapper.builder().build())
sourceColumns
.append(where, args, stage, query.columnFilters(), releaseId, actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
@@ -180,7 +253,7 @@ public class MetricsService {
%s AS bright_processing_scrap,
%s AS bright_diffusion_scrap,
%s AS row_count
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
%s
GROUP BY trend_period
ORDER BY trend_period
@@ -189,7 +262,7 @@ public class MetricsService {
componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"),
componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"),
componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"),
countExpression, table, where);
countExpression, table, factFilterHint(stage,factFallback,query), where);
jdbc.query(sql, (RowCallbackHandler) rs -> {
String period = rs.getString("trend_period");
AggregateStats stats = new AggregateStats().read(rs, stage);
@@ -237,9 +310,17 @@ public class MetricsService {
}
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release) {
return calculateForReleaseSnapshot(query, actor, release, false);
}
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release, boolean tableOnly) {
return calculateForReleaseSnapshot(query,actor,release,tableOnly,null);
}
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release, boolean tableOnly,List<String> productKeys) {
String releaseId = release.id();
boolean hasUnrolledBuilds = hasUnrolledBuilds(releaseId);
AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds);
AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds, null, !tableOnly,productKeys);
String ruleVersion=jdbc.queryForObject("SELECT rule_version FROM data_release WHERE id=?",String.class,releaseId);
ConfigurableRuleEngine rules=rulesForVersion(ruleVersion);
MetricRow summary = rules.applyMetrics(aggregates.summary().toMetricRow("overall", "总体", null, null, null));
@@ -248,7 +329,7 @@ public class MetricsService {
MetricRow adjusted=rules.applyMetrics(new MetricRow(row.key(),row.label(),row.department(),row.group(),row.product(),row.projectGroup(),row.compositeRate(),row.black(),row.bright(),row.coverage(),row.compositeRateWithoutHeadquartersScrap()));
return new TrendMetricRow(row.month(),adjusted.key(),adjusted.label(),adjusted.department(),adjusted.group(),adjusted.product(),adjusted.projectGroup(),adjusted.compositeRate(),adjusted.black(),adjusted.bright(),adjusted.coverage(),adjusted.compositeRateWithoutHeadquartersScrap(),row.period(),row.periodLabel(),row.granularity());
}).toList();
Map<String, Object> coverage = queryCoverage(releaseId, query, actor, hasUnrolledBuilds);
Map<String, Object> coverage = productKeys==null?queryCoverage(releaseId, query, actor, hasUnrolledBuilds):Map.of();
Map<String, Object> context = context(query, query.includeUnassigned());
context.put("ruleVersionId",ruleVersion==null?BusinessRowMapper.RULE_VERSION:ruleVersion);
context.put("calculationRules",rules.calculationRules());context.put("metricFormulaDescriptions",rules.formulaDescriptions());
@@ -351,24 +432,34 @@ public class MetricsService {
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments) {
return queryAggregates(releaseId, query, actor, hasUnrolledBuilds, canonicalGroupDepartments, true);
}
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments, boolean includePeriods) {
return queryAggregates(releaseId,query,actor,hasUnrolledBuilds,canonicalGroupDepartments,includePeriods,null);
}
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments, boolean includePeriods,List<String> productKeys) {
AggregateSet result = new AggregateSet();
boolean salesRollup = salesOrderRollupReady(releaseId, query, actor);
boolean hasSourceFilters = !query.extraFilters().isEmpty() || !query.columnFilters().isEmpty();
if (!hasSourceFilters || salesRollup) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments);
aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods,productKeys);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods,productKeys);
}
if (!salesRollup && (hasUnrolledBuilds || hasSourceFilters)) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments);
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods,productKeys);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods,productKeys);
}
return result;
}
private void aggregateStage(AggregateSet result, String stage, String view, String releaseId,
MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup,
Map<String, String> canonicalGroupDepartments) {
if(query.stage()!=null&&!query.stage().equals(stage))return;
Map<String, String> canonicalGroupDepartments, boolean includePeriods,List<String> productKeys) {
if(query.stage()!=null&&!query.stage().equals(stage)||productKeys!=null&&productKeys.isEmpty())return;
String table = factFallback ? (stage.equals("black") ? "fact_black" : "fact_bright") : (salesRollup ? "sales_order_metric_rollup" : "metric_rollup");
String dimensionColumn = view.equals("overall") ? null : switch (view) {
case "department" -> "f.department";
@@ -382,23 +473,12 @@ public class MetricsService {
boolean canonicalGroupRows = "group".equals(view) && canonicalGroupDepartments != null;
String ownerDepartment = canonicalGroupRows ? ", f.department COLLATE utf8mb4_0900_as_cs AS owner_department" : "";
String aggregateKeys = canonicalGroupRows ? "trend_period, bucket_key, owner_department" : "trend_period, bucket_key";
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
if (salesRollup) new SalesOrderMetricRollups(jdbc).append(where, args, query.extraFilters());
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
String trendBucket = trendBucketExpression(query.trendGranularity());
StageScope scope=stageScope(stage,releaseId,query,actor,factFallback,salesRollup);
List<Object> args=new ArrayList<>(scope.args());
StringBuilder limitedWhere=new StringBuilder(scope.where());
ProductBuckets.append(limitedWhere,args,productKeys);
String where=limitedWhere.toString();
String trendBucket = includePeriods ? trendBucketExpression(query.trendGranularity()) : "'all'";
String sql = """
SELECT %s AS trend_period, %s AS bucket_key%s,
SUM(f.qualified_weight) AS qualified_weight,
@@ -413,7 +493,7 @@ public class MetricsService {
%s AS bright_processing_scrap,
%s AS bright_diffusion_scrap,
%s AS row_count
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
%s
GROUP BY %s
ORDER BY %s
@@ -423,14 +503,14 @@ public class MetricsService {
componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"),
componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"),
componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"),
factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, where,
factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, factFilterHint(stage,factFallback,query), where,
aggregateKeys, aggregateKeys);
jdbc.query(sql, (RowCallbackHandler) rs -> {
String period = rs.getString("trend_period");
String key = rs.getString("bucket_key");
AggregateStats stats = new AggregateStats().read(rs, stage);
result.summary().merge(stage, stats);
result.periods().computeIfAbsent(period, ignored -> new AggregateStats()).merge(stage, stats);
if (includePeriods) result.periods().computeIfAbsent(period, ignored -> new AggregateStats()).merge(stage, stats);
if (dimensionColumn != null) {
if (!canonicalGroupRows || canonicalGroupOwnerMatches(canonicalGroupDepartments, key, rs.getString("owner_department")))
result.dimensions().computeIfAbsent(key, ignored -> new AggregateStats()).merge(stage, stats);
@@ -438,6 +518,29 @@ public class MetricsService {
}, args.toArray());
}
private record StageScope(String stage,String table,String hint,String where,List<Object> args) {}
private StageScope stageScope(String stage,String releaseId,MetricsQuery query,AppUserPrincipal actor,boolean factFallback,boolean salesRollup) {
String table=factFallback?(stage.equals("black")?"fact_black":"fact_bright"):(salesRollup?"sales_order_metric_rollup":"metric_rollup");
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
if (salesRollup) new SalesOrderMetricRollups(jdbc).append(where, args, query.extraFilters());
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
return new StageScope(stage,table,factFilterHint(stage,factFallback,query),where.toString(),List.copyOf(args));
}
private String componentSum(String key) {
String path = "$." + key;
String extracted = "JSON_EXTRACT(f.components, '" + path + "')";
@@ -595,6 +698,16 @@ public class MetricsService {
"sourceMonths", sourceMonths);
}
private String factFilterHint(String stage,boolean factFallback,MetricsQuery query) {
if(sourceColumns.indexed(query.releaseId()))return "";
if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return "";
// 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。
if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null
||query.department()!=null||query.group()!=null||query.product()!=null||query.projectGroup()!=null
||query.productSearch()!=null)return "";
return SourceColumnQueries.factIndexHint(stage);
}
private void coverageStage(Map<String, CoverageMonth> months, String stage, String releaseId,
MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup) {
if(query.stage()!=null&&!query.stage().equals(stage))return;
@@ -609,8 +722,8 @@ public class MetricsService {
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor);
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
String countExpression = factFallback ? "COUNT(*)" : "SUM(f.source_row_count)";
@@ -625,11 +738,11 @@ public class MetricsService {
String sql = """
SELECT f.month_key, %s AS row_count,
%s AS excluded_blank_rows, %s AS unmapped_rows, %s AS project_unconfigured_rows
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
%s
GROUP BY f.month_key
ORDER BY f.month_key
""".formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, where);
""".formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, factFilterHint(stage,factFallback,query), where);
jdbc.query(sql, (RowCallbackHandler) rs -> {
CoverageMonth coverage = months.computeIfAbsent(rs.getString("month_key"), ignored -> new CoverageMonth());
long rowCount = rs.getLong("row_count");
@@ -0,0 +1,27 @@
package cn.yunsheng.quality.metrics;
import java.util.*;
/** 已授权范围内的产品桶限制;空字符串与NULL桶保留各自语义。 */
public final class ProductBuckets {
public static final String EXPRESSION="COALESCE(f.product COLLATE utf8mb4_0900_as_cs, '__UNASSIGNED__') COLLATE utf8mb4_0900_as_cs";
private ProductBuckets() {}
public static List<String> parse(Object input) {
if(input==null)return null;
if(!(input instanceof List<?> values)||values.size()>100)throw new IllegalArgumentException("产品桶必须为不超过100项的列表");
var result=new LinkedHashSet<String>();
for(Object value:values) {
if(!(value instanceof String key)||key.length()>512)throw new IllegalArgumentException("产品桶名称无效");
result.add(key);
}
return List.copyOf(result);
}
public static void append(StringBuilder where,List<Object> args,List<String> keys) {
if(keys==null)return;
if(keys.isEmpty()){where.append(" AND 1=0");return;}
// STRCMP avoids hash-join comparisons dropping canonically equivalent Unicode text.
where.append(" AND (").append(String.join(" OR ",Collections.nCopies(keys.size(),
"STRCMP("+EXPRESSION+",CONVERT(? USING utf8mb4) COLLATE utf8mb4_0900_as_cs)=0"))).append(')');
args.addAll(keys);
}
}
@@ -0,0 +1,47 @@
package cn.yunsheng.quality.metrics;
import java.math.BigDecimal;
import java.util.Comparator;
/** SQL分页与全量导出共用的重量排序约定;比率仍由正式规则计算。 */
public enum ProductTableOrder {
BRIGHT_QUALIFIED("brightQualifiedDesc", "SUM(bright_good)"),
BLACK_TOTAL("blackTotalDesc", "SUM(black_total)"),
BRIGHT_TOTAL("brightTotalDesc", "CASE WHEN SUM(bright_good) IS NULL AND SUM(bright_scrap) IS NULL THEN NULL ELSE COALESCE(SUM(bright_good),0)+COALESCE(SUM(bright_scrap),0) END"),
PRODUCT("productAsc", null);
private final String wire;
private final String expression;
ProductTableOrder(String wire, String expression) { this.wire=wire; this.expression=expression; }
public String wire() { return wire; }
public static ProductTableOrder parse(String value) {
for (var order:values()) if (order.wire.equals(value)) return order;
throw new IllegalArgumentException("产品排序无效");
}
public String rankExpression() { return expression==null?"NULL":expression; }
public String sqlOrder() {
// UTF-16字节序与既有Java对象键次序一致,同重量时固定顺序。
return (expression==null?"":"sort_weight DESC, ")+"CAST(CONVERT(bucket_key USING utf16) AS BINARY) ASC";
}
public ProductTableOrder visibleTo(java.util.Set<String> fields) {
String field=switch(this) {
case BRIGHT_QUALIFIED -> "bright_qualified_weight";
case BLACK_TOTAL -> "black_total_weight";
case BRIGHT_TOTAL -> "bright_total_weight";
case PRODUCT -> "*";
};
return fields.contains("*")||fields.contains(field)?this:PRODUCT;
}
private BigDecimal weight(MetricsService.MetricRow row) {
return switch(this) {
case BRIGHT_QUALIFIED -> row.bright().qualifiedWeight();
case BLACK_TOTAL -> row.black().totalWeight();
case BRIGHT_TOTAL -> row.bright().totalWeight();
case PRODUCT -> null;
};
}
public Comparator<MetricsService.MetricRow> comparator() {
Comparator<MetricsService.MetricRow> keys=Comparator.comparing(row->row.key()==null?"":row.key());
return this==PRODUCT?keys:Comparator.comparing(this::weight,Comparator.nullsLast(Comparator.reverseOrder())).thenComparing(keys);
}
}
@@ -53,6 +53,10 @@ public class RecordController {
private final JsonMapper mapper;
private final GovernanceAccessService governance;
private final RecordsWorkbookExportService workbookExport;
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
@org.springframework.beans.factory.annotation.Autowired
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
public RecordController(JdbcTemplate jdbc, ReleaseService releases, JsonMapper mapper,
GovernanceAccessService governance, RecordsWorkbookExportService workbookExport) {
@@ -61,6 +65,7 @@ public class RecordController {
this.mapper = mapper;
this.governance = governance;
this.workbookExport = workbookExport;
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
}
@@ -156,10 +161,10 @@ public class RecordController {
args.add(search);
args.add(jsonPath(selectedField));
}
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor);
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor,selectedRelease);
var sourceFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters);
for(var rule:sourceFilters)if(!List.of(stage).containsAll((List<?>)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段");
if(!sourceFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,stage,sourceFilters,selectedRelease,actor);
if(!sourceFilters.isEmpty())sourceColumns.append(predicate,args,stage,sourceFilters,selectedRelease,actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
String countSql = """
@@ -52,6 +52,10 @@ public class RecordsWorkbookExportService {
private final ReleaseService releases;
private final GovernanceAccessService governance;
private final JsonMapper mapper;
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
@org.springframework.beans.factory.annotation.Autowired
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
public RecordsWorkbookExportService(JdbcTemplate jdbc, ReleaseService releases,
GovernanceAccessService governance, JsonMapper mapper) {
@@ -59,6 +63,7 @@ public class RecordsWorkbookExportService {
this.releases = releases;
this.governance = governance;
this.mapper = mapper;
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
}
public Path export(Integer year, Integer month, Integer quarter, String department, String group,
@@ -120,8 +125,11 @@ public class RecordsWorkbookExportService {
LinkedHashSet<String> rawFields = new LinkedHashSet<>();
if (releaseId != null) {
QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product,
projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd);
scan(rawScope, rs -> rawFields.addAll(visibleRaw(readJson(rs.getString(1)), allowedFields).keySet()));
projectGroup, includeUnassigned, "CAST(JSON_KEYS(f.raw_values) AS CHAR)", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd);
scan(rawScope, rs -> {
Object decoded=mapper.readValue(rs.getString(1),Object.class);
if(decoded instanceof List<?> keys)for(Object key:keys)if(key instanceof String name&&(allowedFields.contains("*")||allowedFields.contains(name)))rawFields.add(name);
});
}
boolean showOrderType=SalesOrderTypes.allowed(allowedFields);
if (rawFields.size() + HEADER_COUNT + (showOrderType?1:0) > MAX_COLUMNS)
@@ -167,8 +175,8 @@ public class RecordsWorkbookExportService {
SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value);
if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor);
if(!columnFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor,releaseId);
if(!columnFilters.isEmpty())sourceColumns.append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
String columns = select == null ? "f.raw_values" : select;
@@ -15,6 +15,7 @@ import cn.yunsheng.quality.ingestion.ImportWorker;
import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild;
import cn.yunsheng.quality.ingestion.MappingValueCodec;
import cn.yunsheng.quality.metrics.MetricsService;
import cn.yunsheng.quality.metrics.MetricsRollupService;
import cn.yunsheng.quality.release.ReleaseService;
import cn.yunsheng.quality.source.SourceCatalogService;
import cn.yunsheng.quality.source.SourceDeletionService;
@@ -66,6 +67,7 @@ public class DataWorkflowService {
private final TaskExecutor executor;
private final TransactionTemplate transaction;
private final MetricsService metrics;
private MetricsRollupService rollups;
private final AtomicReference<String> runningOperation = new AtomicReference<>();
private final AtomicReference<String> runningLease = new AtomicReference<>();
@@ -108,6 +110,12 @@ public class DataWorkflowService {
this.executor = executor;
this.transaction = new TransactionTemplate(manager);
this.metrics=metrics;
this.rollups=new MetricsRollupService(jdbc);
}
@Autowired
public void setMetricsRollupService(MetricsRollupService rollups) {
this.rollups = rollups;
}
public Map<String, Object> status(AppUserPrincipal actor) {
@@ -376,7 +384,7 @@ public class DataWorkflowService {
sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor);
updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED");
var selectedRules=unclassifiedRules;
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,rollups).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token);
publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false);
jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token);
@@ -0,0 +1,39 @@
CREATE TABLE source_filter_build (
build_id CHAR(36) NOT NULL PRIMARY KEY,
format_version INT NOT NULL,
CONSTRAINT fk_source_filter_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE source_filter_field (
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
build_id CHAR(36) NOT NULL,
stage VARCHAR(16) NOT NULL,
column_name LONGTEXT COLLATE utf8mb4_0900_bin NOT NULL,
name_hash BINARY(32) NOT NULL,
UNIQUE KEY uq_source_filter_field (build_id,stage,name_hash),
CONSTRAINT fk_source_filter_field_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE source_filter_value (
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
field_id BIGINT NOT NULL,
value_hash BINARY(32) NOT NULL,
value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,
UNIQUE KEY uq_source_filter_value_hash (field_id,value_hash),
KEY ix_source_filter_value_text (field_id,value(128)),
CONSTRAINT fk_source_filter_value_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE source_filter_member (
field_id BIGINT NOT NULL,
fact_id BIGINT NOT NULL,
value_id BIGINT NOT NULL,
PRIMARY KEY (field_id,fact_id),
KEY ix_source_filter_member_value (value_id,fact_id),
CONSTRAINT fk_source_filter_member_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id),
CONSTRAINT fk_source_filter_member_value FOREIGN KEY (value_id) REFERENCES source_filter_value(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- Cover representative-row selection without reading every row's large source JSON.
ALTER TABLE fact_black ADD KEY ix_black_source_catalog (build_id,source_revision_id,department,group_name,id);
ALTER TABLE fact_bright ADD KEY ix_bright_source_catalog (build_id,source_revision_id,department,group_name,id);
@@ -0,0 +1,6 @@
-- Keep common candidate authorization/date/product checks off the wide JSON clustered rows.
-- The fact table remains authoritative. Changes to its dimensions update this index automatically.
ALTER TABLE fact_black ADD INDEX ix_black_candidate_scope
(id,build_id,source_revision_id,month_key,department,group_name,product);
ALTER TABLE fact_bright ADD INDEX ix_bright_candidate_scope
(id,build_id,source_revision_id,month_key,department,group_name,product);
@@ -0,0 +1,6 @@
-- Equality on the owning build/revision followed by fact id supports the first visible row.
-- Keep authorization dimensions covered, without reading wide JSON rows while seeking it.
ALTER TABLE fact_black DROP INDEX ix_black_source_catalog,
ADD INDEX ix_black_source_catalog (build_id,source_revision_id,id,department,group_name);
ALTER TABLE fact_bright DROP INDEX ix_bright_source_catalog,
ADD INDEX ix_bright_source_catalog (build_id,source_revision_id,id,department,group_name);
@@ -42,7 +42,7 @@ class FineDefectRollupsMySqlTest {
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)");
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))");
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,sheet_name VARCHAR(80),physical_row_no INT,KEY ix_fact_" + stage + "_month_dims(build_id,month_key),UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))");
for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql", "V42__fine_defect_compact_rollups.sql")) {
try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) {
for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql);
@@ -101,7 +101,7 @@ class SourceColumnQueriesMySqlTest {
assertTrue((Boolean)page.get("pageComplete"));
}
@Test void groupedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() {
@Test void orderedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() {
record Read(String sql,Object[] args,List<?> result) {}
List<Read> reads=new java.util.ArrayList<>();
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
@@ -114,7 +114,8 @@ class SourceColumnQueriesMySqlTest {
assertEquals(2,representatives.size());
for(Read read:representatives) {
String stage=read.sql().contains("fact_black f2")?"black":"bright";
assertTrue(read.sql().contains("GROUP BY f2.build_id,f2.source_revision_id"));
assertTrue(read.sql().contains("ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1"));
assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表");
StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
List<Object> args=new java.util.ArrayList<>();
DataScope.append(scope,args,actor,"f2.department COLLATE utf8mb4_0900_as_cs","f2.group_name COLLATE utf8mb4_0900_as_cs");args.add(RELEASE);
@@ -157,6 +158,23 @@ class SourceColumnQueriesMySqlTest {
assertThrows(ResponseStatusException.class, () -> queries.validate(RELEASE, unauthorized, actor));
}
@Test void sourceNullNormalizationMatchesThePreviousExpressionForEveryOperator() {
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
for(String raw:List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[]}","{\"委托单\":{\"n\":1}}"))
insert("bright","build-bright",BRIGHT_REVISION,"部门A","组A",raw);
String old="CASE WHEN JSON_EXTRACT(f.raw_values,'$.\"委托单\"') IS NULL OR JSON_TYPE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))='NULL' THEN NULL ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))),'') END COLLATE utf8mb4_0900_as_cs";
for(String operator:List.of("in","not_in","is_empty","not_empty")) {
Map<String,Object> rule=new java.util.LinkedHashMap<>(Map.of("field",key,"stages",List.of("bright"),"operator",operator));
List<Object> args=new java.util.ArrayList<>();String condition;
if(operator.equals("in")||operator.equals("not_in")) {
rule.put("values",List.of("null","0007"));args.add("null");args.add("0007");
condition=operator.equals("in")?old+" IN (?,?)":"("+old+" IS NULL OR "+old+" NOT IN (?,?))";
} else condition=old+(operator.equals("is_empty")?" IS NULL":" IS NOT NULL");
Long expected=jdbc.queryForObject("SELECT COUNT(*) FROM fact_bright f WHERE f.build_id='build-bright' AND f.department='部门A' AND f.group_name='组A' AND ("+condition+")",Long.class,args.toArray());
assertEquals(expected,countFiltered("bright",SourceColumnQueries.parse(List.of(rule))),operator);
}
}
@Test void explicitMappingWorksAcrossStagesAndMemberPageCountsMergedTuplesBeforePaging() {
List<Map<String, Object>> catalog = queries.catalog(RELEASE, actor);
String mappedKey = (String) catalog.stream().filter(item -> item.get("label").equals("跨阶段委托号"))
@@ -247,17 +265,27 @@ class SourceColumnQueriesMySqlTest {
return fields.stream().filter(item -> item.get("label").equals(label)).findFirst().orElse(null);
}
@Test void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters() throws Exception {
@org.junit.jupiter.params.ParameterizedTest
@org.junit.jupiter.params.provider.CsvSource({"false,in","true,in","false,contains","true,contains"})
void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed,String operator) throws Exception {
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))");
jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE);
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
for(String stage:List.of("black","bright")) {
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT");
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=?,scrap_weight=?,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id",stage.equals("black")?90:80,stage.equals("black")?10:20);
}
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
var service=new cn.yunsheng.quality.metrics.MetricsService(jdbc,governance);
if(indexed){installFilterIndex();var index=new SourceFilterIndex(jdbc);for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);}
var headerReads=new java.util.concurrent.atomic.AtomicInteger();
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
if(sql.contains("JSON_KEYS(f.raw_values)"))headerReads.incrementAndGet();
return super.query(sql,mapper,args);
}
};
var service=new cn.yunsheng.quality.metrics.MetricsService(capturing,governance);
var releases=mock(cn.yunsheng.quality.release.ReleaseService.class);
when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE);
var mapper=JsonMapper.builder().build();
@@ -265,13 +293,14 @@ class SourceColumnQueriesMySqlTest {
var records=new cn.yunsheng.quality.records.RecordController(jdbc,releases,mapper,governance,exporter);
String fieldKey=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
for(String stage:List.of("black","bright")) {
var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator","in","values",List.of("0007"));
var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator",operator,"values",List.of(operator.equals("contains")?"007":"0007"));
var base=new cn.yunsheng.quality.metrics.MetricsService.MetricsQuery(Integer.valueOf(2026),Integer.valueOf(1),null,"部门A","组A",null,null,"product",false,RELEASE,"month",Map.of()).withColumnFilters(List.of(filter));
var legacy=service.calculate(base,actor);var selected=service.calculate(base.withStage(stage),actor);
assertEquals(1,selected.rows().size());assertEquals(1L,((Number)selected.coverage().get("includedRows")).longValue());
assertEquals(stage.equals("black")?legacy.summary().black():legacy.summary().bright(),stage.equals("black")?selected.summary().black():selected.summary().bright());
assertEquals(0L,selected.summary().coverage().get(stage.equals("black")?"brightRows":"blackRows"));
assertEquals(1,selected.trend().size());
assertEquals(1,service.calculateTrend(base.withStage(stage),actor).trend().size());
var detail=records.recordsFiltered(stage,2026,1,null,"部门A","组A",null,null,null,false,RELEASE,1,50,null,null,null,"asc",null,null,mapper.writeValueAsString(List.of(filter)),actor);
assertEquals(1L,((Number)detail.get("total")).longValue());
java.nio.file.Path file=exporter.exportFiltered(2026,1,null,"部门A","组A",null,null,false,RELEASE,actor,Map.of(),null,stage,List.of(filter));
@@ -282,6 +311,7 @@ class SourceColumnQueriesMySqlTest {
var other=base.withColumnFilters(List.of(filter)).withStage(stage);
assertThrows(ResponseStatusException.class,()->other.withStage(stage.equals("black")?"bright":"black"));
}
assertEquals(2,headerReads.get(),"汇总、趋势与覆盖检查应复用同一份授权字段目录");
}
@Test void continuousRangeDetailsAndBothExcelSheetsExcludeOutsideMonthsAndUnauthorizedRows() throws Exception {
@@ -289,7 +319,7 @@ class SourceColumnQueriesMySqlTest {
when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE);
var mapper=JsonMapper.builder().build();
for(String stage:List.of("black","bright")) {
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT");
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=90,scrap_weight=10,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id");
String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION;
for(String month:List.of("2026-03","2026-09","2026-10","2025-03")) {
@@ -336,7 +366,7 @@ class SourceColumnQueriesMySqlTest {
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
for(String stage:List.of("black","bright")) {
jdbc.execute("DELETE FROM fact_"+stage);
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT");
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
}
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
@@ -390,12 +420,227 @@ class SourceColumnQueriesMySqlTest {
assertThrows(ResponseStatusException.class,()->controller.query(input,actor));
}
@Test void indexedFiltersCandidatesAndTuplesMatchRawSemanticsAndStayAuthorized() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
List<String> samples=List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[1,2]}","{\"委托单\":{\"a\":1}}","{\"委托单\":\"é\"}","{\"委托单\":\"é\"}","{\"委托单\":\"A\"}","{\"委托单\":\"a\"}","{\"委托单\":\" padded \"}","{\"委托单\":\"null\"}");
for(String raw:samples)insert("black","build-black",BLACK_REVISION,"部门A","组A",raw);
String longKey="来源".repeat(300);
insert("black","build-black",BLACK_REVISION,"部门A","组A",JsonMapper.builder().build().writeValueAsString(Map.of(longKey,"long-key-value")));
insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}");
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
List<Map<String,Object>> rules=new java.util.ArrayList<>();
for(String op:List.of("in","contains","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}"))
rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value)));
var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList();
var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor);
var rawMembers=queries.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor);
jdbc.update("INSERT INTO query_filter_field VALUES ('plain','原始委托单','委托单','委托单',TRUE,TRUE,1)");
List<List<String>> legacySelections=List.of(List.of("null"),List.of("0007"),List.of(QueryFilterFields.EMPTY),List.of("0007",QueryFilterFields.EMPTY));
var rawLegacy=legacySelections.stream().map(this::legacyCount).toList();
installFilterIndex();
var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
assertTrue(indexed.indexed(RELEASE));
assertEquals(rawLegacy,legacySelections.stream().map(this::legacyCount).toList());
assertEquals("long-key-value",jdbc.queryForObject("SELECT sv.value FROM source_filter_field sf JOIN source_filter_value sv ON sv.field_id=sf.id WHERE sf.column_name=? AND sf.build_id='build-black'",String.class,longKey));
var combined=List.of(Map.<String,Object>of("field",key,"stages",List.of("black"),"operator","in","values",List.of("0007")),Map.<String,Object>of("field",field(indexed.catalog(RELEASE,actor),"加工单位").get("key"),"stages",List.of("black"),"operator","in","values",List.of("机二")));
assertEquals(0L,indexed.memberPage(RELEASE,Map.of("year",2026,"columnFilters",combined),List.of(key),List.of("black"),50,0,actor).get("totalCount"));
assertEquals(List.of(),indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),"hidden-only-value",50,0,true,actor).get("rows"));
jdbc.update("INSERT INTO transform_build VALUES ('build-child')");
jdbc.update("INSERT INTO fact_black(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT 'build-child',source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no FROM fact_black WHERE build_id='build-black'");
index.ensureBuild("build-child");
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member sm JOIN source_filter_field sf ON sf.id=sm.field_id JOIN fact_black f ON f.id=sm.fact_id WHERE sf.build_id='build-child' AND f.build_id<>'build-child'",Long.class));
assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
assertEquals(rawValues,indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor));
assertEquals(rawMembers,indexed.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor));
var page=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,2,0,true,actor);
assertEquals(2,((List<?>)page.get("rows")).size());assertEquals(true,page.get("hasMore"));assertFalse(page.containsKey("totalCount"));
long members=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class);
index.ensureBuild("build-black");assertEquals(members,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
when(governance.allowedFields(actor)).thenReturn(Set.of("加工单位"));
assertThrows(ResponseStatusException.class,()->indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),null,50,0,true,actor));
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
jdbc.update("DELETE FROM source_filter_build WHERE build_id='build-black'");
assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
}
@Test void containsMatchesLiteralSubstringsWithOrSemanticsInBothStagesAndBothQueryPaths() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
var mapper=JsonMapper.builder().build();
for(String stage:List.of("black","bright")) {
String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION;
for(String value:List.of("xxAlpha%_\\tail","中文日立公司","prefix x' OR 1=1 -- suffix","000123","null"))
insert(stage,"build-"+stage,revision,"部门A","组A",mapper.writeValueAsString(Map.of("委托单",value)));
insert(stage,"build-"+stage,revision,"部门B","组B",mapper.writeValueAsString(Map.of("委托单","日立-Alpha-hidden")));
}
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
List<List<String>> needles=List.of(List.of("Alpha"),List.of("alpha"),List.of("%"),List.of("_"),List.of("\\"),List.of("日立"),List.of("x' OR 1=1 --"),List.of("001"),List.of("null"),List.of("日立","Alpha"));
List<Long> expected=List.of(1L,0L,1L,1L,1L,1L,1L,1L,1L,2L);
for(boolean indexed:List.of(false,true)) {
if(indexed) {
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
queries=new SourceColumnQueries(jdbc,governance,mapper);
}
for(String stage:List.of("black","bright"))for(int n=0;n<needles.size();n++) {
var rule=Map.<String,Object>of("field",key,"stages",List.of(stage),"operator","contains","values",needles.get(n));
assertEquals(expected.get(n),countFiltered(stage,List.of(rule)),stage+" indexed="+indexed+" keywords="+needles.get(n));
}
var canonical=Map.<String,Object>of("field","product","stages",List.of("black"),"operator","contains","values",List.of("P-"));
assertTrue(countFiltered("black",List.of(canonical))>0);
}
}
@Test void memberBuildPlansStartFromTheBoundedStageWindowRatherThanTheGrowingDictionary() throws Exception {
installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black");
jdbc.execute((org.springframework.jdbc.core.ConnectionCallback<Void>)connection->{
try(var statement=connection.createStatement()) {
statement.execute(SourceFilterIndex.CREATE_STAGE_SQL);
try {
statement.execute("INSERT INTO source_filter_staging SELECT v.field_id,m.fact_id,v.value,v.value_hash FROM source_filter_value v JOIN source_filter_member m ON m.value_id=v.id LIMIT 4");
for(String sql:List.of(SourceFilterIndex.MEMBER_INSERT_SQL,SourceFilterIndex.COLLISION_SQL)) {
try(var rows=statement.executeQuery("EXPLAIN "+sql)) {
String first=null;
while(rows.next())if(List.of("s","v").contains(rows.getString("table"))){first=rows.getString("table");break;}
assertEquals("s",first,"每批构建只能从有界临时窗口开始,不能反复扫描历史字典");
}
}
} finally {statement.execute("DROP TEMPORARY TABLE source_filter_staging");}
}
return null;
});
}
@Test void catalogStopsAtTheFirstAuthorizedFactWithoutSortingEveryFactId() throws Exception {
installFilterIndex();
var captured=new java.util.ArrayList<String>();var parameters=new java.util.ArrayList<Object[]>();
var tracking=new JdbcTemplate(jdbc.getDataSource()) {
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
if(sql.startsWith("SELECT CAST(JSON_KEYS")){captured.add(sql);parameters.add(args);}
return super.query(sql,mapper,args);
}
};
var catalog=new SourceColumnQueries(tracking,governance,JsonMapper.builder().build()).catalog(RELEASE,actor);
assertNotNull(field(catalog,"委托单"));
assertNull(field(catalog,"组织外行列"),"首条全局记录不可见时,必须继续寻找当前用户的第一条可见记录");
assertNull(field(catalog,"其他组织列"));
assertEquals(2,captured.size());
for(int i=0;i<captured.size();i++) {
String plan=jdbc.queryForObject("EXPLAIN ANALYZE "+captured.get(i),String.class,parameters.get(i));
assertTrue(plan.contains("Limit: 1 row(s)"),plan);
assertFalse(plan.lines().anyMatch(line->line.contains("Sort:")&&line.contains("f2.")),plan);
assertTrue(plan.contains("Covering index lookup on f2 using ix_"),plan);
}
}
@Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception {
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
var scopeArgs=new java.util.ArrayList<Object>(List.of(RELEASE,"2026-01","2026-12","部门A","组A","P-1"));
var args=new java.util.ArrayList<Object>();
String sql=index.candidateSelect("black","委托单"," WHERE rm.release_id=? AND f.month_key BETWEEN ? AND ? AND f.department=? AND f.group_name=? AND f.product=?",scopeArgs,args);
assertEquals(Set.of("0007","0008"),new java.util.HashSet<>(jdbc.queryForList(sql,String.class,args.toArray())));
boolean covering=false;
for(var row:jdbc.queryForList("EXPLAIN "+sql,args.toArray())) {
if("f".equals(row.get("table"))) {
assertEquals("ix_black_candidate_scope",row.get("key"));
covering=String.valueOf(row.get("Extra")).contains("Using index");
}
}
assertTrue(covering,"候选值的日期、权限和产品校验不应读取宽 JSON 原始行");
}
@Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception {
installFilterIndex();
for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}");
var memberWrites=new java.util.concurrent.atomic.AtomicInteger();
var failing=new JdbcTemplate(jdbc.getDataSource()) {
@Override public int update(String sql) {
if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure");
return super.update(sql);
}
@Override public int update(String sql,Object... args) {
if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure");
return super.update(sql,args);
}
};
var index=new SourceFilterIndex(failing);
assertThrows(IllegalStateException.class,()->index.ensureBuild("build-black"));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build",Long.class));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_field",Long.class));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_value",Long.class));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
assertFalse(index.ready(RELEASE));
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
assertTrue(index.ready(RELEASE));
}
@Test
@EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_FILTER_BENCHMARK",matches="true")
void syntheticWideSourceIndexBenchmark() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
int size=Integer.getInteger("quality.benchmarkRows",10000);
jdbc.execute("CREATE TABLE benchmark_digit(n INT PRIMARY KEY)");for(int n=0;n<10;n++)jdbc.update("INSERT INTO benchmark_digit VALUES (?)",n);
for(String stage:List.of("black","bright")) {
jdbc.execute("DELETE FROM fact_"+stage);
StringBuilder raw=new StringBuilder("JSON_OBJECT('客户名称',CONCAT('客户',MOD(n,300))");
for(int f=0;f<120;f++)raw.append(",'字段").append(f).append("',CONCAT('value-").append(f).append("-',").append(stage.equals("black")?"MOD(n,50)":"n").append(")");
raw.append(')');
String numbers="SELECT a.n+10*b.n+100*c.n+1000*d.n+10000*e.n AS n FROM benchmark_digit a CROSS JOIN benchmark_digit b CROSS JOIN benchmark_digit c CROSS JOIN benchmark_digit d CROSS JOIN benchmark_digit e LIMIT "+size;
jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT ?,?,'2026-01','部门A','组A','P-1','项目A',"+raw+",'data',n+1 FROM ("+numbers+") numbers","build-"+stage,stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION);
}
var rawQuery=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
String field=(String)field(rawQuery.catalog(RELEASE,actor),"客户名称").get("key");
var old=new java.util.LinkedHashMap<String,Object>();
for(String stage:List.of("black","bright")) {
long start=System.nanoTime();var result=rawQuery.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
old.put(stage,Map.of("ms",(System.nanoTime()-start)/1_000_000L,"result",result));
}
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden")) {
long start=System.nanoTime();index.ensureBuild(build);System.out.println("FILTER_BENCH_BUILD "+build+" rows="+size+" ms="+(System.nanoTime()-start)/1_000_000L);
}
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
for(String stage:List.of("black","bright")) {
long start=System.nanoTime();var result=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
long elapsed=(System.nanoTime()-start)/1_000_000L;
assertEquals(((Map<?,?>)old.get(stage)).get("result"),result);
System.out.println("FILTER_BENCH_QUERY "+stage+" rawMs="+((Map<?,?>)old.get(stage)).get("ms")+" indexMs="+elapsed);
}
for(String table:List.of("source_filter_field","source_filter_value","source_filter_member"))jdbc.execute("ANALYZE TABLE "+table);
System.out.println("FILTER_BENCH_SPACE "+jdbc.queryForList("SELECT table_name,table_rows,data_length,index_length FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name LIKE 'source_filter_%'"));
}
private long legacyCount(List<String> selected) {
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
new QueryFilterFields(jdbc,governance).append(where,args,"black",Map.of("plain",selected),actor,RELEASE);
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
}
private long filteredCount(SourceColumnQueries source,Map<String,Object> rule) {
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
source.append(where,args,"black",List.of(rule),RELEASE,actor);
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
}
private void installFilterIndex() throws Exception {
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id);
for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql","V45__source_catalog_first_visible.sql")) {
String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration));
for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement);
}
}
private void schema() {
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))");
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage
+ "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),"
+ "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL)");
+ "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL,sheet_name VARCHAR(80),physical_row_no INT,UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))");
jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64) PRIMARY KEY,label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)");
}
@@ -32,7 +32,7 @@ class SourceColumnQueriesTest {
@Test void rejectsUnknownShapeOperatorsEmptyValuesAndMoreThanFiftyValues() {
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(Map.of()));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("black"), "operator", "contains", "values", List.of("x")))));
"field", SOURCE, "stages", List.of("black"), "operator", "starts_with", "values", List.of("x")))));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("black"), "operator", "in", "values", List.of("")))));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
@@ -42,6 +42,22 @@ class SourceColumnQueriesTest {
"field", SOURCE, "stages", List.of("bright"), "operator", "not_in", "values", tooMany))));
}
@Test void containsKeepsLiteralKeywordsAndBindsEachJsonPathAfterItsNeedle() {
var rule=Map.<String,Object>of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",List.of("000","%_\\","x' OR 1=1 --","000"));
var parsed=SourceColumnQueries.parse(List.of(rule));
assertEquals(List.of("000","%_\\","x' OR 1=1 --"),parsed.getFirst().get("values"));
var source=spy(new SourceColumnQueries(mock(JdbcTemplate.class),governance,JsonMapper.builder().build()));
doReturn(List.of(sourceField())).when(source).scopedCatalog("release",actor);
var sql=new StringBuilder();var args=new ArrayList<Object>();
source.append(sql,args,"black",parsed,"release",actor);
assertEquals(List.of("000","$.\"委托单\"","%_\\","$.\"委托单\"","x' OR 1=1 --","$.\"委托单\""),args);
assertEquals(3,sql.toString().split("LOCATE",-1).length-1);
assertFalse(sql.toString().contains("x' OR"));
for(Object values:List.of(List.of(),List.of(""),List.of(" "),List.of("x".repeat(257)),List.of(7))) {
assertThrows(ResponseStatusException.class,()->SourceColumnQueries.parse(List.of(Map.of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",values))));
}
}
@Test void monthRangeUsesInclusiveYearBoundariesAndKeepsLegacyMonthPredicate() {
Map<String,Object> range=SourceColumnQueries.period(Map.of("period",Map.of("year",2026,"monthStart",3,"monthEnd",9)));
StringBuilder rangeSql=new StringBuilder();List<Object> rangeArgs=new ArrayList<>();
@@ -143,9 +159,9 @@ class SourceColumnQueriesTest {
queries.append(where, args, "black", filters, "release", actor);
assertTrue(where.toString().contains("JSON_EXTRACT(f.raw_values, ?) IS NULL"));
assertTrue(where.toString().contains("NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON))"));
assertTrue(where.toString().contains("COLLATE utf8mb4_0900_as_cs IN (?,?)"));
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "00007", "x' OR 1=1 --"), args);
assertEquals(List.of("$.\"委托单\"", "00007", "x' OR 1=1 --"), args);
assertFalse(where.toString().contains("委托单"));
assertFalse(where.toString().contains("x' OR 1=1"));
}
@@ -160,7 +176,7 @@ class SourceColumnQueriesTest {
StringBuilder where = new StringBuilder();
queries.append(where, args, "black", filters, "release", actor);
assertTrue(where.toString().contains(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL"));
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\""), args);
assertEquals(List.of("$.\"委托单\""), args);
}
List<Map<String, Object>> filters = SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("bright"), "operator", "is_empty")));
@@ -179,8 +195,7 @@ class SourceColumnQueriesTest {
assertTrue(where.toString().contains("IS NULL OR"));
assertTrue(where.toString().contains(" NOT IN (?)"));
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"",
"$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "000123"), args);
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "000123"), args);
}
@Test void rejectsInvalidStageAndMissingFieldsInTheActiveCatalog() {
@@ -223,6 +238,6 @@ class SourceColumnQueriesTest {
columns.put("black", stage.equals("black") ? header : null);
columns.put("bright", stage.equals("bright") ? header : null);
return Map.of("key", key, "label", header, "columns", columns, "stages", List.of(stage),
"type", "source_value", "operators", List.of("in", "not_in", "is_empty", "not_empty"));
"type", "source_value", "operators", List.of("in", "contains", "not_in", "is_empty", "not_empty"));
}
}
@@ -0,0 +1,117 @@
package cn.yunsheng.quality.filters;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import java.util.ArrayDeque;
import java.util.List;
import java.util.concurrent.AbstractExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
class SourceFilterIndexBackfillTest {
@Test
void skipsWhenIndexTablesAreUnavailable() {
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
ManualExecutor worker = new ManualExecutor();
doReturn(false).when(index).available();
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
runner.run(null);
verify(jdbc, never()).query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
verify(index, never()).ensureBuild(anyString());
assertEquals(0, worker.queued());
runner.stopWorker();
}
@Test
void coalescesEventsDuringDrainAndContinuesAfterAnIndividualFailure() {
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
ManualExecutor worker = new ManualExecutor();
doReturn(true).when(index).available();
doReturn(List.of("failed-build", "ready-build")).when(jdbc)
.query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
doAnswer(invocation -> {
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("failed-build"));
throw new IllegalStateException("synthetic index failure");
}).when(index).ensureBuild("failed-build");
runner.run(null);
assertEquals(1, worker.queued());
verify(index, never()).ensureBuild(anyString());
ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
verify(jdbc).query(sql.capture(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
assertTrue(sql.getValue().contains("c.dataset_id='quality'"));
assertTrue(sql.getValue().contains("r.status='PUBLISHED'"));
assertTrue(sql.getValue().contains("r.complete=TRUE"));
assertTrue(sql.getValue().contains("b.status='COMPLETED'"));
worker.runNext();
verify(index).ensureBuild("failed-build");
verify(index).ensureBuild("ready-build");
verify(index).ensureBuild("event-during-drain");
assertEquals(0, worker.queued());
runner.stopWorker();
assertTrue(worker.isShutdown());
}
@Test
void componentHasAnAutowiredProductionConstructorAndCleanShutdown() {
try (var context = new AnnotationConfigApplicationContext()) {
context.registerBean(JdbcTemplate.class, () -> org.mockito.Mockito.mock(JdbcTemplate.class));
context.registerBean(SourceFilterIndex.class, () -> org.mockito.Mockito.mock(SourceFilterIndex.class));
context.register(SourceFilterIndexBackfill.class);
context.refresh();
assertNotNull(context.getBean(SourceFilterIndexBackfill.class));
}
}
private static final class ManualExecutor extends AbstractExecutorService {
private final ArrayDeque<Runnable> tasks = new ArrayDeque<>();
private boolean shutdown;
@Override public void shutdown() { shutdown = true; }
@Override public List<Runnable> shutdownNow() {
shutdown = true;
List<Runnable> pending = List.copyOf(tasks);
tasks.clear();
return pending;
}
@Override public boolean isShutdown() { return shutdown; }
@Override public boolean isTerminated() { return shutdown && tasks.isEmpty(); }
@Override public boolean awaitTermination(long timeout, TimeUnit unit) { return isTerminated(); }
@Override public void execute(Runnable command) {
if (shutdown) throw new RejectedExecutionException("executor is stopped");
tasks.add(command);
}
int queued() { return tasks.size(); }
void runNext() { tasks.removeFirst().run(); }
}
}
@@ -84,4 +84,19 @@ class MetricsResponseAccessTest {
assertTrue(aggregateOnly.trend().isEmpty());
verify(governance, never()).requireBoundAction("QUERY", "query.detail");
}
@Test void productPageRechecksCurrentDetailAndFieldPolicyAndHidesCountsWithoutDetailAccess() {
var page=new ProductPage(Map.of("id","r"),Map.of(),List.of(row),120,2,15,"productAsc");
var access=new MetricsResponseAccess(governance);
when(governance.policy(actor.getId().toString())).thenReturn(Map.of("detailAllowed",true));
when(governance.allowedFields(actor)).thenReturn(Set.of("black_total_weight"));
var visible=access.visibleProductPage(page,actor);
assertEquals(120,visible.total());assertEquals(2,visible.page());
assertEquals(stage.totalWeight(),visible.rows().getFirst().black().totalWeight());
assertNull(visible.rows().getFirst().bright().totalWeight());
verify(governance).requireUserActivePolicy(actor);
when(governance.policy(actor.getId().toString())).thenReturn(Map.of("detailAllowed",false));
var revoked=access.visibleProductPage(page,actor);
assertTrue(revoked.rows().isEmpty());assertEquals(0,revoked.total());
}
}
@@ -1,8 +1,21 @@
package cn.yunsheng.quality.metrics;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import cn.yunsheng.quality.filters.SourceFilterIndex;
import org.junit.jupiter.api.Test;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
class MetricsRollupServiceTest {
@Test
@@ -17,4 +30,65 @@ class MetricsRollupServiceTest {
assertTrue(sql.contains("GROUP BY f.build_id, f.month_key " + collation));
assertTrue(sql.contains("f.group_name " + collation + ", f.project_group " + collation));
}
@Test
void publishesFilterBuildAvailableOnlyAfterTheOuterTransactionCommits() {
var service = new MetricsRollupService(mock(JdbcTemplate.class));
var publisher = mock(ApplicationEventPublisher.class);
service.setApplicationEventPublisher(publisher);
String build = "11111111-1111-4111-8111-111111111111";
TransactionSynchronizationManager.initSynchronization();
try {
service.announceSourceFilterBuild(build);
verifyNoInteractions(publisher);
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
assertEquals(1, synchronizations.size());
for (TransactionSynchronization synchronization : synchronizations) synchronization.afterCommit();
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
} finally {
TransactionSynchronizationManager.clearSynchronization();
}
}
@Test
void publishesImmediatelyWhenThereIsNoTransactionSynchronization() {
var service = new MetricsRollupService(mock(JdbcTemplate.class));
var publisher = mock(ApplicationEventPublisher.class);
service.setApplicationEventPublisher(publisher);
String build = "22222222-2222-4222-8222-222222222222";
service.announceSourceFilterBuild(build);
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
}
@Test
void notificationFailureAfterCommitDoesNotEscapeTheTransactionSynchronization() {
var service = new MetricsRollupService(mock(JdbcTemplate.class));
var publisher = mock(ApplicationEventPublisher.class);
service.setApplicationEventPublisher(publisher);
String build = "33333333-3333-4333-8333-333333333333";
var event = new SourceFilterIndex.BuildAvailable(build);
doThrow(new IllegalStateException("synthetic event delivery failure")).when(publisher).publishEvent(event);
TransactionSynchronizationManager.initSynchronization();
try {
service.announceSourceFilterBuild(build);
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
assertDoesNotThrow(() -> synchronizations.get(0).afterCommit());
} finally {
TransactionSynchronizationManager.clearSynchronization();
}
}
@Test
void directFixtureConstructionKeepsTheSynchronousLegacySchemaFallback() {
JdbcTemplate jdbc = mock(JdbcTemplate.class);
var service = new MetricsRollupService(jdbc);
service.announceSourceFilterBuild("44444444-4444-4444-8444-444444444444");
verify(jdbc).queryForObject(contains("information_schema.tables"), eq(Integer.class));
}
}
@@ -51,4 +51,31 @@ class MetricsTrendControllerTest {
assertEquals(Map.of("customerCode", List.of("000123")), value.extraFilters());
assertEquals("src_any", value.columnFilters().getFirst().get("field"));
}
@Test void tableProjectionKeepsSourceScopeAndUsesTheSameResponseAuthorization() {
MetricsService metrics = mock(MetricsService.class);
MetricsResponseAccess access = mock(MetricsResponseAccess.class);
AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "table", "unused", "表格", true, false,
List.of("USER"), List.of("D"), List.of("G"), List.of());
var response = new MetricsService.MetricSnapshot(Map.of(), Map.of(), null, List.of(), List.of(), Map.of(), Map.of());
when(metrics.calculateTable(any(), eq(actor))).thenReturn(response);
when(access.visible(response, actor)).thenReturn(response);
var actual = new MetricsController(metrics, access).calculateHttp(
2026, null, 3, 9, null, "D", "G", "P", "18X15", "PG", "product", false,
"release-a", "quarter", null, 0, null, "{\"customerCode\":[\"000123\"]}",
"{\"product\":[\"P2\"]}",
"[{\"field\":\"src_any\",\"stages\":[\"black\"],\"operator\":\"contains\",\"values\":[\"000123\"]}]",
"black", "table", actor);
assertEquals(response, actual);
var query = ArgumentCaptor.forClass(MetricsService.MetricsQuery.class);
verify(metrics).calculateTable(query.capture(), eq(actor));
verify(access).visible(response, actor);
var value = query.getValue();
assertEquals(3, value.monthStart()); assertEquals(9, value.monthEnd());
assertEquals("black", value.stage()); assertEquals("P", value.product());
assertEquals("D", value.department()); assertEquals("G", value.group());
assertEquals("18X15", value.productSearch()); assertEquals("PG", value.projectGroup());
assertEquals(Map.of("product", List.of("P2")), value.exclusions());
assertEquals(Map.of("customerCode", List.of("000123")), value.extraFilters());
assertEquals("contains", value.columnFilters().getFirst().get("operator"));
}
}
@@ -49,6 +49,29 @@ class MetricsTrendServiceTest {
assertTrue(trendSql.stream().allMatch(sql -> !sql.contains("bucket_key") && !sql.contains("f.product")));
}
@Test void tableProjectionKeepsFormalRowsSummaryAndCoverageWithoutTimeByProductGrouping() {
JdbcTemplate jdbc = jdbc(true, true);
var governance = mock(GovernanceAccessService.class);
MetricsService service = new MetricsService(jdbc, governance);
var query = new MetricsService.MetricsQuery(2026, null, null, null, null, null, null,
"product", true, RELEASE, "month", Map.of()).withMonthRange(2, 5).withStage("bright");
var full = service.calculate(query, actor);
clearInvocations(jdbc, governance);
var table = service.calculateTable(query, actor);
assertEquals(full.summary(), table.summary());
assertEquals(full.rows(), table.rows());
assertEquals(full.defects(), table.defects());
assertEquals(full.coverage(), table.coverage());
assertEquals(full.context(), table.context());
assertTrue(table.trend().isEmpty());
var sql = capturedSql(jdbc).stream().filter(value -> value.contains("AS bucket_key")).toList();
assertEquals(1, sql.size());
assertTrue(sql.getFirst().contains("SELECT 'all' AS trend_period"));
assertTrue(sql.getFirst().contains("f.month_key BETWEEN ? AND ?"));
verify(governance).requireUserActivePolicy(actor);
verify(governance).requireBoundAction("QUERY", "query.aggregate");
}
@Test void rangeStageAndFiltersReachOnlyTheRequestedTrendAndCoverageQueries() {
JdbcTemplate jdbc = jdbc(true, true);
MetricsService service = new MetricsService(jdbc, mock(GovernanceAccessService.class));
@@ -93,4 +93,59 @@ class SalesOrderMetricRollupsMySqlTest {
rollups.refreshCurrentBuild(build);assertFalse(rollups.ready(release,query.extraFilters(),true,Set.of("*")));
assertEquals(0,jdbc.queryForObject("SELECT COUNT(*) FROM sales_order_metric_rollup_build WHERE version_id=?",Integer.class,version));
}
@Test void databaseProductPagesMatchFormalRowsAcrossRawRollupsFiltersAndStableNullTies() {
// Include blanks, a missing stage, equal weights and UTF-16 ordering edges.
int n=0;
for(String product:List.of("", " ", "edge-😀", "edge-\uE000", "edge-é", "edge-e\u0301", "edge-A", "edge-a")) {
jdbc.update("INSERT INTO fact_bright(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',?,100,1,999,JSON_OBJECT('销售订单号','order-0'),JSON_OBJECT('bright_processing_scrap',1))",build,product);
n++;
}
jdbc.update("INSERT INTO fact_black(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',NULL,NULL,0,0,JSON_OBJECT(),JSON_OBJECT())",build);
// Stage-specific representatives remain distinct exactly as in the existing formal table.
jdbc.update("INSERT INTO fact_black(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',?,50,1,51,JSON_OBJECT(),JSON_OBJECT())",build,"edge-e\u0301");
var base=query("product",true,Map.of());
var scopes=List.of(base,base.withStage("bright"),base.withStage("black"),base.withMonthRange(1,1),base.withProductSearch("edge"),
query("product",false,Map.of(SalesOrderTypes.KEY,List.of("标准销售订单"))),
query("product",true,Map.of(SalesOrderTypes.EXCLUDE_KEY,List.of("样品销售订单"))));
// No rollups -> facts; regular rollups; then sales-order rollups.
for(int path=0;path<3;path++) {
if(path==1) for(String stage:List.of("black","bright")) {
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT build_id,?,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,1,0,0,0 FROM fact_"+stage,stage);
}
if(path==1)jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) VALUES(?,?)",build,22+n+2);
if(path==2)rollups.prepareBuild(version,build);
for(var scope:scopes) for(var order:ProductTableOrder.values()) {
var expected=metrics.calculateTable(scope,actor).rows().stream().sorted(order.comparator()).toList();
var actual=new ArrayList<MetricsService.MetricRow>();
int pages=(expected.size()+2)/3;
for(int page=1;page<=pages+1;page++) {
var result=metrics.calculateProductPage(scope,actor,page,3,order.wire());
assertEquals(expected.size(),result.total(),"total path "+path);
assertTrue(result.rows().size()<=3);
actual.addAll(result.rows());
}
assertEquals(normalized(expected),normalized(actual),"path="+path+" order="+order+" scope="+scope);
}
}
when(actor.getDepartments()).thenReturn(List.of("A"));
var authorized=metrics.calculateTable(base,actor).rows().stream().sorted(ProductTableOrder.PRODUCT.comparator()).toList();
assertEquals(normalized(authorized),normalized(metrics.calculateProductPage(base,actor,1,100,"productAsc").rows()));
}
@Test void pageSelectionDoesNotMultiplyUnicodeEquivalentBucketsAndValidatesLimits() {
var base=query("product",true,Map.of());
var expected=metrics.calculateTable(base,actor).rows().getFirst();
when(governance.allowedFields(actor)).thenReturn(Set.of("black_rate"));
assertEquals(normalized(metrics.calculateTable(base,actor).rows().stream().sorted(ProductTableOrder.PRODUCT.comparator()).toList()),
normalized(metrics.calculateProductPage(base,actor,1,100,"brightQualifiedDesc").rows()),"Hidden weights must not leak through ordering");
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
var limited=metrics.calculatePairedProductSnapshot(base,actor,List.of(expected.key(),expected.key()),()->"source");
assertEquals(normalized(List.of(expected)),normalized(limited.metrics().rows()));
assertTrue(metrics.calculatePairedProductSnapshot(base,actor,List.of(),()->"source").metrics().rows().isEmpty());
for(int page:new int[]{0,-1,1_000_001})assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,page,15,"productAsc"));
for(int size:new int[]{0,101})assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,1,size,"productAsc"));
assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,1,15,"product; DROP TABLE"));
assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(query("overall",true,Map.of()),actor,1,15,"productAsc"));
}
}
+19 -1
View File
@@ -26,6 +26,7 @@ SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED')
BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup',
'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build',
'source_filter_member', 'source_filter_value', 'source_filter_field', 'source_filter_build',
'fact_black', 'fact_bright')
OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion',
'release_action', 'candidate_reconciliation')
@@ -253,7 +254,24 @@ class Database:
self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid))
for t in BUILD_CHILDREN:
if t in self.columns:
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
if t == 'source_filter_member':
# Members have no build_id; drain each owned field in
# order without aliases because the exclusion uses LOCK TABLES.
count = 0
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
count = self.execute('DELETE FROM source_filter_member WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
if count:
break
elif t == 'source_filter_value':
# Values are likewise owned through fields and are deleted
# only after all field memberships have been drained.
count = 0
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
count = self.execute('DELETE FROM source_filter_value WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
if count:
break
else:
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
if count:
return count, 0
self.execute('DELETE FROM transform_build WHERE id=%s', (bid,))
+30 -4
View File
@@ -16,6 +16,7 @@ from unittest.mock import patch
ROOT=Path(__file__).resolve().parents[2]
sys.path.insert(0,str(ROOT/'scripts'))
import backup_retention as br
import managed_test_resources
spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py'))
ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret)
@@ -91,14 +92,18 @@ class DatabaseRetentionTests(unittest.TestCase):
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True)
cls.name='quality_retention_test_'+uuid.uuid4().hex[:12]
managed_test_resources.create(ROOT,cls.name,cls.execute_admin)
with cls.admin.cursor() as c:
c.execute('CREATE DATABASE '+ret.ident(cls.name))
c.execute('USE '+ret.ident(cls.name))
ddl=[
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)',
'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_build(build_id varchar(36) PRIMARY KEY,format_version int,FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
'CREATE TABLE source_filter_member(field_id bigint NOT NULL,fact_id bigint NOT NULL,value_id bigint NOT NULL,PRIMARY KEY(field_id,fact_id),KEY(value_id,fact_id),FOREIGN KEY(field_id) REFERENCES source_filter_field(id),FOREIGN KEY(value_id) REFERENCES source_filter_value(id))',
'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)',
@@ -115,14 +120,25 @@ class DatabaseRetentionTests(unittest.TestCase):
@classmethod
def tearDownClass(cls):
with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name))
managed_test_resources.drop(ROOT,cls.name,cls.execute_admin)
cls.admin.close()
@classmethod
def execute_admin(cls,sql,args=None):
result=''
statements=[part.strip() for part in sql.split(';') if part.strip()]
with cls.admin.cursor() as cursor:
for statement in statements:
cursor.execute(statement,args if len(statements)==1 else None)
result='\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
return result
def setUp(self):
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
self.db=ret.Database(ROOT,self.name);self.db.root=self.root
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table))
for table in self.db.tables:
if table!='__quality_test_owner':self.db.execute('TRUNCATE TABLE '+ret.ident(table))
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')")
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
@@ -187,14 +203,24 @@ class DatabaseRetentionTests(unittest.TestCase):
def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self):
self.retire_old();state={'builds':{'obsolete':1}}
self.db.execute("INSERT INTO source_filter_build VALUES('obsolete',1)")
self.db.execute("INSERT INTO source_filter_field VALUES(9,'obsolete','black','空字段',%s),(10,'obsolete','black','客户',%s),(11,'obsolete','black','工单',%s)", (b'0'*32,b'1'*32,b'2'*32))
self.db.execute("INSERT INTO source_filter_value VALUES(100,10,%s,'客户A'),(101,10,%s,'客户B'),(102,11,%s,'工单A')", (b'1'*32,b'2'*32,b'3'*32))
self.db.execute("INSERT INTO source_filter_member VALUES(10,1,100),(10,2,101),(11,3,102)")
with self.db.frozen('current'):
self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0))
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_member')[0]['n'],2)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_value')[0]['n'],3)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_field')[0]['n'],3)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_build')[0]['n'],1)
old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0]
self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64)
self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,))
for _ in range(4):
for _ in range(20):
with self.db.frozen('current'):self.db.build_step(state,1800,4000,1)
self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'"))
for table in ('source_filter_member','source_filter_value','source_filter_field','source_filter_build'):
self.assertFalse(self.db.rows('SELECT * FROM '+ret.ident(table)))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'"))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'"))
+34 -4
View File
@@ -13,6 +13,7 @@ from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import backup_retention as br
import managed_test_resources
import revision_retention as revisions
spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py'))
ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret)
@@ -26,12 +27,15 @@ class SourceRevisionTests(unittest.TestCase):
cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root',
password=cfg['rootPassword'], autocommit=True)
cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
managed_test_resources.create(ROOT, cls.name, cls.execute_admin)
ddl = [
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))',
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))',
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
"CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)",
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))',
@@ -42,23 +46,34 @@ class SourceRevisionTests(unittest.TestCase):
'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))']
try:
with cls.admin.cursor() as c:
c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name))
c.execute('USE ' + ret.ident(cls.name))
for statement in ddl:
c.execute(statement + ' ENGINE=InnoDB')
except BaseException:
with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name))
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
cls.admin.close(); raise
@classmethod
def tearDownClass(cls):
with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name))
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
cls.admin.close()
@classmethod
def execute_admin(cls, sql, args=None):
result = ''
statements = [part.strip() for part in sql.split(';') if part.strip()]
with cls.admin.cursor() as cursor:
for statement in statements:
cursor.execute(statement, args if len(statements) == 1 else None)
result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
return result
def setUp(self):
self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve()
self.db = ret.Database(ROOT, self.name); self.db.root = self.root
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
for table in self.db.tables:
if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')")
self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')")
@@ -121,6 +136,21 @@ class SourceRevisionTests(unittest.TestCase):
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
self.assertTrue(self.present()); self.assertTrue(path.exists())
def test_indexed_business_values_do_not_count_as_revision_metadata(self):
path = self.add_revision('indexed-only', b'indexed-only workbook')
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,))
self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,))
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists())
def test_indexed_field_names_do_not_count_as_revision_metadata(self):
path = self.add_revision('indexed-field-only', b'field-only workbook')
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,))
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists())
def test_new_reference_resets_continuous_grace(self):
self.add_revision(); self.collect(100)
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500)
+89 -1
View File
@@ -1,7 +1,95 @@
# 当前进度
核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。
## 2026-11-14:数据库分页与测试修复已发布(2026.11.14.2)
用户授权测试通过后发布并同步Gitea。本轮沿用分页独立工作目录,功能提交`8f1b790`,已推送Gitea main,未合入主目录旧分支的无关修改。6项既有失败已复现并修复:报告测试缺少真实辅助函数/钩子绑定4项、SVG趋势测试按固定纵轴校验2项;恢复失败提示按项目静默省略约定校正。未改报告生产逻辑、模型原文或正式指标算法。
前端全套616项全部通过,类型及生产构建通过;常规后端995项(823执行通过、172按条件跳过),另启用隔离MySQL专项10项通过。分页优化、当前页细分和全量导出一并发布;此前本地专项覆盖Unicode、缺失阶段、权限与导出,本次未调用模型或执行私有原表全量回归。
公开版本2026.11.14.2,入口HTML及24个发布文件与本地构建逐个摘要一致。新旧JAR迁移SQL资源完全一致,无新增迁移、表或索引。线上只读基线核对:年度综合7651、黑片6419、亮片6715产品;月度2960、搜索8、来源筛选3产品的正式汇总/全量行/覆盖完全一致,各表首页、第二页和末页核对一致,细分废率及48条来源结果保持。浏览器实际第二页显示16–30行、共6715行,15条数据、无错误;来源抽屉读取48条、121字段,原筛选恢复且99.10%保持。
MySQL原进程、正式版本指针和数据库结构保持;DSH应用/模型/状态路径保持,维护暂停标记恢复原状,原维护兼容保护未解除。完整发布与测试证据位于主目录`output/20261114-pagination-release/`。下方“未发布/6项失败”为之前时点,已由本节结果替代。
## 2026-11-14:分页发布准备与6项测试修复
用户已授权测试通过后发布,并同步Gitea。继续使用分页独立分支,核实远端main仍为`d948e1e`。6项失败已复现:报告组件测试缺少当前真实辅助函数/钩子绑定4项,SVG测试仍假定固定纵轴2项;已补齐依赖并按真实百分比坐标范围校验正式值和像素位置。图表恢复失败的旧告警断言按项目“静默省略、保留原文”约定校正。未改模型原始正文、报告生产逻辑或正式指标算法。
前端全套616项通过;常规后端995项(823执行通过、172按条件跳过)及另启用的隔离MySQL10项通过。本节为发布准备时点,最终发布和同步状态另记后续验收。
## 2026-11-14:数据库分页本地实现(未发布、未同步)
按用户“优化并本地测试”,基于 main `d948e1e` 在独立分支 `codex/database-pagination-local-20261114` 实现,源码位于 `/Users/lyun/.codex/worktrees/quality-database-pagination-20261114`。产品汇总、黑片/亮片宽产品表按数据库页读取,保留正式汇总、趋势、授权与全量导出;当前页细分废率只查相应产品。年度摘要优先,月/季/搜索并行读小页,避免额外串行等待。同重量固定次序、隐藏重量不通过排序泄露;换范围重置与请求取消已覆盖。没有新增表、索引、迁移或持久化副本。
144,016 条合成事实、6,002 产品:年度综合产品表接口三轮中位数 2627→1593ms,数据体积 5,286,041→15,364 字节;同一浏览器旧/新前端表格可用时间中位数 5725→4566ms(约20%),顶部指标/趋势基本不变。分页/细分对齐正式全量接口,9组额外期间、排除及来源筛选一致;浏览器三页查询、换年、来源、第二页全量Excel已验收。常规后端995项(823通过、172跳过),专项隔离MySQL10项和前端相关77项通过,前端构建通过。前端全套616项仍有6个主线既有智能分析报告测试失败,已在未改动基线复现;未修改这些模块。特殊Unicode分组漏计的本轮边界已修正并复验。
仅本地,未提交/推送或发布;未执行私有原表全量回归。自有测试库/账号/进程已清理,正式指针和原进程保持,原浏览器链接恢复。完整报告与截图:主目录 `output/20261114-database-pagination/测试结果.md`。后续如需交付,先核对当前主线和本轮独立差异;不要把主目录旧分支历史改动一并提交。
## 2026-11-14:分步加载已发布(2026.11.14.1)
在用户明确要求发布并同步后,复核 Gitea main 仍为 `3819011`,仅提交本轮查询加载、表格投影、相关测试和使用说明,功能提交 `374f08c`。没有整批提交主工作目录旧分支上的历史改动。
发布前前端77项及类型/生产构建通过;完整后端992项(822执行通过、170按条件跳过)。此前14.4万条隔离合成数据的对照和异常验收继续有效,新旧运行包43个迁移资源逐项摘要一致。发布时业务任务空闲,保留MySQL原进程、正式数据版本、DSH应用/模型/状态路径和固定引擎配置。
线上实际读回核对:年度6715产品、月度2402产品、来源筛选3产品的汇总、表格、趋势、覆盖和来源结果与旧后端完全一致;新table投影除省略趋势外结果与full一致,同范围overview的汇总/趋势也一致。代表年度完整接口约2485ms、table约1313ms、overview约618ms;来源筛选约927/191/210ms,48条来源约358ms。这些是单次只读重放耗时,不是所有筛选或冷启动保证。
前端2026.11.14.1已切换,公开版本、入口HTML和资源与本地生产构建一致。复用现有浏览器,原筛选显示查询完成、合格率99.10%、总重7190.86kg;来源抽屉实际载入48条、121字段,产品钻取的5月来源返回1条,返回上一级正常。恢复页面时纠正了上次手工重写筛选名称的偏差,改用保存的原始链接,没有改业务数据。部署验收脚本的进程检查曾缺少初始化,已修正并重新核对新后端、MySQL身份、健康和数据指针;未回滚业务数据。
仅更新前后端,无新增表、索引或数据库迁移;表格仍返回全量产品,数据库分页尚未实施。现有维护升级兼容保护未解除。私有基线、登录配置、运行包和验收产物不进入Git,完整证据保存在主目录 `output/20261114-progressive-query/`。本轮功能与发布验收记录已同步 Gitea `main`,功能提交 `374f08c`,发布记录 `bfb91ef`;远端分支已读回核验。下方本地试验的“未发布、未同步”为当时时点。
## 2026-11-14:分步加载本地试验(未发布、未同步)
用户要求先本地测试。基于已核实的 Gitea main `3819011` 创建独立分支 `codex/progressive-query-local-20261114`,源码位于 `/Users/lyun/.codex/worktrees/quality-progressive-query-20261114`,保留主工作目录旧分支及原有改动。
本地实现:宽年度产品范围先读取正式小型汇总和可复用趋势,再读取 `part=table` 投影;表格仅省略时间与对象交叉分组,不改正式算法、授权或发布版本。月/季、单产品和小维度复用完整结果,避免额外往返;一般查询解除无关目录等待,沿用已有请求合并/并发上限和底部柏拉图按需加载。刷新代次使旧表格与趋势失去当前资格。
验证:144,016 条合成事实、6,002 产品的同一浏览器三轮中位数,顶部指标约3164→1616ms、趋势1764→1616ms、表格3164→2863ms。接口年度全产品约2506→1040ms提前显示,主要数据全部完成约2526→2332ms;月度基本持平,总体年度未测得耗时改善,不承诺所有筛选提速。18组接口业务结果对照及8组隔离投影范围检查一致,表格失败不阻塞指标/趋势且来源抽屉可读,首批失败可回退。前端77项及类型/生产构建通过;后端992项(822通过、170按条件跳过)。未执行私有原表全量回归或在线模型调用。
仅本地试验,未提交/推送或发布。数据库分页尚未实施,表格仍返回全量产品并保持全量导出;没有新增表、索引或迁移。两个自有合成库、账号及测试/预览进程已清理,正式版本指针未变,共享后端健康UP,原浏览器页面已恢复。完整数据、截图和日志位于主目录 `output/20261114-progressive-query/测试结果.md`。下一步如获发布授权,先核对这批本地变更及当前主线,再决定是否交付第一阶段或继续数据库分页。
核实日期:2026年11月13日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。
## 2026-11-13:Gitea交付(筛选提速与包含条件)
本轮仓库交付整合此前来源筛选超时修复、统一字典/记录索引、后台补建、目录与候选读取优化,以及“包含”条件和输入边界修正;保留11个实施与验收提交,整合提交为 `6983dc3`。目标为 `lyun/quality_analyse` 的 `main`,对应当前已上线2026.11.13.12的功能源码。
整合后功能代码与已验收分支逐文件一致,无冲突或新增业务修改,沿用既有构建、隔离数据库与线上验收结果。本次只同步源码和交付记录,不重复发布、重启服务或运行数据库迁移;私有配置、业务数据、运行包和验收附件不进入仓库。下方“未同步”保留各次实施时点,本节为后续接续的最新仓库交付记录。
## 2026-11-13:来源字段“包含”筛选(已发布)
来源字段操作菜单新增“包含”:自由输入关键词按回车,多个关键词按OR匹配;不同筛选条件之间仍为AND。后台用绑定参数的LOCATE执行字面子串匹配,大小写及正式来源值语义保持,`%`、`_`、反斜杠和引号不是通配或SQL内容。支持1–50个非空白关键词,每个最多256字符;前端控制搜索长度并避免截断代理对,重复关键词失焦后清空草稿。
汇总、趋势、来源明细与Excel统一走同一解析/授权/查询逻辑,兼容原始JSON回退与现有索引。DSH活跃共享契约、图表上下文和历史/证据显示同步接受contains;关键词不做完整候选值的等值核验。打包只替换5个相关DSH业务模块,保留当前已验收的流式助手和原固定发行版;模型配置及状态不变,未调用在线模型。不新增迁移、表或索引,不重建既有52个构建。
验证:前端61项及类型/生产构建通过;完整后端990项(820执行通过、170按条件跳过);专项隔离MySQL及服务45项(44通过、1基准跳过),覆盖两阶段、原表/索引、规范维度、大小写、特殊字符、零/字符串null/空值、组织权限,以及汇总/趋势/明细/Excel。DSH契约/显示Python135项(133通过、2跳过),应用构建16项通过,新打包应用的contains契约及固定运行时检查通过,独立审查无阻断。未执行私有原表全量回归或在线模型验收。
线上关键词命中2个完整客户值,contains与对应in集合的汇总业务数据、趋势及来源结果一致;代表查询约1965/721/465ms。来源29条、121个原始字段,导出29条且ZIP/XML有效;首次导出12907ms、复用后844ms。实际菜单、键盘提交、多关键词字面值、257字符粘贴限制及重复失焦清理已检查,页面无失败提示。前端2026.11.13.12(`f8432b4`),后端功能提交`3c022e8`,后续仅前端输入边界修正,数据库进程保持;DSH固定引擎、模型配置及状态保持。
代码已在独立分支 `codex/source-filter-contains-20261113` 本地提交,本轮未同步Gitea。私有请求、结果及效果截图位于 `output/20261113-source-contains/`,不进入Git。
## 2026-11-13:统一筛选提速(已发布验收)
已发布2026.11.13.10,功能提交 `20f1c7d`;当前正式版52/52个构建的筛选索引全部就绪。新增按构建保存的来源字段字典与稀疏记录索引,来源/配置筛选、候选、指标、趋势、明细及明细导出共用;正式算法、当前字段/组织权限和发布版绑定保持。候选模式只取上限加一,不再为下拉精确计数;常规页面查询不等待无关目录,本字段改选不重复读取自己的候选,明细导出先只读字段名。
生命周期:启动和构建提交后通过同一单线程有界去重队列补索引,完整提交后才启用;失败或尚未就绪保留原查询,导入事务不等待索引。构建过程按500条事实窗口驱动字典关联,并物理重建临时窗口,避免回扫增长字典和长事务临时表累积。保留清理已识别4张新增表,未解除维护兼容保护。
首次加载还存在两条慢路径:候选为校验年份/可见范围反复读取宽原始行,V44新增薄覆盖索引;字段目录扫描约110万事实只为找每构建首条可见记录,V45改索引列序及带完整权限的有序LIMIT1查询。实际计划回归要求覆盖读取、不排序全部事实、找到首条即停;与旧MIN查询直接对照原始字段目录,保留空/隐藏构建和重复成员语义。
验证:前端相关46项及生产构建通过;完整后端988项(819执行通过、169按条件跳过),专项隔离MySQL/服务49项(48通过、1基准跳过),保留清理34项通过,多次独立审查无阻断。隔离场景覆盖黑片/亮片、全部4类来源运算、旧配置null和字面字符串null、Unicode、同一事实行、多字段、当前权限、导出、跨窗口故障回滚与重试。未执行私有原表全量回归。
线上实测在恢复原256MiB数据库缓存、后端重新启动且无并行测试时完成:同一客户筛选首次汇总1366ms、趋势347ms、来源明细513ms、原候选条件381ms、去掉自身约束的完整候选首开3546ms;月度458ms、季度369ms。原始超时场景汇总/趋势约48.3秒;前一版索引已全就绪时首次汇总仍18843ms,此次首条目录优化后降到1366ms。汇总、趋势、29条来源明细的完整规范化结果与原基准一致,月/季与原路径一致。Excel约3479ms,1张表、29条明细、A1:EA30,ZIP/XML有效;页面查询完成,来源抽屉载入121字段、无失败提示,点击产品与来源范围一致。以上是代表场景的只读重放,不是数据库完整冷启动或所有字段组合的固定耗时承诺。
空间实测:新增字典/成员等4表文件共13,727,023,104字节,事实表4个覆盖索引页约372,064,256字节;本口径合计约14.10GB(13.13GiB),不含既有原表、redo/undo和临时文件,也不等同于数据库目录净增差值。此次覆盖全部原始字段,空间显著高于先前紧凑柏拉图汇总。补建期间临时1GiB缓存已恢复256MiB,核验16384页且调整完成。数据库原进程和DSH配置保留,无业务版本指针变更。
实现与验收记录已本地提交;本轮新优化尚未同步Gitea。私有请求、结果校验及页面截图保存在 `output/20261113-filter-speed/`,不进入仓库。主工作目录已有历史改动保持,当前交付在独立分支 `codex/unified-filter-speed-20261113`。
## 2026-11-13:来源字段筛选超时修复
定位:客户等原始字段筛选时,汇总、趋势和覆盖检查重复读取字段目录,数据库对年度查询选择了原表扫描,约48秒超过页面30秒期限。复用服务级字段目录缓存,目录及无窄范围限制的年度来源筛选使用既有构建索引;JSON值每次表达式只提取一次,保留空值、字符串和大小写语义。单月、季度、产品及扩展条件保留原查询计划。查询超时提示改为重试或调整范围,不再误报连接断开。权限和正式计算规则不变,无新增表或索引。
验证:后端完整构建975项(812执行通过、163按条件跳过);隔离MySQL及快照专项34项(33通过、1跳过),前端相关17项及生产构建通过。独立复核无实质问题;已验证主目录本轮6文件与交付目录一致,新旧运行包数据库迁移文件一致。已发布2026.11.13.5(功能提交 `ea84e26`)。线上同条件汇总/趋势从48282/48281ms降至17818/17816ms,完整结果规范化校验值均一致;页面正常加载时两接口约5528/5529ms,无失败提示。实际来源明细返回29条、121个字段,Excel下载有效。数据库迁移文件不变,MySQL原进程和DSH配置保留。只读同条件复测不等于私有原表全量回归。本轮独立分支已本地提交,尚未同步Gitea;私有验收材料位于 `output/20261113-source-filter-timeout/`。
## 2026-11-13:Gitea 同步交付
+5 -3
View File
@@ -144,6 +144,8 @@ def _resolve_source_column_values(task, *, release_id, resolver):
if not isinstance(rules, list):
return None
for rule_index, rule in enumerate(rules):
# Candidate lookup confirms complete values only. A contains keyword is
# intentionally not required to equal a full candidate source value.
if rule.get('operator') not in ('in', 'not_in'):
continue
field_key = rule['field']
@@ -1212,7 +1214,7 @@ def plan_task(*, base_url, api_key, model, question, context, previous=None, can
if aliases and field.get('type')=='source_value':item['aliases']=aliases
compact_fields.append(item)
compact_catalog['sourceFields']=compact_fields
compact_catalog['sourceColumnOperators']=['in','not_in','is_empty','not_empty']
compact_catalog['sourceColumnOperators']=['in','not_in','contains','is_empty','not_empty']
payload['businessCatalog']=compact_catalog
system += ('\n综合以上生成报告应整合previousVerifiedTurns中多个已完成轮次,historyMode=all、freshQuery=false;不得把最新局部追问当作全部历史。'
'只有用户要求更新数据时才freshQuery=true。报告与复述不自动开展新调查。'
@@ -1292,9 +1294,9 @@ JSON字段:goal(完整当前目标),kind(analysis/metrics/members/explain/re
filterUpdates允许department/group/product/projectGroup/extraFilters/exclusions/includeUnassigned。null明确清除该条件,{}表示没有变化;单一维度用目录原名字符串,extraFilters和exclusions用字段到值数组。额外字段仅限businessCatalog.extraFields;客户代码使用extraFilters.customerCode,保留完整字符串。未知字段不能丢弃或扩大范围,应澄清能力限制。汽车交通对应department=新能源汽车磁钢,国内/海外才是其业务分组。
productMatch为ambiguous且未选择时,只需列出授权候选并needsClarification=true,不选择或合并后缀、不换成全部产品。unique使用完整原名;18×15×4不能匹配18×15×4.1。用户选择前轮候选时保留原年份、指标和趋势目标。newTask=true仅用于用户明确另开无关任务,并声明新期间和条件。
今年、上月等使用serverCalendar;最新数据月才用目录,不猜测缺失期间。真正缺少期间或对象有歧义才澄清。普通读取业务指标freshQuery=true。
原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/is_empty/not_empty","values":[完整原文]}。in/not_in要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。大小写、空格、前导零都是字面值,不做归一化。规则AND、values OR;每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。
原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/contains/is_empty/not_empty","values":[完整原文或包含关键字]}。in/not_in/contains要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。规则AND、values OR;文本大小写、空格、前导零均保留原文,不对contains关键字归一化;contains是原文的字面子串筛选,不是通配符或正则。每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。
只问报废率时fields=[scrapRate];同时要求分子分母、计算基数、报废总量或总重时,分别保留对应重量字段;要求分子分母时fields=[scrapRate,scrapWeight,totalWeight]。不能只在goal中提及而漏掉实际fields。分子为现有正式报废总量,分母为该阶段正式总重,黑片使用现有修正总重。利用料、本部报废、过账外废沿用现有分量逻辑,不使用1-合格率,也不平均原表百分比。未指定阶段的报废率返回black和bright两个阶段;没有统一综合报废率定义,用户明确要综合报废率时请求其定义,不转成综合合格率。
按用户原文提出in/not_in文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。
按用户原文提出in/not_in完整文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验in/not_in取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。contains关键字是子串而非完整候选值,不做等值候选核验,按用户原文交由正式查询执行字面子串筛选。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。
kind=members仅用于列出有哪些委托单/批号/规格等唯一成员,不表示原始行;memberFields使用当前目录key且最多5个,必须与用户明确要列出的对象语义相符,不能把筛选字段当输出成员。裸“规格”表示产品规格,要选择目录中真实产品规格列;不能用图号或客户代码代替。明确问图号时选择图号列,明确问客户规格时选择客户规格列。若用户明确给出“成品规格”等目录label,优先精确匹配该label;裸“规格”有多个候选或没有可用字段时,needsClarification=true、memberFields=[],保留原目标并澄清。并列、否定或复合字段意图由规划器按用户表达处理,不能补入已取消字段。stages必须明确为一个或两个阶段,期间与普通filters照常保留。需要澄清时stages可为空,本轮不查询并保留目标。members的fields=[]、verifySource=false、fullDetails=false、freshQuery=true,不计算KPI、不改成metrics;清单由平台直接渲染,无需撰写业务数值。
单期投入重量排名或明确单阶段报废排名:kind=analysis、dimensions为实际product/department/group、fields仅totalWeight/scrapWeight、dimensionStages明确各维度black/bright、trendRequired=false;未限定阶段的报废排名使用queryPlan ranking、fields=[rate,scrapWeight]、stages=[]、dimensionStages对应维度=[];全量正式分母不随Top展示数量改变。普通年度重量排名默认periodCoverage=available,注明已发布月份;只有用户明确要求完整全年齐全时用complete。排名与趋势同时要求时仍保留真实要求,不删除trendRequired。
能力边界必须忠实表达:排名/缺陷保留analysis及真实dimensions和dimensionStages;两个独立期间比较保留comparisonRequired=true和comparisonPeriod;源表核验为source_verify及verifySource=true;原始逐行记录为details及fullDetails=true;报告为report。不得为了适配已接入能力而改写成普通指标。不得把“有哪些”理解成所有原始字段。后端负责说明仍未接入的能力。
+2 -1
View File
@@ -154,7 +154,8 @@ def _scope_label(filters):
if key=='columnFilters':
for rule in value:
stage='、'.join({'black':'黑片','bright':'亮片'}.get(s,s) for s in rule.get('stages',[]))
operation={'in':'为','not_in':'排除','is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'')
operation={'in':'为','not_in':'排除','contains':'包含片段',
'is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'')
parts.append(stage+'来源字段'+operation+'、'.join(rule.get('values',[])))
else: parts.append(names[key]+'='+str(value))
return ';'.join(parts) if parts else '当前授权范围内总体(默认不含未分类)'
+2 -1
View File
@@ -850,7 +850,8 @@ def _history_scope_caption(snapshot, section):
parts.append('包含未分配项')
column_filters = filters.get('columnFilters')
if isinstance(column_filters, list):
operator_labels = {'in': '包含', 'not_in': '不含', 'is_empty': '为空', 'not_empty': '非空'}
operator_labels = {'in': '为', 'not_in': '排除', 'contains': '包含片段',
'is_empty': '为空', 'not_empty': '非空'}
for rule in column_filters:
if not isinstance(rule, dict):
continue
+2 -2
View File
@@ -115,11 +115,11 @@ QUALITY_DATA_SCHEMA: dict[str, Any] = {
"field": {"type": "string", "minLength": 1},
"stages": {"type": "array", "minItems": 1, "maxItems": 2,
"items": {"type": "string", "enum": ["black", "bright"]}},
"operator": {"type": "string", "enum": ["in", "not_in", "is_empty", "not_empty"]},
"operator": {"type": "string", "enum": ["in", "not_in", "contains", "is_empty", "not_empty"]},
"values": {"type": "array", "minItems": 1, "maxItems": 50,
"items": {"type": "string", "maxLength": 256}},
}, "required": ["field", "stages", "operator"]},
"description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR,文本大小写、空格及前导零按原文匹配。is_empty/not_empty不得带values。"},
"description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR;in/not_in匹配完整文本值,contains按原文作字面子串匹配(MySQL LOCATE),不做关键字归一化,大小写、空格及前导零保留原文。每条文本规则提供1到50个不超过256字符的值;is_empty/not_empty不得带values。"},
"view": {"type": "string", "enum": ["overall", "department", "group", "product", "projectGroup"], "description": "query 的正式分组视图;changes 仅支持非 overall 维度。"},
"stage": {"type": ["string", "null"], "enum": ["black", "bright", None], "description": "top/details/defects/investigate必须明确black/bright。rank单阶段传black/bright;rank的metric=scrapWeight且stage省略或null时,合计黑片+亮片报废总重全量排序,并保留正式综合合格率。投入重量rank仍必须指定阶段。changes可选,未传为综合范围。"},
"dimension": {"type": "string", "enum": ["category", "defect", "product", "department", "group", "projectGroup"], "description": "top模式仅category/product/department/group/projectGroup,不能用defect;具体缺陷TOP必须使用mode=defects、dimension=defect。product是规格。changes 使用 product/department/group/projectGroup。defects 使用 defect 或 product;未指定具体 defect 字段时,省略 dimension 默认按 defect 查询阶段具体缺陷。"},
+6 -4
View File
@@ -3,7 +3,7 @@ import copy
DEPARTMENT_ALIASES = {'汽车交通':'新能源汽车磁钢'}
SOURCE_STAGES = {'black', 'bright'}
COLUMN_OPERATORS = {'in', 'not_in', 'is_empty', 'not_empty'}
COLUMN_OPERATORS = {'in', 'not_in', 'contains', 'is_empty', 'not_empty'}
def source_field_labels(source_fields):
@@ -35,11 +35,13 @@ def parse_column_filters(value):
raise ValueError(f'{name}.stages必须是非空black/bright数组。')
if operator not in COLUMN_OPERATORS:raise ValueError(f'{name}.operator不在当前字段支持目录中。')
clean={'field':key,'stages':list(stages),'operator':operator}
if operator in ('in','not_in'):
if operator in ('in','not_in','contains'):
values=rule.get('values')
if (not isinstance(values,list) or not 1<=len(values)<=50
or any(not isinstance(item,str) or item=='' or len(item)>256 for item in values)):
raise ValueError(f'{name}.values必须是1到50个不超过256字符的完整文本值。')
or any(not isinstance(item,str) or item==''
or operator=='contains' and not item.strip()
or len(item)>256 for item in values)):
raise ValueError(f'{name}.values必须是1到50个不超过256字符的文本值或包含关键字。')
clean['values']=list(dict.fromkeys(values))
elif 'values' in rule:raise ValueError(f'{name}的空值操作不接受values。')
result.append(clean)
+6 -4
View File
@@ -9,9 +9,11 @@ import uuid
# These are measured business values, not documents that refer to source IDs.
VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'),
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')}
BUSINESS_VALUE_COLUMNS = {('fact_black', 'raw_values'), ('fact_black', 'components'),
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components'),
('source_filter_field', 'column_name'),
('source_filter_value', 'value')}
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
@@ -54,7 +56,7 @@ class Cleaner:
def metadata_mentions(self, needles, excluded=None):
for table, columns in self.db.column_types.items():
for column, kind in columns.items():
if kind not in TEXT_TYPES or (table, column) in VALUE_JSON:
if kind not in TEXT_TYPES or (table, column) in BUSINESS_VALUE_COLUMNS:
continue
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
args = list(needles)
+9
View File
@@ -34,6 +34,15 @@ def coverage_comparison():
class BoundReportTest(unittest.TestCase):
def test_scope_label_distinguishes_full_value_from_substring_filters(self):
from nao_evidence_report import _scope_label
label = _scope_label({'columnFilters': [
{'field': 'orderNo', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7']},
{'field': 'orderNo', 'stages': ['bright'], 'operator': 'contains', 'values': ['07']},
]})
self.assertIn('黑片来源字段为WO-7', label)
self.assertIn('亮片来源字段包含片段07', label)
def test_priority_summary_keeps_three_signals_and_rate_denominators_separate(self):
signals={'scrapIncrease':[{'label':'放量规格','scrapDelta':120}],
'currentScrap':[{'label':'存量规格','current':{'scrapWeight':500}}],
+8 -4
View File
@@ -456,9 +456,11 @@ class HistoryReportExecutionTest(unittest.TestCase):
first_section = candidate['sections'][0]
first_section['filters']['columnFilters'] = [{
'field': 'src_work_order', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7'],
}, {
'field': 'src_contains_code', 'stages': ['bright'], 'operator': 'contains', 'values': ['07'],
}]
set_source_task_context(candidate, SOURCE_A,
{'sourceFieldLabels': {'src_work_order': '黑片委托单号'}})
{'sourceFieldLabels': {'src_work_order': '黑片委托单号', 'src_contains_code': '客户代码'}})
set_source_task_context(candidate, SOURCE_B,
{'sourceFieldLabels': {'src_work_order': '亮片委托单号'}})
fact = candidate['facts'][0]
@@ -476,12 +478,14 @@ class HistoryReportExecutionTest(unittest.TestCase):
projected = _history_model_fact_projection(candidate, [fact])[0]
self.assertIn('产业部 消费电子产业部', projected['modelScope']['displayScope'])
self.assertIn('业务分组 消费电子-3C', projected['modelScope']['displayScope'])
self.assertIn('黑片委托单号 包含 WO-7', projected['modelScope']['displayScope'])
self.assertIn('黑片委托单号 为 WO-7', projected['modelScope']['displayScope'])
self.assertIn('亮片 客户代码 包含片段 07', projected['modelScope']['displayScope'])
self.assertIn('消费电子产业部', projected['modelText'])
self.assertIn('消费电子-3C', projected['modelText'])
self.assertIn('黑片委托单号', projected['modelText'])
self.assertIn('WO-7', projected['modelText'])
for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:', 'src_work_order'):
for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:',
'src_work_order', 'src_contains_code'):
self.assertNotIn(machine_label, projected['modelText'])
self.assertNotIn(machine_label, json.dumps(projected['modelScope'], ensure_ascii=False))
self.assertEqual(projected['text'], source_text)
@@ -581,7 +585,7 @@ class HistoryReportExecutionTest(unittest.TestCase):
goals = payload['historySourceGoals']
self.assertTrue(any('产业部 消费电子产业部' in goal['displayScopeLabel']
and '业务分组 消费电子-3C' in goal['displayScopeLabel']
and '黑片委托单号 包含 WO-7' in goal['displayScopeLabel']
and '黑片委托单号 为 WO-7' in goal['displayScopeLabel']
for goal in goals))
self.assertTrue(all('filters' not in goal and '_sourceTurnId' not in goal for goal in goals))
self.assertTrue(all('filters' not in scope and 'sourceTurnId' not in scope
+6 -2
View File
@@ -284,6 +284,10 @@ class QualityToolsTests(unittest.TestCase):
self.assertEqual(definition["inputSchema"]["properties"]["mode"]["enum"],
["catalog", "query", "rank", "compare", "top", "changes", "defects", "details", "investigate"])
self.assertIn("defect", definition["inputSchema"]["properties"])
column_filter = definition["inputSchema"]["properties"]["columnFilters"]
self.assertEqual(column_filter["items"]["properties"]["operator"]["enum"],
["in", "not_in", "contains", "is_empty", "not_empty"])
self.assertIn("MySQL LOCATE", column_filter["description"])
self.assertFalse(definition["inputSchema"]["additionalProperties"])
self.assertEqual(definition["inputSchema"]["allOf"][0]["then"]["required"], ["releaseId"])
@@ -529,11 +533,11 @@ class QualityToolsTests(unittest.TestCase):
if path == "/api/analysis/tools/catalog":
response["sourceFields"] = [{"key": "customerCode", "label": "客户代码",
"type": "source_value", "stages": ["black"],
"columns": {"black": "customer_code"}, "operators": ["in"]}]
"columns": {"black": "customer_code"}, "operators": ["in", "contains"]}]
return response
api = AuthorizedColumnApi()
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
rule = {"field": "customerCode", "stages": ["black"], "operator": "in", "values": ["C-17"]}
rule = {"field": "customerCode", "stages": ["black"], "operator": "contains", "values": ["C-17"]}
result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "monthStart": 3, "monthEnd": 9},
"filters": {"exclusions": {"product": ["规格甲"]}}, "columnFilters": [rule],
+29 -2
View File
@@ -25,7 +25,7 @@ from test_nao_quality_tools import FakeApi, RELEASE
def source_field(key, label, black, bright, stages=('black', 'bright')):
return {'key': key, 'label': label, 'columns': {'black': black, 'bright': bright},
'stages': list(stages), 'type': 'source_value',
'operators': ['in', 'not_in', 'is_empty', 'not_empty']}
'operators': ['in', 'not_in', 'contains', 'is_empty', 'not_empty']}
SOURCE_FIELDS = [
@@ -349,6 +349,15 @@ class SourceColumnProtocolTests(unittest.TestCase):
with self.assertRaisesRegex(TaskUnavailable, '不超过256字符'):
protocol_task(columnFilters=[{**rule, 'values': ['X' * 257]}])
def test_contains_keywords_are_preserved_as_literal_or_values(self):
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains',
'values': [' 001', 'A|B', 'abC']}
task = protocol_task(columnFilters=[rule])
self.assertEqual(task['filters']['columnFilters'], [rule])
for invalid in ([], [''], [' '], ['x' * 257], [1], ['one'] * 51):
with self.subTest(invalid=invalid), self.assertRaises(TaskUnavailable):
protocol_task(columnFilters=[{**rule, 'values': invalid}])
def test_protocol_presentation_enums_default_consistently_when_omitted_or_null(self):
base = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3}, comparisonRequired=False,
@@ -396,6 +405,10 @@ class SourceColumnProtocolTests(unittest.TestCase):
self.assertIn('periodCoverage(available/complete,可省略或null', prompt)
self.assertIn('freshQuery(bool)', prompt)
self.assertIn('needsClarification(bool)', prompt)
self.assertIn('in/not_in/contains/is_empty/not_empty', prompt)
self.assertIn('contains关键字是子串而非完整候选值', prompt)
self.assertEqual(model_call.call_args_list[0].kwargs['payload']['businessCatalog']['sourceColumnOperators'],
['in', 'not_in', 'contains', 'is_empty', 'not_empty'])
def test_three_arbitrary_source_keys_keep_literal_values_and_stage_scopes(self):
task = protocol_task()
@@ -429,7 +442,7 @@ class SourceColumnProtocolTests(unittest.TestCase):
def test_unauthorized_unknown_and_missing_stage_columns_fail_closed(self):
for bad in ([{**RULES[0], 'field': 'notInDirectory'}],
[{**RULES[0], 'field': 'blackOnlyCode', 'stages': ['bright']}],
[{**RULES[0], 'operator': 'contains'}]):
[{**RULES[0], 'operator': 'starts_with'}]):
with self.subTest(bad=bad), self.assertRaises(TaskUnavailable):
normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026}, columnFilters=bad), '筛选原表字段',
@@ -529,6 +542,20 @@ class SourceColumnProtocolTests(unittest.TestCase):
self.assertEqual(customer_lookup['filters']['columnFilters'], [
{'field': 'dispatchOrder', 'stages': ['bright'], 'operator': 'not_empty'}])
def test_contains_keyword_skips_exact_candidate_value_validation(self):
from nao_agent_task import _resolve_source_column_values
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains',
'values': ['001']}
task = protocol_task(columnFilters=[rule])
calls = []
def resolver(**contract):
calls.append(contract)
raise AssertionError('contains is a substring, not a complete candidate value')
result = _resolve_source_column_values(task, release_id=RELEASE, resolver=resolver)
self.assertEqual(result, {'status': 'exact'})
self.assertEqual(calls, [])
self.assertEqual(task['filters']['columnFilters'], [rule])
def test_missing_literal_offers_same_scope_alternatives_without_substitution(self):
from nao_agent_task import _resolve_source_column_values
task = protocol_task(columnFilters=[RULES[0]])
+65 -31
View File
@@ -1,3 +1,7 @@
import { useProductPage } from './useProductPage';
import { shouldPageProducts, canReadProductPage } from './productPagination';
import { usePriorityMetrics } from './usePriorityMetrics';
import { overviewTrend, sameTrendScope } from './priorityMetrics';
import { QuerySnapshotButton } from './QuerySnapshotButton';
import { loadedQueryScope, queryForLocation, readQueryLocation } from './queryLoadingScope';
import { AnimatedMetricText, SmoothTableFrame, useAnimatedMetricText, useRetainedResult } from './QueryResultMotion';
@@ -413,11 +417,17 @@ export default function QueryPage({ page, onUnauthorized }: {
setDrilledProduct('');
}, [drilldownContextKey]);
const metricParams = useMemo<QueryParams>(() => ({ ...queryParams, trendGranularity: 'month' }), [queryParams]);
const metricKey = useMemo(() => JSON.stringify(metricParams), [metricParams]);
const metricsReady = !!(initialQuery.releaseId || currentReleaseId !== undefined) && (!metricParams.productSearch || !!options);
const productPagingScope=shouldPageProducts(metricParams);
const overview = usePriorityMetrics(metricParams, active && metricsReady, queryRevision, onUnauthorized,productPagingScope);
const currentOverview = overview.value;
const paging=productPagingScope&&!drilledProduct&&!overview.failed;
const productPage=useProductPage(metricParams,active&&metricsReady&&paging&&canReadProductPage(metricParams,overview.settled),queryRevision,'brightQualifiedDesc',onUnauthorized);
const metricKey = useMemo(() => JSON.stringify([metricParams, queryRevision,productPagingScope?productPage.key:null]), [metricParams, queryRevision,productPagingScope,productPage.key]);
const currentMetricsError = errorKey === metricKey ? error : '';
const currentMetricsData = dataKey === metricKey && !currentMetricsError ? data : null;
const queryPending = !currentMetricsData && !currentMetricsError;
const queryBusy = loading || queryPending;
const queryPending = !drilledProduct && !currentMetricsData && !currentMetricsError;
const queryBusy = !drilledProduct && (loading || queryPending);
const metricKeyRef = useRef(metricKey);
metricKeyRef.current = metricKey;
const trendParams = useMemo<QueryParams>(() => {
@@ -429,7 +439,10 @@ export default function QueryPage({ page, onUnauthorized }: {
}, [queryKey, trendObject, trendGranularity, view]);
const defectQueryParams = useMemo<QueryParams>(() => applyTrendObject(queryParams, trendObject, view), [queryKey, trendObject, view]);
const defectQueryKey = JSON.stringify(defectQueryParams);
const trendKey = JSON.stringify(trendParams);
const trendKey = JSON.stringify([trendParams, queryRevision]);
const reuseOverviewTrend = sameTrendScope(metricParams, trendParams);
const trendGate = !overview.split && !reuseOverviewTrend || overview.settled;
const sharedTrend = reuseOverviewTrend ? currentOverview : null;
const trendKeyRef = useRef(trendKey);
trendKeyRef.current = trendKey;
@@ -447,12 +460,20 @@ export default function QueryPage({ page, onUnauthorized }: {
}, [queryKey, active]);
useEffect(() => {
if (!active || optionsLoading || optionsError) return;
if (!active || !metricsReady || !overview.settled || drilledProduct) return;
if(paging) {
if(productPage.error){setError(productPage.error);setErrorKey(metricKey);setLoading(false);return;}
if(!productPage.value||!currentOverview){setLoading(true);return;}
setData({...currentOverview,rows:productPage.value.rows});setDataKey(metricKey);setLoading(false);setError('');setErrorKey('');return;
}
if (currentOverview && !overview.split) {
setData(currentOverview); setDataKey(metricKey); setLoading(false); setError(''); setErrorKey(''); return;
}
const controller = new AbortController();
setLoading(true);
setError('');
setErrorKey('');
api.metrics(metricParams, controller.signal).then(result => {
(metricParams.view === 'overall' ? api.metrics : api.metricsTable)(metricParams, controller.signal).then(result => {
if (controller.signal.aborted || metricKeyRef.current !== metricKey) return;
setData(result);
setDataKey(metricKey);
@@ -465,7 +486,7 @@ export default function QueryPage({ page, onUnauthorized }: {
setErrorKey(metricKey);
}).finally(() => { if (!controller.signal.aborted) setLoading(false); });
return () => controller.abort();
}, [active, metricKey, queryRevision, onUnauthorized, optionsLoading, optionsError]);
}, [active, metricKey, queryRevision, onUnauthorized, metricsReady, overview.settled, overview.split, currentOverview,drilledProduct,paging,productPage.value,productPage.error]);
useEffect(() => {
if (!active) return;
@@ -479,7 +500,7 @@ export default function QueryPage({ page, onUnauthorized }: {
setTrendObjectMetrics(null);
setTrendObjectMetricsLoading(true);
setTrendObjectMetricsError('');
api.metrics({ ...defectQueryParams, trendGranularity: 'month' }, controller.signal)
api.metrics({ ...defectQueryParams, view: 'overall', trendGranularity: 'month' }, controller.signal)
.then(result => { if (!controller.signal.aborted) setTrendObjectMetrics({ key: defectQueryKey, data: result }); })
.catch(errorValue => {
if (controller.signal.aborted) return;
@@ -490,8 +511,11 @@ export default function QueryPage({ page, onUnauthorized }: {
}, [active, trendObject, defectQueryKey, queryKey, queryRevision, onUnauthorized]);
useEffect(() => {
if (!active || optionsLoading || optionsError) return;
if (!active || !metricsReady || !trendGate) return;
setTrendFailure(null);
if (sharedTrend) {
setTrendData(overviewTrend(sharedTrend).trend); setTrendDataKey(trendKey); setTrendLoading(false); return;
}
const controller = new AbortController();
setTrendLoading(true);
api.trend(trendParams, controller.signal).then(result => {
@@ -503,10 +527,10 @@ export default function QueryPage({ page, onUnauthorized }: {
if (!controller.signal.aborted && !onUnauthorized(errorValue)) setTrendFailure({ key: trendKey, message: chooseError(errorValue) });
}).finally(() => { if (!controller.signal.aborted) setTrendLoading(false); });
return () => controller.abort();
}, [active, trendKey, queryRevision, trendRevision, onUnauthorized, optionsLoading, optionsError]);
}, [active, trendKey, queryRevision, trendRevision, onUnauthorized, metricsReady, trendGate, sharedTrend]);
useEffect(() => {
if (!active || optionsLoading || optionsError) return;
if (!active || !metricsReady || !overview.settled) return;
let alive = true;
setTrendTarget(null);
setTrendTargetKey('');
@@ -525,7 +549,7 @@ export default function QueryPage({ page, onUnauthorized }: {
}
});
return () => { alive = false; };
}, [active, trendKey, queryRevision, trendRevision, onUnauthorized, optionsLoading, optionsError]);
}, [active, trendKey, queryRevision, trendRevision, onUnauthorized, metricsReady, overview.settled]);
const currentTrendTarget = trendTargetKey === trendKey ? trendTarget : null;
const visibleTrendData = useMemo(() => fullYearTrend(trendDataKey === trendKey ? trendData : [], year, trendGranularity), [trendData, trendDataKey, trendKey, year, trendGranularity]);
@@ -554,6 +578,7 @@ export default function QueryPage({ page, onUnauthorized }: {
: page === 'overall' && currentMetricsData
? currentMetricsData.rows.length ? currentMetricsData.rows : [currentMetricsData.summary]
: currentMetricsData?.rows || [];
if(paging)return rawRows;
const scopedRows = view === 'department'
? rawRows.filter(row => !RETIRED_DEPARTMENTS.has(row.department || row.label))
: view === 'group' && department.length
@@ -571,7 +596,7 @@ export default function QueryPage({ page, onUnauthorized }: {
if (!appliedSearch.trim()) return orderedRows;
const term = appliedSearch.trim().toLocaleLowerCase();
return orderedRows.filter(row => [row.label, row.department, row.group, row.product, row.projectGroup].filter(Boolean).join(' ').toLocaleLowerCase().includes(term));
}, [page, period, currentMetricsData, appliedSearch, year, view, department, allowedGroupSet]);
}, [page, period, currentMetricsData, appliedSearch, year, view, department, allowedGroupSet,paging]);
const tableReady = drilledProduct ? trendDataKey === trendKey && !trendLoading : !!currentMetricsData;
const retainedTable = useRetainedResult(tableReady ? {
@@ -593,7 +618,7 @@ export default function QueryPage({ page, onUnauthorized }: {
query:{...defectQueryParams,releaseId:summaryMetricsDataRelease()},trendQuery:trendParams,
tableQuery:drilledProduct?trendParams:queryParams,
tableDimension:tableView==='overall'&&tablePeriod==='year'?'period':tableView,
tableGranularity:drilledProduct?trendGranularity:'month',refreshKey:JSON.stringify([refreshKey,queryRevision]),onUnauthorized,display:defectDisplay,onDisplay:setDefectDisplay});
tableGranularity:drilledProduct?trendGranularity:'month',productKeys:paging?productPage.value?.rows.map(row=>row.key):undefined,tableEnabled:!paging||!!productPage.value,refreshKey:JSON.stringify([refreshKey,queryRevision]),onUnauthorized,display:defectDisplay,onDisplay:setDefectDisplay});
function summaryMetricsDataRelease() { return (drilledProduct?trendObjectMetrics?.data:currentMetricsData)?.release?.id || currentMetricsData?.release?.id || queryParams.releaseId; }
const captureTrendAnchor = useCallback(() => {
@@ -646,8 +671,8 @@ export default function QueryPage({ page, onUnauthorized }: {
);
const activeDefectData = trendObject
? defectQueryKey === queryKey ? currentMetricsData : trendObjectMetrics?.key === defectQueryKey ? trendObjectMetrics.data : null
: currentMetricsData;
? defectQueryKey === queryKey ? currentOverview || currentMetricsData : trendObjectMetrics?.key === defectQueryKey ? trendObjectMetrics.data : null
: currentOverview || currentMetricsData;
const previousDefectSnapshot = useRef<{ data: MetricsResponse; query: QueryParams } | null>(null);
useLayoutEffect(() => {
if (activeDefectData) previousDefectSnapshot.current = { data: activeDefectData, query: defectQueryParams };
@@ -656,7 +681,7 @@ export default function QueryPage({ page, onUnauthorized }: {
const displayedDefectQuery = activeDefectData ? defectQueryParams : previousDefectSnapshot.current?.query || defectQueryParams;
const defectsRefreshing = !activeDefectData && (queryBusy || trendObjectMetricsLoading);
// 产品月度下钻复用当前对象的服务端指标,汇总表仍保留父层结果供返回。
const summaryMetricsData = drilledProduct ? activeDefectData : currentMetricsData;
const summaryMetricsData = drilledProduct ? activeDefectData : currentOverview || currentMetricsData;
const openRecords = useCallback((row: MetricRow | null, stage: Stage, selectedMonth?: number, sourceQuery?: QueryParams, sourceData?: MetricsResponse | null) => {
const source = sourceData === undefined ? summaryMetricsData : sourceData;
if (!source?.release?.id) { message.info('指标数据尚未就绪,请重新查询后打开来源明细。'); return; }
@@ -665,7 +690,7 @@ export default function QueryPage({ page, onUnauthorized }: {
setSelectedRecord(row); setRecordStage(stage); setRecordMonth(selectedMonth); setRecordsOpen(true);
}, [summaryMetricsData, drilledProduct, defectQueryParams, queryParams, view, message]);
const summaryMetricsError = drilledProduct ? trendObjectMetricsError : currentMetricsError;
const summaryBusy = drilledProduct ? trendObjectMetricsLoading || !summaryMetricsData && !summaryMetricsError : queryBusy;
const summaryBusy = drilledProduct ? trendObjectMetricsLoading || !summaryMetricsData && !summaryMetricsError : !summaryMetricsData && queryBusy;
const displayedSummaryData = useRetainedResult(summaryMetricsData);
const rowForSummary = displayedSummaryData?.summary;
const rawFormulaDescriptions = summaryMetricsData?.context.metricFormulaDescriptions;
@@ -758,7 +783,7 @@ export default function QueryPage({ page, onUnauthorized }: {
</div>}
</section>
{currentMetricsError && <Alert showIcon type="error" message={data ? "更新失败,当前显示上次结果" : "质量指标查询失败"} description={currentMetricsError} action={<Button size="small" icon={<ReloadOutlined />} onClick={() => setQueryRevision(value => value + 1)}>重试</Button>} />}
{currentMetricsError && <Alert showIcon type="error" message={currentOverview ? "表格读取失败,指标已更新" : data ? "更新失败,当前显示上次结果" : "质量指标查询失败"} description={currentMetricsError} action={<Button size="small" icon={<ReloadOutlined />} onClick={() => setQueryRevision(value => value + 1)}>重试</Button>} />}
{drilledProduct && summaryMetricsError && <Alert showIcon type="error" message="当前产品指标查询失败" description={summaryMetricsError} action={<Button size="small" onClick={() => setQueryRevision(value => value + 1)}>重试</Button>} />}
<section className={`metric-strip ${!displayedSummaryData ? 'query-awaiting-data' : ''}`} data-snapshot-module="metrics" aria-label="质量指标" aria-busy={summaryBusy} inert={!summaryMetricsData}>
<div className="rate-basis-control"><Segmented size="small" value={rateBasis} options={[{ label: '原口径', value: 'original' }, { label: '去本部报废', value: 'withoutHeadquartersScrap' }]} onChange={value => setRateBasis(value as typeof rateBasis)} aria-label="合格率口径" /></div>
@@ -811,6 +836,9 @@ export default function QueryPage({ page, onUnauthorized }: {
paginationScope={retainedTable?.paginationScope}
exportFilename={`${drilledProduct || scopeName}质量汇总-${year}${drilledProduct || period === 'year' ? '' : period === 'month' ? `-${String(month).padStart(2, '0')}` : `-Q${quarter}`}-${fullFields ? '完整字段' : '关键指标'}`}
rows={tableRows}
columnRows={paging&&currentOverview?[currentOverview.summary]:undefined}
remotePage={paging?productPage.pagination:undefined}
loadExportData={paging?async()=>{const [rows,defectDrilldown]=await Promise.all([productPage.loadAll(),defectDrill.exportTable()]);return {rows,defectDrilldown};}:undefined}
view={tableView}
layoutView={retainedTable?.layoutView || view}
pageSizeOverride={tableProduct ? 12 : undefined}
@@ -858,7 +886,7 @@ export default function QueryPage({ page, onUnauthorized }: {
</section>;
})}
</div>
{displayedDefectData ? <DefectParetoPanel releaseId={displayedDefectData.release?.id} query={displayedDefectQuery} refreshKey={queryRevision} pending={defectsRefreshing} externalScrapWeight={blackDefectItems.find(item => item.name === '外废')?.weight} onUnauthorized={onUnauthorized} /> : !defectsRefreshing && <div className="defect-pareto-status">不良数据暂不可用</div>}
{displayedDefectData ? <DefectParetoPanel releaseId={displayedDefectData.release?.id} query={displayedDefectQuery} refreshKey={queryRevision} pending={defectsRefreshing||queryBusy} externalScrapWeight={blackDefectItems.find(item => item.name === '外废')?.weight} onUnauthorized={onUnauthorized} /> : !defectsRefreshing && <div className="defect-pareto-status">不良数据暂不可用</div>}
</section>
</div>
</>
@@ -947,7 +975,7 @@ export function DefectParetoPanel({ releaseId, query, externalScrapWeight, onUna
const showLoading = useDelayedIndicator(loading);
useEffect(() => {
if (!active || !visible) return;
if (!active || !visible || pending) return;
if (!releaseId) { setResponses(null); setLoading(false); setError(''); return; }
const controller = new AbortController();
const base = { releaseId, ...fineQuery, mode: 'top', dimension: 'defect', limit: 100 };
@@ -964,7 +992,7 @@ export function DefectParetoPanel({ releaseId, query, externalScrapWeight, onUna
})
.finally(() => { if (!controller.signal.aborted) setLoading(false); });
return () => controller.abort();
}, [active, visible, releaseId, fineQueryKey, onUnauthorized, onlyStage, refreshKey, externalScrapWeight]);
}, [active, visible, pending, releaseId, fineQueryKey, onUnauthorized, onlyStage, refreshKey, externalScrapWeight]);
useEffect(() => {
const refresh = () => setThemeRevision(value => value + 1);
@@ -1095,9 +1123,11 @@ const totalWeightTitle = (label: string) => <span className="query-total-weight-
const rateBasisTitle = (label: string, basis: QueryRateBasis, kind: 'bright' | 'composite', onBasis?: () => void) => <span className="query-total-weight-title query-rate-rule-title"><span className="query-rate-label">{onBasis ? <button className="header-link" onClick={onBasis}>{label}</button> : label}{basis !== 'original' && <small className="query-rate-subtitle">(去本部报废)</small>}</span>{basis !== 'original' && <Tooltip trigger={['hover', 'focus']} title={<div>{kind === 'composite' && <>综合合格率=黑片合格率×亮片合格率。<br /></>}原口径:亮片合格重量÷亮片总重量。<br />去本部报废:(亮片合格重量+本部报废)÷亮片总重量,同时从亮片报废重量中扣除本部报废,总重量不变。<br />当前:去本部报废。计算结果以已发布数据版本及生效规则为准。</div>}><InfoCircleOutlined tabIndex={0} role="img" aria-label={`${kind === 'composite' ? '综合合格率' : '亮片合格率'}计算说明`} /></Tooltip>}</span>;
export function MetricTable({
ref, exportFilename, rows, view, defectDrilldown, layoutView = view, pageSizeOverride, paginationScope, period, month, quarter, fullFields, rateBasis, onDetail, onBasis, onDrilldown, onlyStage, defectDisplay = 'weight', emptyText = '当前条件下暂无数据',
ref, exportFilename, rows, view, defectDrilldown, columnRows, remotePage, loadExportData, layoutView = view, pageSizeOverride, paginationScope, period, month, quarter, fullFields, rateBasis, onDetail, onBasis, onDrilldown, onlyStage, defectDisplay = 'weight', emptyText = '当前条件下暂无数据',
}: {
ref?: React.Ref<{ exportExcel: () => Promise<void> }>; exportFilename: string; defectDrilldown?: DefectDrilldownResponse;
columnRows?:MetricRow[];remotePage?:{current:number;pageSize:number;total:number;onChange:(page:number,size:number)=>void};
loadExportData?:()=>Promise<{rows:MetricRow[];defectDrilldown?:DefectDrilldownResponse}>;
rows: MetricRow[]; view: ViewKind; layoutView?: ViewKind; pageSizeOverride?: number; paginationScope?: string; period: PeriodKind; month: number; quarter: number; fullFields: boolean;
rateBasis: 'original' | 'withoutHeadquartersScrap';
onDetail: (row: MetricRow | null, stage: Stage, month?: number) => void; onBasis: () => void; onDrilldown: (row: MetricRow) => void;
@@ -1140,7 +1170,7 @@ export function MetricTable({
headquartersScrapWeight: '本部报废', rate: '合格率',
};
const stageColumns = (stage: Stage): ColumnsType<MetricRow> => {
const base: ColumnsType<MetricRow> = metricStageColumnPlan(rows, stage, defectDrilldown ? false : fullFields, rateBasis).map(column => {
const base: ColumnsType<MetricRow> = metricStageColumnPlan(columnRows||rows, stage, defectDrilldown ? false : fullFields, rateBasis).map(column => {
if (column.kind === 'defect') return defectColumns(stage, [column.name])[0];
const metricColumn = numeric(fieldLabels[column.field], stage, column.field);
return stage === 'bright' && column.field === 'rate'
@@ -1165,11 +1195,14 @@ export function MetricTable({
];
const columns = onlyStage ? allColumns.filter(column => !['compositeRate', `stage-${onlyStage === 'black' ? 'bright' : 'black'}`].includes(String(column.key))) : allColumns;
useImperativeHandle(ref, () => ({
exportExcel: () => downloadMetricWorkbook(exportFilename, columns, rows, (key, row) => {
exportExcel: async () => {
const exported=loadExportData?await loadExportData():{rows,defectDrilldown};
const exportedFine=exported.defectDrilldown;
return downloadMetricWorkbook(exportFilename, columns, exported.rows, (key, row) => {
if (key === 'object') return [title(row), view === 'group' ? row.department : null].filter(Boolean).join(' / ');
if (key === 'compositeRate') return visibleCompositeRate(row, rateBasis);
const stage: Stage = key.startsWith('black-') ? 'black' : 'bright';
if (defectDrilldown && key.startsWith(`${stage}-fine-`)) return defectCell(defectDrilldown,row,key.slice(`${stage}-fine-`.length),defectDisplay);
if (exportedFine && key.startsWith(`${stage}-fine-`)) return defectCell(exportedFine,row,key.slice(`${stage}-fine-`.length),defectDisplay);
if (key.startsWith(`${stage}-defect-`)) {
const weight=defectList(row, stage).find(item => item.name === key.slice(`${stage}-defect-`.length))?.weight ?? null;
return defectDisplay === 'rate' ? defectWeightRate(weight,stageValue(row,stage,'totalWeight')) : weight;
@@ -1177,13 +1210,14 @@ export function MetricTable({
const field = key.slice(stage.length + 1) as 'qualifiedWeight' | 'scrapWeight' | 'totalWeight' | 'headquartersScrapWeight' | 'rate';
if (defectDisplay === 'rate' && (field === 'scrapWeight' || field === 'headquartersScrapWeight')) return defectWeightRate(stageValue(row,stage,field,rateBasis),stageValue(row,stage,'totalWeight'));
return field === 'rate' ? visibleStageRate(row, stage, stage === 'bright' ? rateBasis : 'original') : stageValue(row, stage, field, rateBasis);
}),
});
},
}));
const dataSource = rows.map((row, index) => ({ ...row, sourceMetricKey: row.key, key: row.key || `${title(row)}-${index}` }));
const [currentPage, setCurrentPage] = useState(1);
const [pageSize, setPageSize] = useState(SUMMARY_PAGE_SIZE);
const effectivePage = pageSizeOverride == null ? Math.min(currentPage, Math.max(1, Math.ceil(rows.length / pageSize))) : 1;
const effectivePageSize = pageSizeOverride ?? pageSize;
const effectivePage = remotePage?remotePage.current:pageSizeOverride == null ? Math.min(currentPage, Math.max(1, Math.ceil(rows.length / pageSize))) : 1;
const effectivePageSize = remotePage?.pageSize ?? pageSizeOverride ?? pageSize;
useEffect(() => setCurrentPage(1), [paginationScope ?? rows]);
const columnWidth = (column: ColumnsType<MetricRow>[number]): number => {
if ('children' in column && column.children?.length) {
@@ -1194,9 +1228,9 @@ export function MetricTable({
const scrollWidth = Math.ceil(columns.reduce((total, column) => total + columnWidth(column), 0));
const scroll = { x: scrollWidth };
const displayColumns = compact ? columns.flatMap(column => 'children' in column && column.children ? column.children : [column]) : columns;
const pageRows = dataSource.slice((effectivePage - 1) * effectivePageSize, effectivePage * effectivePageSize);
const pageRows = remotePage?dataSource:dataSource.slice((effectivePage - 1) * effectivePageSize, effectivePage * effectivePageSize);
return <div className="metric-table-scroll"><Table<MetricRow> size="small" bordered tableLayout="fixed" columns={displayColumns} dataSource={pageRows} pagination={false} scroll={scroll} rowKey="key" locale={{ emptyText }} />
{pageSizeOverride == null && <MetricTablePagination current={effectivePage} total={dataSource.length} pageSize={pageSize} onChange={(nextPage, nextPageSize) => { setPageSize(nextPageSize); setCurrentPage(nextPageSize === pageSize ? nextPage : 1); }} />}
{pageSizeOverride == null && <MetricTablePagination current={effectivePage} total={remotePage?.total??dataSource.length} pageSize={effectivePageSize} onChange={remotePage?.onChange||((nextPage, nextPageSize) => { setPageSize(nextPageSize); setCurrentPage(nextPageSize === pageSize ? nextPage : 1); })} />}
</div>;
}
+54 -27
View File
@@ -1,3 +1,7 @@
import { useProductPage } from './useProductPage';
import { shouldPageProducts, canReadProductPage } from './productPagination';
import { usePriorityMetrics } from './usePriorityMetrics';
import { overviewTrend, sameTrendScope } from './priorityMetrics';
import { QuerySnapshotButton } from './QuerySnapshotButton';
import { loadedQueryScope, queryForLocation } from './queryLoadingScope';
import { SmoothTableFrame, useRetainedResult } from './QueryResultMotion';
@@ -37,7 +41,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
const [moreFiltersOpen,setMoreFiltersOpen]=useState(false);
const [extraFilters,setExtraFilters]=useState<Record<string,string[]>>(initial.extraFilters||{});
const [extraFields,setExtraFields]=useState<FilterField[]>([]);
const [options,setOptions]=useState<OptionsResponse|null>(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');const [supportLoading,setSupportLoading]=useState(true);
const [options,setOptions]=useState<OptionsResponse|null>(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');
const [applied,setApplied]=useState<QueryParams|null>(null);const [revision,setRevision]=useState(0);const [response,setResponse]=useState<{key:string;value:MetricsResponse}|null>(null);const [loading,setLoading]=useState(false);const [failure,setFailure]=useState<{key:string;message:string}|null>(null);
const [defectDisplay,setDefectDisplay]=useState<'weight'|'rate'>('weight');
const [fullFields,setFullFields]=useState(true);const [rateBasis,setRateBasis]=useState<'original'|'withoutHeadquartersScrap'>('original');const [basisOpen,setBasisOpen]=useState(false);const [trendMode,setTrendMode]=useState<'production'|'defects'|'data'>('production');
@@ -50,29 +54,47 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
const supportGeneration=useRef(0);const supportController=useRef<AbortController|null>(null);
const loadSupport=useCallback(async()=>{
const generation=++supportGeneration.current;supportController.current?.abort();const controller=new AbortController();supportController.current=controller;
setSupportLoading(true);setSupportError('');
try{const [choices,source,filterFields]=await Promise.all([api.options(controller.signal),api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal),api.filterFields(false,controller.signal)]);if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);setCatalog(source);setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));}
catch(error){if(generation===supportGeneration.current&&!onUnauthorized(error))setSupportError(error instanceof Error?error.message:'查询条件读取失败');}
finally{if(generation===supportGeneration.current)setSupportLoading(false);}
setSupportError('');
const fail=(error:unknown)=>{if(generation===supportGeneration.current&&!controller.signal.aborted&&!onUnauthorized(error))setSupportError(previous=>previous||(error instanceof Error?error.message:'查询条件读取失败'));};
const requests=[
api.options(controller.signal).then(choices=>{if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);}).catch(fail),
api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal).then(source=>{if(generation===supportGeneration.current)setCatalog(source);}).catch(fail),
api.filterFields(false,controller.signal).then(filterFields=>{if(generation===supportGeneration.current)setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));}).catch(fail),
];
await Promise.all(requests);
},[onUnauthorized,initial.releaseId,currentReleaseId]);
useEffect(()=>{if(active)void loadSupport();return()=>{supportGeneration.current++;supportController.current?.abort();};},[active,refreshKey,loadSupport]);
const fields=useMemo(()=>catalog?.sourceFields.filter(field=>field.type==='source_value'&&field.stages.includes(stage))||[],[catalog,stage]);
const exactProductSearch=useMemo(()=>(options?.products||[]).find(name=>name.trim().toLocaleLowerCase()===appliedSearch.trim().toLocaleLowerCase())||'', [options,appliedSearch]);
const queryProducts=useMemo(()=>drilledProduct?[drilledProduct]:exactProductSearch?product.length&&!product.includes(exactProductSearch)?['__codex_no_product_search_match__']:[exactProductSearch]:product,[drilledProduct,exactProductSearch,product]);
useEffect(()=>{setDrilledProduct('');},[department,group,product,projectGroup,sourceFilters,extraFilters]);
const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:initial.releaseId||currentReleaseId||catalog?.releaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,catalog,stage,sourceFilters,extraFilters,initial.releaseId,currentReleaseId]);
const queryReleaseId=initial.releaseId||(currentReleaseId!==undefined?currentReleaseId||undefined:catalog?.releaseId);
const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,queryReleaseId,stage,sourceFilters,extraFilters]);
const catalogMatchesDraft=!!catalog&&!!draft.releaseId&&catalog.releaseId===draft.releaseId;
const queryReady=(query:QueryParams)=>!!queryReleaseId&&query.releaseId===queryReleaseId&&(!query.productSearch||!!options)&&(!(query.columnFilters||[]).length||!!catalog&&catalog.releaseId===query.releaseId);
const draftReady=queryReady(draft);
const appliedReady=!!applied&&queryReady(applied);
// Apply organizational/source filters directly, like the other query pages. Product text search keeps its explicit submit.
useEffect(()=>{if(active&&catalog&&!supportLoading&&!supportError)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,catalog,draft,supportLoading,supportError]);
const queryKey=JSON.stringify({query:applied,revision});const current=response?.key===queryKey?response.value:null;const error=failure?.key===queryKey?failure.message:'';
useEffect(()=>{if(active&&draftReady)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,draft,draftReady]);
const productPagingScope=!!applied&&shouldPageProducts(applied)&&!drilledProduct;
const overview=usePriorityMetrics(applied,active&&appliedReady,JSON.stringify([revision,refreshKey]),onUnauthorized,productPagingScope);
const paging=productPagingScope&&!overview.failed;
const productPage=useProductPage(applied||draft,active&&appliedReady&&paging&&canReadProductPage(applied||draft,overview.settled),JSON.stringify([revision,refreshKey]),stage==='black'?'blackTotalDesc':'brightTotalDesc',onUnauthorized);
const queryKey=JSON.stringify({query:applied,revision,refreshKey,page:productPagingScope?productPage.key:null});const current=response?.key===queryKey?response.value:null;const error=failure?.key===queryKey?failure.message:'';
const currentSummary=overview.value||current;
useEffect(()=>{
if(!active||!applied||supportError)return;const controller=new AbortController();setLoading(true);setFailure(null);
void api.metrics(applied,controller.signal).then(value=>{if(!controller.signal.aborted)setResponse({key:queryKey,value});}).catch(e=>{if(!controller.signal.aborted&&!onUnauthorized(e))setFailure({key:queryKey,message:e instanceof Error?e.message:'查询失败'});}).finally(()=>{if(!controller.signal.aborted)setLoading(false);});
if(!active||!applied||!appliedReady||!overview.settled)return;if(paging){if(productPage.error){setFailure({key:queryKey,message:productPage.error});setLoading(false);return;}if(!productPage.value||!overview.value){setLoading(true);return;}setResponse({key:queryKey,value:{...overview.value,rows:productPage.value.rows}});setLoading(false);setFailure(null);return;}if(overview.value&&!overview.split){setResponse({key:queryKey,value:overview.value});setLoading(false);setFailure(null);return;}const controller=new AbortController();setLoading(true);setFailure(null);
void api.metricsTable(applied,controller.signal).then(value=>{if(!controller.signal.aborted)setResponse({key:queryKey,value});}).catch(e=>{if(!controller.signal.aborted&&!onUnauthorized(e))setFailure({key:queryKey,message:e instanceof Error?e.message:'查询失败'});}).finally(()=>{if(!controller.signal.aborted)setLoading(false);});
return()=>controller.abort();
},[active,queryKey,onUnauthorized,supportError]);
},[active,queryKey,onUnauthorized,appliedReady,overview.settled,overview.split,overview.value,paging,productPage.value,productPage.error]);
useEffect(()=>{if(!active||!applied)return;const bytes=new TextEncoder().encode(JSON.stringify({query:queryForLocation(applied,initial.releaseId)}));const encoded=btoa(Array.from(bytes,b=>String.fromCharCode(b)).join('')).replace(/=/g,'').replace(/\+/g,'-').replace(/\//g,'_');const url=`?page=${stage==='black'?'blackProduct':'brightProduct'}&context=${encoded}`;window.history.replaceState({},'',`${location.pathname}${url}`);onLocationChange(url);},[active,applied,stage]);
const trendQuery=useMemo(()=>applied?{...applied,month:undefined,quarter:undefined,...(trendGranularity==='year'?{year:undefined}:{}),trendGranularity}:null,[applied,trendGranularity]);
const trendKey=JSON.stringify({query:trendQuery,revision});
useEffect(()=>{if(!active||!trendQuery||supportError)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,supportError]);
const trendKey=JSON.stringify({query:trendQuery,revision,refreshKey});
const trendReady=!!trendQuery&&queryReady(trendQuery);
const reuseOverviewTrend=!!applied&&!!trendQuery&&sameTrendScope(applied,trendQuery);
const trendGate=!overview.split&&!reuseOverviewTrend||overview.settled;
const sharedTrend=reuseOverviewTrend?overview.value:null;
useEffect(()=>{if(!active||!trendQuery||!trendReady||!trendGate)return;if(sharedTrend){setTrendResponse({key:trendKey,value:overviewTrend(sharedTrend),query:trendQuery});setTrendLoading(false);setTrendError('');return;}const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,trendReady,trendGate,sharedTrend]);
const visibleTrend=useMemo(()=>fullYearTrend(trendResponse?.key===trendKey?trendResponse.value.trend:[],applied?.year||year,trendGranularity),[trendResponse,trendKey,applied?.year,year,trendGranularity]);
const displayedTrend=useRetainedResult(trendResponse?.key===trendKey?visibleTrend:null)||visibleTrend;
const selectedTrendPeriod=trendGranularity==='month'&&applied?.month
@@ -80,12 +102,14 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
: trendGranularity==='quarter'&&applied?.quarter ? `${applied.year}-Q${applied.quarter}` : undefined;
const selectTrendPeriod=(value:number|string)=>{automaticPeriod.current=false;const next=nextTrendPeriod(value,{period,year,month,quarter});setYear(next.year);setMonth(next.month);setQuarter(next.quarter);setPeriod(next.period);};
const dirty=!!applied&&JSON.stringify(draft)!==JSON.stringify(applied);
const queryBusy=supportLoading||loading||(!error&&!supportError&&(dirty||!current));
const queryFailed=!!(error||supportError);
const metadataPending=!draftReady;
const metadataFailed=!draftReady&&!!supportError;
const queryBusy=!metadataFailed&&(metadataPending||loading||(!error&&(dirty||!current)));
const queryFailed=!!error||metadataFailed;
const submit=()=>{automaticPeriod.current=false;setAppliedSearch(search.trim());setDrilledProduct('');setRevision(v=>v+1);};
const setTrendObject=(value?:string)=>{const selected=value||'';setDrilledProduct('');setSearch(selected);setAppliedSearch(selected);};
const returnToProducts=()=>{setDrilledProduct('');setSearch(drillParentSearch.current);setAppliedSearch(drillParentSearch.current);if(trendMode!=='defects')setTrendMode('production');};
const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:catalog?.releaseId},stage,[]));setRevision(v=>v+1);};
const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,[]));setRevision(v=>v+1);};
const groups=(options?.groups||[]).filter(name=>!department.length||department.includes(options?.groupDepartments?.[name]||''));
const paths=options?.organizationPaths||[];const products=department.length||group.length?[...new Set(paths.filter(path=>(!department.length||department.includes(path.department||''))&&(!group.length||group.includes(path.group||''))).map(path=>path.product).filter((v):v is string=>!!v))]:options?.products||[];
const trendOptions=useMemo(()=>[...products.map(value=>({label:value,value})),...(drilledProduct==='__UNASSIGNED__'?[{label:'未分类',value:drilledProduct}]:[])],[products,drilledProduct]);
@@ -96,16 +120,17 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
const salesSelection=salesOrderTypeSelection(extraFilters);
const moreFilterSummary=[salesSelection.values.length?`销售订单类型${salesSelection.exclude?'(排除)':''}:`+salesSelection.values.map(value=>value==='__EMPTY__'?'未匹配':value).join('、'):'',projectGroup.length?`项目组:${projectGroup.map(v=>v==='__UNASSIGNED__'?'未分类':v).join('、')}`:'',...sourceFilters.map(rule=>fields.find(field=>field.key===rule.field)?.label||rule.field)].filter(Boolean).join(';');
const displayed = useRetainedResult(current && applied ? { data: current, query: applied } : null);
const sortedRows=useMemo(()=>sortStageRowsByTotalWeight(displayed?.data.rows||[],stage),[displayed?.data,stage]);
const sortedRows=useMemo(()=>paging?displayed?.data.rows||[]:sortStageRowsByTotalWeight(displayed?.data.rows||[],stage),[displayed?.data,stage,paging]);
const trendRefreshing=!trendError&&(trendLoading||trendResponse?.key!==trendKey);
const showTrendLoading=useDelayedIndicator(trendRefreshing);
const tableReady=!!current&&(!drilledProduct||(trendResponse?.key===trendKey&&!trendLoading&&!trendError));
const table=useRetainedResult(tableReady&&applied?{rows:drilledProduct?visibleTrend:sortedRows,query:applied,drilledProduct,granularity:trendGranularity,period:drilledProduct?'year' as const:applied.month?'month' as const:applied.quarter?'quarter' as const:'year' as const}:null);
const defectDrill=useDefectDrilldown({active,mode:trendMode,stage,query:{...(applied||draft),releaseId:current?.release?.id||applied?.releaseId},trendQuery:trendQuery||draft,
tableQuery:drilledProduct?(trendQuery||draft):(applied||draft),tableDimension:drilledProduct?'period':'product',tableGranularity:trendGranularity,
productKeys:paging?productPage.value?.rows.map(row=>row.key):undefined,tableEnabled:!paging||!!productPage.value,tableQuery:drilledProduct?(trendQuery||draft):(applied||draft),tableDimension:drilledProduct?'period':'product',tableGranularity:trendGranularity,
refreshKey:JSON.stringify([refreshKey,revision]),onUnauthorized,display:defectDisplay,onDisplay:setDefectDisplay});
const summary=displayed?.data.summary[stage];const stageRate=stage==='bright'&&rateBasis==='withoutHeadquartersScrap'?summary?.rateWithoutHeadquartersScrap:summary?.rate;
const openDetail=(row:MetricRow|null,_stage:Stage,selectedMonth?:number)=>{if(!applied||!current?.release)return;setRecordVisible(true);setRecords({row,query:freezeSourceRecordsQuery(applied,current.release.id,drilledProduct?'overall':'product',row,selectedMonth)});};
const displayedSummary=useRetainedResult(currentSummary);
const summary=displayedSummary?.summary[stage];const stageRate=stage==='bright'&&rateBasis==='withoutHeadquartersScrap'?summary?.rateWithoutHeadquartersScrap:summary?.rate;
const openDetail=(row:MetricRow|null,_stage:Stage,selectedMonth?:number)=>{if(!applied||!currentSummary?.release)return;setRecordVisible(true);setRecords({row,query:freezeSourceRecordsQuery(applied,currentSummary.release.id,drilledProduct?'overall':'product',row,selectedMonth)});};
const runExport=async()=>{setExporting(true);try{await exporter.current?.exportExcel();}catch(e){message.error(e instanceof Error?e.message:'导出失败');}finally{setExporting(false);}};
const formulaKey=stage==='black'?'blackRate':rateBasis==='original'?'brightRate':'brightRateWithoutHeadquartersScrap';
return <div className="query-page stage-product-page">
@@ -124,13 +149,13 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
</div>
<div className="query-toolbar-tail"><span className={`query-status ${queryBusy?'busy':queryFailed?'failed':''}`}><i/>{queryBusy?'正在查询':queryFailed?'查询失败':'查询完成'}</span><QuerySnapshotButton title={`${label}数据`} queryKey={JSON.stringify([draft,trendMode,trendObject,trendGranularity,rateBasis,fullFields,defectDisplay,revision])} disabled={queryBusy || !current} onUnauthorized={onUnauthorized} /><Button type="default" onClick={reset}>重置</Button></div>
{moreFiltersOpen&&<div className="query-extra-fields" id={`stage-more-filters-${stage}`}>
{extraFields.map(field=><SalesOrderTypeSelect key={field.key} field={field} filters={extraFilters} releaseId={draft.releaseId} stage={stage} onUnauthorized={onUnauthorized} onChange={setExtraFilters}/>)}
{draft.releaseId&&extraFields.map(field=><SalesOrderTypeSelect key={field.key} field={field} filters={extraFilters} releaseId={draft.releaseId} stage={stage} onUnauthorized={onUnauthorized} onChange={setExtraFilters}/>)}
{options?.projectGroupStatus==='READY'&&select('项目组',projectGroup,options.projectGroups||[],setProjectGroup,true)}
{catalog&&<StageSourceFilters stage={stage} fields={fields} query={draft} value={sourceFilters} onChange={setSourceFilters} onUnauthorized={onUnauthorized}/>}
{catalogMatchesDraft&&<StageSourceFilters stage={stage} fields={fields} query={draft} value={sourceFilters} onChange={setSourceFilters} onUnauthorized={onUnauthorized}/>}
</div>}
</section>
{error&&<Alert showIcon type="error" message={response?"更新失败,当前显示上次结果":"查询失败"} description={error} action={<Button onClick={()=>setRevision(v=>v+1)}>重试</Button>}/>}
<section className="metric-strip stage-metric-strip" data-snapshot-module="metrics" aria-label="质量指标" aria-busy={loading} inert={!current}>
{error&&<Alert showIcon type="error" message={overview.value?"表格读取失败,指标已更新":response?"更新失败,当前显示上次结果":"查询失败"} description={error} action={<Button onClick={()=>setRevision(v=>v+1)}>重试</Button>}/>}
<section className="metric-strip stage-metric-strip" data-snapshot-module="metrics" aria-label="质量指标" aria-busy={!currentSummary&&!error} inert={!currentSummary}>
{stage==='bright'&&<div className="rate-basis-control"><Segmented size="small" aria-label="合格率口径" value={rateBasis} options={[{label:'原口径',value:'original'},{label:'去本部报废',value:'withoutHeadquartersScrap'}]} onChange={v=>setRateBasis(v as typeof rateBasis)}/></div>}
<MetricCard primary stage={stage} label={`${label}合格率`} value={rate(stageRate)} onClick={()=>setBasisOpen(true)}/>
<MetricCard label="合格重量" value={`${weight(summary?.qualifiedWeight)} kg`} onClick={()=>openDetail(null,stage)}/>
@@ -150,7 +175,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
</div>
</QueryTrendPanel>
<QueryTablePanel heading={<SectionHeading title={table?.drilledProduct?`产品${table.granularity==='quarter'?'季度':table.granularity==='year'?'年度':'月度'}质量数据`:`${label}产品质量汇总`} subtitle={table?.drilledProduct?`${table.query.year} 年 · ${table.drilledProduct==='__UNASSIGNED__'?'未分类':table.drilledProduct} · 重量 / kg 合格率 / %`:'重量 / kg 合格率 / %'}/>}
busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!catalog||!!supportError}
busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!options}
fullFields={fullFields} onFullFields={setFullFields} defectDisplay={defectDisplay} onDefectDisplay={setDefectDisplay}
returnControls={drilledProduct&&<Button className="query-return-button" icon={<ArrowLeftOutlined/>} onClick={returnToProducts}>返回上一级</Button>}
exporting={exporting} exportDisabled={!!defectDrill.category&&(!defectDrill.table||defectDrill.pending||!!defectDrill.error)||!current||loading||!tableReady||!table?.rows.length} onExport={()=>void runExport()}>
@@ -158,7 +183,9 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
<SmoothTableFrame busy={queryBusy||!tableReady||defectDrill.tablePending} blocked={queryBusy||!tableReady||defectDrill.tablePending}>
<MetricTable defectDrilldown={tableReady&&!queryBusy?defectDrill.table:undefined} ref={exporter} exportFilename={`${table?.drilledProduct||label}产品汇总-${table?.query.year||year}-${fullFields?'完整字段':'关键指标'}`}
paginationScope={JSON.stringify({query:table?.query,drilledProduct:table?.drilledProduct})}
rows={table?.rows||[]} view={table?.drilledProduct?'overall':'product'} layoutView="product" pageSizeOverride={table?.drilledProduct?12:undefined}
rows={table?.rows||[]} columnRows={paging&&currentSummary?[currentSummary.summary]:undefined} remotePage={paging?productPage.pagination:undefined}
loadExportData={paging?async()=>{const [rows,defectDrilldown]=await Promise.all([productPage.loadAll(),defectDrill.exportTable()]);return {rows,defectDrilldown};}:undefined}
view={table?.drilledProduct?'overall':'product'} layoutView="product" pageSizeOverride={table?.drilledProduct?12:undefined}
period={table?.period||period} month={table?.query.month||month} quarter={table?.query.quarter||quarter}
fullFields={fullFields} defectDisplay={defectDisplay} rateBasis={rateBasis} onlyStage={stage} onDetail={openDetail} onBasis={()=>setBasisOpen(true)}
emptyText={queryBusy?'正在更新数据…':error?'查询失败,暂无可展示结果':'当前条件下暂无数据'}
@@ -175,7 +202,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
{displayed&&<DefectParetoPanel releaseId={displayed.data.release?.id} query={displayed.query} pending={!current||loading} refreshKey={revision} onlyStage={stage} externalScrapWeight={stage==='black'?displayed.data.defects.black.find(item=>item.name==='外废')?.weight:undefined} onUnauthorized={onUnauthorized}/>}
</div>
</section></div>
<Modal open={basisOpen} title={`${label}计算口径`} onCancel={()=>setBasisOpen(false)} footer={<Button onClick={()=>setBasisOpen(false)}>关闭</Button>}><p>{readableMetricFormula((current?.context.metricFormulaDescriptions as Record<string,unknown>|undefined)?.[formulaKey])}</p></Modal>
<Modal open={basisOpen} title={`${label}计算口径`} onCancel={()=>setBasisOpen(false)} footer={<Button onClick={()=>setBasisOpen(false)}>关闭</Button>}><p>{readableMetricFormula((currentSummary?.context.metricFormulaDescriptions as Record<string,unknown>|undefined)?.[formulaKey])}</p></Modal>
{records&&<RecordsDrawer key={JSON.stringify(records)} open={recordVisible} row={records.row} query={records.query} stage={stage} lockedStage onStage={()=>{}} onClose={()=>setRecordVisible(false)} onUnauthorized={onUnauthorized}/>}
</div>;
}
+12 -4
View File
@@ -11,13 +11,15 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S
const [search,setSearch]=useState('');const [choices,setChoices]=useState<string[]>([]);const [busy,setBusy]=useState(false);const [error,setError]=useState('');const [more,setMore]=useState(false);const [revision,setRevision]=useState(0);
const {active}=usePageCache();
const showBusy=useDelayedIndicator(busy);
const contextKey=JSON.stringify(query);const current=useRef(0);
const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query;
const period={year,...(month?{month}:{}),...(quarter?{quarter}:{})};
const filters={...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)};
const contextKey=JSON.stringify({releaseId,period,filters,field:field.key,stage});const current=useRef(0);
useEffect(()=>{
if(!active)return;
const generation=++current.current,controller=new AbortController();setBusy(true);setError('');setChoices([]);
const timer=window.setTimeout(()=>{
const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query;
void api.sourceColumnValues({releaseId,period:{year,...(month?{month}:{}),...(quarter?{quarter}:{})},filters:{...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)},field:field.key,stages:[stage],search,limit:50,offset:0},controller.signal)
void api.sourceColumnValues({releaseId,period,filters,field:field.key,stages:[stage],search,limit:50,offset:0,candidateOnly:true},controller.signal)
.then(result=>{if(current.current!==generation)return;setChoices(result.values.filter((v):v is string=>typeof v==='string'));setMore(result.hasMore===true||(result.total??0)>50);})
.catch(failure=>{if(controller.signal.aborted||current.current!==generation)return;if(!onUnauthorized(failure))setError(failure instanceof Error?failure.message:'选项读取失败');})
.finally(()=>{if(current.current===generation)setBusy(false);});
@@ -26,10 +28,16 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S
},[active,field.key,stage,contextKey,search,revision,onUnauthorized]);
return <Select {...readableSelectPopup} mode="multiple" allowClear showSearch aria-label={`${field.label}筛选值`} placeholder="选择筛选值" value={value} searchValue={search} filterOption={false} onSearch={setSearch} loading={showBusy} maxTagCount="responsive" onChange={values=>{onChange(values);setSearch('');}} options={[...new Set([...choices,...value])].map(v=>({label:v,value:v}))} notFoundContent={showBusy?<Spin size="small"/>:error?<Button size="small" onClick={()=>setRevision(v=>v+1)}>读取失败,重试</Button>:'暂无匹配值'} popupRender={menu=><>{menu}{more&&<div className="query-options-hint">输入关键词搜索更多值</div>}{error&&<div role="alert" className="query-options-hint">{error}</div>}</>}/>;
}
function ContainsValues({field,value,onChange}:{field:SourceColumnField;value:string[];onChange:(value:string[])=>void}) {
const [search,setSearch]=useState('');
return <Select {...readableSelectPopup} mode="tags" maxCount={50} maxTagCount="responsive" allowClear aria-label={`${field.label}包含文字`} placeholder="输入关键词后按回车" value={value} searchValue={search}
onSearch={text=>setSearch(text.slice(0,256).replace(/[\uD800-\uDBFF]$/u,''))} onBlur={()=>setSearch('')}
onChange={values=>{onChange(values.filter(value=>value.trim().length>0&&value.length<=256));setSearch('');}} options={[]}/>;
}
export function StageSourceFilters({stage,fields,query,value,onChange,onUnauthorized,part='both'}:{part?:'both'|'picker'|'rows';stage:Stage;fields:SourceColumnField[];query:QueryParams;value:SourceColumnFilter[];onChange:(value:SourceColumnFilter[])=>void;onUnauthorized:(error:unknown)=>boolean}) {
const selected=value.map(rule=>rule.field);const patch=(key:string,change:Partial<SourceColumnFilter>)=>onChange(value.map(rule=>rule.field===key?{...rule,...change}:rule));
return <div className="stage-source-filters">
{part!=='rows'&&<Select {...readableSelectPopup} mode="multiple" maxCount={20} maxTagCount="responsive" showSearch optionFilterProp="label" allowClear aria-label="选择筛选字段" placeholder="添加筛选字段" value={selected} options={fields.map(field=>({label:field.label,value:field.key}))} onChange={keys=>onChange(keys.map(key=>value.find(rule=>rule.field===key)||{field:key,stages:[stage],operator:'in',values:[]}))}/>}
{part!=='picker'&&value.map(rule=>{const field=fields.find(item=>item.key===rule.field);return field?<div className="stage-source-filter-row" key={rule.field}><strong>{field.label}</strong><Select {...readableSelectPopup} aria-label={`${field.label}筛选方式`} value={rule.operator} options={[{label:'等于任一值',value:'in'},{label:'不等于这些值',value:'not_in'},{label:'为空',value:'is_empty'},{label:'不为空',value:'not_empty'}]} onChange={operator=>patch(rule.field,{operator,values:[]})}/>{['in','not_in'].includes(rule.operator)?<SourceValues field={field} stage={stage} query={query} value={rule.values||[]} onChange={values=>patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:<span/>}<Button type="text" aria-label={`移除${field.label}筛选`} icon={<CloseOutlined/>} onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}/></div>:<Alert key={rule.field} type="warning" message="所选来源字段当前不可用,请移除后重新选择" action={<Button onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}>移除</Button>}/>;})}
{part!=='picker'&&value.map(rule=>{const field=fields.find(item=>item.key===rule.field);return field?<div className="stage-source-filter-row" key={rule.field}><strong>{field.label}</strong><Select {...readableSelectPopup} aria-label={`${field.label}筛选方式`} value={rule.operator} options={[{label:'等于任一值',value:'in'},{label:'包含',value:'contains'},{label:'不等于这些值',value:'not_in'},{label:'为空',value:'is_empty'},{label:'不为空',value:'not_empty'}]} onChange={operator=>patch(rule.field,{operator,values:[]})}/>{rule.operator==='contains'?<ContainsValues field={field} value={rule.values||[]} onChange={values=>patch(rule.field,{values})}/>:['in','not_in'].includes(rule.operator)?<SourceValues field={field} stage={stage} query={query} value={rule.values||[]} onChange={values=>patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:<span/>}<Button type="text" aria-label={`移除${field.label}筛选`} icon={<CloseOutlined/>} onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}/></div>:<Alert key={rule.field} type="warning" message="所选来源字段当前不可用,请移除后重新选择" action={<Button onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}>移除</Button>}/>;})}
</div>;
}
+4 -4
View File
@@ -7,7 +7,7 @@ export type AnalysisVisualSectionKey = 'overview' | 'stage' | 'department' | 'gr
export type AnalysisVisualDimension = 'department' | 'group' | 'product' | 'category' | 'defect';
export type AnalysisExclusionDimension = 'department' | 'group' | 'product' | 'projectGroup';
export type AnalysisExclusions = Partial<Record<AnalysisExclusionDimension, string[]>>;
export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export type AnalysisVisualContextValue = string | number | boolean | Array<string | number | boolean> | Record<string, string[]> | Record<string, string> | SourceColumnFilter[] | AnalysisExclusions;
export type AnalysisVisualContext = Record<string, AnalysisVisualContextValue>;
@@ -253,8 +253,8 @@ function safeContext(value: unknown): AnalysisVisualContext | null {
|| typeof rule.field !== 'string' || !/^[A-Za-z0-9_]{1,64}$/.test(rule.field)
|| !Array.isArray(rule.stages) || !rule.stages.length || rule.stages.length > 2 || new Set(rule.stages).size !== rule.stages.length
|| rule.stages.some(stage => stage !== 'black' && stage !== 'bright')
|| !['in', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null;
const needsValues = rule.operator === 'in' || rule.operator === 'not_in';
|| !['in', 'contains', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null;
const needsValues = rule.operator === 'in' || rule.operator === 'not_in' || rule.operator === 'contains';
if (needsValues && (!Array.isArray(rule.values) || !rule.values.length || rule.values.length > 50 || rule.values.some(value => typeof value !== 'string' || !value.trim() || value.length > 256))) return null;
if (!needsValues && rule.values !== undefined) return null;
filters.push({ field: rule.field, stages: [...rule.stages] as SourceColumnFilter['stages'], operator: rule.operator as SourceColumnFilter['operator'], ...(needsValues ? { values: [...rule.values as string[]] } : {}) });
@@ -1397,7 +1397,7 @@ export function columnFilterSummary(value: unknown, labelValue: unknown): string
if (!isRecord(rule) || typeof rule.field !== 'string' || !Array.isArray(rule.stages)) return [];
const stage = rule.stages.map(item => item === 'black' ? '黑片' : item === 'bright' ? '亮片' : '').filter(Boolean).join('、');
const label = typeof labels[rule.field] === 'string' ? labels[rule.field] : '来源字段';
const operation = ({ in: '为', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record<string, string>)[String(rule.operator)];
const operation = ({ in: '为', contains: '包含', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record<string, string>)[String(rule.operator)];
if (!operation) return [];
const values = Array.isArray(rule.values) ? rule.values.filter(item => typeof item === 'string').join('、') : '';
return [`${stage} ${label}${operation}${values}`.trim()];
+17 -3
View File
@@ -1,3 +1,5 @@
import type { ProductOrder } from './productPagination';
import type { ProductPageResponse } from './types';
import { WORKSPACE_INVALIDATED } from './pageCache';
import { consumeEventStream, type StreamEvent } from './eventStream';
import { publicErrorMessage } from './publicDataDisplay';
@@ -110,7 +112,7 @@ export async function request<T>(path: string, init: RequestInit = {}, timeoutMs
method,
signal: init.signal ?? undefined,
timeoutMs: timeoutMs ?? requestTimeoutFor(path),
cache: method === 'GET' && isImmutableRead(path),
cache: method === 'GET' && init.cache !== 'reload' && isImmutableRead(path),
tags: requestResourceTags(path),
category: requestCategory(path),
priority: requestPriority(path),
@@ -122,7 +124,10 @@ export async function request<T>(path: string, init: RequestInit = {}, timeoutMs
}
catch (error) {
handleAccessFailure(error);
if (error instanceof Error && error.name === 'RequestTimeoutError') throw new ApiError(408, '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。');
if (error instanceof Error && error.name === 'RequestTimeoutError') {
const query = /^\/api\/(?:metrics(?:\/|$)|records(?:\/|$)|analysis\/(?:source\/|defects(?:\/|$)))/.test(requestPath(path));
throw new ApiError(408, query ? '查询处理超时,请重试或调整筛选范围。' : '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。');
}
throw error;
}
}
@@ -236,7 +241,7 @@ export const api = {
},
activateSalesOrderTypes(versionId:string,expectedVersion:string) { return mutate<{versionId:string}>('/api/sales-order-types/activate',{versionId,expectedVersion}); },
sourceColumnCatalog(releaseId?: string, signal?: AbortSignal) { return request<{releaseId:string;sourceFields:SourceColumnField[]}>(`/api/analysis/source/catalog?${toQuery({releaseId})}`, { signal }); },
sourceColumnValues(body: Record<string,unknown>, signal?: AbortSignal) { return queryPost<{values:Array<string|null>;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); },
sourceColumnValues(body: Record<string,unknown> & {candidateOnly?:true}, signal?: AbortSignal) { return queryPost<{values:Array<string|null>;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); },
workspaceState() { return request<{ authRevision: string; dataRevision: string; currentReleaseId: string | null }>('/api/workspace/state', {}, 8000); },
filterFields(manage = false, signal?: AbortSignal) { return request<FilterField[]>(`/api/filter-fields?manage=${manage}`, { signal }); },
filterFieldOptions(key: string, search = '', signal?: AbortSignal,releaseId?:string,stage?:string) { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search,releaseId,stage})}`, { signal }); },
@@ -268,6 +273,15 @@ export const api = {
metrics(params: QueryParams, signal?: AbortSignal) {
return request<MetricsResponse>(`/api/metrics?${toQuery(params)}`, { signal });
},
productPage(params:QueryParams,page:number,pageSize:number,order:ProductOrder,signal?:AbortSignal) {
return request<ProductPageResponse>(`/api/metrics/products?${toQuery({...params,page,pageSize,order})}`,{signal});
},
exportProductRows(params:QueryParams,rowOrder:ProductOrder,signal?:AbortSignal) {
return request<MetricsResponse>(`/api/metrics?${toQuery({...params,part:'table',rowOrder})}`,{signal,cache:'reload'});
},
metricsTable(params: QueryParams, signal?: AbortSignal) {
return request<MetricsResponse>(`/api/metrics?${toQuery({ ...params, part: 'table' })}`, { signal });
},
trend(params: QueryParams, signal?: AbortSignal) {
return request<TrendResponse>(`/api/metrics/trend?${toQuery(params)}`, { signal });
},
+29
View File
@@ -0,0 +1,29 @@
import type { MetricsResponse, QueryParams, TrendResponse } from './types';
/** 仅调整分组视图,保留全部来源、期间和权限筛选。 */
export function overviewQuery(query: QueryParams): QueryParams {
return { ...query, view: 'overall' };
}
/** 宽范围产品表先取小型汇总;窄范围复用完整结果,避免额外往返。 */
export function splitMetricsLoad(query: QueryParams): boolean {
return query.view === 'product' && !query.month && !query.quarter && !query.product
&& !query.productSearch && !query.monthStart && !query.monthEnd;
}
function canonical(value: unknown): unknown {
if (Array.isArray(value)) return value.map(canonical);
if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value)
.filter(([, item]) => item !== undefined).sort(([a], [b]) => a.localeCompare(b))
.map(([key, item]) => [key, canonical(item)]));
return value;
}
/** 筛选范围和粒度全部一致时,才复用服务端已计算的期间结果。 */
export function sameTrendScope(a: QueryParams, b: QueryParams): boolean {
return JSON.stringify(canonical(overviewQuery(a))) === JSON.stringify(canonical(overviewQuery(b)));
}
export function overviewTrend(value: MetricsResponse): TrendResponse {
return { release: value.release, context: value.context, trend: value.trend, coverage: value.coverage };
}
+19
View File
@@ -0,0 +1,19 @@
import { splitMetricsLoad } from './priorityMetrics.ts';
import type { QueryParams } from './types';
export type ProductOrder = 'brightQualifiedDesc' | 'blackTotalDesc' | 'brightTotalDesc' | 'productAsc';
export const PRODUCT_PAGE_SIZE = 15;
/** 已限定产品的少量结果保留原路径;宽产品表按数据库页读取。 */
export function shouldPageProducts(query: QueryParams): boolean {
return query.view === 'product' && !!query.releaseId && !query.product;
}
export function pageSelection(scope: string, selected: { scope: string; page: number; size: number }) {
return { page: selected.scope === scope ? selected.page : 1, size: selected.size };
}
/** 宽年度先显示摘要;月/季/搜索并行读取小页,避免窄筛选多一次串行等待。 */
export function canReadProductPage(query: QueryParams, overviewSettled: boolean): boolean {
return overviewSettled || !splitMetricsLoad(query);
}
+6 -1
View File
@@ -3,7 +3,7 @@ export type ViewKind = 'overall' | 'department' | 'group' | 'product' | 'project
export type PeriodKind = 'year' | 'quarter' | 'month';
export interface SourceColumnField { key: string; label: string; type: string; stages: Stage[]; columns: Partial<Record<Stage,string>> }
export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export interface QueryParams {
year: number;
@@ -25,6 +25,11 @@ export interface QueryParams {
trendGranularity?: 'month' | 'quarter' | 'year';
}
export interface ProductPageResponse {
release: {id:string}; context:Record<string,unknown>; rows:MetricRow[];
total:number; page:number; pageSize:number; order:string;
}
export interface RecordsParams extends QueryParams {
stage: Stage;
page: number;
+5 -4
View File
@@ -4,8 +4,8 @@ import { api } from './api';
import { defectScope } from './defectDrilldown';
import type { DefectDrilldownResponse, FineDefectCatalogResponse, QueryParams, Stage } from './types';
export function useDefectDrilldown({active,mode,stage,query,trendQuery,tableQuery,tableDimension,tableGranularity,onUnauthorized,display,onDisplay,refreshKey=0}:{
refreshKey?:string|number;active:boolean;mode:string;stage:Stage;query:QueryParams;trendQuery:QueryParams;tableQuery:QueryParams;
export function useDefectDrilldown({active,mode,stage,query,trendQuery,tableQuery,tableDimension,tableGranularity,onUnauthorized,display,onDisplay,refreshKey=0,productKeys,tableEnabled=true}:{
productKeys?:string[];tableEnabled?:boolean;refreshKey?:string|number;active:boolean;mode:string;stage:Stage;query:QueryParams;trendQuery:QueryParams;tableQuery:QueryParams;
tableDimension:string;tableGranularity:string;onUnauthorized:(error:unknown)=>boolean;display:'weight'|'rate';onDisplay:(display:'weight'|'rate')=>void;
}) {
const [selection,setSelection]=useState<{stage:Stage;category:string}|null>(null);
@@ -18,7 +18,8 @@ export function useDefectDrilldown({active,mode,stage,query,trendQuery,tableQuer
const catalogKey=JSON.stringify([enabled,stage,defectScope(query),revision,refreshKey]);
const common={stage,category,releaseId:query.releaseId};
const trend=useDefectPart(enabled&&!!category&&!!query.releaseId,{...defectScope(trendQuery),...common,dimension:'period',trendGranularity:trendQuery.trendGranularity||'month'},[revision,refreshKey],onUnauthorized);
const table=useDefectPart(enabled&&!!category&&!!query.releaseId,{...defectScope(tableQuery),...common,dimension:tableDimension,trendGranularity:tableGranularity},[revision,refreshKey],onUnauthorized);
const exportParams={...defectScope(tableQuery),...common,dimension:tableDimension,trendGranularity:tableGranularity};
const table=useDefectPart(enabled&&!!category&&!!query.releaseId&&tableEnabled,{...exportParams,...(productKeys?{productKeys}:{})},[revision,refreshKey],onUnauthorized);
useEffect(()=>{
if(!enabled||!query.releaseId)return;
const controller=new AbortController();setCatalog(null);setCatalogFailure('');
@@ -39,7 +40,7 @@ export function useDefectDrilldown({active,mode,stage,query,trendQuery,tableQuer
const retry=<Button size="small" onClick={()=>setRevision(v=>v+1)}>重试</Button>;
const controls=category?<div className="defect-drilldown-controls"><Button type="link" onClick={clear}>全部分类</Button><span>› {category}</span>{trend.pending&&<span><Spin size="small"/> 正在读取细分</span>}{trend.error&&<Alert type="error" showIcon message={trend.error} action={retry}/>} {incomplete&&<span className="defect-drilldown-notice">细分来源未完全覆盖分类总量</span>}</div>:catalogFailure&&enabled?<Alert type="error" showIcon message="细分目录读取失败" description={catalogFailure} action={<Button size="small" onClick={()=>setRevision(v=>v+1)}>重试</Button>}/>:null;
const tableControls=table.error?<Alert type="error" showIcon message={table.error} action={retry}/>:null;
return {category,categories,choose,clear,controls,tableControls,pending,error,trend:trend.data,table:table.data,trendPending:trend.pending,tablePending:table.pending,trendError:trend.error,tableError:table.error};
return {category,categories,choose,clear,controls,tableControls,pending,error,exportTable:()=>category?api.defectDrilldown(exportParams):Promise.resolve(undefined),trend:trend.data,table:table.data,trendPending:trend.pending,tablePending:table.pending,trendError:trend.error,tableError:table.error};
}
+24
View File
@@ -0,0 +1,24 @@
import { useEffect, useState } from 'react';
import { api } from './api';
import { overviewQuery, splitMetricsLoad } from './priorityMetrics';
import type { MetricsResponse, QueryParams } from './types';
/** 优先读取正式小型汇总;失败时允许后续正式查询独立回退。 */
export function usePriorityMetrics(query: QueryParams | null, enabled: boolean, refresh: string | number,
onUnauthorized: (error: unknown) => boolean, forceOverview = false) {
const key = JSON.stringify([query, refresh, forceOverview]);
const split = !!query && (forceOverview || splitMetricsLoad(query));
const [result, setResult] = useState<{ key: string; value?: MetricsResponse; failed?: boolean } | null>(null);
useEffect(() => {
if (!enabled || !query) return;
const controller = new AbortController();
api.metrics(split ? overviewQuery(query) : query, controller.signal)
.then(value => { if (!controller.signal.aborted) setResult({ key, value }); })
.catch(error => {
if (!controller.signal.aborted && !onUnauthorized(error)) setResult({ key, failed: true });
});
return () => controller.abort();
}, [enabled, key, onUnauthorized]);
const current = enabled && result?.key === key ? result : null;
return { value: current?.value || null, settled: !!current, failed: !!current?.failed, split };
}
+40
View File
@@ -0,0 +1,40 @@
import { useEffect, useState } from 'react';
import { api } from './api';
import { pageSelection, PRODUCT_PAGE_SIZE, type ProductOrder } from './productPagination';
import type { ProductPageResponse, QueryParams } from './types';
/** 查询范围变化立即使用第一页,避免先发出旧页码请求。 */
export function useProductPage(query: QueryParams, enabled: boolean, refresh: string | number,
order: ProductOrder, onUnauthorized: (error: unknown) => boolean) {
const scope = JSON.stringify([query, refresh, order]);
const [selected, setSelected] = useState({ scope, page: 1, size: PRODUCT_PAGE_SIZE });
const selection = pageSelection(scope, selected);
const key = JSON.stringify([scope, selection.page, selection.size]);
const [result, setResult] = useState<{ key: string; scope: string; value?: ProductPageResponse; error?: string } | null>(null);
useEffect(() => {
if (!enabled) return;
const controller = new AbortController();
api.productPage(query, selection.page, selection.size, order, controller.signal).then(value => {
if (controller.signal.aborted) return;
if (value.release.id !== query.releaseId || value.page !== selection.page || value.pageSize !== selection.size || value.order !== order)
throw Error('分页查询范围已变化,请重试。');
const last = Math.max(1, Math.ceil(value.total / selection.size));
if (selection.page > last) { setSelected({ scope, page: last, size: selection.size }); return; }
setResult({ key, scope, value });
}).catch(error => {
if (!controller.signal.aborted && !onUnauthorized(error)) setResult({ key, scope, error: error instanceof Error ? error.message : '表格读取失败' });
});
return () => controller.abort();
}, [enabled, key, onUnauthorized]);
const current = enabled && result?.key === key ? result : null;
return {
key, value: current?.value || null, error: current?.error || '',
pagination: { current: selection.page, pageSize: selection.size, total: result?.scope === scope ? result.value?.total || 0 : 0,
onChange: (page: number, size: number) => setSelected({ scope, page: size === selection.size ? page : 1, size }) },
loadAll: async () => {
const value = await api.exportProductRows(query, order);
if (value.release?.id !== query.releaseId) throw Error('导出数据版本已变化,请重试。');
return value.rows;
},
};
}
@@ -4,7 +4,7 @@ import fs from 'node:fs';
import ts from 'typescript';
import React from 'react';
import { renderToStaticMarkup } from 'react-dom/server';
import { buildAgentReportLayout, agentReportPrimary, agentReportSupplementary } from '../src/agent/agentReportLayout.ts';
import { buildAgentReportLayout, agentReportPrimary, agentReportSupplementary, partitionAgentReportVisuals } from '../src/agent/agentReportLayout.ts';
import { stripDuplicateFirstMarkdownH1 } from '../src/analysisReportTitle.ts';
import { tableTextAlignment } from '../src/tableTextAlignment.ts';
import { sanitizeAnalysisDisplayText } from '../src/analysisDisplayText.ts';
@@ -26,7 +26,7 @@ const Document=load('../src/agent/AgentDeliverable.tsx','AgentDeliverable',{
ReportVisualFigure:({visual,image})=>image?.svg?React.createElement('figure',{'data-chart':visual.id},visual.title):React.createElement('table',{'data-fallback':visual.id},React.createElement('tbody',null,React.createElement('tr',null,React.createElement('td',null,visual.title)))),
reportVisualImageSource:image=>image?.svg||'',reportVisuals:()=>figures,
useReportVisualImages:()=>({authorized:true,loading:false,images}),analysisSnapshotResult:turn=>turn.result,
reportSnapshotIdentity:()=>'',analysisScopeText:()=>'',stripDuplicateFirstMarkdownH1,buildAgentReportLayout,agentReportPrimary,agentReportSupplementary,
reportSnapshotIdentity:()=>'',analysisScopeText:()=>'',stripDuplicateFirstMarkdownH1,buildAgentReportLayout,agentReportPrimary,agentReportSupplementary,partitionAgentReportVisuals,
captureReportLongImage:()=>{},downloadAuthorizedReport:()=>{},documentCss:'',
});
const answer='# 原始报告\n\n原始导读。\n\n## 月度趋势\n\n原始分析。\n\n| 月份 | 值 |\n|---|---|\n| 10月 | 95.46% |\n\n## 报废大类\n\n原始说明。';
@@ -5,6 +5,8 @@ import ts from 'typescript';
import React from 'react';
import { renderToStaticMarkup } from 'react-dom/server';
import { emptyAnalysisNotice } from '../src/analysisConversationState.ts';
import { analysisChartReferences } from '../src/analysisChartReferences.ts';
import { parseAnalysisVisuals } from '../src/analysisVisualsModel.ts';
import { sanitizeAnalysisDisplayText } from '../src/analysisDisplayText.ts';
import {
analysisMemberListFullyRead,
@@ -116,7 +118,7 @@ test('member-list display is opt-in and historical plain answers do not acquire
const start = source.indexOf('export const AgentConversationTurn');
const code = source.slice(start).replace('export const AgentConversationTurn', 'const AgentConversationTurn');
const view = {
React, memo: value => value, useMemo: callback => callback(), useState: value => [value, () => {}], useEffect: () => {},
React, memo: value => value, useCallback: callback => callback, analysisChartReferences, parseAnalysisVisuals, useMemo: callback => callback(), useState: value => [value, () => {}], useEffect: () => {},
useAnalysisVisualRecovery: turn => ({ turn, available: false, recovering: false, error: turn.recoveryError || '' }),
analysisSnapshotResult: turn => turn.result || turn, partialAnswerIsOnlyProgress: () => false, analysisIsRunning: () => false,
executionEvents: () => [], RunStatus: () => null, authoredReport: () => null, analysisScopeText: () => '', emptyAnalysisNotice,
@@ -140,6 +142,7 @@ return AgentConversationTurn;`, {
assert.match(empty, /<button>重新分析<\/button>/);
assert.match(render({id:'failed-turn',status:'FAILED',answer:'',completion:{reason:'任务理解未完成,请重试'}}), /任务理解未完成,请重试/);
const recovery = render({ id: 'old-turn', status: 'PARTIAL', answer: '原答案', recoveryError: '图表恢复暂不可用' });
assert.match(recovery, /role="alert">图表恢复暂不可用/);
// 图表恢复失败按正式约定静默省略,保留原回答。
assert.doesNotMatch(recovery, /role="alert"|图表恢复暂不可用/);
assert.match(recovery, /原答案/);
});
@@ -1042,3 +1042,12 @@ test('adaptive trends cover every series, preserve missing values, and keep non-
assert.equal(formatVisualValue(.9639601707533932, 'ratio'), '96.40%');
assert.equal(formatVisualValue(3, 'kg'), '3.00 kg');
});
test('contains keywords survive analysis scope and display as substring filters',()=>{
const rule={field:'src_any',stages:['bright'],operator:'contains',values:['日立','%_']};
const parsed=parseAnalysisVisuals([visual({scope:{columnFilters:[rule],columnLabels:{src_any:'客户'}}})]);
assert.equal(parsed.visuals.length,1);
assert.deepEqual(parsed.visuals[0].scope.columnFilters,[rule]);
assert.match(columnFilterSummary([rule],{src_any:'客户'})[0],/包含.*日立/);
assert.equal(parseAnalysisVisuals([visual({scope:{columnFilters:[{...rule,values:[]}]}})]).visuals.length,0);
});
@@ -90,3 +90,12 @@ test('changing only the trend object does not recalculate the unchanged product
assert.equal(f.requests.length,3);assert.equal(f.requests[2].params.dimension,'period');assert.equal(ui.table?.dimension,'product');
assert.equal(ui.trend,undefined);assert.equal(ui.trendPending,true);f.unmount();
});
test('product page changes reload only fine table rows and export remains the complete scope',async()=>{
const f=fixture();f.props.productKeys=['P1'];f.render();f.catalog[0].resolve(catalogData);await tick();f.render().choose('加工废');f.render();
assert.deepEqual(f.requests[1].params.productKeys,['P1']);assert.equal(f.requests[0].params.productKeys,undefined);
f.requests[0].resolve({...response,dimension:'period'});f.requests[1].resolve({...response,dimension:'product'});await tick();f.render();
f.props.productKeys=['P2'];f.render();assert.equal(f.requests.length,3);assert.deepEqual(f.requests[2].params.productKeys,['P2']);assert.equal(f.render().trend?.dimension,'period');
const promise=f.render().exportTable();assert.equal(f.requests[3].params.productKeys,undefined);assert.equal(f.requests[3].params.dimension,'product');f.requests[3].resolve(response);await promise;f.unmount();
});
@@ -158,7 +158,7 @@ test('single-stage detail callback freezes the clicked quarterly/yearly row inst
for(const stage of ['black','bright']) {
const ui=new Function('freezeSourceRecordsQuery','stage',transpile(`
const applied={year:2026,month:10,product:'产品A',stage,columnFilters:[{field:'customer',stages:[stage],operator:'in',values:['001']}]};
const current={release:{id:'release-A'}},drilledProduct='产品A';let records;
const currentSummary={release:{id:'release-A'}},drilledProduct='产品A';let records;
const setRecordVisible=()=>{},setRecords=value=>records=value;
${callback}
return {openDetail,get:()=>records};
+10
View File
@@ -248,3 +248,13 @@ test('fine defect expansion and Excel share authorized parent/child values, pres
assert.equal(fields.some(f=>f.key.startsWith('black-defect-')||f.key.startsWith('black-fine-')),false);
assert.equal(data.groups[0].children[0].rate,0);
});
test('remote page two is not sliced again and full export includes every row',async()=>{
const all=Array.from({length:35},(_,i)=>({...row,key:String(i),label:'产品'+i}));let changed;
const ui=fixture({rows:all.slice(15,30),view:'product',remotePage:{current:2,pageSize:15,total:35,onChange:(...args)=>changed=args},loadExportData:async()=>({rows:all})});
assert.equal(find(ui.tree,n=>n.type==='Table').props.dataSource.length,15);
const pagination=find(ui.tree,n=>n.type==='Pagination');assert.equal(pagination.props.total,35);assert.equal(pagination.props.current,2);
pagination.props.onChange(3,15);assert.deepEqual(changed,[3,15]);
await ui.ref.exportExcel();assert.equal(ui.exported.workbook.getWorksheet('质量汇总').rowCount,37);assert.equal(ui.exported.rows[0].key,'0');assert.equal(ui.exported.rows.at(-1).key,'34');
});
+113
View File
@@ -0,0 +1,113 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
import { overviewQuery, sameTrendScope, overviewTrend, splitMetricsLoad } from '../src/priorityMetrics.ts';
const query = { releaseId: 'r1', year: 2026, view: 'product', trendGranularity: 'month', stage: 'bright',
columnFilters: [{ field: 'customer', operator: 'contains', values: ['Acme'], stages: ['bright'] }] };
const tick = () => new Promise(resolve => setImmediate(resolve));
function fixture() {
const slots = [], effects = [], requests = []; let index = 0, dirty = false, pending = [];
const props = { query, enabled: true, refresh: 0 }; const unauthorized = [];
const hooks = {
useState(initial) { const n = index++; if (!(n in slots)) slots[n] = initial;
return [slots[n], value => { slots[n] = value; dirty = true; }]; },
useEffect(fn, deps) { const n = index++, old = effects[n];
if (!old || deps.some((d, i) => !Object.is(d, old.deps[i]))) pending.push(() => {
old?.cleanup?.(); effects[n] = { deps, cleanup: fn() };
}); },
};
const api = { metrics(params, signal) { let resolve, reject;
const promise = new Promise((r, j) => { resolve = r; reject = j; });
requests.push({ params, signal, resolve, reject }); return promise;
} };
const source = fs.readFileSync(new URL('../src/usePriorityMetrics.ts', import.meta.url), 'utf8');
const code = ts.transpileModule(source.slice(source.indexOf('export function')).replace('export function', 'function'),
{ compilerOptions: { target: ts.ScriptTarget.ES2022 } }).outputText;
const hook = new Function('hooks', 'api', 'overviewQuery', 'splitMetricsLoad', `const {useState,useEffect}=hooks;${code};return usePriorityMetrics;`)(hooks, api, overviewQuery, splitMetricsLoad);
const onUnauthorized = error => { unauthorized.push(error); return error.status === 403; };
return { props, requests, unauthorized, render() { let result, count = 0; do {
dirty = false; index = 0; pending = []; result = hook(props.query, props.enabled, props.refresh, onUnauthorized, props.forceOverview);
pending.forEach(run => run()); assert.ok(++count < 10);
} while (dirty); return result; }, unmount() { effects.forEach(e => e?.cleanup?.()); } };
}
test('overview retains every source predicate and does not mutate the table query', () => {
const small = overviewQuery(query);
assert.deepEqual(small, { ...query, view: 'overall' }); assert.equal(query.view, 'product');
assert.equal(small.columnFilters, query.columnFilters);
});
test('trend reuse requires the same release, time range, grain, stage and all filters', () => {
assert.equal(sameTrendScope(query, { ...query, view: 'overall', month: undefined }), true);
for (const changed of [{ month: 1 }, { quarter: 2 }, { monthStart: 2, monthEnd: 5 }, { year: undefined },
{ releaseId: 'r2' }, { trendGranularity: 'quarter' }, { stage: 'black' }, { includeUnassigned: false },
{ product: 'P1' }, { extraFilters: { order: ['X'] } }, { columnFilters: [] }]) {
assert.equal(sameTrendScope(query, { ...query, ...changed }), false, JSON.stringify(changed));
}
assert.equal(sameTrendScope(query, Object.fromEntries(Object.entries(query).reverse())), true);
});
test('trend adapter uses the exact formal values and performs no rate arithmetic', () => {
const value = { release: { id: 'r1' }, context: {}, trend: [{ compositeRate: .123456789 }], coverage: {} };
assert.equal(overviewTrend(value).trend, value.trend);
});
test('priority gate opens immediately after the small server aggregate and stays closed while loading', async () => {
const f = fixture(); assert.equal(f.render().settled, false);
assert.equal(f.requests.length, 1); assert.equal(f.requests[0].params.view, 'overall');
const value = { release: { id: 'r1' }, summary: { compositeRate: .9 }, trend: [] };
f.requests[0].resolve(value); await tick(); const result = f.render();
assert.equal(result.settled, true); assert.equal(result.value, value);
f.render(); assert.equal(f.requests.length, 1); f.unmount();
});
test('year switches and refresh invalidate the gate and late old replies cannot replace the new result', async () => {
const f = fixture(); f.render(); f.props.query = { ...query, year: 2025 };
assert.equal(f.render().settled, false); assert.equal(f.requests[0].signal.aborted, true);
f.requests[0].resolve({ summary: 'old' }); await tick(); assert.equal(f.render().value, null);
f.requests[1].resolve({ summary: 'new' }); await tick(); assert.equal(f.render().value.summary, 'new');
f.props.refresh++; assert.equal(f.render().settled, false); assert.equal(f.requests.length, 3); f.unmount();
});
test('overview errors allow the regular query to retry independently', async () => {
const f = fixture(); f.render(); f.requests[0].reject(Error('temporary failure')); await tick();
assert.deepEqual(f.render(), { value: null, settled: true, failed: true, split: true }); f.unmount();
});
test('authorization failures do not unlock further requests', async () => {
const f = fixture(); f.render(); f.requests[0].reject({ status: 403 }); await tick();
assert.equal(f.render().settled, false); assert.equal(f.unauthorized.length, 1); f.unmount();
});
test('inactive pages neither start requests nor accept late results', async () => {
const f = fixture(); f.props.enabled = false; f.render(); assert.equal(f.requests.length, 0);
f.props.enabled = true; f.render(); f.props.enabled = false; f.render();
assert.equal(f.requests[0].signal.aborted, true); f.requests[0].resolve({ summary: 'late' });
await tick(); assert.equal(f.render().value, null); f.unmount();
});
test('narrow queries reuse a complete response rather than adding a summary round trip', async () => {
for (const change of [{month:2},{quarter:1},{product:'P'},{productSearch:'P'},{view:'department'},{view:'overall'}]) {
const f=fixture();f.props.query={...query,...change};const first=f.render();
assert.equal(first.split,false);assert.deepEqual(f.requests[0].params,f.props.query);
f.requests[0].resolve({summary:'full',rows:[1],trend:[2]});await tick();
assert.equal(f.render().value.rows.length,1);assert.equal(f.requests.length,1);f.unmount();
}
});
test('refresh generations cannot qualify retained old table and trend results as current', () => {
const source=name=>fs.readFileSync(new URL(`../src/${name}`,import.meta.url),'utf8');
const general=source('QueryPage.tsx');
const metricExpression=general.match(/const metricKey = useMemo\(\(\) => ([^;]+), \[metricParams, queryRevision[^\]]*\]\);/)[1];
const key=new Function('metricParams','queryRevision',`const productPagingScope=true,productPage={key:'page2'};return ${metricExpression}`);
assert.notEqual(key(query,0),key(query,1));
const stage=source('StageProductPage.tsx');
const expression=stage.match(/const queryKey=([^;]+);/)[1];
const stageKey=new Function('applied','revision','refreshKey',`const productPagingScope=true,productPage={key:'page2'};return ${expression}`);
assert.notEqual(stageKey(query,0,0),stageKey(query,0,1));
assert.notEqual(stageKey(query,0,0),stageKey(query,1,0));
});
test('paged month and search tables keep their first response small',async()=>{
for(const change of [{month:2},{quarter:1},{productSearch:'P'}]) {
const f=fixture();f.props.query={...query,...change};f.props.forceOverview=true;
assert.equal(f.render().split,true);assert.equal(f.requests[0].params.view,'overall');
for(const key of Object.keys(change))assert.equal(f.requests[0].params[key],change[key]);f.unmount();
}
});
@@ -0,0 +1,67 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
import {shouldPageProducts,pageSelection,PRODUCT_PAGE_SIZE,canReadProductPage} from '../src/productPagination.ts';
const query={releaseId:'r1',year:2026,view:'product'};
const tick = () => new Promise(resolve => setImmediate(resolve));
function fixture() {
const slots = [], effects = [], requests = []; let index = 0, dirty = false, pending = [];
const props = { query, enabled: true, refresh: 0 }; const unauthorized = [];
const hooks = {
useState(initial) { const n = index++; if (!(n in slots)) slots[n] = initial;
return [slots[n], value => { slots[n] = value; dirty = true; }]; },
useEffect(fn, deps) { const n = index++, old = effects[n];
if (!old || deps.some((d, i) => !Object.is(d, old.deps[i]))) pending.push(() => {
old?.cleanup?.(); effects[n] = { deps, cleanup: fn() };
}); },
};
const api = { productPage(params,page,pageSize,order,signal) { let resolve, reject;
const promise = new Promise((r, j) => { resolve = r; reject = j; });
requests.push({ params, page, pageSize, order, signal, resolve, reject }); return promise;
} };
const source = fs.readFileSync(new URL('../src/useProductPage.ts', import.meta.url), 'utf8');
const code = ts.transpileModule(source.slice(source.indexOf('export function')).replace('export function', 'function'),
{ compilerOptions: { target: ts.ScriptTarget.ES2022 } }).outputText;
const hook = new Function('hooks', 'api', 'pageSelection', 'PRODUCT_PAGE_SIZE', `const {useState,useEffect}=hooks;${code};return useProductPage;`)(hooks, api, pageSelection, PRODUCT_PAGE_SIZE);
const onUnauthorized = error => { unauthorized.push(error); return error.status === 403; };
return { props, requests, unauthorized, render() { let result, count = 0; do {
dirty = false; index = 0; pending = []; result = hook(props.query, props.enabled, props.refresh, 'brightQualifiedDesc', onUnauthorized);
pending.forEach(run => run()); assert.ok(++count < 10);
} while (dirty); return result; }, unmount() { effects.forEach(e => e?.cleanup?.()); } };
}
const response=(request,total=80)=>({release:{id:request.params.releaseId},rows:[{key:'P'}],total,page:request.page,pageSize:request.pageSize,order:request.order});
test('only unpinned product tables use server pagination',()=>{
assert.equal(shouldPageProducts(query),true);
for(const extra of [{view:'overall'},{product:'P'},{releaseId:undefined}])assert.equal(shouldPageProducts({...query,...extra}),false);
assert.equal(shouldPageProducts({...query,month:2,productSearch:'x'}),true);
});
test('page changes fetch only the selected page; year and size changes reset immediately',async()=>{
const f=fixture();let value=f.render();assert.equal(f.requests[0].page,1);
f.requests[0].resolve(response(f.requests[0]));await tick();value=f.render();assert.equal(value.pagination.total,80);
value.pagination.onChange(3,15);f.render();assert.equal(f.requests[1].page,3);assert.equal(f.render().value,null);
f.requests[1].resolve(response(f.requests[1]));await tick();value=f.render();
value.pagination.onChange(3,50);f.render();assert.equal(f.requests[2].page,1);assert.equal(f.requests[2].pageSize,50);
f.props.query={...query,year:2025};f.render();assert.equal(f.requests[3].page,1);assert.equal(f.requests[3].pageSize,50);
assert.equal(f.requests[2].signal.aborted,true);f.requests[2].resolve(response(f.requests[2]));await tick();assert.equal(f.render().value,null);f.unmount();
});
test('empty out-of-range pages recover to the valid last page',async()=>{
const f=fixture();f.render().pagination.onChange(9,15);f.render();f.requests[1].resolve(response(f.requests[1],20));await tick();f.render();
assert.equal(f.requests[2].page,2);f.requests[2].resolve(response(f.requests[2],20));await tick();assert.equal(f.render().pagination.current,2);f.unmount();
});
test('refresh, inactive pages, errors and authorization cannot qualify stale rows',async()=>{
const f=fixture();f.render();f.requests[0].resolve(response(f.requests[0]));await tick();assert.ok(f.render().value);
f.props.refresh++;assert.equal(f.render().value,null);f.requests[1].reject(Error('retry'));await tick();assert.equal(f.render().error,'retry');
f.props.refresh++;f.render();f.requests[2].reject({status:403});await tick();assert.equal(f.unauthorized.filter(e=>e.status===403).length,1);assert.equal(f.render().value,null);
f.props.enabled=false;f.render();assert.equal(f.requests[2].signal.aborted,true);f.unmount();
});
test('wrong release replies are rejected instead of displayed',async()=>{
const f=fixture();f.render();f.requests[0].resolve({...response(f.requests[0]),release:{id:'old'}});await tick();assert.match(f.render().error,/范围已变化/);f.unmount();
});
test('annual aggregate keeps priority while narrow pages avoid a serial round trip',()=>{
assert.equal(canReadProductPage(query,false),false);assert.equal(canReadProductPage(query,true),true);
for(const narrow of [{month:2},{quarter:1},{productSearch:'P'},{monthStart:1,monthEnd:5}])assert.equal(canReadProductPage({...query,...narrow},false),true);
});
+36
View File
@@ -0,0 +1,36 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
const source=fs.readFileSync(new URL('../src/StageSourceFilters.tsx',import.meta.url),'utf8');
const output=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX,target:ts.ScriptTarget.ES2022}}).outputText;
let lastSearch;
const jsx=(type,props)=>({type,props});const exports={};
new Function('require','exports',output)(name=>{
if(name==='react/jsx-runtime')return {jsx,jsxs:jsx};
if(name==='react')return {useState:initial=>[initial,value=>{lastSearch=value;} ]};
if(name==='antd')return {Select:'Select',Button:'Button',Alert:'Alert',Spin:'Spin'};
if(name==='@ant-design/icons')return {CloseOutlined:'CloseOutlined'};
if(name==='./readableSelectPopup')return {readableSelectPopup:{}};
return {};
},exports);
const nodes=value=>!value||typeof value!=='object'?[]:[value,...[value.props?.children].flat(Infinity).flatMap(nodes)];
const field={key:'customer',label:'客户名称',type:'source_value',stages:['bright'],columns:{bright:'客户名称'}};
test('contains is selectable and accepts bounded free keywords without a value-candidate lookup',()=>{
let changed;
const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立']};
const root=exports.StageSourceFilters({stage:'bright',fields:[field],query:{year:2026},value:[rule],onChange:value=>{changed=value;},onUnauthorized:()=>false});
const all=nodes(root);const operation=all.find(x=>x.type==='Select'&&x.props['aria-label']==='客户名称筛选方式');
assert.ok(operation.props.options.some(x=>x.value==='contains'&&x.label==='包含'));
const free=all.find(x=>typeof x.type==='function'&&x.type.name==='ContainsValues');
const keywords=free.type(free.props);
assert.equal(keywords.props.mode,'tags');assert.equal(keywords.props.maxCount,50);assert.deepEqual(keywords.props.options,[]);
keywords.props.onSearch('x'.repeat(257));assert.equal(lastSearch.length,256);
keywords.props.onSearch('x'.repeat(255)+'😀');assert.equal(lastSearch,'x'.repeat(255));
keywords.props.onSearch('日立');keywords.props.onBlur();assert.equal(lastSearch,'');
keywords.props.onChange(['日立','%_',' ']);
assert.deepEqual(changed,[{...rule,values:['日立','%_']}]);
operation.props.onChange('in');assert.deepEqual(changed,[{...rule,operator:'in',values:[]}]);
});
@@ -0,0 +1,53 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
const read=name=>fs.readFileSync(new URL(`../src/${name}`,import.meta.url),'utf8');
test('stage product common queries use a resolved release and do not wait for source catalog when unfiltered',()=>{
const source=read('StageProductPage.tsx');
const releaseExpression=source.match(/const queryReleaseId=([^;]+);/)[1];
const resolvedRelease=new Function('initial','currentReleaseId','catalog',`return ${releaseExpression}`);
assert.equal(resolvedRelease({releaseId:'historical'},'current',{releaseId:'catalog'}),'historical');
assert.equal(resolvedRelease({},'current',{releaseId:'catalog'}),'current');
assert.equal(resolvedRelease({},null,{releaseId:'stale'}),undefined);
assert.equal(resolvedRelease({},undefined,{releaseId:'resolved'}),'resolved');
const expression=source.match(/const queryReady=\(query:QueryParams\)=>((?:[^;]|\n)+);/)[1];
const ready=new Function('query','queryReleaseId','catalog','options',`return ${expression}`);
assert.equal(ready({releaseId:'r1',columnFilters:[]},'r1',null,null),true);
assert.equal(ready({releaseId:'r1',columnFilters:[]},undefined,null,null),false);
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',null,null),false);
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',{releaseId:'r1'},null),true);
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r2',{releaseId:'r1'},null),false);
assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,null),false);
assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,{}),true);
assert.match(source,/if\(active&&draftReady\)setApplied/);
assert.doesNotMatch(source,/if\(!active\|\|!applied\|\|supportError\)/);
});
test('source option request context excludes its own selected values and tracks other filters',()=>{
const source=read('StageSourceFilters.tsx');
const declarations=source.match(/ const \{year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters\}=query;\n const period=.*;\n const filters=.*;\n const contextKey=[^;]+;/)[0];
const run=new Function('query','field','stage',`${ts.transpileModule(declarations,{compilerOptions:{target:ts.ScriptTarget.ES2022}}).outputText};return {filters,contextKey};`);
const base={year:2026,month:4,releaseId:'r1',department:'D1',columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['North']}]};
const first=run(base,{key:'customer'},'black');
const ownValueChanged=run({...base,columnFilters:[{field:'customer',values:['OTHER']},{field:'region',values:['North']}]},{key:'customer'},'black');
const otherValueChanged=run({...base,columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['South']}]},{key:'customer'},'black');
assert.deepEqual(first.filters.columnFilters,[{field:'region',values:['North']}]);
assert.equal(first.contextKey,ownValueChanged.contextKey);
assert.notEqual(first.contextKey,otherValueChanged.contextKey);
});
test('source value dropdowns request candidate-only results while preserving search and debounce behavior',()=>{
const source=read('StageSourceFilters.tsx');
assert.match(source,/sourceColumnValues\(\{releaseId,period,filters,field:field\.key,stages:\[stage\],search,limit:50,offset:0,candidateOnly:true\}/);
assert.match(source,/search\?300:0/);
assert.match(source,/result\.values\.filter\(\(v\):v is string=>typeof v==='string'\)/);
assert.match(read('api.ts'),/sourceColumnValues\(body: Record<string,unknown> & \{candidateOnly\?:true\}/);
});
test('ordinary product search only depends on product options', () => {
assert.match(read('StageProductPage.tsx'), /searchUnavailable=\{!options\}/);
});
@@ -30,3 +30,9 @@ test('stage query includes unclassified source rows even when an old link exclud
assert.equal(base.includeUnassigned,false);
}
});
test('contains keywords are retained while an unfilled contains rule is inactive',()=>{
const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立','%_']};
assert.deepEqual(activeStageFilters([rule],'bright'),[rule]);
assert.deepEqual(activeStageFilters([{...rule,values:[]}],'bright'),[]);
});
+10 -1
View File
@@ -27,7 +27,16 @@ for(const type of ['line','area'])test(`900px report ${type} keeps a positive pl
assert.equal(points.length,18);
const ys=points.filter((_,i)=>i%2===1);
assert.ok(Math.max(...ys)-Math.min(...ys)>8,'different rates must not become a flat line');
ys.forEach((y,i)=>assert.ok(Math.abs(y-(rect.y+rect.height*(1-v.series[0].values[i])))<.001));
const series=chart.getModel().getSeriesByIndex(0).getData();
const [low,high]=chart.getModel().getComponent('yAxis').axis.scale.getExtent();
assert.ok(low>0 && high-low<20,'rate trends retain their focused percentage axis');
ys.forEach((y,i)=>{
const expected=v.series[0].values[i]*100;
assert.ok(Math.abs(series.get('y',i)-expected)<1e-10,'the plotted value preserves the formal rate');
assert.ok(expected>=low&&expected<=high,'the value must fit inside the actual axis');
const pixel=rect.y+rect.height*(high-expected)/(high-low);
assert.ok(Math.abs(y-pixel)<.001,'SVG coordinates match the formal rate on the actual axis');
});
assert.match(chart.renderToSVGString(),/<svg/);
} finally {chart.dispose();}
});