18 Commits
Author SHA1 Message Date
lyun ebb722df36 文档:记录分页发布与全量测试验收 2026-11-14 14:44:52 +08:00
lyun 8f1b790b1d 优化:产品查询数据库分页并修复报告测试契约 2026-11-14 14:34:56 +08:00
lyun d948e1e2a4 文档:确认分步加载已交付主线 2026-11-14 12:46:19 +08:00
lyun bfb91efc2b 文档:记录分步加载发布与线上验收 2026-11-14 12:44:51 +08:00
lyun 374f08c526 优化:查询按范围分步加载并精简表格聚合 2026-11-14 12:34:50 +08:00
lyun 381901184c 文档:记录筛选提速与包含条件的仓库交付 2026-11-13 22:16:57 +08:00
lyun 6983dc32eb 同步:来源筛选提速与包含条件 2026-11-13 22:15:58 +08:00
lyun f47341f3f0 Document contains filter usage and live acceptance 2026-11-13 22:10:50 +08:00
lyun f8432b4e02 Bound contains keyword input and clear duplicate search drafts 2026-11-13 21:51:31 +08:00
lyun 3c022e8614 Add literal contains filters across source queries and analysis context 2026-11-13 21:23:12 +08:00
lyun 7cae4cfbdf Record accepted filter-speed release and measured index storage 2026-11-13 20:32:16 +08:00
lyun 20f1c7d0e2 Seek the first visible source row without scanning all fact ids 2026-11-13 20:14:51 +08:00
lyun fa70dc6504 Cover source candidate scope checks without wide fact row reads 2026-11-13 19:19:18 +08:00
lyun 7bd9e51004 优化:按批次重置筛选索引临时窗口 2026-11-13 18:37:11 +08:00
lyun 5e02c6ad66 优化:筛选索引按暂存窗口读取字典 2026-11-13 18:20:32 +08:00
lyun 7782e33bec 优化:统一来源筛选索引与后台构建 2026-11-13 17:56:18 +08:00
lyun 5068b8fbc9 文档:记录来源筛选修复的线上验收 2026-11-13 16:56:38 +08:00
lyun ea84e26b2f 修复:来源字段筛选超时与重复扫描 2026-11-13 16:52:27 +08:00
71 changed files with 2285 additions and 229 deletions
+10
View File
@@ -58,6 +58,16 @@ DSH CLI 固定为 `0.2.2-rc.1`,依赖与官方 Python SDK 校验记录在 `run
底部表格同步展开当前分类的细项,重量/废率切换与 Excel 导出跟随当前显示;产品和期间行维度保持不变。细项使用正式阶段同期总重量作分母,月、季、年均先汇总分量再计算;没有来源或分母时显示缺失,真实零显示零。细项与正式分类总量不一致时提示覆盖不足,不补造分类。 底部表格同步展开当前分类的细项,重量/废率切换与 Excel 导出跟随当前显示;产品和期间行维度保持不变。细项使用正式阶段同期总重量作分母,月、季、年均先汇总分量再计算;没有来源或分母时显示缺失,真实零显示零。细项与正式分类总量不一致时提示覆盖不足,不补造分类。
## 筛选索引与查询加载
来源字段支持等于、包含、排除和空值筛选;“包含”输入关键词后按回车生效,多个关键词任一命中,区分大小写,`%`、`_` 按普通文字匹配,每个关键词最多256字符、最多50个。各类筛选共用按构建冻结的值字典及记录索引;汇总、趋势、来源明细和导出沿用当前授权与正式指标计算。源字段值保留空值、大小写及前导零语义,旧配置筛选的字符串 `null` 约定不变。下拉框使用只读候选模式,只取一页及是否还有;需要精确总数的原接口继续保留。
启动时在单线程后台补建当前完整正式版本的索引,导入及归类在事务提交后进入同一去重队列。完整构建标记提交后才启用;未完成、失败或未补建的历史构建沿用原始查询,不阻止启动和正式数据发布。索引增加持久存储及后台构建成本,规模取决于非空字段数和不同值数量;高基数宽表应先测容量。保留清理随无引用构建回收索引;升级后原维护兼容保护继续生效。
年度全产品查询先显示正式指标和同范围趋势,再读取产品表格;月/季及产品搜索并行读取摘要和产品页,单产品及总体/产业部等范围复用完整结果。不同范围的趋势独立加载,底部柏拉图接近视口且主查询就绪后读取。刷新和筛选变化会使旧结果失去当前资格;表格失败时,已就绪的指标与趋势仍可使用。
普通查询在数据版本已解析后开始,来源字段条件仍等待对应版本的字段目录;产品文本搜索等待必要产品选项,不因无关元数据失败而禁用。产品汇总、黑片和亮片的宽产品表由数据库分页,默认每页15行,可选择10/15/20/50/100行;换年、筛选和修改每页数量时回到第一页。指标与趋势继续统计完整范围,细分废率随当前页读取。Excel 重新读取完整授权范围并全量导出,产品钻取后的期间表保持原有展示。
## 旧版本时间迁移 ## 旧版本时间迁移
连接与 MySQL 会话现在统一使用 UTC,界面仍按本地时区显示。V25 只校正旧代码按 UTC 墙上时间写入的截止/有效期、报告调度及报告租约列(前者由 Java 写入,租约使用 SQL UTC_TIMESTAMP),不改 SQL CURRENT_TIMESTAMP 自动生成的创建时间,也不修改业务日期或指标。若旧库存在此类值,迁移要求先备份并核实旧会话相对 UTC 的偏移,然后设置 `MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS`(旧会话 UTC 为 `0`,旧会话中国标准时间为 `28800`)。本机启动可在私有 local-config 中记录 `legacyTimestampOffsetSeconds`;不能仅凭电脑当前时区猜测。全新空库不需要该参数;迁移标记和历史值修改在同一事务提交,防止提交确认丢失后重复平移。还原旧备份时应重新核实其原运行时区。 连接与 MySQL 会话现在统一使用 UTC,界面仍按本地时区显示。V25 只校正旧代码按 UTC 墙上时间写入的截止/有效期、报告调度及报告租约列(前者由 Java 写入,租约使用 SQL UTC_TIMESTAMP),不改 SQL CURRENT_TIMESTAMP 自动生成的创建时间,也不修改业务日期或指标。若旧库存在此类值,迁移要求先备份并核实旧会话相对 UTC 的偏移,然后设置 `MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS`(旧会话 UTC 为 `0`,旧会话中国标准时间为 `28800`)。本机启动可在私有 local-config 中记录 `legacyTimestampOffsetSeconds`;不能仅凭电脑当前时区猜测。全新空库不需要该参数;迁移标记和历史值修改在同一事务提交,防止提交确认丢失后重复平移。还原旧备份时应重新核实其原运行时区。
@@ -369,7 +369,7 @@ public class AnalysisExplorationTools {
SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value); SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value);
SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value); SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value);
appendDetailExclusions(sql, args, query); appendDetailExclusions(sql, args, query);
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor); new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor,releaseId);
DataScope.append(sql, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", DataScope.append(sql, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs"); "f.group_name COLLATE utf8mb4_0900_as_cs");
boolean includeUnassigned = query.includeUnassigned() boolean includeUnassigned = query.includeUnassigned()
@@ -158,6 +158,12 @@ public class AnalysisFineDefectsService {
/** One authorized scan (or rollup query) for every period/object and all visible children. */ /** One authorized scan (or rollup query) for every period/object and all visible children. */
@Transactional(readOnly = true, isolation = Isolation.REPEATABLE_READ) @Transactional(readOnly = true, isolation = Isolation.REPEATABLE_READ)
public Map<String, Object> grouped(Map<String, Object> input, AppUserPrincipal actor, String dimension, String granularity) { public Map<String, Object> grouped(Map<String, Object> input, AppUserPrincipal actor, String dimension, String granularity) {
return grouped(input,actor,dimension,granularity,null);
}
public Map<String,Object> grouped(Map<String,Object> input,AppUserPrincipal actor,String dimension,String granularity,List<String> productKeys) {
productKeys=cn.yunsheng.quality.metrics.ProductBuckets.parse(productKeys);
if(productKeys!=null&&!dimension.equals("product"))throw bad("产品桶只能用于产品细分表");
Set<String> allowed = governance.allowedFields(actor); Set<String> allowed = governance.allowedFields(actor);
Request request = parse(input, allowed, true); Request request = parse(input, allowed, true);
String bucket = switch (dimension) { String bucket = switch (dimension) {
@@ -184,6 +190,7 @@ public class AnalysisFineDefectsService {
if (request.productSearch() != null) { if (request.productSearch() != null) {
predicate.append(" AND LOCATE(LOWER(?),LOWER(COALESCE(f.product,'')))>0"); args.add(request.productSearch()); predicate.append(" AND LOCATE(LOWER(?),LOWER(COALESCE(f.product,'')))>0"); args.add(request.productSearch());
} }
cn.yunsheng.quality.metrics.ProductBuckets.append(predicate,args,productKeys);
List<DefectField> fields = applicableFields(request,allowed); List<DefectField> fields = applicableFields(request,allowed);
List<Category> categories = categories(request.stage()); List<Category> categories = categories(request.stage());
Set<String> components = new LinkedHashSet<>(); Set<String> components = new LinkedHashSet<>();
@@ -276,7 +283,7 @@ public class AnalysisFineDefectsService {
SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value); SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value);
if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__")) if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')"); predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor); new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor,releaseId);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs"); "f.group_name COLLATE utf8mb4_0900_as_cs");
} }
@@ -90,7 +90,7 @@ public class AnalysisInvestigationService {
SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x); SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x);
SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x); SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x);
if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')"); if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor); new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor,release);
DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
} }
if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());} if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());}
@@ -891,7 +891,7 @@ final class AnalysisProtocolHistoryFacts {
String label=text(columnLabels.get(field),historyFieldLabel(section,sources,field)); String label=text(columnLabels.get(field),historyFieldLabel(section,sources,field));
if(label.isBlank())label="授权列"; if(label.isBlank())label="授权列";
List<String> prefix=strings(rule.get("stages")).stream().map(AnalysisProtocolHistoryFacts::stageLabel).toList(); List<String> prefix=strings(rule.get("stages")).stream().map(AnalysisProtocolHistoryFacts::stageLabel).toList();
String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";}; String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "contains"->"包含文字";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";};
List<String> values=scopeLiterals(rule.get("values")); List<String> values=scopeLiterals(rule.get("values"));
StringBuilder part=new StringBuilder();if(!prefix.isEmpty())part.append(String.join("、",prefix)).append(' '); StringBuilder part=new StringBuilder();if(!prefix.isEmpty())part.append(String.join("、",prefix)).append(' ');
part.append(label).append(' ').append(operator);if(!values.isEmpty())part.append(' ').append(String.join("、",values));parts.add(part.toString()); part.append(label).append(' ').append(operator);if(!values.isEmpty())part.append(' ').append(String.join("、",values));parts.add(part.toString());
@@ -43,12 +43,16 @@ public class AnalysisSourceQueryController {
@PostMapping("/values") @PostMapping("/values")
public Map<String,Object> values(@RequestBody Map<String,Object> input,@AuthenticationPrincipal AppUserPrincipal actor) { public Map<String,Object> values(@RequestBody Map<String,Object> input,@AuthenticationPrincipal AppUserPrincipal actor) {
String scope=captureScope(actor); String scope=captureScope(actor);
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset").containsAll(input.keySet()))throw bad("候选值请求包含未知字段"); if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset","candidateOnly").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null)); String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null));
Map<String,Object> context=context(input); Map<String,Object> context=context(input);
if(!(input.get("field") instanceof String field)||field.isBlank()||input.get("search")!=null&&!(input.get("search") instanceof String))throw bad("候选字段或搜索值无效"); if(!(input.get("field") instanceof String field)||field.isBlank()||input.get("search")!=null&&!(input.get("search") instanceof String))throw bad("候选字段或搜索值无效");
Map<String,Object> result=new LinkedHashMap<>(columns.values(release,context,field,strings(input.get("stages"),2),Objects.toString(input.get("search"),null), if(input.containsKey("candidateOnly")&&!(input.get("candidateOnly") instanceof Boolean))throw bad("候选分页参数无效");
integer(input.getOrDefault("limit",50),1,200),integer(input.getOrDefault("offset",0),0,100000),actor)); List<String> stages=strings(input.get("stages"),2);String search=Objects.toString(input.get("search"),null);
int limit=integer(input.getOrDefault("limit",50),1,200),offset=integer(input.getOrDefault("offset",0),0,100000);
Map<String,Object> result=new LinkedHashMap<>(Boolean.TRUE.equals(input.get("candidateOnly"))
? columns.values(release,context,field,stages,search,limit,offset,true,actor)
: columns.values(release,context,field,stages,search,limit,offset,actor));
result.put("period",input.get("period"));result.put("filters",input.get("filters")); result.put("period",input.get("period"));result.put("filters",input.get("filters"));
result.put("values",AnalysisProtocolService.list(result.get("rows")).stream().map(row->AnalysisProtocolService.map(row).get("value")).toList()); result.put("values",AnalysisProtocolService.list(result.get("rows")).stream().map(row->AnalysisProtocolService.map(row).get("value")).toList());
result.put("candidateOnly",true);reauthorize(actor,scope,false);return result; result.put("candidateOnly",true);reauthorize(actor,scope,false);return result;
@@ -32,13 +32,17 @@ public class DefectDrilldownController {
String granularity=String.valueOf(input.getOrDefault("trendGranularity","month")); String granularity=String.valueOf(input.getOrDefault("trendGranularity","month"));
if(!Set.of("period","overall","department","group","product","projectGroup").contains(dimension))throw bad("细分维度无效"); if(!Set.of("period","overall","department","group","product","projectGroup").contains(dimension))throw bad("细分维度无效");
if(!Set.of("month","quarter","year").contains(granularity))throw bad("趋势粒度无效"); if(!Set.of("month","quarter","year").contains(granularity))throw bad("趋势粒度无效");
Map<String,Object> scope=new LinkedHashMap<>(input);scope.remove("trendGranularity");scope.remove("view");scope.put("dimension","defect");scope.put("mode","top");scope.put("limit",100); List<String> productKeys=cn.yunsheng.quality.metrics.ProductBuckets.parse(input.get("productKeys"));
if(productKeys!=null&&!dimension.equals("product"))throw bad("产品桶只能用于产品细分表");
Map<String,Object> scope=new LinkedHashMap<>(input);scope.remove("productKeys");scope.remove("trendGranularity");scope.remove("view");scope.put("dimension","defect");scope.put("mode","top");scope.put("limit",100);
// Reuse the source parser to reject unknown filters and validate the selected category/stage. // Reuse the source parser to reject unknown filters and validate the selected category/stage.
AnalysisFineDefectsService.Request request=AnalysisFineDefectsService.parse(scope,governance.allowedFields(actor),true); AnalysisFineDefectsService.Request request=AnalysisFineDefectsService.parse(scope,governance.allowedFields(actor),true);
if(request.category()==null)throw bad("必须指定不良分类"); if(request.category()==null)throw bad("必须指定不良分类");
var query=new MetricsService.MetricsQuery(number(scope,"year"),number(scope,"month"),number(scope,"quarter"),request.department(),request.group(),request.product(),request.projectGroup(),dimension.equals("period")?"overall":dimension,request.includeUnassigned(),request.releaseId(),granularity,AnalysisExplorationTools.exclusions(scope.get("exclusions"))) var query=new MetricsService.MetricsQuery(number(scope,"year"),number(scope,"month"),number(scope,"quarter"),request.department(),request.group(),request.product(),request.projectGroup(),dimension.equals("period")?"overall":dimension,request.includeUnassigned(),request.releaseId(),granularity,AnalysisExplorationTools.exclusions(scope.get("exclusions")))
.withProductSearch(request.productSearch()).withExtraFilters(scope.get("extraFilters")).withColumnFilters(scope.get("columnFilters")).withMonthRange(number(scope,"monthStart"),number(scope,"monthEnd")).withStage(request.stage()); .withProductSearch(request.productSearch()).withExtraFilters(scope.get("extraFilters")).withColumnFilters(scope.get("columnFilters")).withMonthRange(number(scope,"monthStart"),number(scope,"monthEnd")).withStage(request.stage());
var paired=metrics.calculatePairedSnapshot(query,actor,dimension.equals("period"),() -> fine.grouped(scope,actor,dimension,granularity)); var paired=productKeys==null
? metrics.calculatePairedSnapshot(query,actor,dimension.equals("period"),() -> fine.grouped(scope,actor,dimension,granularity))
: metrics.calculatePairedProductSnapshot(query,actor,productKeys,() -> fine.grouped(scope,actor,dimension,granularity,productKeys));
// Refresh the visible fields after the snapshot; a concurrent policy revocation cannot release old fields. // Refresh the visible fields after the snapshot; a concurrent policy revocation cannot release old fields.
governance.requireDetail(actor);governance.requireBoundAction("QUERY","query.detail"); governance.requireDetail(actor);governance.requireBoundAction("QUERY","query.detail");
Set<String> currentAllowed=governance.allowedFields(actor); Set<String> currentAllowed=governance.allowedFields(actor);
@@ -111,7 +111,7 @@ public class AnalyticsController {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前字段授权不包含所选报废类别"); throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前字段授权不包含所选报废类别");
List<Object> args = new ArrayList<>(extra.isEmpty()?List.of(selected, selected):List.of(selected)); List<Object> args = new ArrayList<>(extra.isEmpty()?List.of(selected, selected):List.of(selected));
StringBuilder where = baseWhere(args, year, month, quarter, department, group, product, projectGroup, includeUnassigned, actor); StringBuilder where = baseWhere(args, year, month, quarter, department, group, product, projectGroup, includeUnassigned, actor);
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor); new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor,selected);
List<AggregateTop> aggregated; List<AggregateTop> aggregated;
if (dimension.equals("category")) { if (dimension.equals("category")) {
List<Component> selectedComponents = selectedCategory == null ? visibleComponents : List.of(selectedCategory); List<Component> selectedComponents = selectedCategory == null ? visibleComponents : List.of(selectedCategory);
@@ -36,9 +36,11 @@ public class FilterFieldsController {
governance.requireUserActivePolicy(actor); governance.requireUserActivePolicy(actor);
Map<String,List<String>> result=new LinkedHashMap<>(); Map<String,List<String>> result=new LinkedHashMap<>();
String release=releases.resolvePublishedRelease(null); String release=releases.resolvePublishedRelease(null);
SourceFilterIndex index=new SourceFilterIndex(jdbc);
for(String stage:List.of("black","bright")) { for(String stage:List.of("black","bright")) {
// Raw source headers are stable within each build. Read a representative row of every current build. // Raw source headers are stable within each build. Read a representative row of every current build.
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2 WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release); String hint=index.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":"";
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2"+hint+" WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
Set<String> columns=new TreeSet<>(); Set<String> columns=new TreeSet<>();
var json=tools.jackson.databind.json.JsonMapper.builder().build(); var json=tools.jackson.databind.json.JsonMapper.builder().build();
Set<String> allowed=governance.allowedFields(actor); Set<String> allowed=governance.allowedFields(actor);
@@ -48,6 +48,9 @@ public class QueryFilterFields {
return Collections.unmodifiableMap(result); return Collections.unmodifiableMap(result);
} }
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor) { public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor) {
append(where,args,stage,filters,actor,null);
}
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor,String releaseId) {
if(filters==null || filters.isEmpty())return; if(filters==null || filters.isEmpty())return;
Map<String,Field> fields=new HashMap<>(); visible(actor).forEach(f->fields.put(f.key(),f)); Map<String,Field> fields=new HashMap<>(); visible(actor).forEach(f->fields.put(f.key(),f));
for(var entry:filters.entrySet()) { for(var entry:filters.entrySet()) {
@@ -61,6 +64,7 @@ public class QueryFilterFields {
boolean empty=entry.getValue().contains(EMPTY); boolean empty=entry.getValue().contains(EMPTY);
if(empty && !f.allowEmpty())throw bad(f.label()+"不允许筛选空值"); if(empty && !f.allowEmpty())throw bad(f.label()+"不允许筛选空值");
String column=stage.equals("black")?f.blackColumn():f.brightColumn(); String column=stage.equals("black")?f.blackColumn():f.brightColumn();
if(releaseId!=null&&new SourceFilterIndex(jdbc).appendLegacy(where,args,stage,releaseId,column,entry.getValue()))continue;
// Paths and selected values are bound parameters; never interpolate configured identifiers. // Paths and selected values are bound parameters; never interpolate configured identifiers.
String expression="NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs"; String expression="NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs";
List<String> values=entry.getValue().stream().filter(v->!v.equals(EMPTY)).toList(); List<String> values=entry.getValue().stream().filter(v->!v.equals(EMPTY)).toList();
@@ -90,10 +94,15 @@ public class QueryFilterFields {
StringBuilder scope=new StringBuilder(" WHERE rm.release_id=?"); StringBuilder scope=new StringBuilder(" WHERE rm.release_id=?");
List<Object> scoped=new ArrayList<>();scoped.add(releaseId); List<Object> scoped=new ArrayList<>();scoped.add(releaseId);
DataScope.append(scope,scoped,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); DataScope.append(scope,scoped,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
var index=new SourceFilterIndex(jdbc);
if(index.ready(releaseId)) {
selects.add(index.candidateSelect(stage,column,scope.toString(),scoped,args).replace("AS candidate_value","AS value"));
continue;
}
selects.add("SELECT NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs AS value FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+scope); selects.add("SELECT NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs AS value FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+scope);
args.add(path(column));args.addAll(scoped); args.add(path(column));args.addAll(scoped);
} }
String predicate=" WHERE value IS NOT NULL"; String predicate=" WHERE value IS NOT NULL AND CAST(value AS BINARY)<>CAST('null' AS BINARY)";
if(search!=null && !search.isBlank()) { predicate+=" AND LOCATE(LOWER(?),LOWER(value))>0";args.add(search); } if(search!=null && !search.isBlank()) { predicate+=" AND LOCATE(LOWER(?),LOWER(value))>0";args.add(search); }
List<String> found=jdbc.query("SELECT DISTINCT value FROM ("+String.join(" UNION ALL ",selects)+") choices"+predicate+" ORDER BY value LIMIT 201",(rs,n)->rs.getString(1),args.toArray()); List<String> found=jdbc.query("SELECT DISTINCT value FROM ("+String.join(" UNION ALL ",selects)+") choices"+predicate+" ORDER BY value LIMIT 201",(rs,n)->rs.getString(1),args.toArray());
return Map.of("values",found.stream().limit(200).toList(),"hasMore",found.size()>200); return Map.of("values",found.stream().limit(200).toList(),"hasMore",found.size()>200);
@@ -60,8 +60,18 @@ public class SourceColumnQueries {
} }
return List.copyOf(result); return List.copyOf(result);
} }
// 只提取一次,并区分 JSON null 与字符串“null”,供筛选、候选值和成员读取共用。
private static final String SOURCE_TEXT = "JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON)))";
private static final String SOURCE_VALUE = "NULLIF(TRIM("+SOURCE_TEXT+"), '') COLLATE utf8mb4_0900_as_cs";
/** 原始字段筛选按发布构建读取,避免优化器从整年月份范围扫描大 JSON 原表。 */
public static String factIndexHint(String stage) {
if(!List.of("black","bright").contains(stage))throw bad("数据阶段无效");
return " FORCE INDEX (uq_fact_"+stage+"_origin)";
}
private static final List<String> STAGES = List.of("black", "bright"); private static final List<String> STAGES = List.of("black", "bright");
private static final List<String> OPERATORS = List.of("in", "not_in", "is_empty", "not_empty"); private static final List<String> OPERATORS = List.of("in", "contains", "not_in", "is_empty", "not_empty");
private static final Map<String, String> CANONICAL_COLUMNS = Map.of( private static final Map<String, String> CANONICAL_COLUMNS = Map.of(
"product", "product", "department", "department", "group", "group_name", "projectGroup", "project_group"); "product", "product", "department", "department", "group", "group_name", "projectGroup", "project_group");
private static final Map<String, String> CANONICAL_LABELS = Map.of( private static final Map<String, String> CANONICAL_LABELS = Map.of(
@@ -70,11 +80,17 @@ public class SourceColumnQueries {
private final JdbcTemplate jdbc; private final JdbcTemplate jdbc;
private final GovernanceAccessService governance; private final GovernanceAccessService governance;
private final JsonMapper json; private final JsonMapper json;
private SourceFilterIndex filterIndex;
@org.springframework.beans.factory.annotation.Autowired
public void setFilterIndex(SourceFilterIndex index) { this.filterIndex=index; }
public boolean indexed(String release) { return filterIndex.ready(release); }
public SourceColumnQueries(JdbcTemplate jdbc, GovernanceAccessService governance, JsonMapper json) { public SourceColumnQueries(JdbcTemplate jdbc, GovernanceAccessService governance, JsonMapper json) {
this.jdbc = jdbc; this.jdbc = jdbc;
this.governance = governance; this.governance = governance;
this.json = json; this.json = json;
this.filterIndex = new SourceFilterIndex(jdbc);
} }
/** /**
@@ -94,16 +110,16 @@ public class SourceColumnQueries {
Map<String, Boolean> stagePresent = new LinkedHashMap<>(); Map<String, Boolean> stagePresent = new LinkedHashMap<>();
for (String stage : STAGES) { for (String stage : STAGES) {
Set<String> headers = new LinkedHashSet<>(); Set<String> headers = new LinkedHashSet<>();
StringBuilder representativeScope = new StringBuilder(" WHERE rm.release_id=?"); StringBuilder representativeScope = new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
List<Object> args = new ArrayList<>(List.of(releaseId)); List<Object> args = new ArrayList<>(List.of(releaseId));
DataScope.append(representativeScope, args, actor, DataScope.append(representativeScope, args, actor,
"f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs"); "f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs");
// Stop at the first authorized fact of each build instead of aggregating every fact id.
String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) " String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) "
+ "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id " + "FROM (SELECT DISTINCT transform_build_id,source_revision_id FROM release_member WHERE release_id=?) rm "
+ "FROM release_member rm JOIN fact_" + stage + " f2 " + "JOIN fact_" + stage + " f ON f.id=(SELECT f2.id FROM fact_" + stage + " f2"
+ "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id" + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage))
+ representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative " + representativeScope + " ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1) ORDER BY rm.transform_build_id";
+ "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id";
List<String> keyLists = rawHeaders.get(new QueryKey(sql,args,"raw-headers"), List<String> keyLists = rawHeaders.get(new QueryKey(sql,args,"raw-headers"),
() -> Collections.unmodifiableList(new ArrayList<>(jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray())))); () -> Collections.unmodifiableList(new ArrayList<>(jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray()))));
stagePresent.put(stage, !keyLists.isEmpty()); stagePresent.put(stage, !keyLists.isEmpty());
@@ -192,12 +208,13 @@ public class SourceColumnQueries {
normalizedStages.add(value); normalizedStages.add(value);
} }
List<String> values = new ArrayList<>(); List<String> values = new ArrayList<>();
if (operator.equals("in") || operator.equals("not_in")) { if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) {
if (!(rule.get("values") instanceof List<?> rawValues) || rawValues.isEmpty() || rawValues.size() > 50) if (!(rule.get("values") instanceof List<?> rawValues) || rawValues.isEmpty() || rawValues.size() > 50)
throw bad("in/not_in 筛选必须提供1到50个字符串值"); throw bad("文本筛选必须提供1到50个字符串值");
for (Object value : rawValues) { for (Object value : rawValues) {
if (!(value instanceof String text) || text.isEmpty() || text.length() > 256) if (!(value instanceof String text) || text.isEmpty() || text.length() > 256)
throw bad("来源列筛选值必须是1到256字符的完整字符串;空值请使用空值操作符"); throw bad("来源列筛选值必须是1到256字符的完整字符串;空值请使用空值操作符");
if(operator.equals("contains")&&text.isBlank())throw bad("包含筛选的关键词不能全为空白");
if (!values.contains(text)) values.add(text); if (!values.contains(text)) values.add(text);
} }
} else if (rule.containsKey("values")) { } else if (rule.containsKey("values")) {
@@ -207,7 +224,7 @@ public class SourceColumnQueries {
normalized.put("field", field); normalized.put("field", field);
normalized.put("stages", List.copyOf(normalizedStages)); normalized.put("stages", List.copyOf(normalizedStages));
normalized.put("operator", operator); normalized.put("operator", operator);
if (operator.equals("in") || operator.equals("not_in")) normalized.put("values", List.copyOf(values)); if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) normalized.put("values", List.copyOf(values));
parsed.add(Collections.unmodifiableMap(normalized)); parsed.add(Collections.unmodifiableMap(normalized));
} }
return List.copyOf(parsed); return List.copyOf(parsed);
@@ -239,11 +256,10 @@ public class SourceColumnQueries {
String operator = (String) rule.get("operator"); String operator = (String) rule.get("operator");
String expression; String expression;
Object column = ((Map<?, ?>) field.get("columns")).get(stage); Object column = ((Map<?, ?>) field.get("columns")).get(stage);
@SuppressWarnings("unchecked") List<String> selected = (List<String>) rule.getOrDefault("values",List.of());
if("source_value".equals(field.get("type")) && filterIndex.append(where,args,stage,releaseId,String.valueOf(column),operator,selected))continue;
if ("source_value".equals(field.get("type"))) { if ("source_value".equals(field.get("type"))) {
expression = "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL " expression = SOURCE_VALUE;
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
+ "COLLATE utf8mb4_0900_as_cs";
} else { } else {
String factColumn = CANONICAL_COLUMNS.get(key); String factColumn = CANONICAL_COLUMNS.get(key);
if (factColumn == null) throw bad("规范维度无效"); if (factColumn == null) throw bad("规范维度无效");
@@ -255,19 +271,26 @@ public class SourceColumnQueries {
if (operator.equals("in")) { if (operator.equals("in")) {
where.append(expression).append(" IN (") where.append(expression).append(" IN (")
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append(')'); .append(String.join(",", Collections.nCopies(values.size(), "?"))).append(')');
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column)); if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
} else { } else {
where.append('(').append(expression).append(" IS NULL OR ").append(expression).append(" NOT IN (") where.append('(').append(expression).append(" IS NULL OR ").append(expression).append(" NOT IN (")
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append("))"); .append(String.join(",", Collections.nCopies(values.size(), "?"))).append("))");
if ("source_value".equals(field.get("type"))) { if ("source_value".equals(field.get("type"))) {
bindSourcePaths(args, String.valueOf(column)); bindSourcePath(args, String.valueOf(column));
bindSourcePaths(args, String.valueOf(column)); bindSourcePath(args, String.valueOf(column));
} }
} }
args.addAll(values); args.addAll(values);
} else if (operator.equals("contains")) {
for(int i=0;i<selected.size();i++) {
if(i>0)where.append(" OR ");
where.append("LOCATE(?,").append(expression).append(")>0");
args.add(selected.get(i));
if("source_value".equals(field.get("type")))bindSourcePath(args,String.valueOf(column));
}
} else { } else {
where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL"); where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL");
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column)); if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column));
} }
where.append(')'); where.append(')');
} }
@@ -376,12 +399,16 @@ public class SourceColumnQueries {
public Map<String, Object> values(String releaseId, Map<String, Object> context, String fieldKey, public Map<String, Object> values(String releaseId, Map<String, Object> context, String fieldKey,
List<String> stages, String search, int limit, int offset, List<String> stages, String search, int limit, int offset,
AppUserPrincipal actor) { AppUserPrincipal actor) {
return values(releaseId,context,fieldKey,stages,search,limit,offset,false,actor);
}
public Map<String,Object> values(String releaseId,Map<String,Object> context,String fieldKey,List<String> stages,
String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
requireQueryAccess(actor); requireQueryAccess(actor);
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,actor)); return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,candidateOnly,actor));
} }
private Map<String,Object> valuesSnapshot(String releaseId,Map<String,Object> context,String fieldKey, private Map<String,Object> valuesSnapshot(String releaseId,Map<String,Object> context,String fieldKey,
List<String> stages,String search,int limit,int offset,AppUserPrincipal actor) { List<String> stages,String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效"); if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效");
if (fieldKey == null || fieldKey.isBlank()) throw bad("来源列标识无效"); if (fieldKey == null || fieldKey.isBlank()) throw bad("来源列标识无效");
if (stages == null || stages.isEmpty() || stages.size() > STAGES.size() if (stages == null || stages.isEmpty() || stages.size() > STAGES.size()
@@ -412,6 +439,10 @@ public class SourceColumnQueries {
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?"); StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> whereArgs = new ArrayList<>(List.of(releaseId)); List<Object> whereArgs = new ArrayList<>(List.of(releaseId));
appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor); appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor);
if("source_value".equals(field.get("type")) && filterIndex.ready(releaseId)) {
selects.add(filterIndex.candidateSelect(stage,String.valueOf(columns.get(stage)),where.toString(),whereArgs,unionArgs));
continue;
}
String fact = stage.equals("black") ? "fact_black" : "fact_bright"; String fact = stage.equals("black") ? "fact_black" : "fact_bright";
selects.add("SELECT " + expression + " AS candidate_value " selects.add("SELECT " + expression + " AS candidate_value "
+ "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id " + "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id "
@@ -430,6 +461,13 @@ public class SourceColumnQueries {
} }
String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition; String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition;
String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value"; String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value";
if(candidateOnly) {
String sql="SELECT candidate_value FROM ("+candidates+") matches GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?";
List<Object> parameters=combine(unionArgs,searchArgs);parameters.add(limit+1);parameters.add(offset);
List<Map<String,Object>> found=pages.get(new QueryKey(sql,parameters,"candidates:"+fieldKey),()->List.copyOf(jdbc.query(sql,(rs,row)->Map.of("value",rs.getString(1)),parameters.toArray())));
boolean more=found.size()>limit;
return Map.of("rows",List.copyOf(found.subList(0,Math.min(limit,found.size()))),"hasMore",more,"limit",limit,"offset",offset,"pageComplete",true,"complete",offset==0&&!more,"releaseId",releaseId,"field",fieldKey,"selectedStages",List.copyOf(stages));
}
long total = total("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", combine(unionArgs,searchArgs)); long total = total("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", combine(unionArgs,searchArgs));
List<Map<String, Object>> rows = page("SELECT candidate_value FROM (" + candidates + ") matches " List<Map<String, Object>> rows = page("SELECT candidate_value FROM (" + candidates + ") matches "
+ "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?", + "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?",
@@ -556,7 +594,7 @@ public class SourceColumnQueries {
List<String> groups = parseSelections(context.dimensions().get("group"), "group"); List<String> groups = parseSelections(context.dimensions().get("group"), "group");
if (!context.includeUnassigned() && !groups.contains("__UNASSIGNED__")) if (!context.includeUnassigned() && !groups.contains("__UNASSIGNED__"))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')"); where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor); new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor,releaseId);
if (!context.columnFilters().isEmpty()) append(where, args, stage, context.columnFilters(), releaseId, actor); if (!context.columnFilters().isEmpty()) append(where, args, stage, context.columnFilters(), releaseId, actor);
DataScope.append(where, args, actor, DataScope.append(where, args, actor,
"f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
@@ -578,12 +616,7 @@ public class SourceColumnQueries {
} }
String path = jsonPath(column); String path = jsonPath(column);
args.add(path); args.add(path);
args.add(path); return SOURCE_VALUE;
args.add(path);
return "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL "
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
+ "COLLATE utf8mb4_0900_as_cs";
} }
private static List<Object> combine(List<Object> first, List<Object> second) { private static List<Object> combine(List<Object> first, List<Object> second) {
@@ -601,9 +634,7 @@ public class SourceColumnQueries {
} }
String path = jsonPath(column); String path = jsonPath(column);
args.add(path); args.add(path);
args.add(path); return SOURCE_TEXT+" COLLATE utf8mb4_0900_as_cs";
return "CASE WHEN JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?)) END COLLATE utf8mb4_0900_as_cs";
} }
private static String tupleColumns(int count) { private static String tupleColumns(int count) {
@@ -814,11 +845,9 @@ public class SourceColumnQueries {
catch (Exception e) { throw new IllegalArgumentException("来源列名无法编码", e); } catch (Exception e) { throw new IllegalArgumentException("来源列名无法编码", e); }
} }
private void bindSourcePaths(List<Object> args, String header) { private void bindSourcePath(List<Object> args, String header) {
String path = jsonPath(header); String path = jsonPath(header);
args.add(path); args.add(path);
args.add(path);
args.add(path);
} }
private static ResponseStatusException bad(String message) { private static ResponseStatusException bad(String message) {
@@ -0,0 +1,143 @@
package cn.yunsheng.quality.filters;
import java.util.List;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
/** Sparse, build-owned source values. Every consumer still authorizes and scopes the actual facts. */
@Service
public class SourceFilterIndex {
public record BuildAvailable(String buildId) {}
private static final int FORMAT = 1;
static final String CREATE_STAGE_SQL="CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB";
static final String MEMBER_INSERT_SQL="INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash";
static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)";
private final JdbcTemplate jdbc;
private volatile Boolean available;
private volatile Boolean candidateScopeAvailable;
public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; }
public boolean available() {
if (available == null) {
Integer count=jdbc.queryForObject("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('source_filter_build','source_filter_field','source_filter_value','source_filter_member')",Integer.class);
available=count!=null&&count==4;
}
return available;
}
private String candidateFactHint(String stage) {
if (candidateScopeAvailable == null) {
Integer count=jdbc.queryForObject("SELECT COUNT(DISTINCT table_name) FROM information_schema.statistics WHERE table_schema=DATABASE() AND ((table_name='fact_black' AND index_name='ix_black_candidate_scope') OR (table_name='fact_bright' AND index_name='ix_bright_candidate_scope'))",Integer.class);
candidateScopeAvailable=count!=null&&count==2;
}
return candidateScopeAvailable ? " FORCE INDEX (ix_"+stage+"_candidate_scope)" : "";
}
public boolean ready(String release) {
if(release==null||!available())return false;
// Only successful immutable builds are retained; a pending backfill is checked again later.
Integer missing=jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm LEFT JOIN source_filter_build sb ON sb.build_id=rm.transform_build_id AND sb.format_version=? WHERE rm.release_id=? AND sb.build_id IS NULL",Integer.class,FORMAT,release);
if(missing==null||missing!=0)return false;
return true;
}
/** Idempotent and atomic with fact construction, including callers which instantiate this helper. */
public void ensureBuild(String build) {
if(!available())return;
var tx=new TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource()));
tx.executeWithoutResult(status->{
jdbc.queryForObject("SELECT id FROM transform_build WHERE id=? FOR UPDATE",String.class,build);
Integer done=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build WHERE build_id=? AND format_version=?",Integer.class,build,FORMAT);
if(done!=null&&done>0)return;
jdbc.update("DELETE m FROM source_filter_member m JOIN source_filter_field sf ON sf.id=m.field_id WHERE sf.build_id=?",build);
jdbc.update("DELETE v FROM source_filter_value v JOIN source_filter_field sf ON sf.id=v.field_id WHERE sf.build_id=?",build);
jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build);
jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build);
// Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source.
try {
for(String stage:List.of("black","bright"))buildStage(build,stage);
jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT);
} finally { jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging"); }
});
}
private void buildStage(String build,String stage) {
var bounds=jdbc.queryForMap("SELECT MIN(id) AS first_id,MAX(id) AS last_id FROM fact_"+stage+" FORCE INDEX(uq_fact_"+stage+"_origin) WHERE build_id=?",build);
if(bounds.get("first_id")==null)return;
long after=((Number)bounds.get("first_id")).longValue()-1;
long last=((Number)bounds.get("last_id")).longValue();
while(after<last) {
if(Thread.currentThread().isInterrupted())throw new IllegalStateException("来源筛选索引构建已中断");
List<Long> ids=jdbc.queryForList("SELECT id FROM fact_"+stage+" FORCE INDEX(PRIMARY) WHERE build_id=? AND id>? AND id<=? ORDER BY id LIMIT 500",Long.class,build,after,last);
if(ids.isEmpty())return;
long end=ids.getLast();
String rows=" FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE WHERE f.build_id=? AND f.id>? AND f.id<=?";
jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end);
String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')";
String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?";
// DROP/CREATE TEMPORARY do not commit the owning build transaction. Reset the
// physical table too: DELETE retains undo/deleted records until that transaction ends.
jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging");
jdbc.execute(CREATE_STAGE_SQL);
jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end);
jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash");
Integer collisions=jdbc.queryForObject(COLLISION_SQL,Integer.class);
if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布");
jdbc.update(MEMBER_INSERT_SQL);
after=end;
}
}
/** Same-row membership is shared by metrics, trends, details and exports; empty values are sparse. */
public boolean append(StringBuilder where,List<Object> args,String stage,String release,String column,
String operator,List<String> values) {
if(!ready(release))return false;
where.append(" AND ");
membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"),
operator.equals("in")||operator.equals("not_in")||operator.equals("contains")?values:null,false,operator.equals("contains"));
return true;
}
/** Legacy configured filters classify only the byte-exact literal string "null" as empty. */
public boolean appendLegacy(StringBuilder where,List<Object> args,String stage,String release,String column,List<String> selected) {
if(!ready(release))return false;
var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList();
where.append(" AND (");
if(values.isEmpty())where.append("FALSE");
else membership(where,args,stage,release,column,false,values,true,false);
if(selected.contains(QueryFilterFields.EMPTY)) {
where.append(" OR ");membership(where,args,stage,release,column,true,null,true,false);
}
where.append(')');return true;
}
private void membership(StringBuilder where,List<Object> args,String stage,String release,String column,
boolean negative,List<String> values,boolean legacy,boolean contains) {
where.append("f.id ").append(negative?"NOT IN (":"IN (")
.append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin");
args.add(release);args.add(stage);args.add(column);
if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)");
if(contains) {
where.append(" AND (");
for(int i=0;i<values.size();i++) {
if(i>0)where.append(" OR ");
where.append("LOCATE(?,sv.value COLLATE utf8mb4_0900_as_cs)>0");args.add(values.get(i));
}
where.append(')');
} else if(values!=null) {
where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')');
args.addAll(values);
}
where.append(')');
}
/** Dictionary-first candidates short-circuit after one authorized matching member per value. */
public String candidateSelect(String stage,String column,String scope,List<Object> scopeArgs,List<Object> args) {
args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs);
return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id "
+"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f"+candidateFactHint(stage)+" ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id "
+scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)";
}
}
@@ -0,0 +1,194 @@
package cn.yunsheng.quality.filters;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.context.event.EventListener;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
/** Asynchronously builds current and newly completed snapshots; unindexed queries retain their raw-value fallback. */
@Component
public class SourceFilterIndexBackfill implements ApplicationRunner {
private static final Logger LOG = LoggerFactory.getLogger(SourceFilterIndexBackfill.class);
private static final int MAX_PENDING_BUILDS = 1024;
private final JdbcTemplate jdbc;
private final SourceFilterIndex index;
private final ExecutorService worker;
private final Object pendingLock = new Object();
private final LinkedHashSet<String> pending = new LinkedHashSet<>();
private boolean drainScheduled;
private String inFlight;
@Autowired
public SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index) {
this(jdbc, index, createWorker());
}
SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index, ExecutorService worker) {
this.jdbc = jdbc;
this.index = index;
this.worker = worker;
}
@Override
public void run(ApplicationArguments args) {
List<String> currentBuilds;
try {
if (!index.available()) return;
currentBuilds = jdbc.query("""
SELECT DISTINCT b.id
FROM current_release c
JOIN data_release r ON r.id=c.release_id
JOIN release_member rm ON rm.release_id=r.id
JOIN transform_build b ON b.id=rm.transform_build_id
WHERE c.dataset_id='quality'
AND r.status='PUBLISHED'
AND r.complete=TRUE
AND b.status='COMPLETED'
ORDER BY b.id
""", (rs, row) -> rs.getString(1));
} catch (RuntimeException failure) {
LOG.error("Source filter index backfill discovery failed; raw-value fallback remains available", failure);
return;
}
if (currentBuilds.isEmpty()) return;
int accepted = enqueue(currentBuilds);
LOG.info("Source filter index backfill queued: discovered={}, accepted={}", currentBuilds.size(), accepted);
}
@EventListener
public void onBuildAvailable(SourceFilterIndex.BuildAvailable event) {
if (event == null || event.buildId() == null || event.buildId().isBlank()) return;
int accepted = enqueue(List.of(event.buildId()));
if (accepted > 0) LOG.info("Source filter index build queued after commit: build={}", event.buildId());
}
private int enqueue(List<String> builds) {
boolean schedule = false;
int accepted = 0;
int deferred = 0;
synchronized (pendingLock) {
for (String build : builds) {
if (build == null || build.isBlank() || build.equals(inFlight) || pending.contains(build)) continue;
if (pending.size() >= MAX_PENDING_BUILDS) {
deferred++;
continue;
}
pending.add(build);
accepted++;
}
if (!pending.isEmpty() && !drainScheduled) {
drainScheduled = true;
schedule = true;
}
}
if (deferred > 0) {
LOG.warn("Source filter index queue is full; {} builds keep raw-value fallback", deferred);
}
if (schedule) scheduleDrain();
return accepted;
}
private void scheduleDrain() {
try {
worker.execute(this::drainPending);
} catch (RejectedExecutionException failure) {
int queued;
synchronized (pendingLock) {
drainScheduled = false;
queued = pending.size();
}
LOG.error("Source filter index drain could not be scheduled; {} builds keep raw-value fallback", queued, failure);
}
}
private void drainPending() {
long started = System.nanoTime();
int succeeded = 0;
int failed = 0;
while (true) {
String buildId;
synchronized (pendingLock) {
if (pending.isEmpty()) {
drainScheduled = false;
break;
}
var iterator = pending.iterator();
buildId = iterator.next();
iterator.remove();
inFlight = buildId;
}
if (Thread.currentThread().isInterrupted()) {
synchronized (pendingLock) {
pending.add(buildId);
inFlight = null;
drainScheduled = false;
}
LOG.info("Source filter index drain interrupted; {} builds remain queued", pendingCount());
return;
}
long buildStarted = System.nanoTime();
try {
index.ensureBuild(buildId);
succeeded++;
LOG.info("Source filter index build completed: build={}, queuedRemaining={}, elapsedMs={}", buildId,
pendingCount(), (System.nanoTime() - buildStarted) / 1_000_000L);
} catch (RuntimeException failure) {
if (Thread.currentThread().isInterrupted()) {
synchronized (pendingLock) {
pending.add(buildId);
inFlight = null;
drainScheduled = false;
}
LOG.info("Source filter index drain interrupted while building {}; {} builds remain queued", buildId,
pendingCount());
return;
}
failed++;
LOG.error("Source filter index build failed: build={}; raw-value fallback remains available, continuing",
buildId, failure);
} finally {
synchronized (pendingLock) { inFlight = null; }
}
}
LOG.info("Source filter index drain finished: succeeded={}, failed={}, elapsedMs={}", succeeded, failed,
(System.nanoTime() - started) / 1_000_000L);
}
private int pendingCount() {
synchronized (pendingLock) { return pending.size(); }
}
@PreDestroy
public void stopWorker() {
worker.shutdownNow();
try {
if (!worker.awaitTermination(2, TimeUnit.SECONDS)) {
LOG.warn("Source filter index backfill worker did not stop within two seconds");
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
}
}
private static ExecutorService createWorker() {
return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1), runnable -> {
Thread thread = new Thread(runnable, "source-filter-index-backfill");
thread.setDaemon(true);
return thread;
}, new ThreadPoolExecutor.AbortPolicy());
}
}
@@ -70,10 +70,12 @@ public class MetricsController {
@RequestParam(required=false) String exclusions, @RequestParam(required=false) String exclusions,
@RequestParam(required=false) String columnFilters, @RequestParam(required=false) String columnFilters,
@RequestParam(required=false) String stage, @RequestParam(required=false) String stage,
@RequestParam(defaultValue="full") String part,
@AuthenticationPrincipal AppUserPrincipal actor) { @AuthenticationPrincipal AppUserPrincipal actor) {
if (!List.of("full", "table").contains(part)) throw new IllegalArgumentException("part 必须是 full 或 table");
return calculateRangeStage(year, month, monthStart, monthEnd, quarter, department, group, product, productSearch, return calculateRangeStage(year, month, monthStart, monthEnd, quarter, department, group, product, productSearch,
projectGroup, view, includeUnassigned, releaseId, trendGranularity, rowLimit, rowOffset, rowOrder, projectGroup, view, includeUnassigned, releaseId, trendGranularity, rowLimit, rowOffset, rowOrder,
extraFilters, exclusions, columnFilters, stage, actor); extraFilters, exclusions, columnFilters, stage, actor, "table".equals(part));
} }
/** 轻量趋势接口:查询参数与正式指标接口一致,但只返回版本、上下文、趋势和覆盖信息。 */ /** 轻量趋势接口:查询参数与正式指标接口一致,但只返回版本、上下文、趋势和覆盖信息。 */
@@ -107,6 +109,40 @@ public class MetricsController {
return responseAccess.visibleTrend(metrics.calculateTrend(query, actor), actor); return responseAccess.visibleTrend(metrics.calculateTrend(query, actor), actor);
} }
/** 产品分页接口:范围、授权和正式规则与主查询一致。 */
@GetMapping("/products")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN', 'DATA_ADMIN', 'USER')")
public MetricsService.ProductPage productsHttp(@RequestParam(required = false) Integer year,
@RequestParam(required = false) Integer month,
@RequestParam(required = false) Integer monthStart,
@RequestParam(required = false) Integer monthEnd,
@RequestParam(required = false) String quarter,
@RequestParam(required = false) String department,
@RequestParam(required = false) String group,
@RequestParam(required = false) String product,
@RequestParam(required = false) String productSearch,
@RequestParam(required = false) String projectGroup,
@RequestParam(defaultValue = "product") String view,
@RequestParam(defaultValue = "true") boolean includeUnassigned,
@RequestParam(required = false) String releaseId,
@RequestParam(defaultValue = "month") String trendGranularity,
@RequestParam(required = false) String extraFilters,
@RequestParam(required = false) String exclusions,
@RequestParam(required = false) String columnFilters,
@RequestParam(required = false) String stage,
@RequestParam(defaultValue="1") int page,
@RequestParam(defaultValue="15") int pageSize,
@RequestParam(defaultValue="brightQualifiedDesc") String order,
@AuthenticationPrincipal AppUserPrincipal actor) {
Integer quarterNumber = parseQuarter(quarter);
MetricsService.MetricsQuery query = new MetricsService.MetricsQuery(year, month, quarterNumber, department,
group, product, projectGroup, view, includeUnassigned, releaseId, trendGranularity,
parseExclusions(exclusions));
query = query.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters)
.withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage);
return responseAccess.visibleProductPage(metrics.calculateProductPage(query,actor,page,pageSize,order),actor);
}
/** Existing direct callers retain the pre-range controller signature. */ /** Existing direct callers retain the pre-range controller signature. */
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter, public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter,
String department, String group, String product, String productSearch, String department, String group, String product, String productSearch,
@@ -131,19 +167,25 @@ public class MetricsController {
} }
private MetricsService.MetricSnapshot calculateRangeStage(Integer year,Integer month,Integer monthStart,Integer monthEnd,String quarter,String department,String group,String product,String productSearch,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Integer rowLimit,int rowOffset,String rowOrder,String extraFilters,String exclusions,String columnFilters,String stage,AppUserPrincipal actor) { private MetricsService.MetricSnapshot calculateRangeStage(Integer year,Integer month,Integer monthStart,Integer monthEnd,String quarter,String department,String group,String product,String productSearch,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Integer rowLimit,int rowOffset,String rowOrder,String extraFilters,String exclusions,String columnFilters,String stage,AppUserPrincipal actor) {
return calculateRangeStage(year,month,monthStart,monthEnd,quarter,department,group,product,productSearch,projectGroup,view,includeUnassigned,releaseId,trendGranularity,rowLimit,rowOffset,rowOrder,extraFilters,exclusions,columnFilters,stage,actor,false);
}
private MetricsService.MetricSnapshot calculateRangeStage(Integer year,Integer month,Integer monthStart,Integer monthEnd,String quarter,String department,String group,String product,String productSearch,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Integer rowLimit,int rowOffset,String rowOrder,String extraFilters,String exclusions,String columnFilters,String stage,AppUserPrincipal actor,boolean tableOnly) {
Integer quarterNumber = parseQuarter(quarter); Integer quarterNumber = parseQuarter(quarter);
MetricsService.MetricSnapshot visible = responseAccess.visible(metrics.calculate(new MetricsService.MetricsQuery(year, month, quarterNumber, department, group, MetricsService.MetricsQuery query = new MetricsService.MetricsQuery(year, month, quarterNumber, department, group,
product, projectGroup, view, includeUnassigned, releaseId, trendGranularity, parseExclusions(exclusions)) product, projectGroup, view, includeUnassigned, releaseId, trendGranularity, parseExclusions(exclusions))
.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters).withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage), actor), actor); .withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters).withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage);
if (rowLimit == null) return visible; MetricsService.MetricSnapshot visible = responseAccess.visible(tableOnly ? metrics.calculateTable(query,actor) : metrics.calculate(query,actor), actor);
if (rowLimit < 1 || rowLimit > 500 || rowOffset < 0) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数"); if (rowLimit == null && rowOrder == null) return visible;
if (rowOrder != null && !rowOrder.equals("scrapDesc")) throw new IllegalArgumentException("rowOrder 无效"); if (rowLimit != null && (rowLimit < 1 || rowLimit > 500 || rowOffset < 0)) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
ProductTableOrder productOrder=rowOrder==null||rowOrder.equals("scrapDesc")?null:ProductTableOrder.parse(rowOrder);
List<MetricsService.MetricRow> ordered = new ArrayList<>(visible.rows()); List<MetricsService.MetricRow> ordered = new ArrayList<>(visible.rows());
if ("scrapDesc".equals(rowOrder)) ordered.sort(Comparator if ("scrapDesc".equals(rowOrder)) ordered.sort(Comparator
.comparing(this::totalScrap).reversed() .comparing(this::totalScrap).reversed()
.thenComparing(row -> row.key() == null ? "" : row.key())); .thenComparing(row -> row.key() == null ? "" : row.key()));
int start = Math.min(rowOffset, ordered.size()); if(productOrder!=null)ordered.sort(productOrder.comparator());
int end = (int) Math.min((long) start + rowLimit, ordered.size()); int start = rowLimit==null?0:Math.min(rowOffset, ordered.size());
int end = rowLimit==null?ordered.size():(int) Math.min((long) start + rowLimit, ordered.size());
Map<String, Object> context = new LinkedHashMap<>(visible.context()); Map<String, Object> context = new LinkedHashMap<>(visible.context());
context.put("rowCount", ordered.size()); context.put("rowCount", ordered.size());
return new MetricsService.MetricSnapshot(visible.release(), context, visible.summary(), ordered.subList(start, end), return new MetricsService.MetricSnapshot(visible.release(), context, visible.summary(), ordered.subList(start, end),
@@ -31,6 +31,17 @@ public class MetricsResponseAccess {
return new MetricSnapshot(filtered.release(), filtered.context(), hideDefects(filtered.summary()), List.of(), List.of(), defects, filtered.coverage()); return new MetricSnapshot(filtered.release(), filtered.context(), hideDefects(filtered.summary()), List.of(), List.of(), defects, filtered.coverage());
} }
/** 分页结果返回前重新核验当前用户、字段和明细服务绑定。 */
public MetricsService.ProductPage visibleProductPage(MetricsService.ProductPage page,AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);
if(!Boolean.TRUE.equals(governance.policy(actor.getId().toString()).get("detailAllowed")))
return new MetricsService.ProductPage(page.release(),page.context(),List.of(),0,page.page(),page.pageSize(),page.order());
governance.requireBoundAction("QUERY","query.detail");
Set<String> allowed=governance.allowedFields(actor);
var rows=allowed.contains("*")?page.rows():page.rows().stream().map(row->restrictDefects(row,allowed)).toList();
return new MetricsService.ProductPage(page.release(),page.context(),rows,page.total(),page.page(),page.pageSize(),page.order());
}
/** 趋势查询沿用明细授权和指标字段可见性边界。 */ /** 趋势查询沿用明细授权和指标字段可见性边界。 */
public MetricTrendSnapshot visibleTrend(MetricTrendSnapshot snapshot, AppUserPrincipal actor) { public MetricTrendSnapshot visibleTrend(MetricTrendSnapshot snapshot, AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor); governance.requireUserActivePolicy(actor);
@@ -1,13 +1,21 @@
package cn.yunsheng.quality.metrics; package cn.yunsheng.quality.metrics;
import cn.yunsheng.quality.filters.SourceFilterIndex;
import java.util.List; import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
/** Builds immutable, per-transform-build source-component rollups for read-only metric queries. */ /** Builds immutable, per-transform-build source-component rollups for read-only metric queries. */
@Service @Service
public class MetricsRollupService { public class MetricsRollupService {
private static final Logger LOG = LoggerFactory.getLogger(MetricsRollupService.class);
static final String DIMENSION_COLLATION = "utf8mb4_0900_as_cs"; static final String DIMENSION_COLLATION = "utf8mb4_0900_as_cs";
private static final List<String> FACT_TABLES = List.of("fact_black", "fact_bright"); private static final List<String> FACT_TABLES = List.of("fact_black", "fact_bright");
private static final String COMPONENTS = "JSON_OBJECT(" + private static final String COMPONENTS = "JSON_OBJECT(" +
@@ -22,10 +30,18 @@ public class MetricsRollupService {
private final JdbcTemplate jdbc; private final JdbcTemplate jdbc;
private final cn.yunsheng.quality.analysis.FineDefectRollups fineDefects; private final cn.yunsheng.quality.analysis.FineDefectRollups fineDefects;
private final SourceFilterIndex sourceFilterIndex;
private ApplicationEventPublisher eventPublisher;
public MetricsRollupService(JdbcTemplate jdbc) { public MetricsRollupService(JdbcTemplate jdbc) {
this.jdbc = jdbc; this.jdbc = jdbc;
this.fineDefects = new cn.yunsheng.quality.analysis.FineDefectRollups(jdbc, tools.jackson.databind.json.JsonMapper.builder().build()); this.fineDefects = new cn.yunsheng.quality.analysis.FineDefectRollups(jdbc, tools.jackson.databind.json.JsonMapper.builder().build());
this.sourceFilterIndex = new SourceFilterIndex(jdbc);
}
@Autowired
public void setApplicationEventPublisher(ApplicationEventPublisher eventPublisher) {
this.eventPublisher = eventPublisher;
} }
public record GroupKey(String stage,String month,String department,String group,String project,String product) {} public record GroupKey(String stage,String month,String department,String group,String project,String product) {}
@@ -36,6 +52,7 @@ public class MetricsRollupService {
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent); jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent);
if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总"); if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总");
fineDefects.copyBuild(parent, child); fineDefects.copyBuild(parent, child);
announceSourceFilterBuild(child);
} }
@Transactional @Transactional
@@ -74,6 +91,30 @@ public class MetricsRollupService {
""", buildId, buildId, buildId); """, buildId, buildId, buildId);
fineDefects.refreshBuild(buildId); fineDefects.refreshBuild(buildId);
new SalesOrderMetricRollups(jdbc).refreshCurrentBuild(buildId); new SalesOrderMetricRollups(jdbc).refreshCurrentBuild(buildId);
announceSourceFilterBuild(buildId);
}
void announceSourceFilterBuild(String buildId) {
if (eventPublisher == null) {
// Directly constructed fixtures have no application event bus; retain
// their explicit synchronous behavior (legacy schemas no-op safely).
sourceFilterIndex.ensureBuild(buildId);
return;
}
Runnable publish = () -> {
try {
eventPublisher.publishEvent(new SourceFilterIndex.BuildAvailable(buildId));
} catch (RuntimeException failure) {
LOG.error("Source filter index notification failed for {}; raw-value fallback remains available", buildId, failure);
}
};
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override public void afterCommit() { publish.run(); }
});
} else {
publish.run();
}
} }
static String rollupInsertSql(String table) { static String rollupInsertSql(String table) {
@@ -39,6 +39,10 @@ public class MetricsService {
private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128; private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128;
private final JdbcTemplate jdbc; private final JdbcTemplate jdbc;
private final GovernanceAccessService governance; private final GovernanceAccessService governance;
private SourceColumnQueries sourceColumns;
@Autowired
public void setSourceColumns(SourceColumnQueries sourceColumns) { this.sourceColumns = sourceColumns; }
private final TransactionTemplate readSnapshot; private final TransactionTemplate readSnapshot;
private final TransactionTemplate governanceSnapshot; private final TransactionTemplate governanceSnapshot;
@@ -46,6 +50,7 @@ public class MetricsService {
public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance, PlatformTransactionManager transactionManager) { public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance, PlatformTransactionManager transactionManager) {
this.jdbc = jdbc; this.jdbc = jdbc;
this.governance = governance; this.governance = governance;
this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
this.readSnapshot = new TransactionTemplate(transactionManager); this.readSnapshot = new TransactionTemplate(transactionManager);
this.readSnapshot.setIsolationLevel(TransactionDefinition.ISOLATION_REPEATABLE_READ); this.readSnapshot.setIsolationLevel(TransactionDefinition.ISOLATION_REPEATABLE_READ);
this.readSnapshot.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); this.readSnapshot.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
@@ -56,10 +61,21 @@ public class MetricsService {
/** Keeps package-level in-memory aggregation tests independent of a database transaction manager. */ /** Keeps package-level in-memory aggregation tests independent of a database transaction manager. */
public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance) { public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance) {
this.jdbc = jdbc; this.governance = governance; this.readSnapshot = null; this.governanceSnapshot = null; this.jdbc = jdbc; this.governance = governance;
this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
this.readSnapshot = null; this.governanceSnapshot = null;
} }
public MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor) { public MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor) {
return calculate(query, actor, false);
}
/** 仅按对象汇总表格,省去时间与对象交叉分组的中间行。 */
public MetricSnapshot calculateTable(MetricsQuery query, AppUserPrincipal actor) {
return calculate(query, actor, true);
}
private MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor, boolean tableOnly) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "指标查询必须绑定当前有效用户"); if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor); governance.requireUserActivePolicy(actor);
governance.requireBoundAction("QUERY", "query.aggregate"); governance.requireBoundAction("QUERY", "query.aggregate");
@@ -68,7 +84,55 @@ public class MetricsService {
? currentReleaseId() : query.releaseId(); ? currentReleaseId() : query.releaseId();
if (releaseId == null) return emptySnapshot(query); if (releaseId == null) return emptySnapshot(query);
ReleaseInfo release = release(releaseId, actor); ReleaseInfo release = release(releaseId, actor);
return calculateForRelease(query, actor, release); return withConsistentSnapshot(() -> calculateForReleaseSnapshot(query, actor, release, tableOnly));
}
public record ProductPage(Map<String,Object> release,Map<String,Object> context,List<MetricRow> rows,long total,int page,int pageSize,String order) {}
/** 数据库选出当前页的产品桶,再按正式规则计算这些桶的完整分量。 */
public ProductPage calculateProductPage(MetricsQuery query,AppUserPrincipal actor,int page,int pageSize,String orderName) {
if(actor==null)throw new ResponseStatusException(HttpStatus.FORBIDDEN,"指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor);governance.requireBoundAction("QUERY","query.aggregate");validate(query);
if(!"product".equals(query.view()))throw new IllegalArgumentException("产品分页只支持product视图");
if(page<1||page>1_000_000||pageSize<1||pageSize>100)throw new IllegalArgumentException("页码或每页数量无效");
ProductTableOrder requestedOrder=ProductTableOrder.parse(orderName);
ProductTableOrder order=requestedOrder.visibleTo(governance.allowedFields(actor));
String releaseId=query.releaseId()==null||query.releaseId().isBlank()?currentReleaseId():query.releaseId();
if(releaseId==null){var empty=emptySnapshot(query);return new ProductPage(empty.release(),empty.context(),List.of(),0,page,pageSize,requestedOrder.wire());}
ReleaseInfo release=release(releaseId,actor);
return withConsistentSnapshot(()->{
boolean unrolled=hasUnrolledBuilds(releaseId),sales=salesOrderRollupReady(releaseId,query,actor);
boolean source=!query.extraFilters().isEmpty()||!query.columnFilters().isEmpty();
var branches=new ArrayList<String>();var args=new ArrayList<Object>();
for(String stage:List.of("black","bright")) {
if(query.stage()!=null&&!query.stage().equals(stage))continue;
if(!source||sales)addProductRankBranch(branches,args,stageScope(stage,releaseId,query,actor,false,sales));
if(!sales&&(source||unrolled))addProductRankBranch(branches,args,stageScope(stage,releaseId,query,actor,true,false));
}
String candidates="WITH candidates AS ("+String.join(" UNION ALL ",branches)+") ";
String ranked="SELECT ANY_VALUE(bucket_key) AS bucket_key,"+order.rankExpression()+" AS sort_weight FROM candidates GROUP BY CAST(bucket_key AS BINARY)";
String sql=candidates+", ranked AS ("+ranked+") SELECT bucket_key,COUNT(*) OVER() AS row_count FROM ranked ORDER BY "+order.sqlOrder()+" LIMIT ? OFFSET ?";
var pageArgs=new ArrayList<Object>(args);pageArgs.add(pageSize);pageArgs.add((long)(page-1)*pageSize);
var keys=new ArrayList<String>();long[] count={0};
jdbc.query(sql,(RowCallbackHandler)rs->{keys.add(rs.getString("bucket_key"));count[0]=rs.getLong("row_count");},pageArgs.toArray());
if(keys.isEmpty()) {
Long total=jdbc.queryForObject(candidates+"SELECT COUNT(*) FROM (SELECT CAST(bucket_key AS BINARY) FROM candidates GROUP BY CAST(bucket_key AS BINARY)) all_products",Long.class,args.toArray());
count[0]=total==null?0:total;
}
var snapshot=calculateForReleaseSnapshot(query,actor,release,true,keys);
var byKey=new LinkedHashMap<String,MetricRow>();snapshot.rows().forEach(row->byKey.put(row.key(),row));
var rows=keys.stream().map(byKey::get).filter(Objects::nonNull).toList();
return new ProductPage(snapshot.release(),snapshot.context(),rows,count[0],page,pageSize,requestedOrder.wire());
});
}
private void addProductRankBranch(List<String> branches,List<Object> args,StageScope scope) {
boolean black=scope.stage().equals("black");
String nil="CAST(NULL AS DECIMAL(38,8))";
branches.add("SELECT "+ProductBuckets.EXPRESSION+" AS bucket_key,"+(black?"SUM(f.total_weight)":nil)+" AS black_total,"+
(black?nil:"SUM(f.qualified_weight)")+" AS bright_good,"+(black?nil:"SUM(f.scrap_weight)")+" AS bright_scrap "+
"FROM release_member rm JOIN "+scope.table()+" f"+scope.hint()+" ON f.build_id=rm.transform_build_id"+scope.where()+" GROUP BY bucket_key");
args.addAll(scope.args());
} }
/** 只按时间桶生成趋势,不构造总体摘要或产品/分组维度行。 */ /** 只按时间桶生成趋势,不构造总体摘要或产品/分组维度行。 */
@@ -89,6 +153,15 @@ public class MetricsService {
/** Authorize before opening the read snapshot, so service-binding locks do not span source scans. */ /** Authorize before opening the read snapshot, so service-binding locks do not span source scans. */
public <T> PairedSnapshot<T> calculatePairedSnapshot(MetricsQuery query, AppUserPrincipal actor, public <T> PairedSnapshot<T> calculatePairedSnapshot(MetricsQuery query, AppUserPrincipal actor,
boolean trendOnly, Supplier<T> source) { boolean trendOnly, Supplier<T> source) {
return calculatePairedSnapshot(query,actor,trendOnly,source,null);
}
public <T> PairedSnapshot<T> calculatePairedProductSnapshot(MetricsQuery query,AppUserPrincipal actor,List<String> productKeys,Supplier<T> source) {
if(!"product".equals(query.view()))throw new IllegalArgumentException("产品桶范围无效");
return calculatePairedSnapshot(query,actor,false,source,ProductBuckets.parse(productKeys));
}
private <T> PairedSnapshot<T> calculatePairedSnapshot(MetricsQuery query,AppUserPrincipal actor,boolean trendOnly,Supplier<T> source,List<String> productKeys) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN,"指标查询必须绑定当前有效用户"); if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN,"指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor); governance.requireUserActivePolicy(actor);
governance.requireBoundAction("QUERY","query.aggregate"); governance.requireBoundAction("QUERY","query.aggregate");
@@ -98,7 +171,7 @@ public class MetricsService {
ReleaseInfo release=release(releaseId,actor); ReleaseInfo release=release(releaseId,actor);
return withConsistentSnapshot(() -> trendOnly return withConsistentSnapshot(() -> trendOnly
? new PairedSnapshot<T>(null,calculateTrendForReleaseSnapshot(query,actor,release),source.get()) ? new PairedSnapshot<T>(null,calculateTrendForReleaseSnapshot(query,actor,release),source.get())
: new PairedSnapshot<T>(calculateForReleaseSnapshot(query,actor,release),null,source.get())); : new PairedSnapshot<T>(calculateForReleaseSnapshot(query,actor,release,productKeys!=null,productKeys),null,source.get()));
} }
private MetricTrendSnapshot calculateTrendForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, private MetricTrendSnapshot calculateTrendForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor,
@@ -156,9 +229,9 @@ public class MetricsService {
appendDimensions(where, args, query); appendDimensions(where, args, query);
appendExclusions(where, args, query); appendExclusions(where, args, query);
if (factFallback) { if (factFallback) {
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor); new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor,releaseId);
if (!query.columnFilters().isEmpty()) if (!query.columnFilters().isEmpty())
new SourceColumnQueries(jdbc, governance, JsonMapper.builder().build()) sourceColumns
.append(where, args, stage, query.columnFilters(), releaseId, actor); .append(where, args, stage, query.columnFilters(), releaseId, actor);
} }
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
@@ -180,7 +253,7 @@ public class MetricsService {
%s AS bright_processing_scrap, %s AS bright_processing_scrap,
%s AS bright_diffusion_scrap, %s AS bright_diffusion_scrap,
%s AS row_count %s AS row_count
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
%s %s
GROUP BY trend_period GROUP BY trend_period
ORDER BY trend_period ORDER BY trend_period
@@ -189,7 +262,7 @@ public class MetricsService {
componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"), componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"),
componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"), componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"),
componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"), componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"),
countExpression, table, where); countExpression, table, factFilterHint(stage,factFallback,query), where);
jdbc.query(sql, (RowCallbackHandler) rs -> { jdbc.query(sql, (RowCallbackHandler) rs -> {
String period = rs.getString("trend_period"); String period = rs.getString("trend_period");
AggregateStats stats = new AggregateStats().read(rs, stage); AggregateStats stats = new AggregateStats().read(rs, stage);
@@ -237,9 +310,17 @@ public class MetricsService {
} }
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release) { private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release) {
return calculateForReleaseSnapshot(query, actor, release, false);
}
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release, boolean tableOnly) {
return calculateForReleaseSnapshot(query,actor,release,tableOnly,null);
}
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release, boolean tableOnly,List<String> productKeys) {
String releaseId = release.id(); String releaseId = release.id();
boolean hasUnrolledBuilds = hasUnrolledBuilds(releaseId); boolean hasUnrolledBuilds = hasUnrolledBuilds(releaseId);
AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds); AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds, null, !tableOnly,productKeys);
String ruleVersion=jdbc.queryForObject("SELECT rule_version FROM data_release WHERE id=?",String.class,releaseId); String ruleVersion=jdbc.queryForObject("SELECT rule_version FROM data_release WHERE id=?",String.class,releaseId);
ConfigurableRuleEngine rules=rulesForVersion(ruleVersion); ConfigurableRuleEngine rules=rulesForVersion(ruleVersion);
MetricRow summary = rules.applyMetrics(aggregates.summary().toMetricRow("overall", "总体", null, null, null)); MetricRow summary = rules.applyMetrics(aggregates.summary().toMetricRow("overall", "总体", null, null, null));
@@ -248,7 +329,7 @@ public class MetricsService {
MetricRow adjusted=rules.applyMetrics(new MetricRow(row.key(),row.label(),row.department(),row.group(),row.product(),row.projectGroup(),row.compositeRate(),row.black(),row.bright(),row.coverage(),row.compositeRateWithoutHeadquartersScrap())); MetricRow adjusted=rules.applyMetrics(new MetricRow(row.key(),row.label(),row.department(),row.group(),row.product(),row.projectGroup(),row.compositeRate(),row.black(),row.bright(),row.coverage(),row.compositeRateWithoutHeadquartersScrap()));
return new TrendMetricRow(row.month(),adjusted.key(),adjusted.label(),adjusted.department(),adjusted.group(),adjusted.product(),adjusted.projectGroup(),adjusted.compositeRate(),adjusted.black(),adjusted.bright(),adjusted.coverage(),adjusted.compositeRateWithoutHeadquartersScrap(),row.period(),row.periodLabel(),row.granularity()); return new TrendMetricRow(row.month(),adjusted.key(),adjusted.label(),adjusted.department(),adjusted.group(),adjusted.product(),adjusted.projectGroup(),adjusted.compositeRate(),adjusted.black(),adjusted.bright(),adjusted.coverage(),adjusted.compositeRateWithoutHeadquartersScrap(),row.period(),row.periodLabel(),row.granularity());
}).toList(); }).toList();
Map<String, Object> coverage = queryCoverage(releaseId, query, actor, hasUnrolledBuilds); Map<String, Object> coverage = productKeys==null?queryCoverage(releaseId, query, actor, hasUnrolledBuilds):Map.of();
Map<String, Object> context = context(query, query.includeUnassigned()); Map<String, Object> context = context(query, query.includeUnassigned());
context.put("ruleVersionId",ruleVersion==null?BusinessRowMapper.RULE_VERSION:ruleVersion); context.put("ruleVersionId",ruleVersion==null?BusinessRowMapper.RULE_VERSION:ruleVersion);
context.put("calculationRules",rules.calculationRules());context.put("metricFormulaDescriptions",rules.formulaDescriptions()); context.put("calculationRules",rules.calculationRules());context.put("metricFormulaDescriptions",rules.formulaDescriptions());
@@ -351,24 +432,34 @@ public class MetricsService {
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor, private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments) { boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments) {
return queryAggregates(releaseId, query, actor, hasUnrolledBuilds, canonicalGroupDepartments, true);
}
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments, boolean includePeriods) {
return queryAggregates(releaseId,query,actor,hasUnrolledBuilds,canonicalGroupDepartments,includePeriods,null);
}
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments, boolean includePeriods,List<String> productKeys) {
AggregateSet result = new AggregateSet(); AggregateSet result = new AggregateSet();
boolean salesRollup = salesOrderRollupReady(releaseId, query, actor); boolean salesRollup = salesOrderRollupReady(releaseId, query, actor);
boolean hasSourceFilters = !query.extraFilters().isEmpty() || !query.columnFilters().isEmpty(); boolean hasSourceFilters = !query.extraFilters().isEmpty() || !query.columnFilters().isEmpty();
if (!hasSourceFilters || salesRollup) { if (!hasSourceFilters || salesRollup) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments); aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods,productKeys);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments); aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods,productKeys);
} }
if (!salesRollup && (hasUnrolledBuilds || hasSourceFilters)) { if (!salesRollup && (hasUnrolledBuilds || hasSourceFilters)) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments); aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods,productKeys);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments); aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods,productKeys);
} }
return result; return result;
} }
private void aggregateStage(AggregateSet result, String stage, String view, String releaseId, private void aggregateStage(AggregateSet result, String stage, String view, String releaseId,
MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup, MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup,
Map<String, String> canonicalGroupDepartments) { Map<String, String> canonicalGroupDepartments, boolean includePeriods,List<String> productKeys) {
if(query.stage()!=null&&!query.stage().equals(stage))return; if(query.stage()!=null&&!query.stage().equals(stage)||productKeys!=null&&productKeys.isEmpty())return;
String table = factFallback ? (stage.equals("black") ? "fact_black" : "fact_bright") : (salesRollup ? "sales_order_metric_rollup" : "metric_rollup"); String table = factFallback ? (stage.equals("black") ? "fact_black" : "fact_bright") : (salesRollup ? "sales_order_metric_rollup" : "metric_rollup");
String dimensionColumn = view.equals("overall") ? null : switch (view) { String dimensionColumn = view.equals("overall") ? null : switch (view) {
case "department" -> "f.department"; case "department" -> "f.department";
@@ -382,23 +473,12 @@ public class MetricsService {
boolean canonicalGroupRows = "group".equals(view) && canonicalGroupDepartments != null; boolean canonicalGroupRows = "group".equals(view) && canonicalGroupDepartments != null;
String ownerDepartment = canonicalGroupRows ? ", f.department COLLATE utf8mb4_0900_as_cs AS owner_department" : ""; String ownerDepartment = canonicalGroupRows ? ", f.department COLLATE utf8mb4_0900_as_cs AS owner_department" : "";
String aggregateKeys = canonicalGroupRows ? "trend_period, bucket_key, owner_department" : "trend_period, bucket_key"; String aggregateKeys = canonicalGroupRows ? "trend_period, bucket_key, owner_department" : "trend_period, bucket_key";
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?"); StageScope scope=stageScope(stage,releaseId,query,actor,factFallback,salesRollup);
List<Object> args = new ArrayList<>(); List<Object> args=new ArrayList<>(scope.args());
args.add(releaseId); StringBuilder limitedWhere=new StringBuilder(scope.where());
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)"); ProductBuckets.append(limitedWhere,args,productKeys);
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); } String where=limitedWhere.toString();
if (salesRollup) new SalesOrderMetricRollups(jdbc).append(where, args, query.extraFilters()); String trendBucket = includePeriods ? trendBucketExpression(query.trendGranularity()) : "'all'";
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
String trendBucket = trendBucketExpression(query.trendGranularity());
String sql = """ String sql = """
SELECT %s AS trend_period, %s AS bucket_key%s, SELECT %s AS trend_period, %s AS bucket_key%s,
SUM(f.qualified_weight) AS qualified_weight, SUM(f.qualified_weight) AS qualified_weight,
@@ -413,7 +493,7 @@ public class MetricsService {
%s AS bright_processing_scrap, %s AS bright_processing_scrap,
%s AS bright_diffusion_scrap, %s AS bright_diffusion_scrap,
%s AS row_count %s AS row_count
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
%s %s
GROUP BY %s GROUP BY %s
ORDER BY %s ORDER BY %s
@@ -423,14 +503,14 @@ public class MetricsService {
componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"), componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"),
componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"), componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"),
componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"), componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"),
factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, where, factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, factFilterHint(stage,factFallback,query), where,
aggregateKeys, aggregateKeys); aggregateKeys, aggregateKeys);
jdbc.query(sql, (RowCallbackHandler) rs -> { jdbc.query(sql, (RowCallbackHandler) rs -> {
String period = rs.getString("trend_period"); String period = rs.getString("trend_period");
String key = rs.getString("bucket_key"); String key = rs.getString("bucket_key");
AggregateStats stats = new AggregateStats().read(rs, stage); AggregateStats stats = new AggregateStats().read(rs, stage);
result.summary().merge(stage, stats); result.summary().merge(stage, stats);
result.periods().computeIfAbsent(period, ignored -> new AggregateStats()).merge(stage, stats); if (includePeriods) result.periods().computeIfAbsent(period, ignored -> new AggregateStats()).merge(stage, stats);
if (dimensionColumn != null) { if (dimensionColumn != null) {
if (!canonicalGroupRows || canonicalGroupOwnerMatches(canonicalGroupDepartments, key, rs.getString("owner_department"))) if (!canonicalGroupRows || canonicalGroupOwnerMatches(canonicalGroupDepartments, key, rs.getString("owner_department")))
result.dimensions().computeIfAbsent(key, ignored -> new AggregateStats()).merge(stage, stats); result.dimensions().computeIfAbsent(key, ignored -> new AggregateStats()).merge(stage, stats);
@@ -438,6 +518,29 @@ public class MetricsService {
}, args.toArray()); }, args.toArray());
} }
private record StageScope(String stage,String table,String hint,String where,List<Object> args) {}
private StageScope stageScope(String stage,String releaseId,MetricsQuery query,AppUserPrincipal actor,boolean factFallback,boolean salesRollup) {
String table=factFallback?(stage.equals("black")?"fact_black":"fact_bright"):(salesRollup?"sales_order_metric_rollup":"metric_rollup");
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
if (salesRollup) new SalesOrderMetricRollups(jdbc).append(where, args, query.extraFilters());
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
return new StageScope(stage,table,factFilterHint(stage,factFallback,query),where.toString(),List.copyOf(args));
}
private String componentSum(String key) { private String componentSum(String key) {
String path = "$." + key; String path = "$." + key;
String extracted = "JSON_EXTRACT(f.components, '" + path + "')"; String extracted = "JSON_EXTRACT(f.components, '" + path + "')";
@@ -595,6 +698,16 @@ public class MetricsService {
"sourceMonths", sourceMonths); "sourceMonths", sourceMonths);
} }
private String factFilterHint(String stage,boolean factFallback,MetricsQuery query) {
if(sourceColumns.indexed(query.releaseId()))return "";
if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return "";
// 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。
if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null
||query.department()!=null||query.group()!=null||query.product()!=null||query.projectGroup()!=null
||query.productSearch()!=null)return "";
return SourceColumnQueries.factIndexHint(stage);
}
private void coverageStage(Map<String, CoverageMonth> months, String stage, String releaseId, private void coverageStage(Map<String, CoverageMonth> months, String stage, String releaseId,
MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup) { MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup) {
if(query.stage()!=null&&!query.stage().equals(stage))return; if(query.stage()!=null&&!query.stage().equals(stage))return;
@@ -609,8 +722,8 @@ public class MetricsService {
appendDimensions(where, args, query); appendDimensions(where, args, query);
appendExclusions(where, args, query); appendExclusions(where, args, query);
if(factFallback) { if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor); new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor); if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
} }
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
String countExpression = factFallback ? "COUNT(*)" : "SUM(f.source_row_count)"; String countExpression = factFallback ? "COUNT(*)" : "SUM(f.source_row_count)";
@@ -625,11 +738,11 @@ public class MetricsService {
String sql = """ String sql = """
SELECT f.month_key, %s AS row_count, SELECT f.month_key, %s AS row_count,
%s AS excluded_blank_rows, %s AS unmapped_rows, %s AS project_unconfigured_rows %s AS excluded_blank_rows, %s AS unmapped_rows, %s AS project_unconfigured_rows
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id
%s %s
GROUP BY f.month_key GROUP BY f.month_key
ORDER BY f.month_key ORDER BY f.month_key
""".formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, where); """.formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, factFilterHint(stage,factFallback,query), where);
jdbc.query(sql, (RowCallbackHandler) rs -> { jdbc.query(sql, (RowCallbackHandler) rs -> {
CoverageMonth coverage = months.computeIfAbsent(rs.getString("month_key"), ignored -> new CoverageMonth()); CoverageMonth coverage = months.computeIfAbsent(rs.getString("month_key"), ignored -> new CoverageMonth());
long rowCount = rs.getLong("row_count"); long rowCount = rs.getLong("row_count");
@@ -0,0 +1,27 @@
package cn.yunsheng.quality.metrics;
import java.util.*;
/** 已授权范围内的产品桶限制;空字符串与NULL桶保留各自语义。 */
public final class ProductBuckets {
public static final String EXPRESSION="COALESCE(f.product COLLATE utf8mb4_0900_as_cs, '__UNASSIGNED__') COLLATE utf8mb4_0900_as_cs";
private ProductBuckets() {}
public static List<String> parse(Object input) {
if(input==null)return null;
if(!(input instanceof List<?> values)||values.size()>100)throw new IllegalArgumentException("产品桶必须为不超过100项的列表");
var result=new LinkedHashSet<String>();
for(Object value:values) {
if(!(value instanceof String key)||key.length()>512)throw new IllegalArgumentException("产品桶名称无效");
result.add(key);
}
return List.copyOf(result);
}
public static void append(StringBuilder where,List<Object> args,List<String> keys) {
if(keys==null)return;
if(keys.isEmpty()){where.append(" AND 1=0");return;}
// STRCMP avoids hash-join comparisons dropping canonically equivalent Unicode text.
where.append(" AND (").append(String.join(" OR ",Collections.nCopies(keys.size(),
"STRCMP("+EXPRESSION+",CONVERT(? USING utf8mb4) COLLATE utf8mb4_0900_as_cs)=0"))).append(')');
args.addAll(keys);
}
}
@@ -0,0 +1,47 @@
package cn.yunsheng.quality.metrics;
import java.math.BigDecimal;
import java.util.Comparator;
/** SQL分页与全量导出共用的重量排序约定;比率仍由正式规则计算。 */
public enum ProductTableOrder {
BRIGHT_QUALIFIED("brightQualifiedDesc", "SUM(bright_good)"),
BLACK_TOTAL("blackTotalDesc", "SUM(black_total)"),
BRIGHT_TOTAL("brightTotalDesc", "CASE WHEN SUM(bright_good) IS NULL AND SUM(bright_scrap) IS NULL THEN NULL ELSE COALESCE(SUM(bright_good),0)+COALESCE(SUM(bright_scrap),0) END"),
PRODUCT("productAsc", null);
private final String wire;
private final String expression;
ProductTableOrder(String wire, String expression) { this.wire=wire; this.expression=expression; }
public String wire() { return wire; }
public static ProductTableOrder parse(String value) {
for (var order:values()) if (order.wire.equals(value)) return order;
throw new IllegalArgumentException("产品排序无效");
}
public String rankExpression() { return expression==null?"NULL":expression; }
public String sqlOrder() {
// UTF-16字节序与既有Java对象键次序一致,同重量时固定顺序。
return (expression==null?"":"sort_weight DESC, ")+"CAST(CONVERT(bucket_key USING utf16) AS BINARY) ASC";
}
public ProductTableOrder visibleTo(java.util.Set<String> fields) {
String field=switch(this) {
case BRIGHT_QUALIFIED -> "bright_qualified_weight";
case BLACK_TOTAL -> "black_total_weight";
case BRIGHT_TOTAL -> "bright_total_weight";
case PRODUCT -> "*";
};
return fields.contains("*")||fields.contains(field)?this:PRODUCT;
}
private BigDecimal weight(MetricsService.MetricRow row) {
return switch(this) {
case BRIGHT_QUALIFIED -> row.bright().qualifiedWeight();
case BLACK_TOTAL -> row.black().totalWeight();
case BRIGHT_TOTAL -> row.bright().totalWeight();
case PRODUCT -> null;
};
}
public Comparator<MetricsService.MetricRow> comparator() {
Comparator<MetricsService.MetricRow> keys=Comparator.comparing(row->row.key()==null?"":row.key());
return this==PRODUCT?keys:Comparator.comparing(this::weight,Comparator.nullsLast(Comparator.reverseOrder())).thenComparing(keys);
}
}
@@ -53,6 +53,10 @@ public class RecordController {
private final JsonMapper mapper; private final JsonMapper mapper;
private final GovernanceAccessService governance; private final GovernanceAccessService governance;
private final RecordsWorkbookExportService workbookExport; private final RecordsWorkbookExportService workbookExport;
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
@org.springframework.beans.factory.annotation.Autowired
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
public RecordController(JdbcTemplate jdbc, ReleaseService releases, JsonMapper mapper, public RecordController(JdbcTemplate jdbc, ReleaseService releases, JsonMapper mapper,
GovernanceAccessService governance, RecordsWorkbookExportService workbookExport) { GovernanceAccessService governance, RecordsWorkbookExportService workbookExport) {
@@ -61,6 +65,7 @@ public class RecordController {
this.mapper = mapper; this.mapper = mapper;
this.governance = governance; this.governance = governance;
this.workbookExport = workbookExport; this.workbookExport = workbookExport;
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
} }
@@ -156,10 +161,10 @@ public class RecordController {
args.add(search); args.add(search);
args.add(jsonPath(selectedField)); args.add(jsonPath(selectedField));
} }
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor); new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor,selectedRelease);
var sourceFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters); var sourceFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters);
for(var rule:sourceFilters)if(!List.of(stage).containsAll((List<?>)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段"); for(var rule:sourceFilters)if(!List.of(stage).containsAll((List<?>)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段");
if(!sourceFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,stage,sourceFilters,selectedRelease,actor); if(!sourceFilters.isEmpty())sourceColumns.append(predicate,args,stage,sourceFilters,selectedRelease,actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
String countSql = """ String countSql = """
@@ -52,6 +52,10 @@ public class RecordsWorkbookExportService {
private final ReleaseService releases; private final ReleaseService releases;
private final GovernanceAccessService governance; private final GovernanceAccessService governance;
private final JsonMapper mapper; private final JsonMapper mapper;
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
@org.springframework.beans.factory.annotation.Autowired
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
public RecordsWorkbookExportService(JdbcTemplate jdbc, ReleaseService releases, public RecordsWorkbookExportService(JdbcTemplate jdbc, ReleaseService releases,
GovernanceAccessService governance, JsonMapper mapper) { GovernanceAccessService governance, JsonMapper mapper) {
@@ -59,6 +63,7 @@ public class RecordsWorkbookExportService {
this.releases = releases; this.releases = releases;
this.governance = governance; this.governance = governance;
this.mapper = mapper; this.mapper = mapper;
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
} }
public Path export(Integer year, Integer month, Integer quarter, String department, String group, public Path export(Integer year, Integer month, Integer quarter, String department, String group,
@@ -120,8 +125,11 @@ public class RecordsWorkbookExportService {
LinkedHashSet<String> rawFields = new LinkedHashSet<>(); LinkedHashSet<String> rawFields = new LinkedHashSet<>();
if (releaseId != null) { if (releaseId != null) {
QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product, QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product,
projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd); projectGroup, includeUnassigned, "CAST(JSON_KEYS(f.raw_values) AS CHAR)", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd);
scan(rawScope, rs -> rawFields.addAll(visibleRaw(readJson(rs.getString(1)), allowedFields).keySet())); scan(rawScope, rs -> {
Object decoded=mapper.readValue(rs.getString(1),Object.class);
if(decoded instanceof List<?> keys)for(Object key:keys)if(key instanceof String name&&(allowedFields.contains("*")||allowedFields.contains(name)))rawFields.add(name);
});
} }
boolean showOrderType=SalesOrderTypes.allowed(allowedFields); boolean showOrderType=SalesOrderTypes.allowed(allowedFields);
if (rawFields.size() + HEADER_COUNT + (showOrderType?1:0) > MAX_COLUMNS) if (rawFields.size() + HEADER_COUNT + (showOrderType?1:0) > MAX_COLUMNS)
@@ -167,8 +175,8 @@ public class RecordsWorkbookExportService {
SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value); SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value);
if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__")) if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')"); predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor); new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor,releaseId);
if(!columnFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor); if(!columnFilters.isEmpty())sourceColumns.append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs"); "f.group_name COLLATE utf8mb4_0900_as_cs");
String columns = select == null ? "f.raw_values" : select; String columns = select == null ? "f.raw_values" : select;
@@ -15,6 +15,7 @@ import cn.yunsheng.quality.ingestion.ImportWorker;
import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild; import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild;
import cn.yunsheng.quality.ingestion.MappingValueCodec; import cn.yunsheng.quality.ingestion.MappingValueCodec;
import cn.yunsheng.quality.metrics.MetricsService; import cn.yunsheng.quality.metrics.MetricsService;
import cn.yunsheng.quality.metrics.MetricsRollupService;
import cn.yunsheng.quality.release.ReleaseService; import cn.yunsheng.quality.release.ReleaseService;
import cn.yunsheng.quality.source.SourceCatalogService; import cn.yunsheng.quality.source.SourceCatalogService;
import cn.yunsheng.quality.source.SourceDeletionService; import cn.yunsheng.quality.source.SourceDeletionService;
@@ -66,6 +67,7 @@ public class DataWorkflowService {
private final TaskExecutor executor; private final TaskExecutor executor;
private final TransactionTemplate transaction; private final TransactionTemplate transaction;
private final MetricsService metrics; private final MetricsService metrics;
private MetricsRollupService rollups;
private final AtomicReference<String> runningOperation = new AtomicReference<>(); private final AtomicReference<String> runningOperation = new AtomicReference<>();
private final AtomicReference<String> runningLease = new AtomicReference<>(); private final AtomicReference<String> runningLease = new AtomicReference<>();
@@ -108,6 +110,12 @@ public class DataWorkflowService {
this.executor = executor; this.executor = executor;
this.transaction = new TransactionTemplate(manager); this.transaction = new TransactionTemplate(manager);
this.metrics=metrics; this.metrics=metrics;
this.rollups=new MetricsRollupService(jdbc);
}
@Autowired
public void setMetricsRollupService(MetricsRollupService rollups) {
this.rollups = rollups;
} }
public Map<String, Object> status(AppUserPrincipal actor) { public Map<String, Object> status(AppUserPrincipal actor) {
@@ -376,7 +384,7 @@ public class DataWorkflowService {
sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor); sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor);
updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED"); updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED");
var selectedRules=unclassifiedRules; var selectedRules=unclassifiedRules;
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor)); var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,rollups).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token); jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token);
publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false); publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false);
jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token); jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token);
@@ -0,0 +1,39 @@
CREATE TABLE source_filter_build (
build_id CHAR(36) NOT NULL PRIMARY KEY,
format_version INT NOT NULL,
CONSTRAINT fk_source_filter_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE source_filter_field (
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
build_id CHAR(36) NOT NULL,
stage VARCHAR(16) NOT NULL,
column_name LONGTEXT COLLATE utf8mb4_0900_bin NOT NULL,
name_hash BINARY(32) NOT NULL,
UNIQUE KEY uq_source_filter_field (build_id,stage,name_hash),
CONSTRAINT fk_source_filter_field_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE source_filter_value (
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
field_id BIGINT NOT NULL,
value_hash BINARY(32) NOT NULL,
value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,
UNIQUE KEY uq_source_filter_value_hash (field_id,value_hash),
KEY ix_source_filter_value_text (field_id,value(128)),
CONSTRAINT fk_source_filter_value_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE source_filter_member (
field_id BIGINT NOT NULL,
fact_id BIGINT NOT NULL,
value_id BIGINT NOT NULL,
PRIMARY KEY (field_id,fact_id),
KEY ix_source_filter_member_value (value_id,fact_id),
CONSTRAINT fk_source_filter_member_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id),
CONSTRAINT fk_source_filter_member_value FOREIGN KEY (value_id) REFERENCES source_filter_value(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- Cover representative-row selection without reading every row's large source JSON.
ALTER TABLE fact_black ADD KEY ix_black_source_catalog (build_id,source_revision_id,department,group_name,id);
ALTER TABLE fact_bright ADD KEY ix_bright_source_catalog (build_id,source_revision_id,department,group_name,id);
@@ -0,0 +1,6 @@
-- Keep common candidate authorization/date/product checks off the wide JSON clustered rows.
-- The fact table remains authoritative. Changes to its dimensions update this index automatically.
ALTER TABLE fact_black ADD INDEX ix_black_candidate_scope
(id,build_id,source_revision_id,month_key,department,group_name,product);
ALTER TABLE fact_bright ADD INDEX ix_bright_candidate_scope
(id,build_id,source_revision_id,month_key,department,group_name,product);
@@ -0,0 +1,6 @@
-- Equality on the owning build/revision followed by fact id supports the first visible row.
-- Keep authorization dimensions covered, without reading wide JSON rows while seeking it.
ALTER TABLE fact_black DROP INDEX ix_black_source_catalog,
ADD INDEX ix_black_source_catalog (build_id,source_revision_id,id,department,group_name);
ALTER TABLE fact_bright DROP INDEX ix_bright_source_catalog,
ADD INDEX ix_bright_source_catalog (build_id,source_revision_id,id,department,group_name);
@@ -42,7 +42,7 @@ class FineDefectRollupsMySqlTest {
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)"); jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)");
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))"); jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))");
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))"); jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))"); for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,sheet_name VARCHAR(80),physical_row_no INT,KEY ix_fact_" + stage + "_month_dims(build_id,month_key),UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))");
for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql", "V42__fine_defect_compact_rollups.sql")) { for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql", "V42__fine_defect_compact_rollups.sql")) {
try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) { try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) {
for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql); for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql);
@@ -101,7 +101,7 @@ class SourceColumnQueriesMySqlTest {
assertTrue((Boolean)page.get("pageComplete")); assertTrue((Boolean)page.get("pageComplete"));
} }
@Test void groupedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() { @Test void orderedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() {
record Read(String sql,Object[] args,List<?> result) {} record Read(String sql,Object[] args,List<?> result) {}
List<Read> reads=new java.util.ArrayList<>(); List<Read> reads=new java.util.ArrayList<>();
var capturing=new JdbcTemplate(jdbc.getDataSource()) { var capturing=new JdbcTemplate(jdbc.getDataSource()) {
@@ -114,7 +114,8 @@ class SourceColumnQueriesMySqlTest {
assertEquals(2,representatives.size()); assertEquals(2,representatives.size());
for(Read read:representatives) { for(Read read:representatives) {
String stage=read.sql().contains("fact_black f2")?"black":"bright"; String stage=read.sql().contains("fact_black f2")?"black":"bright";
assertTrue(read.sql().contains("GROUP BY f2.build_id,f2.source_revision_id")); assertTrue(read.sql().contains("ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1"));
assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表");
StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"); StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id");
List<Object> args=new java.util.ArrayList<>(); List<Object> args=new java.util.ArrayList<>();
DataScope.append(scope,args,actor,"f2.department COLLATE utf8mb4_0900_as_cs","f2.group_name COLLATE utf8mb4_0900_as_cs");args.add(RELEASE); DataScope.append(scope,args,actor,"f2.department COLLATE utf8mb4_0900_as_cs","f2.group_name COLLATE utf8mb4_0900_as_cs");args.add(RELEASE);
@@ -157,6 +158,23 @@ class SourceColumnQueriesMySqlTest {
assertThrows(ResponseStatusException.class, () -> queries.validate(RELEASE, unauthorized, actor)); assertThrows(ResponseStatusException.class, () -> queries.validate(RELEASE, unauthorized, actor));
} }
@Test void sourceNullNormalizationMatchesThePreviousExpressionForEveryOperator() {
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
for(String raw:List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[]}","{\"委托单\":{\"n\":1}}"))
insert("bright","build-bright",BRIGHT_REVISION,"部门A","组A",raw);
String old="CASE WHEN JSON_EXTRACT(f.raw_values,'$.\"委托单\"') IS NULL OR JSON_TYPE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))='NULL' THEN NULL ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))),'') END COLLATE utf8mb4_0900_as_cs";
for(String operator:List.of("in","not_in","is_empty","not_empty")) {
Map<String,Object> rule=new java.util.LinkedHashMap<>(Map.of("field",key,"stages",List.of("bright"),"operator",operator));
List<Object> args=new java.util.ArrayList<>();String condition;
if(operator.equals("in")||operator.equals("not_in")) {
rule.put("values",List.of("null","0007"));args.add("null");args.add("0007");
condition=operator.equals("in")?old+" IN (?,?)":"("+old+" IS NULL OR "+old+" NOT IN (?,?))";
} else condition=old+(operator.equals("is_empty")?" IS NULL":" IS NOT NULL");
Long expected=jdbc.queryForObject("SELECT COUNT(*) FROM fact_bright f WHERE f.build_id='build-bright' AND f.department='部门A' AND f.group_name='组A' AND ("+condition+")",Long.class,args.toArray());
assertEquals(expected,countFiltered("bright",SourceColumnQueries.parse(List.of(rule))),operator);
}
}
@Test void explicitMappingWorksAcrossStagesAndMemberPageCountsMergedTuplesBeforePaging() { @Test void explicitMappingWorksAcrossStagesAndMemberPageCountsMergedTuplesBeforePaging() {
List<Map<String, Object>> catalog = queries.catalog(RELEASE, actor); List<Map<String, Object>> catalog = queries.catalog(RELEASE, actor);
String mappedKey = (String) catalog.stream().filter(item -> item.get("label").equals("跨阶段委托号")) String mappedKey = (String) catalog.stream().filter(item -> item.get("label").equals("跨阶段委托号"))
@@ -247,17 +265,27 @@ class SourceColumnQueriesMySqlTest {
return fields.stream().filter(item -> item.get("label").equals(label)).findFirst().orElse(null); return fields.stream().filter(item -> item.get("label").equals(label)).findFirst().orElse(null);
} }
@Test void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters() throws Exception { @org.junit.jupiter.params.ParameterizedTest
@org.junit.jupiter.params.provider.CsvSource({"false,in","true,in","false,contains","true,contains"})
void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed,String operator) throws Exception {
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))"); jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))");
jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE); jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE);
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))"); jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
for(String stage:List.of("black","bright")) { for(String stage:List.of("black","bright")) {
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT"); jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=?,scrap_weight=?,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id",stage.equals("black")?90:80,stage.equals("black")?10:20); jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=?,scrap_weight=?,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id",stage.equals("black")?90:80,stage.equals("black")?10:20);
} }
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black"); jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT"); jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
var service=new cn.yunsheng.quality.metrics.MetricsService(jdbc,governance); if(indexed){installFilterIndex();var index=new SourceFilterIndex(jdbc);for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);}
var headerReads=new java.util.concurrent.atomic.AtomicInteger();
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
if(sql.contains("JSON_KEYS(f.raw_values)"))headerReads.incrementAndGet();
return super.query(sql,mapper,args);
}
};
var service=new cn.yunsheng.quality.metrics.MetricsService(capturing,governance);
var releases=mock(cn.yunsheng.quality.release.ReleaseService.class); var releases=mock(cn.yunsheng.quality.release.ReleaseService.class);
when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE); when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE);
var mapper=JsonMapper.builder().build(); var mapper=JsonMapper.builder().build();
@@ -265,13 +293,14 @@ class SourceColumnQueriesMySqlTest {
var records=new cn.yunsheng.quality.records.RecordController(jdbc,releases,mapper,governance,exporter); var records=new cn.yunsheng.quality.records.RecordController(jdbc,releases,mapper,governance,exporter);
String fieldKey=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key"); String fieldKey=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
for(String stage:List.of("black","bright")) { for(String stage:List.of("black","bright")) {
var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator","in","values",List.of("0007")); var filter=Map.<String,Object>of("field",fieldKey,"stages",List.of(stage),"operator",operator,"values",List.of(operator.equals("contains")?"007":"0007"));
var base=new cn.yunsheng.quality.metrics.MetricsService.MetricsQuery(Integer.valueOf(2026),Integer.valueOf(1),null,"部门A","组A",null,null,"product",false,RELEASE,"month",Map.of()).withColumnFilters(List.of(filter)); var base=new cn.yunsheng.quality.metrics.MetricsService.MetricsQuery(Integer.valueOf(2026),Integer.valueOf(1),null,"部门A","组A",null,null,"product",false,RELEASE,"month",Map.of()).withColumnFilters(List.of(filter));
var legacy=service.calculate(base,actor);var selected=service.calculate(base.withStage(stage),actor); var legacy=service.calculate(base,actor);var selected=service.calculate(base.withStage(stage),actor);
assertEquals(1,selected.rows().size());assertEquals(1L,((Number)selected.coverage().get("includedRows")).longValue()); assertEquals(1,selected.rows().size());assertEquals(1L,((Number)selected.coverage().get("includedRows")).longValue());
assertEquals(stage.equals("black")?legacy.summary().black():legacy.summary().bright(),stage.equals("black")?selected.summary().black():selected.summary().bright()); assertEquals(stage.equals("black")?legacy.summary().black():legacy.summary().bright(),stage.equals("black")?selected.summary().black():selected.summary().bright());
assertEquals(0L,selected.summary().coverage().get(stage.equals("black")?"brightRows":"blackRows")); assertEquals(0L,selected.summary().coverage().get(stage.equals("black")?"brightRows":"blackRows"));
assertEquals(1,selected.trend().size()); assertEquals(1,selected.trend().size());
assertEquals(1,service.calculateTrend(base.withStage(stage),actor).trend().size());
var detail=records.recordsFiltered(stage,2026,1,null,"部门A","组A",null,null,null,false,RELEASE,1,50,null,null,null,"asc",null,null,mapper.writeValueAsString(List.of(filter)),actor); var detail=records.recordsFiltered(stage,2026,1,null,"部门A","组A",null,null,null,false,RELEASE,1,50,null,null,null,"asc",null,null,mapper.writeValueAsString(List.of(filter)),actor);
assertEquals(1L,((Number)detail.get("total")).longValue()); assertEquals(1L,((Number)detail.get("total")).longValue());
java.nio.file.Path file=exporter.exportFiltered(2026,1,null,"部门A","组A",null,null,false,RELEASE,actor,Map.of(),null,stage,List.of(filter)); java.nio.file.Path file=exporter.exportFiltered(2026,1,null,"部门A","组A",null,null,false,RELEASE,actor,Map.of(),null,stage,List.of(filter));
@@ -282,6 +311,7 @@ class SourceColumnQueriesMySqlTest {
var other=base.withColumnFilters(List.of(filter)).withStage(stage); var other=base.withColumnFilters(List.of(filter)).withStage(stage);
assertThrows(ResponseStatusException.class,()->other.withStage(stage.equals("black")?"bright":"black")); assertThrows(ResponseStatusException.class,()->other.withStage(stage.equals("black")?"bright":"black"));
} }
assertEquals(2,headerReads.get(),"汇总、趋势与覆盖检查应复用同一份授权字段目录");
} }
@Test void continuousRangeDetailsAndBothExcelSheetsExcludeOutsideMonthsAndUnauthorizedRows() throws Exception { @Test void continuousRangeDetailsAndBothExcelSheetsExcludeOutsideMonthsAndUnauthorizedRows() throws Exception {
@@ -289,7 +319,7 @@ class SourceColumnQueriesMySqlTest {
when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE); when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE);
var mapper=JsonMapper.builder().build(); var mapper=JsonMapper.builder().build();
for(String stage:List.of("black","bright")) { for(String stage:List.of("black","bright")) {
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT"); jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=90,scrap_weight=10,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id"); jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=90,scrap_weight=10,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id");
String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION; String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION;
for(String month:List.of("2026-03","2026-09","2026-10","2025-03")) { for(String month:List.of("2026-03","2026-09","2026-10","2025-03")) {
@@ -336,7 +366,7 @@ class SourceColumnQueriesMySqlTest {
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))"); jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
for(String stage:List.of("black","bright")) { for(String stage:List.of("black","bright")) {
jdbc.execute("DELETE FROM fact_"+stage); jdbc.execute("DELETE FROM fact_"+stage);
jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT"); jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON");
} }
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black"); jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT"); jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
@@ -390,12 +420,227 @@ class SourceColumnQueriesMySqlTest {
assertThrows(ResponseStatusException.class,()->controller.query(input,actor)); assertThrows(ResponseStatusException.class,()->controller.query(input,actor));
} }
@Test void indexedFiltersCandidatesAndTuplesMatchRawSemanticsAndStayAuthorized() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
List<String> samples=List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[1,2]}","{\"委托单\":{\"a\":1}}","{\"委托单\":\"é\"}","{\"委托单\":\"é\"}","{\"委托单\":\"A\"}","{\"委托单\":\"a\"}","{\"委托单\":\" padded \"}","{\"委托单\":\"null\"}");
for(String raw:samples)insert("black","build-black",BLACK_REVISION,"部门A","组A",raw);
String longKey="来源".repeat(300);
insert("black","build-black",BLACK_REVISION,"部门A","组A",JsonMapper.builder().build().writeValueAsString(Map.of(longKey,"long-key-value")));
insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}");
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
List<Map<String,Object>> rules=new java.util.ArrayList<>();
for(String op:List.of("in","contains","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}"))
rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value)));
var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList();
var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor);
var rawMembers=queries.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor);
jdbc.update("INSERT INTO query_filter_field VALUES ('plain','原始委托单','委托单','委托单',TRUE,TRUE,1)");
List<List<String>> legacySelections=List.of(List.of("null"),List.of("0007"),List.of(QueryFilterFields.EMPTY),List.of("0007",QueryFilterFields.EMPTY));
var rawLegacy=legacySelections.stream().map(this::legacyCount).toList();
installFilterIndex();
var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
assertTrue(indexed.indexed(RELEASE));
assertEquals(rawLegacy,legacySelections.stream().map(this::legacyCount).toList());
assertEquals("long-key-value",jdbc.queryForObject("SELECT sv.value FROM source_filter_field sf JOIN source_filter_value sv ON sv.field_id=sf.id WHERE sf.column_name=? AND sf.build_id='build-black'",String.class,longKey));
var combined=List.of(Map.<String,Object>of("field",key,"stages",List.of("black"),"operator","in","values",List.of("0007")),Map.<String,Object>of("field",field(indexed.catalog(RELEASE,actor),"加工单位").get("key"),"stages",List.of("black"),"operator","in","values",List.of("机二")));
assertEquals(0L,indexed.memberPage(RELEASE,Map.of("year",2026,"columnFilters",combined),List.of(key),List.of("black"),50,0,actor).get("totalCount"));
assertEquals(List.of(),indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),"hidden-only-value",50,0,true,actor).get("rows"));
jdbc.update("INSERT INTO transform_build VALUES ('build-child')");
jdbc.update("INSERT INTO fact_black(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT 'build-child',source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no FROM fact_black WHERE build_id='build-black'");
index.ensureBuild("build-child");
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member sm JOIN source_filter_field sf ON sf.id=sm.field_id JOIN fact_black f ON f.id=sm.fact_id WHERE sf.build_id='build-child' AND f.build_id<>'build-child'",Long.class));
assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
assertEquals(rawValues,indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor));
assertEquals(rawMembers,indexed.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor));
var page=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,2,0,true,actor);
assertEquals(2,((List<?>)page.get("rows")).size());assertEquals(true,page.get("hasMore"));assertFalse(page.containsKey("totalCount"));
long members=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class);
index.ensureBuild("build-black");assertEquals(members,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
when(governance.allowedFields(actor)).thenReturn(Set.of("加工单位"));
assertThrows(ResponseStatusException.class,()->indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),null,50,0,true,actor));
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
jdbc.update("DELETE FROM source_filter_build WHERE build_id='build-black'");
assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
}
@Test void containsMatchesLiteralSubstringsWithOrSemanticsInBothStagesAndBothQueryPaths() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
var mapper=JsonMapper.builder().build();
for(String stage:List.of("black","bright")) {
String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION;
for(String value:List.of("xxAlpha%_\\tail","中文日立公司","prefix x' OR 1=1 -- suffix","000123","null"))
insert(stage,"build-"+stage,revision,"部门A","组A",mapper.writeValueAsString(Map.of("委托单",value)));
insert(stage,"build-"+stage,revision,"部门B","组B",mapper.writeValueAsString(Map.of("委托单","日立-Alpha-hidden")));
}
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
List<List<String>> needles=List.of(List.of("Alpha"),List.of("alpha"),List.of("%"),List.of("_"),List.of("\\"),List.of("日立"),List.of("x' OR 1=1 --"),List.of("001"),List.of("null"),List.of("日立","Alpha"));
List<Long> expected=List.of(1L,0L,1L,1L,1L,1L,1L,1L,1L,2L);
for(boolean indexed:List.of(false,true)) {
if(indexed) {
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
queries=new SourceColumnQueries(jdbc,governance,mapper);
}
for(String stage:List.of("black","bright"))for(int n=0;n<needles.size();n++) {
var rule=Map.<String,Object>of("field",key,"stages",List.of(stage),"operator","contains","values",needles.get(n));
assertEquals(expected.get(n),countFiltered(stage,List.of(rule)),stage+" indexed="+indexed+" keywords="+needles.get(n));
}
var canonical=Map.<String,Object>of("field","product","stages",List.of("black"),"operator","contains","values",List.of("P-"));
assertTrue(countFiltered("black",List.of(canonical))>0);
}
}
@Test void memberBuildPlansStartFromTheBoundedStageWindowRatherThanTheGrowingDictionary() throws Exception {
installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black");
jdbc.execute((org.springframework.jdbc.core.ConnectionCallback<Void>)connection->{
try(var statement=connection.createStatement()) {
statement.execute(SourceFilterIndex.CREATE_STAGE_SQL);
try {
statement.execute("INSERT INTO source_filter_staging SELECT v.field_id,m.fact_id,v.value,v.value_hash FROM source_filter_value v JOIN source_filter_member m ON m.value_id=v.id LIMIT 4");
for(String sql:List.of(SourceFilterIndex.MEMBER_INSERT_SQL,SourceFilterIndex.COLLISION_SQL)) {
try(var rows=statement.executeQuery("EXPLAIN "+sql)) {
String first=null;
while(rows.next())if(List.of("s","v").contains(rows.getString("table"))){first=rows.getString("table");break;}
assertEquals("s",first,"每批构建只能从有界临时窗口开始,不能反复扫描历史字典");
}
}
} finally {statement.execute("DROP TEMPORARY TABLE source_filter_staging");}
}
return null;
});
}
@Test void catalogStopsAtTheFirstAuthorizedFactWithoutSortingEveryFactId() throws Exception {
installFilterIndex();
var captured=new java.util.ArrayList<String>();var parameters=new java.util.ArrayList<Object[]>();
var tracking=new JdbcTemplate(jdbc.getDataSource()) {
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
if(sql.startsWith("SELECT CAST(JSON_KEYS")){captured.add(sql);parameters.add(args);}
return super.query(sql,mapper,args);
}
};
var catalog=new SourceColumnQueries(tracking,governance,JsonMapper.builder().build()).catalog(RELEASE,actor);
assertNotNull(field(catalog,"委托单"));
assertNull(field(catalog,"组织外行列"),"首条全局记录不可见时,必须继续寻找当前用户的第一条可见记录");
assertNull(field(catalog,"其他组织列"));
assertEquals(2,captured.size());
for(int i=0;i<captured.size();i++) {
String plan=jdbc.queryForObject("EXPLAIN ANALYZE "+captured.get(i),String.class,parameters.get(i));
assertTrue(plan.contains("Limit: 1 row(s)"),plan);
assertFalse(plan.lines().anyMatch(line->line.contains("Sort:")&&line.contains("f2.")),plan);
assertTrue(plan.contains("Covering index lookup on f2 using ix_"),plan);
}
}
@Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception {
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
var scopeArgs=new java.util.ArrayList<Object>(List.of(RELEASE,"2026-01","2026-12","部门A","组A","P-1"));
var args=new java.util.ArrayList<Object>();
String sql=index.candidateSelect("black","委托单"," WHERE rm.release_id=? AND f.month_key BETWEEN ? AND ? AND f.department=? AND f.group_name=? AND f.product=?",scopeArgs,args);
assertEquals(Set.of("0007","0008"),new java.util.HashSet<>(jdbc.queryForList(sql,String.class,args.toArray())));
boolean covering=false;
for(var row:jdbc.queryForList("EXPLAIN "+sql,args.toArray())) {
if("f".equals(row.get("table"))) {
assertEquals("ix_black_candidate_scope",row.get("key"));
covering=String.valueOf(row.get("Extra")).contains("Using index");
}
}
assertTrue(covering,"候选值的日期、权限和产品校验不应读取宽 JSON 原始行");
}
@Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception {
installFilterIndex();
for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}");
var memberWrites=new java.util.concurrent.atomic.AtomicInteger();
var failing=new JdbcTemplate(jdbc.getDataSource()) {
@Override public int update(String sql) {
if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure");
return super.update(sql);
}
@Override public int update(String sql,Object... args) {
if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure");
return super.update(sql,args);
}
};
var index=new SourceFilterIndex(failing);
assertThrows(IllegalStateException.class,()->index.ensureBuild("build-black"));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build",Long.class));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_field",Long.class));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_value",Long.class));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
assertFalse(index.ready(RELEASE));
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
assertTrue(index.ready(RELEASE));
}
@Test
@EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_FILTER_BENCHMARK",matches="true")
void syntheticWideSourceIndexBenchmark() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
int size=Integer.getInteger("quality.benchmarkRows",10000);
jdbc.execute("CREATE TABLE benchmark_digit(n INT PRIMARY KEY)");for(int n=0;n<10;n++)jdbc.update("INSERT INTO benchmark_digit VALUES (?)",n);
for(String stage:List.of("black","bright")) {
jdbc.execute("DELETE FROM fact_"+stage);
StringBuilder raw=new StringBuilder("JSON_OBJECT('客户名称',CONCAT('客户',MOD(n,300))");
for(int f=0;f<120;f++)raw.append(",'字段").append(f).append("',CONCAT('value-").append(f).append("-',").append(stage.equals("black")?"MOD(n,50)":"n").append(")");
raw.append(')');
String numbers="SELECT a.n+10*b.n+100*c.n+1000*d.n+10000*e.n AS n FROM benchmark_digit a CROSS JOIN benchmark_digit b CROSS JOIN benchmark_digit c CROSS JOIN benchmark_digit d CROSS JOIN benchmark_digit e LIMIT "+size;
jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT ?,?,'2026-01','部门A','组A','P-1','项目A',"+raw+",'data',n+1 FROM ("+numbers+") numbers","build-"+stage,stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION);
}
var rawQuery=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
String field=(String)field(rawQuery.catalog(RELEASE,actor),"客户名称").get("key");
var old=new java.util.LinkedHashMap<String,Object>();
for(String stage:List.of("black","bright")) {
long start=System.nanoTime();var result=rawQuery.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
old.put(stage,Map.of("ms",(System.nanoTime()-start)/1_000_000L,"result",result));
}
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden")) {
long start=System.nanoTime();index.ensureBuild(build);System.out.println("FILTER_BENCH_BUILD "+build+" rows="+size+" ms="+(System.nanoTime()-start)/1_000_000L);
}
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
for(String stage:List.of("black","bright")) {
long start=System.nanoTime();var result=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
long elapsed=(System.nanoTime()-start)/1_000_000L;
assertEquals(((Map<?,?>)old.get(stage)).get("result"),result);
System.out.println("FILTER_BENCH_QUERY "+stage+" rawMs="+((Map<?,?>)old.get(stage)).get("ms")+" indexMs="+elapsed);
}
for(String table:List.of("source_filter_field","source_filter_value","source_filter_member"))jdbc.execute("ANALYZE TABLE "+table);
System.out.println("FILTER_BENCH_SPACE "+jdbc.queryForList("SELECT table_name,table_rows,data_length,index_length FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name LIKE 'source_filter_%'"));
}
private long legacyCount(List<String> selected) {
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
new QueryFilterFields(jdbc,governance).append(where,args,"black",Map.of("plain",selected),actor,RELEASE);
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
}
private long filteredCount(SourceColumnQueries source,Map<String,Object> rule) {
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
source.append(where,args,"black",List.of(rule),RELEASE,actor);
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
}
private void installFilterIndex() throws Exception {
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id);
for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql","V45__source_catalog_first_visible.sql")) {
String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration));
for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement);
}
}
private void schema() { private void schema() {
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))"); jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))");
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))"); jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage
+ "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7)," + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),"
+ "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL)"); + "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL,sheet_name VARCHAR(80),physical_row_no INT,UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))");
jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64) PRIMARY KEY,label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)"); jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64) PRIMARY KEY,label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)");
} }
@@ -32,7 +32,7 @@ class SourceColumnQueriesTest {
@Test void rejectsUnknownShapeOperatorsEmptyValuesAndMoreThanFiftyValues() { @Test void rejectsUnknownShapeOperatorsEmptyValuesAndMoreThanFiftyValues() {
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(Map.of())); assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(Map.of()));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of( assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("black"), "operator", "contains", "values", List.of("x"))))); "field", SOURCE, "stages", List.of("black"), "operator", "starts_with", "values", List.of("x")))));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of( assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("black"), "operator", "in", "values", List.of(""))))); "field", SOURCE, "stages", List.of("black"), "operator", "in", "values", List.of("")))));
assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of( assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of(
@@ -42,6 +42,22 @@ class SourceColumnQueriesTest {
"field", SOURCE, "stages", List.of("bright"), "operator", "not_in", "values", tooMany)))); "field", SOURCE, "stages", List.of("bright"), "operator", "not_in", "values", tooMany))));
} }
@Test void containsKeepsLiteralKeywordsAndBindsEachJsonPathAfterItsNeedle() {
var rule=Map.<String,Object>of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",List.of("000","%_\\","x' OR 1=1 --","000"));
var parsed=SourceColumnQueries.parse(List.of(rule));
assertEquals(List.of("000","%_\\","x' OR 1=1 --"),parsed.getFirst().get("values"));
var source=spy(new SourceColumnQueries(mock(JdbcTemplate.class),governance,JsonMapper.builder().build()));
doReturn(List.of(sourceField())).when(source).scopedCatalog("release",actor);
var sql=new StringBuilder();var args=new ArrayList<Object>();
source.append(sql,args,"black",parsed,"release",actor);
assertEquals(List.of("000","$.\"委托单\"","%_\\","$.\"委托单\"","x' OR 1=1 --","$.\"委托单\""),args);
assertEquals(3,sql.toString().split("LOCATE",-1).length-1);
assertFalse(sql.toString().contains("x' OR"));
for(Object values:List.of(List.of(),List.of(""),List.of(" "),List.of("x".repeat(257)),List.of(7))) {
assertThrows(ResponseStatusException.class,()->SourceColumnQueries.parse(List.of(Map.of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",values))));
}
}
@Test void monthRangeUsesInclusiveYearBoundariesAndKeepsLegacyMonthPredicate() { @Test void monthRangeUsesInclusiveYearBoundariesAndKeepsLegacyMonthPredicate() {
Map<String,Object> range=SourceColumnQueries.period(Map.of("period",Map.of("year",2026,"monthStart",3,"monthEnd",9))); Map<String,Object> range=SourceColumnQueries.period(Map.of("period",Map.of("year",2026,"monthStart",3,"monthEnd",9)));
StringBuilder rangeSql=new StringBuilder();List<Object> rangeArgs=new ArrayList<>(); StringBuilder rangeSql=new StringBuilder();List<Object> rangeArgs=new ArrayList<>();
@@ -143,9 +159,9 @@ class SourceColumnQueriesTest {
queries.append(where, args, "black", filters, "release", actor); queries.append(where, args, "black", filters, "release", actor);
assertTrue(where.toString().contains("JSON_EXTRACT(f.raw_values, ?) IS NULL")); assertTrue(where.toString().contains("NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON))"));
assertTrue(where.toString().contains("COLLATE utf8mb4_0900_as_cs IN (?,?)")); assertTrue(where.toString().contains("COLLATE utf8mb4_0900_as_cs IN (?,?)"));
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "00007", "x' OR 1=1 --"), args); assertEquals(List.of("$.\"委托单\"", "00007", "x' OR 1=1 --"), args);
assertFalse(where.toString().contains("委托单")); assertFalse(where.toString().contains("委托单"));
assertFalse(where.toString().contains("x' OR 1=1")); assertFalse(where.toString().contains("x' OR 1=1"));
} }
@@ -160,7 +176,7 @@ class SourceColumnQueriesTest {
StringBuilder where = new StringBuilder(); StringBuilder where = new StringBuilder();
queries.append(where, args, "black", filters, "release", actor); queries.append(where, args, "black", filters, "release", actor);
assertTrue(where.toString().contains(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL")); assertTrue(where.toString().contains(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL"));
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\""), args); assertEquals(List.of("$.\"委托单\""), args);
} }
List<Map<String, Object>> filters = SourceColumnQueries.parse(List.of(Map.of( List<Map<String, Object>> filters = SourceColumnQueries.parse(List.of(Map.of(
"field", SOURCE, "stages", List.of("bright"), "operator", "is_empty"))); "field", SOURCE, "stages", List.of("bright"), "operator", "is_empty")));
@@ -179,8 +195,7 @@ class SourceColumnQueriesTest {
assertTrue(where.toString().contains("IS NULL OR")); assertTrue(where.toString().contains("IS NULL OR"));
assertTrue(where.toString().contains(" NOT IN (?)")); assertTrue(where.toString().contains(" NOT IN (?)"));
assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "000123"), args);
"$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "000123"), args);
} }
@Test void rejectsInvalidStageAndMissingFieldsInTheActiveCatalog() { @Test void rejectsInvalidStageAndMissingFieldsInTheActiveCatalog() {
@@ -223,6 +238,6 @@ class SourceColumnQueriesTest {
columns.put("black", stage.equals("black") ? header : null); columns.put("black", stage.equals("black") ? header : null);
columns.put("bright", stage.equals("bright") ? header : null); columns.put("bright", stage.equals("bright") ? header : null);
return Map.of("key", key, "label", header, "columns", columns, "stages", List.of(stage), return Map.of("key", key, "label", header, "columns", columns, "stages", List.of(stage),
"type", "source_value", "operators", List.of("in", "not_in", "is_empty", "not_empty")); "type", "source_value", "operators", List.of("in", "contains", "not_in", "is_empty", "not_empty"));
} }
} }
@@ -0,0 +1,117 @@
package cn.yunsheng.quality.filters;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import java.util.ArrayDeque;
import java.util.List;
import java.util.concurrent.AbstractExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
class SourceFilterIndexBackfillTest {
@Test
void skipsWhenIndexTablesAreUnavailable() {
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
ManualExecutor worker = new ManualExecutor();
doReturn(false).when(index).available();
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
runner.run(null);
verify(jdbc, never()).query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
verify(index, never()).ensureBuild(anyString());
assertEquals(0, worker.queued());
runner.stopWorker();
}
@Test
void coalescesEventsDuringDrainAndContinuesAfterAnIndividualFailure() {
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
ManualExecutor worker = new ManualExecutor();
doReturn(true).when(index).available();
doReturn(List.of("failed-build", "ready-build")).when(jdbc)
.query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
doAnswer(invocation -> {
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("failed-build"));
throw new IllegalStateException("synthetic index failure");
}).when(index).ensureBuild("failed-build");
runner.run(null);
assertEquals(1, worker.queued());
verify(index, never()).ensureBuild(anyString());
ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
verify(jdbc).query(sql.capture(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
assertTrue(sql.getValue().contains("c.dataset_id='quality'"));
assertTrue(sql.getValue().contains("r.status='PUBLISHED'"));
assertTrue(sql.getValue().contains("r.complete=TRUE"));
assertTrue(sql.getValue().contains("b.status='COMPLETED'"));
worker.runNext();
verify(index).ensureBuild("failed-build");
verify(index).ensureBuild("ready-build");
verify(index).ensureBuild("event-during-drain");
assertEquals(0, worker.queued());
runner.stopWorker();
assertTrue(worker.isShutdown());
}
@Test
void componentHasAnAutowiredProductionConstructorAndCleanShutdown() {
try (var context = new AnnotationConfigApplicationContext()) {
context.registerBean(JdbcTemplate.class, () -> org.mockito.Mockito.mock(JdbcTemplate.class));
context.registerBean(SourceFilterIndex.class, () -> org.mockito.Mockito.mock(SourceFilterIndex.class));
context.register(SourceFilterIndexBackfill.class);
context.refresh();
assertNotNull(context.getBean(SourceFilterIndexBackfill.class));
}
}
private static final class ManualExecutor extends AbstractExecutorService {
private final ArrayDeque<Runnable> tasks = new ArrayDeque<>();
private boolean shutdown;
@Override public void shutdown() { shutdown = true; }
@Override public List<Runnable> shutdownNow() {
shutdown = true;
List<Runnable> pending = List.copyOf(tasks);
tasks.clear();
return pending;
}
@Override public boolean isShutdown() { return shutdown; }
@Override public boolean isTerminated() { return shutdown && tasks.isEmpty(); }
@Override public boolean awaitTermination(long timeout, TimeUnit unit) { return isTerminated(); }
@Override public void execute(Runnable command) {
if (shutdown) throw new RejectedExecutionException("executor is stopped");
tasks.add(command);
}
int queued() { return tasks.size(); }
void runNext() { tasks.removeFirst().run(); }
}
}
@@ -84,4 +84,19 @@ class MetricsResponseAccessTest {
assertTrue(aggregateOnly.trend().isEmpty()); assertTrue(aggregateOnly.trend().isEmpty());
verify(governance, never()).requireBoundAction("QUERY", "query.detail"); verify(governance, never()).requireBoundAction("QUERY", "query.detail");
} }
@Test void productPageRechecksCurrentDetailAndFieldPolicyAndHidesCountsWithoutDetailAccess() {
var page=new ProductPage(Map.of("id","r"),Map.of(),List.of(row),120,2,15,"productAsc");
var access=new MetricsResponseAccess(governance);
when(governance.policy(actor.getId().toString())).thenReturn(Map.of("detailAllowed",true));
when(governance.allowedFields(actor)).thenReturn(Set.of("black_total_weight"));
var visible=access.visibleProductPage(page,actor);
assertEquals(120,visible.total());assertEquals(2,visible.page());
assertEquals(stage.totalWeight(),visible.rows().getFirst().black().totalWeight());
assertNull(visible.rows().getFirst().bright().totalWeight());
verify(governance).requireUserActivePolicy(actor);
when(governance.policy(actor.getId().toString())).thenReturn(Map.of("detailAllowed",false));
var revoked=access.visibleProductPage(page,actor);
assertTrue(revoked.rows().isEmpty());assertEquals(0,revoked.total());
}
} }
@@ -1,8 +1,21 @@
package cn.yunsheng.quality.metrics; package cn.yunsheng.quality.metrics;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import cn.yunsheng.quality.filters.SourceFilterIndex;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
class MetricsRollupServiceTest { class MetricsRollupServiceTest {
@Test @Test
@@ -17,4 +30,65 @@ class MetricsRollupServiceTest {
assertTrue(sql.contains("GROUP BY f.build_id, f.month_key " + collation)); assertTrue(sql.contains("GROUP BY f.build_id, f.month_key " + collation));
assertTrue(sql.contains("f.group_name " + collation + ", f.project_group " + collation)); assertTrue(sql.contains("f.group_name " + collation + ", f.project_group " + collation));
} }
@Test
void publishesFilterBuildAvailableOnlyAfterTheOuterTransactionCommits() {
var service = new MetricsRollupService(mock(JdbcTemplate.class));
var publisher = mock(ApplicationEventPublisher.class);
service.setApplicationEventPublisher(publisher);
String build = "11111111-1111-4111-8111-111111111111";
TransactionSynchronizationManager.initSynchronization();
try {
service.announceSourceFilterBuild(build);
verifyNoInteractions(publisher);
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
assertEquals(1, synchronizations.size());
for (TransactionSynchronization synchronization : synchronizations) synchronization.afterCommit();
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
} finally {
TransactionSynchronizationManager.clearSynchronization();
}
}
@Test
void publishesImmediatelyWhenThereIsNoTransactionSynchronization() {
var service = new MetricsRollupService(mock(JdbcTemplate.class));
var publisher = mock(ApplicationEventPublisher.class);
service.setApplicationEventPublisher(publisher);
String build = "22222222-2222-4222-8222-222222222222";
service.announceSourceFilterBuild(build);
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
}
@Test
void notificationFailureAfterCommitDoesNotEscapeTheTransactionSynchronization() {
var service = new MetricsRollupService(mock(JdbcTemplate.class));
var publisher = mock(ApplicationEventPublisher.class);
service.setApplicationEventPublisher(publisher);
String build = "33333333-3333-4333-8333-333333333333";
var event = new SourceFilterIndex.BuildAvailable(build);
doThrow(new IllegalStateException("synthetic event delivery failure")).when(publisher).publishEvent(event);
TransactionSynchronizationManager.initSynchronization();
try {
service.announceSourceFilterBuild(build);
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
assertDoesNotThrow(() -> synchronizations.get(0).afterCommit());
} finally {
TransactionSynchronizationManager.clearSynchronization();
}
}
@Test
void directFixtureConstructionKeepsTheSynchronousLegacySchemaFallback() {
JdbcTemplate jdbc = mock(JdbcTemplate.class);
var service = new MetricsRollupService(jdbc);
service.announceSourceFilterBuild("44444444-4444-4444-8444-444444444444");
verify(jdbc).queryForObject(contains("information_schema.tables"), eq(Integer.class));
}
} }
@@ -51,4 +51,31 @@ class MetricsTrendControllerTest {
assertEquals(Map.of("customerCode", List.of("000123")), value.extraFilters()); assertEquals(Map.of("customerCode", List.of("000123")), value.extraFilters());
assertEquals("src_any", value.columnFilters().getFirst().get("field")); assertEquals("src_any", value.columnFilters().getFirst().get("field"));
} }
@Test void tableProjectionKeepsSourceScopeAndUsesTheSameResponseAuthorization() {
MetricsService metrics = mock(MetricsService.class);
MetricsResponseAccess access = mock(MetricsResponseAccess.class);
AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "table", "unused", "表格", true, false,
List.of("USER"), List.of("D"), List.of("G"), List.of());
var response = new MetricsService.MetricSnapshot(Map.of(), Map.of(), null, List.of(), List.of(), Map.of(), Map.of());
when(metrics.calculateTable(any(), eq(actor))).thenReturn(response);
when(access.visible(response, actor)).thenReturn(response);
var actual = new MetricsController(metrics, access).calculateHttp(
2026, null, 3, 9, null, "D", "G", "P", "18X15", "PG", "product", false,
"release-a", "quarter", null, 0, null, "{\"customerCode\":[\"000123\"]}",
"{\"product\":[\"P2\"]}",
"[{\"field\":\"src_any\",\"stages\":[\"black\"],\"operator\":\"contains\",\"values\":[\"000123\"]}]",
"black", "table", actor);
assertEquals(response, actual);
var query = ArgumentCaptor.forClass(MetricsService.MetricsQuery.class);
verify(metrics).calculateTable(query.capture(), eq(actor));
verify(access).visible(response, actor);
var value = query.getValue();
assertEquals(3, value.monthStart()); assertEquals(9, value.monthEnd());
assertEquals("black", value.stage()); assertEquals("P", value.product());
assertEquals("D", value.department()); assertEquals("G", value.group());
assertEquals("18X15", value.productSearch()); assertEquals("PG", value.projectGroup());
assertEquals(Map.of("product", List.of("P2")), value.exclusions());
assertEquals(Map.of("customerCode", List.of("000123")), value.extraFilters());
assertEquals("contains", value.columnFilters().getFirst().get("operator"));
}
} }
@@ -49,6 +49,29 @@ class MetricsTrendServiceTest {
assertTrue(trendSql.stream().allMatch(sql -> !sql.contains("bucket_key") && !sql.contains("f.product"))); assertTrue(trendSql.stream().allMatch(sql -> !sql.contains("bucket_key") && !sql.contains("f.product")));
} }
@Test void tableProjectionKeepsFormalRowsSummaryAndCoverageWithoutTimeByProductGrouping() {
JdbcTemplate jdbc = jdbc(true, true);
var governance = mock(GovernanceAccessService.class);
MetricsService service = new MetricsService(jdbc, governance);
var query = new MetricsService.MetricsQuery(2026, null, null, null, null, null, null,
"product", true, RELEASE, "month", Map.of()).withMonthRange(2, 5).withStage("bright");
var full = service.calculate(query, actor);
clearInvocations(jdbc, governance);
var table = service.calculateTable(query, actor);
assertEquals(full.summary(), table.summary());
assertEquals(full.rows(), table.rows());
assertEquals(full.defects(), table.defects());
assertEquals(full.coverage(), table.coverage());
assertEquals(full.context(), table.context());
assertTrue(table.trend().isEmpty());
var sql = capturedSql(jdbc).stream().filter(value -> value.contains("AS bucket_key")).toList();
assertEquals(1, sql.size());
assertTrue(sql.getFirst().contains("SELECT 'all' AS trend_period"));
assertTrue(sql.getFirst().contains("f.month_key BETWEEN ? AND ?"));
verify(governance).requireUserActivePolicy(actor);
verify(governance).requireBoundAction("QUERY", "query.aggregate");
}
@Test void rangeStageAndFiltersReachOnlyTheRequestedTrendAndCoverageQueries() { @Test void rangeStageAndFiltersReachOnlyTheRequestedTrendAndCoverageQueries() {
JdbcTemplate jdbc = jdbc(true, true); JdbcTemplate jdbc = jdbc(true, true);
MetricsService service = new MetricsService(jdbc, mock(GovernanceAccessService.class)); MetricsService service = new MetricsService(jdbc, mock(GovernanceAccessService.class));
@@ -93,4 +93,59 @@ class SalesOrderMetricRollupsMySqlTest {
rollups.refreshCurrentBuild(build);assertFalse(rollups.ready(release,query.extraFilters(),true,Set.of("*"))); rollups.refreshCurrentBuild(build);assertFalse(rollups.ready(release,query.extraFilters(),true,Set.of("*")));
assertEquals(0,jdbc.queryForObject("SELECT COUNT(*) FROM sales_order_metric_rollup_build WHERE version_id=?",Integer.class,version)); assertEquals(0,jdbc.queryForObject("SELECT COUNT(*) FROM sales_order_metric_rollup_build WHERE version_id=?",Integer.class,version));
} }
@Test void databaseProductPagesMatchFormalRowsAcrossRawRollupsFiltersAndStableNullTies() {
// Include blanks, a missing stage, equal weights and UTF-16 ordering edges.
int n=0;
for(String product:List.of("", " ", "edge-😀", "edge-\uE000", "edge-é", "edge-e\u0301", "edge-A", "edge-a")) {
jdbc.update("INSERT INTO fact_bright(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',?,100,1,999,JSON_OBJECT('销售订单号','order-0'),JSON_OBJECT('bright_processing_scrap',1))",build,product);
n++;
}
jdbc.update("INSERT INTO fact_black(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',NULL,NULL,0,0,JSON_OBJECT(),JSON_OBJECT())",build);
// Stage-specific representatives remain distinct exactly as in the existing formal table.
jdbc.update("INSERT INTO fact_black(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',?,50,1,51,JSON_OBJECT(),JSON_OBJECT())",build,"edge-e\u0301");
var base=query("product",true,Map.of());
var scopes=List.of(base,base.withStage("bright"),base.withStage("black"),base.withMonthRange(1,1),base.withProductSearch("edge"),
query("product",false,Map.of(SalesOrderTypes.KEY,List.of("标准销售订单"))),
query("product",true,Map.of(SalesOrderTypes.EXCLUDE_KEY,List.of("样品销售订单"))));
// No rollups -> facts; regular rollups; then sales-order rollups.
for(int path=0;path<3;path++) {
if(path==1) for(String stage:List.of("black","bright")) {
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT build_id,?,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,1,0,0,0 FROM fact_"+stage,stage);
}
if(path==1)jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) VALUES(?,?)",build,22+n+2);
if(path==2)rollups.prepareBuild(version,build);
for(var scope:scopes) for(var order:ProductTableOrder.values()) {
var expected=metrics.calculateTable(scope,actor).rows().stream().sorted(order.comparator()).toList();
var actual=new ArrayList<MetricsService.MetricRow>();
int pages=(expected.size()+2)/3;
for(int page=1;page<=pages+1;page++) {
var result=metrics.calculateProductPage(scope,actor,page,3,order.wire());
assertEquals(expected.size(),result.total(),"total path "+path);
assertTrue(result.rows().size()<=3);
actual.addAll(result.rows());
}
assertEquals(normalized(expected),normalized(actual),"path="+path+" order="+order+" scope="+scope);
}
}
when(actor.getDepartments()).thenReturn(List.of("A"));
var authorized=metrics.calculateTable(base,actor).rows().stream().sorted(ProductTableOrder.PRODUCT.comparator()).toList();
assertEquals(normalized(authorized),normalized(metrics.calculateProductPage(base,actor,1,100,"productAsc").rows()));
}
@Test void pageSelectionDoesNotMultiplyUnicodeEquivalentBucketsAndValidatesLimits() {
var base=query("product",true,Map.of());
var expected=metrics.calculateTable(base,actor).rows().getFirst();
when(governance.allowedFields(actor)).thenReturn(Set.of("black_rate"));
assertEquals(normalized(metrics.calculateTable(base,actor).rows().stream().sorted(ProductTableOrder.PRODUCT.comparator()).toList()),
normalized(metrics.calculateProductPage(base,actor,1,100,"brightQualifiedDesc").rows()),"Hidden weights must not leak through ordering");
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
var limited=metrics.calculatePairedProductSnapshot(base,actor,List.of(expected.key(),expected.key()),()->"source");
assertEquals(normalized(List.of(expected)),normalized(limited.metrics().rows()));
assertTrue(metrics.calculatePairedProductSnapshot(base,actor,List.of(),()->"source").metrics().rows().isEmpty());
for(int page:new int[]{0,-1,1_000_001})assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,page,15,"productAsc"));
for(int size:new int[]{0,101})assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,1,size,"productAsc"));
assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,1,15,"product; DROP TABLE"));
assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(query("overall",true,Map.of()),actor,1,15,"productAsc"));
}
} }
+18
View File
@@ -26,6 +26,7 @@ SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED') TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED')
BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup', BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup',
'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build', 'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build',
'source_filter_member', 'source_filter_value', 'source_filter_field', 'source_filter_build',
'fact_black', 'fact_bright') 'fact_black', 'fact_bright')
OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion', OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion',
'release_action', 'candidate_reconciliation') 'release_action', 'candidate_reconciliation')
@@ -253,6 +254,23 @@ class Database:
self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid)) self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid))
for t in BUILD_CHILDREN: for t in BUILD_CHILDREN:
if t in self.columns: if t in self.columns:
if t == 'source_filter_member':
# Members have no build_id; drain each owned field in
# order without aliases because the exclusion uses LOCK TABLES.
count = 0
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
count = self.execute('DELETE FROM source_filter_member WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
if count:
break
elif t == 'source_filter_value':
# Values are likewise owned through fields and are deleted
# only after all field memberships have been drained.
count = 0
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
count = self.execute('DELETE FROM source_filter_value WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
if count:
break
else:
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,)) count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
if count: if count:
return count, 0 return count, 0
+30 -4
View File
@@ -16,6 +16,7 @@ from unittest.mock import patch
ROOT=Path(__file__).resolve().parents[2] ROOT=Path(__file__).resolve().parents[2]
sys.path.insert(0,str(ROOT/'scripts')) sys.path.insert(0,str(ROOT/'scripts'))
import backup_retention as br import backup_retention as br
import managed_test_resources
spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py')) spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py'))
ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret) ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret)
@@ -91,14 +92,18 @@ class DatabaseRetentionTests(unittest.TestCase):
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text()) cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True) cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True)
cls.name='quality_retention_test_'+uuid.uuid4().hex[:12] cls.name='quality_retention_test_'+uuid.uuid4().hex[:12]
managed_test_resources.create(ROOT,cls.name,cls.execute_admin)
with cls.admin.cursor() as c: with cls.admin.cursor() as c:
c.execute('CREATE DATABASE '+ret.ident(cls.name))
c.execute('USE '+ret.ident(cls.name)) c.execute('USE '+ret.ident(cls.name))
ddl=[ ddl=[
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))', 'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))', 'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)', 'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)',
'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))', 'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_build(build_id varchar(36) PRIMARY KEY,format_version int,FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
'CREATE TABLE source_filter_member(field_id bigint NOT NULL,fact_id bigint NOT NULL,value_id bigint NOT NULL,PRIMARY KEY(field_id,fact_id),KEY(value_id,fact_id),FOREIGN KEY(field_id) REFERENCES source_filter_field(id),FOREIGN KEY(value_id) REFERENCES source_filter_value(id))',
'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))', 'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))', 'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)', 'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)',
@@ -115,14 +120,25 @@ class DatabaseRetentionTests(unittest.TestCase):
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name)) managed_test_resources.drop(ROOT,cls.name,cls.execute_admin)
cls.admin.close() cls.admin.close()
@classmethod
def execute_admin(cls,sql,args=None):
result=''
statements=[part.strip() for part in sql.split(';') if part.strip()]
with cls.admin.cursor() as cursor:
for statement in statements:
cursor.execute(statement,args if len(statements)==1 else None)
result='\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
return result
def setUp(self): def setUp(self):
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name) self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
self.db=ret.Database(ROOT,self.name);self.db.root=self.root self.db=ret.Database(ROOT,self.name);self.db.root=self.root
self.db.execute('SET FOREIGN_KEY_CHECKS=0') self.db.execute('SET FOREIGN_KEY_CHECKS=0')
for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table)) for table in self.db.tables:
if table!='__quality_test_owner':self.db.execute('TRUNCATE TABLE '+ret.ident(table))
self.db.execute('SET FOREIGN_KEY_CHECKS=1') self.db.execute('SET FOREIGN_KEY_CHECKS=1')
self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')") self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')")
self.db.execute("INSERT INTO current_release VALUES('quality','current')") self.db.execute("INSERT INTO current_release VALUES('quality','current')")
@@ -187,14 +203,24 @@ class DatabaseRetentionTests(unittest.TestCase):
def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self): def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self):
self.retire_old();state={'builds':{'obsolete':1}} self.retire_old();state={'builds':{'obsolete':1}}
self.db.execute("INSERT INTO source_filter_build VALUES('obsolete',1)")
self.db.execute("INSERT INTO source_filter_field VALUES(9,'obsolete','black','空字段',%s),(10,'obsolete','black','客户',%s),(11,'obsolete','black','工单',%s)", (b'0'*32,b'1'*32,b'2'*32))
self.db.execute("INSERT INTO source_filter_value VALUES(100,10,%s,'客户A'),(101,10,%s,'客户B'),(102,11,%s,'工单A')", (b'1'*32,b'2'*32,b'3'*32))
self.db.execute("INSERT INTO source_filter_member VALUES(10,1,100),(10,2,101),(11,3,102)")
with self.db.frozen('current'): with self.db.frozen('current'):
self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0)) self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0))
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_member')[0]['n'],2)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_value')[0]['n'],3)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_field')[0]['n'],3)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_build')[0]['n'],1)
old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0] old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0]
self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64) self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64)
self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,)) self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,))
for _ in range(4): for _ in range(20):
with self.db.frozen('current'):self.db.build_step(state,1800,4000,1) with self.db.frozen('current'):self.db.build_step(state,1800,4000,1)
self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")) self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'"))
for table in ('source_filter_member','source_filter_value','source_filter_field','source_filter_build'):
self.assertFalse(self.db.rows('SELECT * FROM '+ret.ident(table)))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'")) self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'"))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'")) self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'"))
+34 -4
View File
@@ -13,6 +13,7 @@ from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2] ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts')) sys.path.insert(0, str(ROOT / 'scripts'))
import backup_retention as br import backup_retention as br
import managed_test_resources
import revision_retention as revisions import revision_retention as revisions
spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py')) spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py'))
ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret) ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret)
@@ -26,12 +27,15 @@ class SourceRevisionTests(unittest.TestCase):
cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root', cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root',
password=cfg['rootPassword'], autocommit=True) password=cfg['rootPassword'], autocommit=True)
cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12] cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
managed_test_resources.create(ROOT, cls.name, cls.execute_admin)
ddl = [ ddl = [
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))', 'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))',
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))', 'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))',
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))', 'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
"CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)", "CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)",
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', 'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', 'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', 'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))', 'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))',
@@ -42,23 +46,34 @@ class SourceRevisionTests(unittest.TestCase):
'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))'] 'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))']
try: try:
with cls.admin.cursor() as c: with cls.admin.cursor() as c:
c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name)) c.execute('USE ' + ret.ident(cls.name))
for statement in ddl: for statement in ddl:
c.execute(statement + ' ENGINE=InnoDB') c.execute(statement + ' ENGINE=InnoDB')
except BaseException: except BaseException:
with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name)) managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
cls.admin.close(); raise cls.admin.close(); raise
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name)) managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
cls.admin.close() cls.admin.close()
@classmethod
def execute_admin(cls, sql, args=None):
result = ''
statements = [part.strip() for part in sql.split(';') if part.strip()]
with cls.admin.cursor() as cursor:
for statement in statements:
cursor.execute(statement, args if len(statements) == 1 else None)
result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
return result
def setUp(self): def setUp(self):
self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve() self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve()
self.db = ret.Database(ROOT, self.name); self.db.root = self.root self.db = ret.Database(ROOT, self.name); self.db.root = self.root
self.db.execute('SET FOREIGN_KEY_CHECKS=0') self.db.execute('SET FOREIGN_KEY_CHECKS=0')
for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table)) for table in self.db.tables:
if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
self.db.execute('SET FOREIGN_KEY_CHECKS=1') self.db.execute('SET FOREIGN_KEY_CHECKS=1')
self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')") self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')")
self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')") self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')")
@@ -121,6 +136,21 @@ class SourceRevisionTests(unittest.TestCase):
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0) self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
self.assertTrue(self.present()); self.assertTrue(path.exists()) self.assertTrue(self.present()); self.assertTrue(path.exists())
def test_indexed_business_values_do_not_count_as_revision_metadata(self):
path = self.add_revision('indexed-only', b'indexed-only workbook')
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,))
self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,))
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists())
def test_indexed_field_names_do_not_count_as_revision_metadata(self):
path = self.add_revision('indexed-field-only', b'field-only workbook')
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,))
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists())
def test_new_reference_resets_continuous_grace(self): def test_new_reference_resets_continuous_grace(self):
self.add_revision(); self.collect(100) self.add_revision(); self.collect(100)
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500) self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500)
+89 -1
View File
@@ -1,7 +1,95 @@
# 当前进度 # 当前进度
核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。 ## 2026-11-14:数据库分页与测试修复已发布(2026.11.14.2)
用户授权测试通过后发布并同步Gitea。本轮沿用分页独立工作目录,功能提交`8f1b790`,已推送Gitea main,未合入主目录旧分支的无关修改。6项既有失败已复现并修复:报告测试缺少真实辅助函数/钩子绑定4项、SVG趋势测试按固定纵轴校验2项;恢复失败提示按项目静默省略约定校正。未改报告生产逻辑、模型原文或正式指标算法。
前端全套616项全部通过,类型及生产构建通过;常规后端995项(823执行通过、172按条件跳过),另启用隔离MySQL专项10项通过。分页优化、当前页细分和全量导出一并发布;此前本地专项覆盖Unicode、缺失阶段、权限与导出,本次未调用模型或执行私有原表全量回归。
公开版本2026.11.14.2,入口HTML及24个发布文件与本地构建逐个摘要一致。新旧JAR迁移SQL资源完全一致,无新增迁移、表或索引。线上只读基线核对:年度综合7651、黑片6419、亮片6715产品;月度2960、搜索8、来源筛选3产品的正式汇总/全量行/覆盖完全一致,各表首页、第二页和末页核对一致,细分废率及48条来源结果保持。浏览器实际第二页显示16–30行、共6715行,15条数据、无错误;来源抽屉读取48条、121字段,原筛选恢复且99.10%保持。
MySQL原进程、正式版本指针和数据库结构保持;DSH应用/模型/状态路径保持,维护暂停标记恢复原状,原维护兼容保护未解除。完整发布与测试证据位于主目录`output/20261114-pagination-release/`。下方“未发布/6项失败”为之前时点,已由本节结果替代。
## 2026-11-14:分页发布准备与6项测试修复
用户已授权测试通过后发布,并同步Gitea。继续使用分页独立分支,核实远端main仍为`d948e1e`。6项失败已复现:报告组件测试缺少当前真实辅助函数/钩子绑定4项,SVG测试仍假定固定纵轴2项;已补齐依赖并按真实百分比坐标范围校验正式值和像素位置。图表恢复失败的旧告警断言按项目“静默省略、保留原文”约定校正。未改模型原始正文、报告生产逻辑或正式指标算法。
前端全套616项通过;常规后端995项(823执行通过、172按条件跳过)及另启用的隔离MySQL10项通过。本节为发布准备时点,最终发布和同步状态另记后续验收。
## 2026-11-14:数据库分页本地实现(未发布、未同步)
按用户“优化并本地测试”,基于 main `d948e1e` 在独立分支 `codex/database-pagination-local-20261114` 实现,源码位于 `/Users/lyun/.codex/worktrees/quality-database-pagination-20261114`。产品汇总、黑片/亮片宽产品表按数据库页读取,保留正式汇总、趋势、授权与全量导出;当前页细分废率只查相应产品。年度摘要优先,月/季/搜索并行读小页,避免额外串行等待。同重量固定次序、隐藏重量不通过排序泄露;换范围重置与请求取消已覆盖。没有新增表、索引、迁移或持久化副本。
144,016 条合成事实、6,002 产品:年度综合产品表接口三轮中位数 2627→1593ms,数据体积 5,286,041→15,364 字节;同一浏览器旧/新前端表格可用时间中位数 5725→4566ms(约20%),顶部指标/趋势基本不变。分页/细分对齐正式全量接口,9组额外期间、排除及来源筛选一致;浏览器三页查询、换年、来源、第二页全量Excel已验收。常规后端995项(823通过、172跳过),专项隔离MySQL10项和前端相关77项通过,前端构建通过。前端全套616项仍有6个主线既有智能分析报告测试失败,已在未改动基线复现;未修改这些模块。特殊Unicode分组漏计的本轮边界已修正并复验。
仅本地,未提交/推送或发布;未执行私有原表全量回归。自有测试库/账号/进程已清理,正式指针和原进程保持,原浏览器链接恢复。完整报告与截图:主目录 `output/20261114-database-pagination/测试结果.md`。后续如需交付,先核对当前主线和本轮独立差异;不要把主目录旧分支历史改动一并提交。
## 2026-11-14:分步加载已发布(2026.11.14.1)
在用户明确要求发布并同步后,复核 Gitea main 仍为 `3819011`,仅提交本轮查询加载、表格投影、相关测试和使用说明,功能提交 `374f08c`。没有整批提交主工作目录旧分支上的历史改动。
发布前前端77项及类型/生产构建通过;完整后端992项(822执行通过、170按条件跳过)。此前14.4万条隔离合成数据的对照和异常验收继续有效,新旧运行包43个迁移资源逐项摘要一致。发布时业务任务空闲,保留MySQL原进程、正式数据版本、DSH应用/模型/状态路径和固定引擎配置。
线上实际读回核对:年度6715产品、月度2402产品、来源筛选3产品的汇总、表格、趋势、覆盖和来源结果与旧后端完全一致;新table投影除省略趋势外结果与full一致,同范围overview的汇总/趋势也一致。代表年度完整接口约2485ms、table约1313ms、overview约618ms;来源筛选约927/191/210ms,48条来源约358ms。这些是单次只读重放耗时,不是所有筛选或冷启动保证。
前端2026.11.14.1已切换,公开版本、入口HTML和资源与本地生产构建一致。复用现有浏览器,原筛选显示查询完成、合格率99.10%、总重7190.86kg;来源抽屉实际载入48条、121字段,产品钻取的5月来源返回1条,返回上一级正常。恢复页面时纠正了上次手工重写筛选名称的偏差,改用保存的原始链接,没有改业务数据。部署验收脚本的进程检查曾缺少初始化,已修正并重新核对新后端、MySQL身份、健康和数据指针;未回滚业务数据。
仅更新前后端,无新增表、索引或数据库迁移;表格仍返回全量产品,数据库分页尚未实施。现有维护升级兼容保护未解除。私有基线、登录配置、运行包和验收产物不进入Git,完整证据保存在主目录 `output/20261114-progressive-query/`。本轮功能与发布验收记录已同步 Gitea `main`,功能提交 `374f08c`,发布记录 `bfb91ef`;远端分支已读回核验。下方本地试验的“未发布、未同步”为当时时点。
## 2026-11-14:分步加载本地试验(未发布、未同步)
用户要求先本地测试。基于已核实的 Gitea main `3819011` 创建独立分支 `codex/progressive-query-local-20261114`,源码位于 `/Users/lyun/.codex/worktrees/quality-progressive-query-20261114`,保留主工作目录旧分支及原有改动。
本地实现:宽年度产品范围先读取正式小型汇总和可复用趋势,再读取 `part=table` 投影;表格仅省略时间与对象交叉分组,不改正式算法、授权或发布版本。月/季、单产品和小维度复用完整结果,避免额外往返;一般查询解除无关目录等待,沿用已有请求合并/并发上限和底部柏拉图按需加载。刷新代次使旧表格与趋势失去当前资格。
验证:144,016 条合成事实、6,002 产品的同一浏览器三轮中位数,顶部指标约3164→1616ms、趋势1764→1616ms、表格3164→2863ms。接口年度全产品约2506→1040ms提前显示,主要数据全部完成约2526→2332ms;月度基本持平,总体年度未测得耗时改善,不承诺所有筛选提速。18组接口业务结果对照及8组隔离投影范围检查一致,表格失败不阻塞指标/趋势且来源抽屉可读,首批失败可回退。前端77项及类型/生产构建通过;后端992项(822通过、170按条件跳过)。未执行私有原表全量回归或在线模型调用。
仅本地试验,未提交/推送或发布。数据库分页尚未实施,表格仍返回全量产品并保持全量导出;没有新增表、索引或迁移。两个自有合成库、账号及测试/预览进程已清理,正式版本指针未变,共享后端健康UP,原浏览器页面已恢复。完整数据、截图和日志位于主目录 `output/20261114-progressive-query/测试结果.md`。下一步如获发布授权,先核对这批本地变更及当前主线,再决定是否交付第一阶段或继续数据库分页。
核实日期:2026年11月13日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。
## 2026-11-13:Gitea交付(筛选提速与包含条件)
本轮仓库交付整合此前来源筛选超时修复、统一字典/记录索引、后台补建、目录与候选读取优化,以及“包含”条件和输入边界修正;保留11个实施与验收提交,整合提交为 `6983dc3`。目标为 `lyun/quality_analyse` 的 `main`,对应当前已上线2026.11.13.12的功能源码。
整合后功能代码与已验收分支逐文件一致,无冲突或新增业务修改,沿用既有构建、隔离数据库与线上验收结果。本次只同步源码和交付记录,不重复发布、重启服务或运行数据库迁移;私有配置、业务数据、运行包和验收附件不进入仓库。下方“未同步”保留各次实施时点,本节为后续接续的最新仓库交付记录。
## 2026-11-13:来源字段“包含”筛选(已发布)
来源字段操作菜单新增“包含”:自由输入关键词按回车,多个关键词按OR匹配;不同筛选条件之间仍为AND。后台用绑定参数的LOCATE执行字面子串匹配,大小写及正式来源值语义保持,`%`、`_`、反斜杠和引号不是通配或SQL内容。支持1–50个非空白关键词,每个最多256字符;前端控制搜索长度并避免截断代理对,重复关键词失焦后清空草稿。
汇总、趋势、来源明细与Excel统一走同一解析/授权/查询逻辑,兼容原始JSON回退与现有索引。DSH活跃共享契约、图表上下文和历史/证据显示同步接受contains;关键词不做完整候选值的等值核验。打包只替换5个相关DSH业务模块,保留当前已验收的流式助手和原固定发行版;模型配置及状态不变,未调用在线模型。不新增迁移、表或索引,不重建既有52个构建。
验证:前端61项及类型/生产构建通过;完整后端990项(820执行通过、170按条件跳过);专项隔离MySQL及服务45项(44通过、1基准跳过),覆盖两阶段、原表/索引、规范维度、大小写、特殊字符、零/字符串null/空值、组织权限,以及汇总/趋势/明细/Excel。DSH契约/显示Python135项(133通过、2跳过),应用构建16项通过,新打包应用的contains契约及固定运行时检查通过,独立审查无阻断。未执行私有原表全量回归或在线模型验收。
线上关键词命中2个完整客户值,contains与对应in集合的汇总业务数据、趋势及来源结果一致;代表查询约1965/721/465ms。来源29条、121个原始字段,导出29条且ZIP/XML有效;首次导出12907ms、复用后844ms。实际菜单、键盘提交、多关键词字面值、257字符粘贴限制及重复失焦清理已检查,页面无失败提示。前端2026.11.13.12(`f8432b4`),后端功能提交`3c022e8`,后续仅前端输入边界修正,数据库进程保持;DSH固定引擎、模型配置及状态保持。
代码已在独立分支 `codex/source-filter-contains-20261113` 本地提交,本轮未同步Gitea。私有请求、结果及效果截图位于 `output/20261113-source-contains/`,不进入Git。
## 2026-11-13:统一筛选提速(已发布验收)
已发布2026.11.13.10,功能提交 `20f1c7d`;当前正式版52/52个构建的筛选索引全部就绪。新增按构建保存的来源字段字典与稀疏记录索引,来源/配置筛选、候选、指标、趋势、明细及明细导出共用;正式算法、当前字段/组织权限和发布版绑定保持。候选模式只取上限加一,不再为下拉精确计数;常规页面查询不等待无关目录,本字段改选不重复读取自己的候选,明细导出先只读字段名。
生命周期:启动和构建提交后通过同一单线程有界去重队列补索引,完整提交后才启用;失败或尚未就绪保留原查询,导入事务不等待索引。构建过程按500条事实窗口驱动字典关联,并物理重建临时窗口,避免回扫增长字典和长事务临时表累积。保留清理已识别4张新增表,未解除维护兼容保护。
首次加载还存在两条慢路径:候选为校验年份/可见范围反复读取宽原始行,V44新增薄覆盖索引;字段目录扫描约110万事实只为找每构建首条可见记录,V45改索引列序及带完整权限的有序LIMIT1查询。实际计划回归要求覆盖读取、不排序全部事实、找到首条即停;与旧MIN查询直接对照原始字段目录,保留空/隐藏构建和重复成员语义。
验证:前端相关46项及生产构建通过;完整后端988项(819执行通过、169按条件跳过),专项隔离MySQL/服务49项(48通过、1基准跳过),保留清理34项通过,多次独立审查无阻断。隔离场景覆盖黑片/亮片、全部4类来源运算、旧配置null和字面字符串null、Unicode、同一事实行、多字段、当前权限、导出、跨窗口故障回滚与重试。未执行私有原表全量回归。
线上实测在恢复原256MiB数据库缓存、后端重新启动且无并行测试时完成:同一客户筛选首次汇总1366ms、趋势347ms、来源明细513ms、原候选条件381ms、去掉自身约束的完整候选首开3546ms;月度458ms、季度369ms。原始超时场景汇总/趋势约48.3秒;前一版索引已全就绪时首次汇总仍18843ms,此次首条目录优化后降到1366ms。汇总、趋势、29条来源明细的完整规范化结果与原基准一致,月/季与原路径一致。Excel约3479ms,1张表、29条明细、A1:EA30,ZIP/XML有效;页面查询完成,来源抽屉载入121字段、无失败提示,点击产品与来源范围一致。以上是代表场景的只读重放,不是数据库完整冷启动或所有字段组合的固定耗时承诺。
空间实测:新增字典/成员等4表文件共13,727,023,104字节,事实表4个覆盖索引页约372,064,256字节;本口径合计约14.10GB(13.13GiB),不含既有原表、redo/undo和临时文件,也不等同于数据库目录净增差值。此次覆盖全部原始字段,空间显著高于先前紧凑柏拉图汇总。补建期间临时1GiB缓存已恢复256MiB,核验16384页且调整完成。数据库原进程和DSH配置保留,无业务版本指针变更。
实现与验收记录已本地提交;本轮新优化尚未同步Gitea。私有请求、结果校验及页面截图保存在 `output/20261113-filter-speed/`,不进入仓库。主工作目录已有历史改动保持,当前交付在独立分支 `codex/unified-filter-speed-20261113`。
## 2026-11-13:来源字段筛选超时修复
定位:客户等原始字段筛选时,汇总、趋势和覆盖检查重复读取字段目录,数据库对年度查询选择了原表扫描,约48秒超过页面30秒期限。复用服务级字段目录缓存,目录及无窄范围限制的年度来源筛选使用既有构建索引;JSON值每次表达式只提取一次,保留空值、字符串和大小写语义。单月、季度、产品及扩展条件保留原查询计划。查询超时提示改为重试或调整范围,不再误报连接断开。权限和正式计算规则不变,无新增表或索引。
验证:后端完整构建975项(812执行通过、163按条件跳过);隔离MySQL及快照专项34项(33通过、1跳过),前端相关17项及生产构建通过。独立复核无实质问题;已验证主目录本轮6文件与交付目录一致,新旧运行包数据库迁移文件一致。已发布2026.11.13.5(功能提交 `ea84e26`)。线上同条件汇总/趋势从48282/48281ms降至17818/17816ms,完整结果规范化校验值均一致;页面正常加载时两接口约5528/5529ms,无失败提示。实际来源明细返回29条、121个字段,Excel下载有效。数据库迁移文件不变,MySQL原进程和DSH配置保留。只读同条件复测不等于私有原表全量回归。本轮独立分支已本地提交,尚未同步Gitea;私有验收材料位于 `output/20261113-source-filter-timeout/`。
## 2026-11-13:Gitea 同步交付 ## 2026-11-13:Gitea 同步交付
+5 -3
View File
@@ -144,6 +144,8 @@ def _resolve_source_column_values(task, *, release_id, resolver):
if not isinstance(rules, list): if not isinstance(rules, list):
return None return None
for rule_index, rule in enumerate(rules): for rule_index, rule in enumerate(rules):
# Candidate lookup confirms complete values only. A contains keyword is
# intentionally not required to equal a full candidate source value.
if rule.get('operator') not in ('in', 'not_in'): if rule.get('operator') not in ('in', 'not_in'):
continue continue
field_key = rule['field'] field_key = rule['field']
@@ -1212,7 +1214,7 @@ def plan_task(*, base_url, api_key, model, question, context, previous=None, can
if aliases and field.get('type')=='source_value':item['aliases']=aliases if aliases and field.get('type')=='source_value':item['aliases']=aliases
compact_fields.append(item) compact_fields.append(item)
compact_catalog['sourceFields']=compact_fields compact_catalog['sourceFields']=compact_fields
compact_catalog['sourceColumnOperators']=['in','not_in','is_empty','not_empty'] compact_catalog['sourceColumnOperators']=['in','not_in','contains','is_empty','not_empty']
payload['businessCatalog']=compact_catalog payload['businessCatalog']=compact_catalog
system += ('\n综合以上生成报告应整合previousVerifiedTurns中多个已完成轮次,historyMode=all、freshQuery=false;不得把最新局部追问当作全部历史。' system += ('\n综合以上生成报告应整合previousVerifiedTurns中多个已完成轮次,historyMode=all、freshQuery=false;不得把最新局部追问当作全部历史。'
'只有用户要求更新数据时才freshQuery=true。报告与复述不自动开展新调查。' '只有用户要求更新数据时才freshQuery=true。报告与复述不自动开展新调查。'
@@ -1292,9 +1294,9 @@ JSON字段:goal(完整当前目标),kind(analysis/metrics/members/explain/re
filterUpdates允许department/group/product/projectGroup/extraFilters/exclusions/includeUnassigned。null明确清除该条件,{}表示没有变化;单一维度用目录原名字符串,extraFilters和exclusions用字段到值数组。额外字段仅限businessCatalog.extraFields;客户代码使用extraFilters.customerCode,保留完整字符串。未知字段不能丢弃或扩大范围,应澄清能力限制。汽车交通对应department=新能源汽车磁钢,国内/海外才是其业务分组。 filterUpdates允许department/group/product/projectGroup/extraFilters/exclusions/includeUnassigned。null明确清除该条件,{}表示没有变化;单一维度用目录原名字符串,extraFilters和exclusions用字段到值数组。额外字段仅限businessCatalog.extraFields;客户代码使用extraFilters.customerCode,保留完整字符串。未知字段不能丢弃或扩大范围,应澄清能力限制。汽车交通对应department=新能源汽车磁钢,国内/海外才是其业务分组。
productMatch为ambiguous且未选择时,只需列出授权候选并needsClarification=true,不选择或合并后缀、不换成全部产品。unique使用完整原名;18×15×4不能匹配18×15×4.1。用户选择前轮候选时保留原年份、指标和趋势目标。newTask=true仅用于用户明确另开无关任务,并声明新期间和条件。 productMatch为ambiguous且未选择时,只需列出授权候选并needsClarification=true,不选择或合并后缀、不换成全部产品。unique使用完整原名;18×15×4不能匹配18×15×4.1。用户选择前轮候选时保留原年份、指标和趋势目标。newTask=true仅用于用户明确另开无关任务,并声明新期间和条件。
今年、上月等使用serverCalendar;最新数据月才用目录,不猜测缺失期间。真正缺少期间或对象有歧义才澄清。普通读取业务指标freshQuery=true。 今年、上月等使用serverCalendar;最新数据月才用目录,不猜测缺失期间。真正缺少期间或对象有歧义才澄清。普通读取业务指标freshQuery=true。
原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/is_empty/not_empty","values":[完整原文]}。in/not_in要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。大小写、空格、前导零都是字面值,不做归一化。规则AND、values OR;每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。 原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/contains/is_empty/not_empty","values":[完整原文或包含关键字]}。in/not_in/contains要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。规则AND、values OR;文本大小写、空格、前导零均保留原文,不对contains关键字归一化;contains是原文的字面子串筛选,不是通配符或正则。每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。
只问报废率时fields=[scrapRate];同时要求分子分母、计算基数、报废总量或总重时,分别保留对应重量字段;要求分子分母时fields=[scrapRate,scrapWeight,totalWeight]。不能只在goal中提及而漏掉实际fields。分子为现有正式报废总量,分母为该阶段正式总重,黑片使用现有修正总重。利用料、本部报废、过账外废沿用现有分量逻辑,不使用1-合格率,也不平均原表百分比。未指定阶段的报废率返回black和bright两个阶段;没有统一综合报废率定义,用户明确要综合报废率时请求其定义,不转成综合合格率。 只问报废率时fields=[scrapRate];同时要求分子分母、计算基数、报废总量或总重时,分别保留对应重量字段;要求分子分母时fields=[scrapRate,scrapWeight,totalWeight]。不能只在goal中提及而漏掉实际fields。分子为现有正式报废总量,分母为该阶段正式总重,黑片使用现有修正总重。利用料、本部报废、过账外废沿用现有分量逻辑,不使用1-合格率,也不平均原表百分比。未指定阶段的报废率返回black和bright两个阶段;没有统一综合报废率定义,用户明确要综合报废率时请求其定义,不转成综合合格率。
按用户原文提出in/not_in文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。 按用户原文提出in/not_in完整文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验in/not_in取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。contains关键字是子串而非完整候选值,不做等值候选核验,按用户原文交由正式查询执行字面子串筛选。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。
kind=members仅用于列出有哪些委托单/批号/规格等唯一成员,不表示原始行;memberFields使用当前目录key且最多5个,必须与用户明确要列出的对象语义相符,不能把筛选字段当输出成员。裸“规格”表示产品规格,要选择目录中真实产品规格列;不能用图号或客户代码代替。明确问图号时选择图号列,明确问客户规格时选择客户规格列。若用户明确给出“成品规格”等目录label,优先精确匹配该label;裸“规格”有多个候选或没有可用字段时,needsClarification=true、memberFields=[],保留原目标并澄清。并列、否定或复合字段意图由规划器按用户表达处理,不能补入已取消字段。stages必须明确为一个或两个阶段,期间与普通filters照常保留。需要澄清时stages可为空,本轮不查询并保留目标。members的fields=[]、verifySource=false、fullDetails=false、freshQuery=true,不计算KPI、不改成metrics;清单由平台直接渲染,无需撰写业务数值。 kind=members仅用于列出有哪些委托单/批号/规格等唯一成员,不表示原始行;memberFields使用当前目录key且最多5个,必须与用户明确要列出的对象语义相符,不能把筛选字段当输出成员。裸“规格”表示产品规格,要选择目录中真实产品规格列;不能用图号或客户代码代替。明确问图号时选择图号列,明确问客户规格时选择客户规格列。若用户明确给出“成品规格”等目录label,优先精确匹配该label;裸“规格”有多个候选或没有可用字段时,needsClarification=true、memberFields=[],保留原目标并澄清。并列、否定或复合字段意图由规划器按用户表达处理,不能补入已取消字段。stages必须明确为一个或两个阶段,期间与普通filters照常保留。需要澄清时stages可为空,本轮不查询并保留目标。members的fields=[]、verifySource=false、fullDetails=false、freshQuery=true,不计算KPI、不改成metrics;清单由平台直接渲染,无需撰写业务数值。
单期投入重量排名或明确单阶段报废排名:kind=analysis、dimensions为实际product/department/group、fields仅totalWeight/scrapWeight、dimensionStages明确各维度black/bright、trendRequired=false;未限定阶段的报废排名使用queryPlan ranking、fields=[rate,scrapWeight]、stages=[]、dimensionStages对应维度=[];全量正式分母不随Top展示数量改变。普通年度重量排名默认periodCoverage=available,注明已发布月份;只有用户明确要求完整全年齐全时用complete。排名与趋势同时要求时仍保留真实要求,不删除trendRequired。 单期投入重量排名或明确单阶段报废排名:kind=analysis、dimensions为实际product/department/group、fields仅totalWeight/scrapWeight、dimensionStages明确各维度black/bright、trendRequired=false;未限定阶段的报废排名使用queryPlan ranking、fields=[rate,scrapWeight]、stages=[]、dimensionStages对应维度=[];全量正式分母不随Top展示数量改变。普通年度重量排名默认periodCoverage=available,注明已发布月份;只有用户明确要求完整全年齐全时用complete。排名与趋势同时要求时仍保留真实要求,不删除trendRequired。
能力边界必须忠实表达:排名/缺陷保留analysis及真实dimensions和dimensionStages;两个独立期间比较保留comparisonRequired=true和comparisonPeriod;源表核验为source_verify及verifySource=true;原始逐行记录为details及fullDetails=true;报告为report。不得为了适配已接入能力而改写成普通指标。不得把“有哪些”理解成所有原始字段。后端负责说明仍未接入的能力。 能力边界必须忠实表达:排名/缺陷保留analysis及真实dimensions和dimensionStages;两个独立期间比较保留comparisonRequired=true和comparisonPeriod;源表核验为source_verify及verifySource=true;原始逐行记录为details及fullDetails=true;报告为report。不得为了适配已接入能力而改写成普通指标。不得把“有哪些”理解成所有原始字段。后端负责说明仍未接入的能力。
+2 -1
View File
@@ -154,7 +154,8 @@ def _scope_label(filters):
if key=='columnFilters': if key=='columnFilters':
for rule in value: for rule in value:
stage='、'.join({'black':'黑片','bright':'亮片'}.get(s,s) for s in rule.get('stages',[])) stage='、'.join({'black':'黑片','bright':'亮片'}.get(s,s) for s in rule.get('stages',[]))
operation={'in':'为','not_in':'排除','is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'') operation={'in':'为','not_in':'排除','contains':'包含片段',
'is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'')
parts.append(stage+'来源字段'+operation+'、'.join(rule.get('values',[]))) parts.append(stage+'来源字段'+operation+'、'.join(rule.get('values',[])))
else: parts.append(names[key]+'='+str(value)) else: parts.append(names[key]+'='+str(value))
return ';'.join(parts) if parts else '当前授权范围内总体(默认不含未分类)' return ';'.join(parts) if parts else '当前授权范围内总体(默认不含未分类)'
+2 -1
View File
@@ -850,7 +850,8 @@ def _history_scope_caption(snapshot, section):
parts.append('包含未分配项') parts.append('包含未分配项')
column_filters = filters.get('columnFilters') column_filters = filters.get('columnFilters')
if isinstance(column_filters, list): if isinstance(column_filters, list):
operator_labels = {'in': '包含', 'not_in': '不含', 'is_empty': '为空', 'not_empty': '非空'} operator_labels = {'in': '为', 'not_in': '排除', 'contains': '包含片段',
'is_empty': '为空', 'not_empty': '非空'}
for rule in column_filters: for rule in column_filters:
if not isinstance(rule, dict): if not isinstance(rule, dict):
continue continue
+2 -2
View File
@@ -115,11 +115,11 @@ QUALITY_DATA_SCHEMA: dict[str, Any] = {
"field": {"type": "string", "minLength": 1}, "field": {"type": "string", "minLength": 1},
"stages": {"type": "array", "minItems": 1, "maxItems": 2, "stages": {"type": "array", "minItems": 1, "maxItems": 2,
"items": {"type": "string", "enum": ["black", "bright"]}}, "items": {"type": "string", "enum": ["black", "bright"]}},
"operator": {"type": "string", "enum": ["in", "not_in", "is_empty", "not_empty"]}, "operator": {"type": "string", "enum": ["in", "not_in", "contains", "is_empty", "not_empty"]},
"values": {"type": "array", "minItems": 1, "maxItems": 50, "values": {"type": "array", "minItems": 1, "maxItems": 50,
"items": {"type": "string", "maxLength": 256}}, "items": {"type": "string", "maxLength": 256}},
}, "required": ["field", "stages", "operator"]}, }, "required": ["field", "stages", "operator"]},
"description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR,文本大小写、空格及前导零按原文匹配。is_empty/not_empty不得带values。"}, "description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR;in/not_in匹配完整文本值,contains按原文作字面子串匹配(MySQL LOCATE),不做关键字归一化,大小写、空格及前导零保留原文。每条文本规则提供1到50个不超过256字符的值;is_empty/not_empty不得带values。"},
"view": {"type": "string", "enum": ["overall", "department", "group", "product", "projectGroup"], "description": "query 的正式分组视图;changes 仅支持非 overall 维度。"}, "view": {"type": "string", "enum": ["overall", "department", "group", "product", "projectGroup"], "description": "query 的正式分组视图;changes 仅支持非 overall 维度。"},
"stage": {"type": ["string", "null"], "enum": ["black", "bright", None], "description": "top/details/defects/investigate必须明确black/bright。rank单阶段传black/bright;rank的metric=scrapWeight且stage省略或null时,合计黑片+亮片报废总重全量排序,并保留正式综合合格率。投入重量rank仍必须指定阶段。changes可选,未传为综合范围。"}, "stage": {"type": ["string", "null"], "enum": ["black", "bright", None], "description": "top/details/defects/investigate必须明确black/bright。rank单阶段传black/bright;rank的metric=scrapWeight且stage省略或null时,合计黑片+亮片报废总重全量排序,并保留正式综合合格率。投入重量rank仍必须指定阶段。changes可选,未传为综合范围。"},
"dimension": {"type": "string", "enum": ["category", "defect", "product", "department", "group", "projectGroup"], "description": "top模式仅category/product/department/group/projectGroup,不能用defect;具体缺陷TOP必须使用mode=defects、dimension=defect。product是规格。changes 使用 product/department/group/projectGroup。defects 使用 defect 或 product;未指定具体 defect 字段时,省略 dimension 默认按 defect 查询阶段具体缺陷。"}, "dimension": {"type": "string", "enum": ["category", "defect", "product", "department", "group", "projectGroup"], "description": "top模式仅category/product/department/group/projectGroup,不能用defect;具体缺陷TOP必须使用mode=defects、dimension=defect。product是规格。changes 使用 product/department/group/projectGroup。defects 使用 defect 或 product;未指定具体 defect 字段时,省略 dimension 默认按 defect 查询阶段具体缺陷。"},
+6 -4
View File
@@ -3,7 +3,7 @@ import copy
DEPARTMENT_ALIASES = {'汽车交通':'新能源汽车磁钢'} DEPARTMENT_ALIASES = {'汽车交通':'新能源汽车磁钢'}
SOURCE_STAGES = {'black', 'bright'} SOURCE_STAGES = {'black', 'bright'}
COLUMN_OPERATORS = {'in', 'not_in', 'is_empty', 'not_empty'} COLUMN_OPERATORS = {'in', 'not_in', 'contains', 'is_empty', 'not_empty'}
def source_field_labels(source_fields): def source_field_labels(source_fields):
@@ -35,11 +35,13 @@ def parse_column_filters(value):
raise ValueError(f'{name}.stages必须是非空black/bright数组。') raise ValueError(f'{name}.stages必须是非空black/bright数组。')
if operator not in COLUMN_OPERATORS:raise ValueError(f'{name}.operator不在当前字段支持目录中。') if operator not in COLUMN_OPERATORS:raise ValueError(f'{name}.operator不在当前字段支持目录中。')
clean={'field':key,'stages':list(stages),'operator':operator} clean={'field':key,'stages':list(stages),'operator':operator}
if operator in ('in','not_in'): if operator in ('in','not_in','contains'):
values=rule.get('values') values=rule.get('values')
if (not isinstance(values,list) or not 1<=len(values)<=50 if (not isinstance(values,list) or not 1<=len(values)<=50
or any(not isinstance(item,str) or item=='' or len(item)>256 for item in values)): or any(not isinstance(item,str) or item==''
raise ValueError(f'{name}.values必须是1到50个不超过256字符的完整文本值。') or operator=='contains' and not item.strip()
or len(item)>256 for item in values)):
raise ValueError(f'{name}.values必须是1到50个不超过256字符的文本值或包含关键字。')
clean['values']=list(dict.fromkeys(values)) clean['values']=list(dict.fromkeys(values))
elif 'values' in rule:raise ValueError(f'{name}的空值操作不接受values。') elif 'values' in rule:raise ValueError(f'{name}的空值操作不接受values。')
result.append(clean) result.append(clean)
+5 -3
View File
@@ -9,9 +9,11 @@ import uuid
# These are measured business values, not documents that refer to source IDs. # These are measured business values, not documents that refer to source IDs.
VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'), BUSINESS_VALUE_COLUMNS = {('fact_black', 'raw_values'), ('fact_black', 'components'),
('fact_bright', 'raw_values'), ('fact_bright', 'components'), ('fact_bright', 'raw_values'), ('fact_bright', 'components'),
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')} ('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components'),
('source_filter_field', 'column_name'),
('source_filter_value', 'value')}
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'} TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
@@ -54,7 +56,7 @@ class Cleaner:
def metadata_mentions(self, needles, excluded=None): def metadata_mentions(self, needles, excluded=None):
for table, columns in self.db.column_types.items(): for table, columns in self.db.column_types.items():
for column, kind in columns.items(): for column, kind in columns.items():
if kind not in TEXT_TYPES or (table, column) in VALUE_JSON: if kind not in TEXT_TYPES or (table, column) in BUSINESS_VALUE_COLUMNS:
continue continue
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles] conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
args = list(needles) args = list(needles)
+9
View File
@@ -34,6 +34,15 @@ def coverage_comparison():
class BoundReportTest(unittest.TestCase): class BoundReportTest(unittest.TestCase):
def test_scope_label_distinguishes_full_value_from_substring_filters(self):
from nao_evidence_report import _scope_label
label = _scope_label({'columnFilters': [
{'field': 'orderNo', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7']},
{'field': 'orderNo', 'stages': ['bright'], 'operator': 'contains', 'values': ['07']},
]})
self.assertIn('黑片来源字段为WO-7', label)
self.assertIn('亮片来源字段包含片段07', label)
def test_priority_summary_keeps_three_signals_and_rate_denominators_separate(self): def test_priority_summary_keeps_three_signals_and_rate_denominators_separate(self):
signals={'scrapIncrease':[{'label':'放量规格','scrapDelta':120}], signals={'scrapIncrease':[{'label':'放量规格','scrapDelta':120}],
'currentScrap':[{'label':'存量规格','current':{'scrapWeight':500}}], 'currentScrap':[{'label':'存量规格','current':{'scrapWeight':500}}],
+8 -4
View File
@@ -456,9 +456,11 @@ class HistoryReportExecutionTest(unittest.TestCase):
first_section = candidate['sections'][0] first_section = candidate['sections'][0]
first_section['filters']['columnFilters'] = [{ first_section['filters']['columnFilters'] = [{
'field': 'src_work_order', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7'], 'field': 'src_work_order', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7'],
}, {
'field': 'src_contains_code', 'stages': ['bright'], 'operator': 'contains', 'values': ['07'],
}] }]
set_source_task_context(candidate, SOURCE_A, set_source_task_context(candidate, SOURCE_A,
{'sourceFieldLabels': {'src_work_order': '黑片委托单号'}}) {'sourceFieldLabels': {'src_work_order': '黑片委托单号', 'src_contains_code': '客户代码'}})
set_source_task_context(candidate, SOURCE_B, set_source_task_context(candidate, SOURCE_B,
{'sourceFieldLabels': {'src_work_order': '亮片委托单号'}}) {'sourceFieldLabels': {'src_work_order': '亮片委托单号'}})
fact = candidate['facts'][0] fact = candidate['facts'][0]
@@ -476,12 +478,14 @@ class HistoryReportExecutionTest(unittest.TestCase):
projected = _history_model_fact_projection(candidate, [fact])[0] projected = _history_model_fact_projection(candidate, [fact])[0]
self.assertIn('产业部 消费电子产业部', projected['modelScope']['displayScope']) self.assertIn('产业部 消费电子产业部', projected['modelScope']['displayScope'])
self.assertIn('业务分组 消费电子-3C', projected['modelScope']['displayScope']) self.assertIn('业务分组 消费电子-3C', projected['modelScope']['displayScope'])
self.assertIn('黑片委托单号 包含 WO-7', projected['modelScope']['displayScope']) self.assertIn('黑片委托单号 为 WO-7', projected['modelScope']['displayScope'])
self.assertIn('亮片 客户代码 包含片段 07', projected['modelScope']['displayScope'])
self.assertIn('消费电子产业部', projected['modelText']) self.assertIn('消费电子产业部', projected['modelText'])
self.assertIn('消费电子-3C', projected['modelText']) self.assertIn('消费电子-3C', projected['modelText'])
self.assertIn('黑片委托单号', projected['modelText']) self.assertIn('黑片委托单号', projected['modelText'])
self.assertIn('WO-7', projected['modelText']) self.assertIn('WO-7', projected['modelText'])
for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:', 'src_work_order'): for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:',
'src_work_order', 'src_contains_code'):
self.assertNotIn(machine_label, projected['modelText']) self.assertNotIn(machine_label, projected['modelText'])
self.assertNotIn(machine_label, json.dumps(projected['modelScope'], ensure_ascii=False)) self.assertNotIn(machine_label, json.dumps(projected['modelScope'], ensure_ascii=False))
self.assertEqual(projected['text'], source_text) self.assertEqual(projected['text'], source_text)
@@ -581,7 +585,7 @@ class HistoryReportExecutionTest(unittest.TestCase):
goals = payload['historySourceGoals'] goals = payload['historySourceGoals']
self.assertTrue(any('产业部 消费电子产业部' in goal['displayScopeLabel'] self.assertTrue(any('产业部 消费电子产业部' in goal['displayScopeLabel']
and '业务分组 消费电子-3C' in goal['displayScopeLabel'] and '业务分组 消费电子-3C' in goal['displayScopeLabel']
and '黑片委托单号 包含 WO-7' in goal['displayScopeLabel'] and '黑片委托单号 为 WO-7' in goal['displayScopeLabel']
for goal in goals)) for goal in goals))
self.assertTrue(all('filters' not in goal and '_sourceTurnId' not in goal for goal in goals)) self.assertTrue(all('filters' not in goal and '_sourceTurnId' not in goal for goal in goals))
self.assertTrue(all('filters' not in scope and 'sourceTurnId' not in scope self.assertTrue(all('filters' not in scope and 'sourceTurnId' not in scope
+6 -2
View File
@@ -284,6 +284,10 @@ class QualityToolsTests(unittest.TestCase):
self.assertEqual(definition["inputSchema"]["properties"]["mode"]["enum"], self.assertEqual(definition["inputSchema"]["properties"]["mode"]["enum"],
["catalog", "query", "rank", "compare", "top", "changes", "defects", "details", "investigate"]) ["catalog", "query", "rank", "compare", "top", "changes", "defects", "details", "investigate"])
self.assertIn("defect", definition["inputSchema"]["properties"]) self.assertIn("defect", definition["inputSchema"]["properties"])
column_filter = definition["inputSchema"]["properties"]["columnFilters"]
self.assertEqual(column_filter["items"]["properties"]["operator"]["enum"],
["in", "not_in", "contains", "is_empty", "not_empty"])
self.assertIn("MySQL LOCATE", column_filter["description"])
self.assertFalse(definition["inputSchema"]["additionalProperties"]) self.assertFalse(definition["inputSchema"]["additionalProperties"])
self.assertEqual(definition["inputSchema"]["allOf"][0]["then"]["required"], ["releaseId"]) self.assertEqual(definition["inputSchema"]["allOf"][0]["then"]["required"], ["releaseId"])
@@ -529,11 +533,11 @@ class QualityToolsTests(unittest.TestCase):
if path == "/api/analysis/tools/catalog": if path == "/api/analysis/tools/catalog":
response["sourceFields"] = [{"key": "customerCode", "label": "客户代码", response["sourceFields"] = [{"key": "customerCode", "label": "客户代码",
"type": "source_value", "stages": ["black"], "type": "source_value", "stages": ["black"],
"columns": {"black": "customer_code"}, "operators": ["in"]}] "columns": {"black": "customer_code"}, "operators": ["in", "contains"]}]
return response return response
api = AuthorizedColumnApi() api = AuthorizedColumnApi()
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name) tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
rule = {"field": "customerCode", "stages": ["black"], "operator": "in", "values": ["C-17"]} rule = {"field": "customerCode", "stages": ["black"], "operator": "contains", "values": ["C-17"]}
result = tools.quality_data({"mode": "defects", "releaseId": RELEASE, result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "monthStart": 3, "monthEnd": 9}, "period": {"year": 2026, "monthStart": 3, "monthEnd": 9},
"filters": {"exclusions": {"product": ["规格甲"]}}, "columnFilters": [rule], "filters": {"exclusions": {"product": ["规格甲"]}}, "columnFilters": [rule],
+29 -2
View File
@@ -25,7 +25,7 @@ from test_nao_quality_tools import FakeApi, RELEASE
def source_field(key, label, black, bright, stages=('black', 'bright')): def source_field(key, label, black, bright, stages=('black', 'bright')):
return {'key': key, 'label': label, 'columns': {'black': black, 'bright': bright}, return {'key': key, 'label': label, 'columns': {'black': black, 'bright': bright},
'stages': list(stages), 'type': 'source_value', 'stages': list(stages), 'type': 'source_value',
'operators': ['in', 'not_in', 'is_empty', 'not_empty']} 'operators': ['in', 'not_in', 'contains', 'is_empty', 'not_empty']}
SOURCE_FIELDS = [ SOURCE_FIELDS = [
@@ -349,6 +349,15 @@ class SourceColumnProtocolTests(unittest.TestCase):
with self.assertRaisesRegex(TaskUnavailable, '不超过256字符'): with self.assertRaisesRegex(TaskUnavailable, '不超过256字符'):
protocol_task(columnFilters=[{**rule, 'values': ['X' * 257]}]) protocol_task(columnFilters=[{**rule, 'values': ['X' * 257]}])
def test_contains_keywords_are_preserved_as_literal_or_values(self):
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains',
'values': [' 001', 'A|B', 'abC']}
task = protocol_task(columnFilters=[rule])
self.assertEqual(task['filters']['columnFilters'], [rule])
for invalid in ([], [''], [' '], ['x' * 257], [1], ['one'] * 51):
with self.subTest(invalid=invalid), self.assertRaises(TaskUnavailable):
protocol_task(columnFilters=[{**rule, 'values': invalid}])
def test_protocol_presentation_enums_default_consistently_when_omitted_or_null(self): def test_protocol_presentation_enums_default_consistently_when_omitted_or_null(self):
base = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'], base = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3}, comparisonRequired=False, period={'year': 2026, 'month': 3}, comparisonRequired=False,
@@ -396,6 +405,10 @@ class SourceColumnProtocolTests(unittest.TestCase):
self.assertIn('periodCoverage(available/complete,可省略或null', prompt) self.assertIn('periodCoverage(available/complete,可省略或null', prompt)
self.assertIn('freshQuery(bool)', prompt) self.assertIn('freshQuery(bool)', prompt)
self.assertIn('needsClarification(bool)', prompt) self.assertIn('needsClarification(bool)', prompt)
self.assertIn('in/not_in/contains/is_empty/not_empty', prompt)
self.assertIn('contains关键字是子串而非完整候选值', prompt)
self.assertEqual(model_call.call_args_list[0].kwargs['payload']['businessCatalog']['sourceColumnOperators'],
['in', 'not_in', 'contains', 'is_empty', 'not_empty'])
def test_three_arbitrary_source_keys_keep_literal_values_and_stage_scopes(self): def test_three_arbitrary_source_keys_keep_literal_values_and_stage_scopes(self):
task = protocol_task() task = protocol_task()
@@ -429,7 +442,7 @@ class SourceColumnProtocolTests(unittest.TestCase):
def test_unauthorized_unknown_and_missing_stage_columns_fail_closed(self): def test_unauthorized_unknown_and_missing_stage_columns_fail_closed(self):
for bad in ([{**RULES[0], 'field': 'notInDirectory'}], for bad in ([{**RULES[0], 'field': 'notInDirectory'}],
[{**RULES[0], 'field': 'blackOnlyCode', 'stages': ['bright']}], [{**RULES[0], 'field': 'blackOnlyCode', 'stages': ['bright']}],
[{**RULES[0], 'operator': 'contains'}]): [{**RULES[0], 'operator': 'starts_with'}]):
with self.subTest(bad=bad), self.assertRaises(TaskUnavailable): with self.subTest(bad=bad), self.assertRaises(TaskUnavailable):
normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'], normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026}, columnFilters=bad), '筛选原表字段', period={'year': 2026}, columnFilters=bad), '筛选原表字段',
@@ -529,6 +542,20 @@ class SourceColumnProtocolTests(unittest.TestCase):
self.assertEqual(customer_lookup['filters']['columnFilters'], [ self.assertEqual(customer_lookup['filters']['columnFilters'], [
{'field': 'dispatchOrder', 'stages': ['bright'], 'operator': 'not_empty'}]) {'field': 'dispatchOrder', 'stages': ['bright'], 'operator': 'not_empty'}])
def test_contains_keyword_skips_exact_candidate_value_validation(self):
from nao_agent_task import _resolve_source_column_values
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains',
'values': ['001']}
task = protocol_task(columnFilters=[rule])
calls = []
def resolver(**contract):
calls.append(contract)
raise AssertionError('contains is a substring, not a complete candidate value')
result = _resolve_source_column_values(task, release_id=RELEASE, resolver=resolver)
self.assertEqual(result, {'status': 'exact'})
self.assertEqual(calls, [])
self.assertEqual(task['filters']['columnFilters'], [rule])
def test_missing_literal_offers_same_scope_alternatives_without_substitution(self): def test_missing_literal_offers_same_scope_alternatives_without_substitution(self):
from nao_agent_task import _resolve_source_column_values from nao_agent_task import _resolve_source_column_values
task = protocol_task(columnFilters=[RULES[0]]) task = protocol_task(columnFilters=[RULES[0]])
+65 -31
View File
@@ -1,3 +1,7 @@
import { useProductPage } from './useProductPage';
import { shouldPageProducts, canReadProductPage } from './productPagination';
import { usePriorityMetrics } from './usePriorityMetrics';
import { overviewTrend, sameTrendScope } from './priorityMetrics';
import { QuerySnapshotButton } from './QuerySnapshotButton'; import { QuerySnapshotButton } from './QuerySnapshotButton';
import { loadedQueryScope, queryForLocation, readQueryLocation } from './queryLoadingScope'; import { loadedQueryScope, queryForLocation, readQueryLocation } from './queryLoadingScope';
import { AnimatedMetricText, SmoothTableFrame, useAnimatedMetricText, useRetainedResult } from './QueryResultMotion'; import { AnimatedMetricText, SmoothTableFrame, useAnimatedMetricText, useRetainedResult } from './QueryResultMotion';
@@ -413,11 +417,17 @@ export default function QueryPage({ page, onUnauthorized }: {
setDrilledProduct(''); setDrilledProduct('');
}, [drilldownContextKey]); }, [drilldownContextKey]);
const metricParams = useMemo<QueryParams>(() => ({ ...queryParams, trendGranularity: 'month' }), [queryParams]); const metricParams = useMemo<QueryParams>(() => ({ ...queryParams, trendGranularity: 'month' }), [queryParams]);
const metricKey = useMemo(() => JSON.stringify(metricParams), [metricParams]); const metricsReady = !!(initialQuery.releaseId || currentReleaseId !== undefined) && (!metricParams.productSearch || !!options);
const productPagingScope=shouldPageProducts(metricParams);
const overview = usePriorityMetrics(metricParams, active && metricsReady, queryRevision, onUnauthorized,productPagingScope);
const currentOverview = overview.value;
const paging=productPagingScope&&!drilledProduct&&!overview.failed;
const productPage=useProductPage(metricParams,active&&metricsReady&&paging&&canReadProductPage(metricParams,overview.settled),queryRevision,'brightQualifiedDesc',onUnauthorized);
const metricKey = useMemo(() => JSON.stringify([metricParams, queryRevision,productPagingScope?productPage.key:null]), [metricParams, queryRevision,productPagingScope,productPage.key]);
const currentMetricsError = errorKey === metricKey ? error : ''; const currentMetricsError = errorKey === metricKey ? error : '';
const currentMetricsData = dataKey === metricKey && !currentMetricsError ? data : null; const currentMetricsData = dataKey === metricKey && !currentMetricsError ? data : null;
const queryPending = !currentMetricsData && !currentMetricsError; const queryPending = !drilledProduct && !currentMetricsData && !currentMetricsError;
const queryBusy = loading || queryPending; const queryBusy = !drilledProduct && (loading || queryPending);
const metricKeyRef = useRef(metricKey); const metricKeyRef = useRef(metricKey);
metricKeyRef.current = metricKey; metricKeyRef.current = metricKey;
const trendParams = useMemo<QueryParams>(() => { const trendParams = useMemo<QueryParams>(() => {
@@ -429,7 +439,10 @@ export default function QueryPage({ page, onUnauthorized }: {
}, [queryKey, trendObject, trendGranularity, view]); }, [queryKey, trendObject, trendGranularity, view]);
const defectQueryParams = useMemo<QueryParams>(() => applyTrendObject(queryParams, trendObject, view), [queryKey, trendObject, view]); const defectQueryParams = useMemo<QueryParams>(() => applyTrendObject(queryParams, trendObject, view), [queryKey, trendObject, view]);
const defectQueryKey = JSON.stringify(defectQueryParams); const defectQueryKey = JSON.stringify(defectQueryParams);
const trendKey = JSON.stringify(trendParams); const trendKey = JSON.stringify([trendParams, queryRevision]);
const reuseOverviewTrend = sameTrendScope(metricParams, trendParams);
const trendGate = !overview.split && !reuseOverviewTrend || overview.settled;
const sharedTrend = reuseOverviewTrend ? currentOverview : null;
const trendKeyRef = useRef(trendKey); const trendKeyRef = useRef(trendKey);
trendKeyRef.current = trendKey; trendKeyRef.current = trendKey;
@@ -447,12 +460,20 @@ export default function QueryPage({ page, onUnauthorized }: {
}, [queryKey, active]); }, [queryKey, active]);
useEffect(() => { useEffect(() => {
if (!active || optionsLoading || optionsError) return; if (!active || !metricsReady || !overview.settled || drilledProduct) return;
if(paging) {
if(productPage.error){setError(productPage.error);setErrorKey(metricKey);setLoading(false);return;}
if(!productPage.value||!currentOverview){setLoading(true);return;}
setData({...currentOverview,rows:productPage.value.rows});setDataKey(metricKey);setLoading(false);setError('');setErrorKey('');return;
}
if (currentOverview && !overview.split) {
setData(currentOverview); setDataKey(metricKey); setLoading(false); setError(''); setErrorKey(''); return;
}
const controller = new AbortController(); const controller = new AbortController();
setLoading(true); setLoading(true);
setError(''); setError('');
setErrorKey(''); setErrorKey('');
api.metrics(metricParams, controller.signal).then(result => { (metricParams.view === 'overall' ? api.metrics : api.metricsTable)(metricParams, controller.signal).then(result => {
if (controller.signal.aborted || metricKeyRef.current !== metricKey) return; if (controller.signal.aborted || metricKeyRef.current !== metricKey) return;
setData(result); setData(result);
setDataKey(metricKey); setDataKey(metricKey);
@@ -465,7 +486,7 @@ export default function QueryPage({ page, onUnauthorized }: {
setErrorKey(metricKey); setErrorKey(metricKey);
}).finally(() => { if (!controller.signal.aborted) setLoading(false); }); }).finally(() => { if (!controller.signal.aborted) setLoading(false); });
return () => controller.abort(); return () => controller.abort();
}, [active, metricKey, queryRevision, onUnauthorized, optionsLoading, optionsError]); }, [active, metricKey, queryRevision, onUnauthorized, metricsReady, overview.settled, overview.split, currentOverview,drilledProduct,paging,productPage.value,productPage.error]);
useEffect(() => { useEffect(() => {
if (!active) return; if (!active) return;
@@ -479,7 +500,7 @@ export default function QueryPage({ page, onUnauthorized }: {
setTrendObjectMetrics(null); setTrendObjectMetrics(null);
setTrendObjectMetricsLoading(true); setTrendObjectMetricsLoading(true);
setTrendObjectMetricsError(''); setTrendObjectMetricsError('');
api.metrics({ ...defectQueryParams, trendGranularity: 'month' }, controller.signal) api.metrics({ ...defectQueryParams, view: 'overall', trendGranularity: 'month' }, controller.signal)
.then(result => { if (!controller.signal.aborted) setTrendObjectMetrics({ key: defectQueryKey, data: result }); }) .then(result => { if (!controller.signal.aborted) setTrendObjectMetrics({ key: defectQueryKey, data: result }); })
.catch(errorValue => { .catch(errorValue => {
if (controller.signal.aborted) return; if (controller.signal.aborted) return;
@@ -490,8 +511,11 @@ export default function QueryPage({ page, onUnauthorized }: {
}, [active, trendObject, defectQueryKey, queryKey, queryRevision, onUnauthorized]); }, [active, trendObject, defectQueryKey, queryKey, queryRevision, onUnauthorized]);
useEffect(() => { useEffect(() => {
if (!active || optionsLoading || optionsError) return; if (!active || !metricsReady || !trendGate) return;
setTrendFailure(null); setTrendFailure(null);
if (sharedTrend) {
setTrendData(overviewTrend(sharedTrend).trend); setTrendDataKey(trendKey); setTrendLoading(false); return;
}
const controller = new AbortController(); const controller = new AbortController();
setTrendLoading(true); setTrendLoading(true);
api.trend(trendParams, controller.signal).then(result => { api.trend(trendParams, controller.signal).then(result => {
@@ -503,10 +527,10 @@ export default function QueryPage({ page, onUnauthorized }: {
if (!controller.signal.aborted && !onUnauthorized(errorValue)) setTrendFailure({ key: trendKey, message: chooseError(errorValue) }); if (!controller.signal.aborted && !onUnauthorized(errorValue)) setTrendFailure({ key: trendKey, message: chooseError(errorValue) });
}).finally(() => { if (!controller.signal.aborted) setTrendLoading(false); }); }).finally(() => { if (!controller.signal.aborted) setTrendLoading(false); });
return () => controller.abort(); return () => controller.abort();
}, [active, trendKey, queryRevision, trendRevision, onUnauthorized, optionsLoading, optionsError]); }, [active, trendKey, queryRevision, trendRevision, onUnauthorized, metricsReady, trendGate, sharedTrend]);
useEffect(() => { useEffect(() => {
if (!active || optionsLoading || optionsError) return; if (!active || !metricsReady || !overview.settled) return;
let alive = true; let alive = true;
setTrendTarget(null); setTrendTarget(null);
setTrendTargetKey(''); setTrendTargetKey('');
@@ -525,7 +549,7 @@ export default function QueryPage({ page, onUnauthorized }: {
} }
}); });
return () => { alive = false; }; return () => { alive = false; };
}, [active, trendKey, queryRevision, trendRevision, onUnauthorized, optionsLoading, optionsError]); }, [active, trendKey, queryRevision, trendRevision, onUnauthorized, metricsReady, overview.settled]);
const currentTrendTarget = trendTargetKey === trendKey ? trendTarget : null; const currentTrendTarget = trendTargetKey === trendKey ? trendTarget : null;
const visibleTrendData = useMemo(() => fullYearTrend(trendDataKey === trendKey ? trendData : [], year, trendGranularity), [trendData, trendDataKey, trendKey, year, trendGranularity]); const visibleTrendData = useMemo(() => fullYearTrend(trendDataKey === trendKey ? trendData : [], year, trendGranularity), [trendData, trendDataKey, trendKey, year, trendGranularity]);
@@ -554,6 +578,7 @@ export default function QueryPage({ page, onUnauthorized }: {
: page === 'overall' && currentMetricsData : page === 'overall' && currentMetricsData
? currentMetricsData.rows.length ? currentMetricsData.rows : [currentMetricsData.summary] ? currentMetricsData.rows.length ? currentMetricsData.rows : [currentMetricsData.summary]
: currentMetricsData?.rows || []; : currentMetricsData?.rows || [];
if(paging)return rawRows;
const scopedRows = view === 'department' const scopedRows = view === 'department'
? rawRows.filter(row => !RETIRED_DEPARTMENTS.has(row.department || row.label)) ? rawRows.filter(row => !RETIRED_DEPARTMENTS.has(row.department || row.label))
: view === 'group' && department.length : view === 'group' && department.length
@@ -571,7 +596,7 @@ export default function QueryPage({ page, onUnauthorized }: {
if (!appliedSearch.trim()) return orderedRows; if (!appliedSearch.trim()) return orderedRows;
const term = appliedSearch.trim().toLocaleLowerCase(); const term = appliedSearch.trim().toLocaleLowerCase();
return orderedRows.filter(row => [row.label, row.department, row.group, row.product, row.projectGroup].filter(Boolean).join(' ').toLocaleLowerCase().includes(term)); return orderedRows.filter(row => [row.label, row.department, row.group, row.product, row.projectGroup].filter(Boolean).join(' ').toLocaleLowerCase().includes(term));
}, [page, period, currentMetricsData, appliedSearch, year, view, department, allowedGroupSet]); }, [page, period, currentMetricsData, appliedSearch, year, view, department, allowedGroupSet,paging]);
const tableReady = drilledProduct ? trendDataKey === trendKey && !trendLoading : !!currentMetricsData; const tableReady = drilledProduct ? trendDataKey === trendKey && !trendLoading : !!currentMetricsData;
const retainedTable = useRetainedResult(tableReady ? { const retainedTable = useRetainedResult(tableReady ? {
@@ -593,7 +618,7 @@ export default function QueryPage({ page, onUnauthorized }: {
query:{...defectQueryParams,releaseId:summaryMetricsDataRelease()},trendQuery:trendParams, query:{...defectQueryParams,releaseId:summaryMetricsDataRelease()},trendQuery:trendParams,
tableQuery:drilledProduct?trendParams:queryParams, tableQuery:drilledProduct?trendParams:queryParams,
tableDimension:tableView==='overall'&&tablePeriod==='year'?'period':tableView, tableDimension:tableView==='overall'&&tablePeriod==='year'?'period':tableView,
tableGranularity:drilledProduct?trendGranularity:'month',refreshKey:JSON.stringify([refreshKey,queryRevision]),onUnauthorized,display:defectDisplay,onDisplay:setDefectDisplay}); tableGranularity:drilledProduct?trendGranularity:'month',productKeys:paging?productPage.value?.rows.map(row=>row.key):undefined,tableEnabled:!paging||!!productPage.value,refreshKey:JSON.stringify([refreshKey,queryRevision]),onUnauthorized,display:defectDisplay,onDisplay:setDefectDisplay});
function summaryMetricsDataRelease() { return (drilledProduct?trendObjectMetrics?.data:currentMetricsData)?.release?.id || currentMetricsData?.release?.id || queryParams.releaseId; } function summaryMetricsDataRelease() { return (drilledProduct?trendObjectMetrics?.data:currentMetricsData)?.release?.id || currentMetricsData?.release?.id || queryParams.releaseId; }
const captureTrendAnchor = useCallback(() => { const captureTrendAnchor = useCallback(() => {
@@ -646,8 +671,8 @@ export default function QueryPage({ page, onUnauthorized }: {
); );
const activeDefectData = trendObject const activeDefectData = trendObject
? defectQueryKey === queryKey ? currentMetricsData : trendObjectMetrics?.key === defectQueryKey ? trendObjectMetrics.data : null ? defectQueryKey === queryKey ? currentOverview || currentMetricsData : trendObjectMetrics?.key === defectQueryKey ? trendObjectMetrics.data : null
: currentMetricsData; : currentOverview || currentMetricsData;
const previousDefectSnapshot = useRef<{ data: MetricsResponse; query: QueryParams } | null>(null); const previousDefectSnapshot = useRef<{ data: MetricsResponse; query: QueryParams } | null>(null);
useLayoutEffect(() => { useLayoutEffect(() => {
if (activeDefectData) previousDefectSnapshot.current = { data: activeDefectData, query: defectQueryParams }; if (activeDefectData) previousDefectSnapshot.current = { data: activeDefectData, query: defectQueryParams };
@@ -656,7 +681,7 @@ export default function QueryPage({ page, onUnauthorized }: {
const displayedDefectQuery = activeDefectData ? defectQueryParams : previousDefectSnapshot.current?.query || defectQueryParams; const displayedDefectQuery = activeDefectData ? defectQueryParams : previousDefectSnapshot.current?.query || defectQueryParams;
const defectsRefreshing = !activeDefectData && (queryBusy || trendObjectMetricsLoading); const defectsRefreshing = !activeDefectData && (queryBusy || trendObjectMetricsLoading);
// 产品月度下钻复用当前对象的服务端指标,汇总表仍保留父层结果供返回。 // 产品月度下钻复用当前对象的服务端指标,汇总表仍保留父层结果供返回。
const summaryMetricsData = drilledProduct ? activeDefectData : currentMetricsData; const summaryMetricsData = drilledProduct ? activeDefectData : currentOverview || currentMetricsData;
const openRecords = useCallback((row: MetricRow | null, stage: Stage, selectedMonth?: number, sourceQuery?: QueryParams, sourceData?: MetricsResponse | null) => { const openRecords = useCallback((row: MetricRow | null, stage: Stage, selectedMonth?: number, sourceQuery?: QueryParams, sourceData?: MetricsResponse | null) => {
const source = sourceData === undefined ? summaryMetricsData : sourceData; const source = sourceData === undefined ? summaryMetricsData : sourceData;
if (!source?.release?.id) { message.info('指标数据尚未就绪,请重新查询后打开来源明细。'); return; } if (!source?.release?.id) { message.info('指标数据尚未就绪,请重新查询后打开来源明细。'); return; }
@@ -665,7 +690,7 @@ export default function QueryPage({ page, onUnauthorized }: {
setSelectedRecord(row); setRecordStage(stage); setRecordMonth(selectedMonth); setRecordsOpen(true); setSelectedRecord(row); setRecordStage(stage); setRecordMonth(selectedMonth); setRecordsOpen(true);
}, [summaryMetricsData, drilledProduct, defectQueryParams, queryParams, view, message]); }, [summaryMetricsData, drilledProduct, defectQueryParams, queryParams, view, message]);
const summaryMetricsError = drilledProduct ? trendObjectMetricsError : currentMetricsError; const summaryMetricsError = drilledProduct ? trendObjectMetricsError : currentMetricsError;
const summaryBusy = drilledProduct ? trendObjectMetricsLoading || !summaryMetricsData && !summaryMetricsError : queryBusy; const summaryBusy = drilledProduct ? trendObjectMetricsLoading || !summaryMetricsData && !summaryMetricsError : !summaryMetricsData && queryBusy;
const displayedSummaryData = useRetainedResult(summaryMetricsData); const displayedSummaryData = useRetainedResult(summaryMetricsData);
const rowForSummary = displayedSummaryData?.summary; const rowForSummary = displayedSummaryData?.summary;
const rawFormulaDescriptions = summaryMetricsData?.context.metricFormulaDescriptions; const rawFormulaDescriptions = summaryMetricsData?.context.metricFormulaDescriptions;
@@ -758,7 +783,7 @@ export default function QueryPage({ page, onUnauthorized }: {
</div>} </div>}
</section> </section>
{currentMetricsError && <Alert showIcon type="error" message={data ? "更新失败,当前显示上次结果" : "质量指标查询失败"} description={currentMetricsError} action={<Button size="small" icon={<ReloadOutlined />} onClick={() => setQueryRevision(value => value + 1)}>重试</Button>} />} {currentMetricsError && <Alert showIcon type="error" message={currentOverview ? "表格读取失败,指标已更新" : data ? "更新失败,当前显示上次结果" : "质量指标查询失败"} description={currentMetricsError} action={<Button size="small" icon={<ReloadOutlined />} onClick={() => setQueryRevision(value => value + 1)}>重试</Button>} />}
{drilledProduct && summaryMetricsError && <Alert showIcon type="error" message="当前产品指标查询失败" description={summaryMetricsError} action={<Button size="small" onClick={() => setQueryRevision(value => value + 1)}>重试</Button>} />} {drilledProduct && summaryMetricsError && <Alert showIcon type="error" message="当前产品指标查询失败" description={summaryMetricsError} action={<Button size="small" onClick={() => setQueryRevision(value => value + 1)}>重试</Button>} />}
<section className={`metric-strip ${!displayedSummaryData ? 'query-awaiting-data' : ''}`} data-snapshot-module="metrics" aria-label="质量指标" aria-busy={summaryBusy} inert={!summaryMetricsData}> <section className={`metric-strip ${!displayedSummaryData ? 'query-awaiting-data' : ''}`} data-snapshot-module="metrics" aria-label="质量指标" aria-busy={summaryBusy} inert={!summaryMetricsData}>
<div className="rate-basis-control"><Segmented size="small" value={rateBasis} options={[{ label: '原口径', value: 'original' }, { label: '去本部报废', value: 'withoutHeadquartersScrap' }]} onChange={value => setRateBasis(value as typeof rateBasis)} aria-label="合格率口径" /></div> <div className="rate-basis-control"><Segmented size="small" value={rateBasis} options={[{ label: '原口径', value: 'original' }, { label: '去本部报废', value: 'withoutHeadquartersScrap' }]} onChange={value => setRateBasis(value as typeof rateBasis)} aria-label="合格率口径" /></div>
@@ -811,6 +836,9 @@ export default function QueryPage({ page, onUnauthorized }: {
paginationScope={retainedTable?.paginationScope} paginationScope={retainedTable?.paginationScope}
exportFilename={`${drilledProduct || scopeName}质量汇总-${year}${drilledProduct || period === 'year' ? '' : period === 'month' ? `-${String(month).padStart(2, '0')}` : `-Q${quarter}`}-${fullFields ? '完整字段' : '关键指标'}`} exportFilename={`${drilledProduct || scopeName}质量汇总-${year}${drilledProduct || period === 'year' ? '' : period === 'month' ? `-${String(month).padStart(2, '0')}` : `-Q${quarter}`}-${fullFields ? '完整字段' : '关键指标'}`}
rows={tableRows} rows={tableRows}
columnRows={paging&&currentOverview?[currentOverview.summary]:undefined}
remotePage={paging?productPage.pagination:undefined}
loadExportData={paging?async()=>{const [rows,defectDrilldown]=await Promise.all([productPage.loadAll(),defectDrill.exportTable()]);return {rows,defectDrilldown};}:undefined}
view={tableView} view={tableView}
layoutView={retainedTable?.layoutView || view} layoutView={retainedTable?.layoutView || view}
pageSizeOverride={tableProduct ? 12 : undefined} pageSizeOverride={tableProduct ? 12 : undefined}
@@ -858,7 +886,7 @@ export default function QueryPage({ page, onUnauthorized }: {
</section>; </section>;
})} })}
</div> </div>
{displayedDefectData ? <DefectParetoPanel releaseId={displayedDefectData.release?.id} query={displayedDefectQuery} refreshKey={queryRevision} pending={defectsRefreshing} externalScrapWeight={blackDefectItems.find(item => item.name === '外废')?.weight} onUnauthorized={onUnauthorized} /> : !defectsRefreshing && <div className="defect-pareto-status">不良数据暂不可用</div>} {displayedDefectData ? <DefectParetoPanel releaseId={displayedDefectData.release?.id} query={displayedDefectQuery} refreshKey={queryRevision} pending={defectsRefreshing||queryBusy} externalScrapWeight={blackDefectItems.find(item => item.name === '外废')?.weight} onUnauthorized={onUnauthorized} /> : !defectsRefreshing && <div className="defect-pareto-status">不良数据暂不可用</div>}
</section> </section>
</div> </div>
</> </>
@@ -947,7 +975,7 @@ export function DefectParetoPanel({ releaseId, query, externalScrapWeight, onUna
const showLoading = useDelayedIndicator(loading); const showLoading = useDelayedIndicator(loading);
useEffect(() => { useEffect(() => {
if (!active || !visible) return; if (!active || !visible || pending) return;
if (!releaseId) { setResponses(null); setLoading(false); setError(''); return; } if (!releaseId) { setResponses(null); setLoading(false); setError(''); return; }
const controller = new AbortController(); const controller = new AbortController();
const base = { releaseId, ...fineQuery, mode: 'top', dimension: 'defect', limit: 100 }; const base = { releaseId, ...fineQuery, mode: 'top', dimension: 'defect', limit: 100 };
@@ -964,7 +992,7 @@ export function DefectParetoPanel({ releaseId, query, externalScrapWeight, onUna
}) })
.finally(() => { if (!controller.signal.aborted) setLoading(false); }); .finally(() => { if (!controller.signal.aborted) setLoading(false); });
return () => controller.abort(); return () => controller.abort();
}, [active, visible, releaseId, fineQueryKey, onUnauthorized, onlyStage, refreshKey, externalScrapWeight]); }, [active, visible, pending, releaseId, fineQueryKey, onUnauthorized, onlyStage, refreshKey, externalScrapWeight]);
useEffect(() => { useEffect(() => {
const refresh = () => setThemeRevision(value => value + 1); const refresh = () => setThemeRevision(value => value + 1);
@@ -1095,9 +1123,11 @@ const totalWeightTitle = (label: string) => <span className="query-total-weight-
const rateBasisTitle = (label: string, basis: QueryRateBasis, kind: 'bright' | 'composite', onBasis?: () => void) => <span className="query-total-weight-title query-rate-rule-title"><span className="query-rate-label">{onBasis ? <button className="header-link" onClick={onBasis}>{label}</button> : label}{basis !== 'original' && <small className="query-rate-subtitle">(去本部报废)</small>}</span>{basis !== 'original' && <Tooltip trigger={['hover', 'focus']} title={<div>{kind === 'composite' && <>综合合格率=黑片合格率×亮片合格率。<br /></>}原口径:亮片合格重量÷亮片总重量。<br />去本部报废:(亮片合格重量+本部报废)÷亮片总重量,同时从亮片报废重量中扣除本部报废,总重量不变。<br />当前:去本部报废。计算结果以已发布数据版本及生效规则为准。</div>}><InfoCircleOutlined tabIndex={0} role="img" aria-label={`${kind === 'composite' ? '综合合格率' : '亮片合格率'}计算说明`} /></Tooltip>}</span>; const rateBasisTitle = (label: string, basis: QueryRateBasis, kind: 'bright' | 'composite', onBasis?: () => void) => <span className="query-total-weight-title query-rate-rule-title"><span className="query-rate-label">{onBasis ? <button className="header-link" onClick={onBasis}>{label}</button> : label}{basis !== 'original' && <small className="query-rate-subtitle">(去本部报废)</small>}</span>{basis !== 'original' && <Tooltip trigger={['hover', 'focus']} title={<div>{kind === 'composite' && <>综合合格率=黑片合格率×亮片合格率。<br /></>}原口径:亮片合格重量÷亮片总重量。<br />去本部报废:(亮片合格重量+本部报废)÷亮片总重量,同时从亮片报废重量中扣除本部报废,总重量不变。<br />当前:去本部报废。计算结果以已发布数据版本及生效规则为准。</div>}><InfoCircleOutlined tabIndex={0} role="img" aria-label={`${kind === 'composite' ? '综合合格率' : '亮片合格率'}计算说明`} /></Tooltip>}</span>;
export function MetricTable({ export function MetricTable({
ref, exportFilename, rows, view, defectDrilldown, layoutView = view, pageSizeOverride, paginationScope, period, month, quarter, fullFields, rateBasis, onDetail, onBasis, onDrilldown, onlyStage, defectDisplay = 'weight', emptyText = '当前条件下暂无数据', ref, exportFilename, rows, view, defectDrilldown, columnRows, remotePage, loadExportData, layoutView = view, pageSizeOverride, paginationScope, period, month, quarter, fullFields, rateBasis, onDetail, onBasis, onDrilldown, onlyStage, defectDisplay = 'weight', emptyText = '当前条件下暂无数据',
}: { }: {
ref?: React.Ref<{ exportExcel: () => Promise<void> }>; exportFilename: string; defectDrilldown?: DefectDrilldownResponse; ref?: React.Ref<{ exportExcel: () => Promise<void> }>; exportFilename: string; defectDrilldown?: DefectDrilldownResponse;
columnRows?:MetricRow[];remotePage?:{current:number;pageSize:number;total:number;onChange:(page:number,size:number)=>void};
loadExportData?:()=>Promise<{rows:MetricRow[];defectDrilldown?:DefectDrilldownResponse}>;
rows: MetricRow[]; view: ViewKind; layoutView?: ViewKind; pageSizeOverride?: number; paginationScope?: string; period: PeriodKind; month: number; quarter: number; fullFields: boolean; rows: MetricRow[]; view: ViewKind; layoutView?: ViewKind; pageSizeOverride?: number; paginationScope?: string; period: PeriodKind; month: number; quarter: number; fullFields: boolean;
rateBasis: 'original' | 'withoutHeadquartersScrap'; rateBasis: 'original' | 'withoutHeadquartersScrap';
onDetail: (row: MetricRow | null, stage: Stage, month?: number) => void; onBasis: () => void; onDrilldown: (row: MetricRow) => void; onDetail: (row: MetricRow | null, stage: Stage, month?: number) => void; onBasis: () => void; onDrilldown: (row: MetricRow) => void;
@@ -1140,7 +1170,7 @@ export function MetricTable({
headquartersScrapWeight: '本部报废', rate: '合格率', headquartersScrapWeight: '本部报废', rate: '合格率',
}; };
const stageColumns = (stage: Stage): ColumnsType<MetricRow> => { const stageColumns = (stage: Stage): ColumnsType<MetricRow> => {
const base: ColumnsType<MetricRow> = metricStageColumnPlan(rows, stage, defectDrilldown ? false : fullFields, rateBasis).map(column => { const base: ColumnsType<MetricRow> = metricStageColumnPlan(columnRows||rows, stage, defectDrilldown ? false : fullFields, rateBasis).map(column => {
if (column.kind === 'defect') return defectColumns(stage, [column.name])[0]; if (column.kind === 'defect') return defectColumns(stage, [column.name])[0];
const metricColumn = numeric(fieldLabels[column.field], stage, column.field); const metricColumn = numeric(fieldLabels[column.field], stage, column.field);
return stage === 'bright' && column.field === 'rate' return stage === 'bright' && column.field === 'rate'
@@ -1165,11 +1195,14 @@ export function MetricTable({
]; ];
const columns = onlyStage ? allColumns.filter(column => !['compositeRate', `stage-${onlyStage === 'black' ? 'bright' : 'black'}`].includes(String(column.key))) : allColumns; const columns = onlyStage ? allColumns.filter(column => !['compositeRate', `stage-${onlyStage === 'black' ? 'bright' : 'black'}`].includes(String(column.key))) : allColumns;
useImperativeHandle(ref, () => ({ useImperativeHandle(ref, () => ({
exportExcel: () => downloadMetricWorkbook(exportFilename, columns, rows, (key, row) => { exportExcel: async () => {
const exported=loadExportData?await loadExportData():{rows,defectDrilldown};
const exportedFine=exported.defectDrilldown;
return downloadMetricWorkbook(exportFilename, columns, exported.rows, (key, row) => {
if (key === 'object') return [title(row), view === 'group' ? row.department : null].filter(Boolean).join(' / '); if (key === 'object') return [title(row), view === 'group' ? row.department : null].filter(Boolean).join(' / ');
if (key === 'compositeRate') return visibleCompositeRate(row, rateBasis); if (key === 'compositeRate') return visibleCompositeRate(row, rateBasis);
const stage: Stage = key.startsWith('black-') ? 'black' : 'bright'; const stage: Stage = key.startsWith('black-') ? 'black' : 'bright';
if (defectDrilldown && key.startsWith(`${stage}-fine-`)) return defectCell(defectDrilldown,row,key.slice(`${stage}-fine-`.length),defectDisplay); if (exportedFine && key.startsWith(`${stage}-fine-`)) return defectCell(exportedFine,row,key.slice(`${stage}-fine-`.length),defectDisplay);
if (key.startsWith(`${stage}-defect-`)) { if (key.startsWith(`${stage}-defect-`)) {
const weight=defectList(row, stage).find(item => item.name === key.slice(`${stage}-defect-`.length))?.weight ?? null; const weight=defectList(row, stage).find(item => item.name === key.slice(`${stage}-defect-`.length))?.weight ?? null;
return defectDisplay === 'rate' ? defectWeightRate(weight,stageValue(row,stage,'totalWeight')) : weight; return defectDisplay === 'rate' ? defectWeightRate(weight,stageValue(row,stage,'totalWeight')) : weight;
@@ -1177,13 +1210,14 @@ export function MetricTable({
const field = key.slice(stage.length + 1) as 'qualifiedWeight' | 'scrapWeight' | 'totalWeight' | 'headquartersScrapWeight' | 'rate'; const field = key.slice(stage.length + 1) as 'qualifiedWeight' | 'scrapWeight' | 'totalWeight' | 'headquartersScrapWeight' | 'rate';
if (defectDisplay === 'rate' && (field === 'scrapWeight' || field === 'headquartersScrapWeight')) return defectWeightRate(stageValue(row,stage,field,rateBasis),stageValue(row,stage,'totalWeight')); if (defectDisplay === 'rate' && (field === 'scrapWeight' || field === 'headquartersScrapWeight')) return defectWeightRate(stageValue(row,stage,field,rateBasis),stageValue(row,stage,'totalWeight'));
return field === 'rate' ? visibleStageRate(row, stage, stage === 'bright' ? rateBasis : 'original') : stageValue(row, stage, field, rateBasis); return field === 'rate' ? visibleStageRate(row, stage, stage === 'bright' ? rateBasis : 'original') : stageValue(row, stage, field, rateBasis);
}), });
},
})); }));
const dataSource = rows.map((row, index) => ({ ...row, sourceMetricKey: row.key, key: row.key || `${title(row)}-${index}` })); const dataSource = rows.map((row, index) => ({ ...row, sourceMetricKey: row.key, key: row.key || `${title(row)}-${index}` }));
const [currentPage, setCurrentPage] = useState(1); const [currentPage, setCurrentPage] = useState(1);
const [pageSize, setPageSize] = useState(SUMMARY_PAGE_SIZE); const [pageSize, setPageSize] = useState(SUMMARY_PAGE_SIZE);
const effectivePage = pageSizeOverride == null ? Math.min(currentPage, Math.max(1, Math.ceil(rows.length / pageSize))) : 1; const effectivePage = remotePage?remotePage.current:pageSizeOverride == null ? Math.min(currentPage, Math.max(1, Math.ceil(rows.length / pageSize))) : 1;
const effectivePageSize = pageSizeOverride ?? pageSize; const effectivePageSize = remotePage?.pageSize ?? pageSizeOverride ?? pageSize;
useEffect(() => setCurrentPage(1), [paginationScope ?? rows]); useEffect(() => setCurrentPage(1), [paginationScope ?? rows]);
const columnWidth = (column: ColumnsType<MetricRow>[number]): number => { const columnWidth = (column: ColumnsType<MetricRow>[number]): number => {
if ('children' in column && column.children?.length) { if ('children' in column && column.children?.length) {
@@ -1194,9 +1228,9 @@ export function MetricTable({
const scrollWidth = Math.ceil(columns.reduce((total, column) => total + columnWidth(column), 0)); const scrollWidth = Math.ceil(columns.reduce((total, column) => total + columnWidth(column), 0));
const scroll = { x: scrollWidth }; const scroll = { x: scrollWidth };
const displayColumns = compact ? columns.flatMap(column => 'children' in column && column.children ? column.children : [column]) : columns; const displayColumns = compact ? columns.flatMap(column => 'children' in column && column.children ? column.children : [column]) : columns;
const pageRows = dataSource.slice((effectivePage - 1) * effectivePageSize, effectivePage * effectivePageSize); const pageRows = remotePage?dataSource:dataSource.slice((effectivePage - 1) * effectivePageSize, effectivePage * effectivePageSize);
return <div className="metric-table-scroll"><Table<MetricRow> size="small" bordered tableLayout="fixed" columns={displayColumns} dataSource={pageRows} pagination={false} scroll={scroll} rowKey="key" locale={{ emptyText }} /> return <div className="metric-table-scroll"><Table<MetricRow> size="small" bordered tableLayout="fixed" columns={displayColumns} dataSource={pageRows} pagination={false} scroll={scroll} rowKey="key" locale={{ emptyText }} />
{pageSizeOverride == null && <MetricTablePagination current={effectivePage} total={dataSource.length} pageSize={pageSize} onChange={(nextPage, nextPageSize) => { setPageSize(nextPageSize); setCurrentPage(nextPageSize === pageSize ? nextPage : 1); }} />} {pageSizeOverride == null && <MetricTablePagination current={effectivePage} total={remotePage?.total??dataSource.length} pageSize={effectivePageSize} onChange={remotePage?.onChange||((nextPage, nextPageSize) => { setPageSize(nextPageSize); setCurrentPage(nextPageSize === pageSize ? nextPage : 1); })} />}
</div>; </div>;
} }
+54 -27
View File
@@ -1,3 +1,7 @@
import { useProductPage } from './useProductPage';
import { shouldPageProducts, canReadProductPage } from './productPagination';
import { usePriorityMetrics } from './usePriorityMetrics';
import { overviewTrend, sameTrendScope } from './priorityMetrics';
import { QuerySnapshotButton } from './QuerySnapshotButton'; import { QuerySnapshotButton } from './QuerySnapshotButton';
import { loadedQueryScope, queryForLocation } from './queryLoadingScope'; import { loadedQueryScope, queryForLocation } from './queryLoadingScope';
import { SmoothTableFrame, useRetainedResult } from './QueryResultMotion'; import { SmoothTableFrame, useRetainedResult } from './QueryResultMotion';
@@ -37,7 +41,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
const [moreFiltersOpen,setMoreFiltersOpen]=useState(false); const [moreFiltersOpen,setMoreFiltersOpen]=useState(false);
const [extraFilters,setExtraFilters]=useState<Record<string,string[]>>(initial.extraFilters||{}); const [extraFilters,setExtraFilters]=useState<Record<string,string[]>>(initial.extraFilters||{});
const [extraFields,setExtraFields]=useState<FilterField[]>([]); const [extraFields,setExtraFields]=useState<FilterField[]>([]);
const [options,setOptions]=useState<OptionsResponse|null>(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');const [supportLoading,setSupportLoading]=useState(true); const [options,setOptions]=useState<OptionsResponse|null>(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');
const [applied,setApplied]=useState<QueryParams|null>(null);const [revision,setRevision]=useState(0);const [response,setResponse]=useState<{key:string;value:MetricsResponse}|null>(null);const [loading,setLoading]=useState(false);const [failure,setFailure]=useState<{key:string;message:string}|null>(null); const [applied,setApplied]=useState<QueryParams|null>(null);const [revision,setRevision]=useState(0);const [response,setResponse]=useState<{key:string;value:MetricsResponse}|null>(null);const [loading,setLoading]=useState(false);const [failure,setFailure]=useState<{key:string;message:string}|null>(null);
const [defectDisplay,setDefectDisplay]=useState<'weight'|'rate'>('weight'); const [defectDisplay,setDefectDisplay]=useState<'weight'|'rate'>('weight');
const [fullFields,setFullFields]=useState(true);const [rateBasis,setRateBasis]=useState<'original'|'withoutHeadquartersScrap'>('original');const [basisOpen,setBasisOpen]=useState(false);const [trendMode,setTrendMode]=useState<'production'|'defects'|'data'>('production'); const [fullFields,setFullFields]=useState(true);const [rateBasis,setRateBasis]=useState<'original'|'withoutHeadquartersScrap'>('original');const [basisOpen,setBasisOpen]=useState(false);const [trendMode,setTrendMode]=useState<'production'|'defects'|'data'>('production');
@@ -50,29 +54,47 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
const supportGeneration=useRef(0);const supportController=useRef<AbortController|null>(null); const supportGeneration=useRef(0);const supportController=useRef<AbortController|null>(null);
const loadSupport=useCallback(async()=>{ const loadSupport=useCallback(async()=>{
const generation=++supportGeneration.current;supportController.current?.abort();const controller=new AbortController();supportController.current=controller; const generation=++supportGeneration.current;supportController.current?.abort();const controller=new AbortController();supportController.current=controller;
setSupportLoading(true);setSupportError(''); setSupportError('');
try{const [choices,source,filterFields]=await Promise.all([api.options(controller.signal),api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal),api.filterFields(false,controller.signal)]);if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);setCatalog(source);setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));} const fail=(error:unknown)=>{if(generation===supportGeneration.current&&!controller.signal.aborted&&!onUnauthorized(error))setSupportError(previous=>previous||(error instanceof Error?error.message:'查询条件读取失败'));};
catch(error){if(generation===supportGeneration.current&&!onUnauthorized(error))setSupportError(error instanceof Error?error.message:'查询条件读取失败');} const requests=[
finally{if(generation===supportGeneration.current)setSupportLoading(false);} api.options(controller.signal).then(choices=>{if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);}).catch(fail),
api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal).then(source=>{if(generation===supportGeneration.current)setCatalog(source);}).catch(fail),
api.filterFields(false,controller.signal).then(filterFields=>{if(generation===supportGeneration.current)setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));}).catch(fail),
];
await Promise.all(requests);
},[onUnauthorized,initial.releaseId,currentReleaseId]); },[onUnauthorized,initial.releaseId,currentReleaseId]);
useEffect(()=>{if(active)void loadSupport();return()=>{supportGeneration.current++;supportController.current?.abort();};},[active,refreshKey,loadSupport]); useEffect(()=>{if(active)void loadSupport();return()=>{supportGeneration.current++;supportController.current?.abort();};},[active,refreshKey,loadSupport]);
const fields=useMemo(()=>catalog?.sourceFields.filter(field=>field.type==='source_value'&&field.stages.includes(stage))||[],[catalog,stage]); const fields=useMemo(()=>catalog?.sourceFields.filter(field=>field.type==='source_value'&&field.stages.includes(stage))||[],[catalog,stage]);
const exactProductSearch=useMemo(()=>(options?.products||[]).find(name=>name.trim().toLocaleLowerCase()===appliedSearch.trim().toLocaleLowerCase())||'', [options,appliedSearch]); const exactProductSearch=useMemo(()=>(options?.products||[]).find(name=>name.trim().toLocaleLowerCase()===appliedSearch.trim().toLocaleLowerCase())||'', [options,appliedSearch]);
const queryProducts=useMemo(()=>drilledProduct?[drilledProduct]:exactProductSearch?product.length&&!product.includes(exactProductSearch)?['__codex_no_product_search_match__']:[exactProductSearch]:product,[drilledProduct,exactProductSearch,product]); const queryProducts=useMemo(()=>drilledProduct?[drilledProduct]:exactProductSearch?product.length&&!product.includes(exactProductSearch)?['__codex_no_product_search_match__']:[exactProductSearch]:product,[drilledProduct,exactProductSearch,product]);
useEffect(()=>{setDrilledProduct('');},[department,group,product,projectGroup,sourceFilters,extraFilters]); useEffect(()=>{setDrilledProduct('');},[department,group,product,projectGroup,sourceFilters,extraFilters]);
const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:initial.releaseId||currentReleaseId||catalog?.releaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,catalog,stage,sourceFilters,extraFilters,initial.releaseId,currentReleaseId]); const queryReleaseId=initial.releaseId||(currentReleaseId!==undefined?currentReleaseId||undefined:catalog?.releaseId);
const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,queryReleaseId,stage,sourceFilters,extraFilters]);
const catalogMatchesDraft=!!catalog&&!!draft.releaseId&&catalog.releaseId===draft.releaseId;
const queryReady=(query:QueryParams)=>!!queryReleaseId&&query.releaseId===queryReleaseId&&(!query.productSearch||!!options)&&(!(query.columnFilters||[]).length||!!catalog&&catalog.releaseId===query.releaseId);
const draftReady=queryReady(draft);
const appliedReady=!!applied&&queryReady(applied);
// Apply organizational/source filters directly, like the other query pages. Product text search keeps its explicit submit. // Apply organizational/source filters directly, like the other query pages. Product text search keeps its explicit submit.
useEffect(()=>{if(active&&catalog&&!supportLoading&&!supportError)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,catalog,draft,supportLoading,supportError]); useEffect(()=>{if(active&&draftReady)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,draft,draftReady]);
const queryKey=JSON.stringify({query:applied,revision});const current=response?.key===queryKey?response.value:null;const error=failure?.key===queryKey?failure.message:''; const productPagingScope=!!applied&&shouldPageProducts(applied)&&!drilledProduct;
const overview=usePriorityMetrics(applied,active&&appliedReady,JSON.stringify([revision,refreshKey]),onUnauthorized,productPagingScope);
const paging=productPagingScope&&!overview.failed;
const productPage=useProductPage(applied||draft,active&&appliedReady&&paging&&canReadProductPage(applied||draft,overview.settled),JSON.stringify([revision,refreshKey]),stage==='black'?'blackTotalDesc':'brightTotalDesc',onUnauthorized);
const queryKey=JSON.stringify({query:applied,revision,refreshKey,page:productPagingScope?productPage.key:null});const current=response?.key===queryKey?response.value:null;const error=failure?.key===queryKey?failure.message:'';
const currentSummary=overview.value||current;
useEffect(()=>{ useEffect(()=>{
if(!active||!applied||supportError)return;const controller=new AbortController();setLoading(true);setFailure(null); if(!active||!applied||!appliedReady||!overview.settled)return;if(paging){if(productPage.error){setFailure({key:queryKey,message:productPage.error});setLoading(false);return;}if(!productPage.value||!overview.value){setLoading(true);return;}setResponse({key:queryKey,value:{...overview.value,rows:productPage.value.rows}});setLoading(false);setFailure(null);return;}if(overview.value&&!overview.split){setResponse({key:queryKey,value:overview.value});setLoading(false);setFailure(null);return;}const controller=new AbortController();setLoading(true);setFailure(null);
void api.metrics(applied,controller.signal).then(value=>{if(!controller.signal.aborted)setResponse({key:queryKey,value});}).catch(e=>{if(!controller.signal.aborted&&!onUnauthorized(e))setFailure({key:queryKey,message:e instanceof Error?e.message:'查询失败'});}).finally(()=>{if(!controller.signal.aborted)setLoading(false);}); void api.metricsTable(applied,controller.signal).then(value=>{if(!controller.signal.aborted)setResponse({key:queryKey,value});}).catch(e=>{if(!controller.signal.aborted&&!onUnauthorized(e))setFailure({key:queryKey,message:e instanceof Error?e.message:'查询失败'});}).finally(()=>{if(!controller.signal.aborted)setLoading(false);});
return()=>controller.abort(); return()=>controller.abort();
},[active,queryKey,onUnauthorized,supportError]); },[active,queryKey,onUnauthorized,appliedReady,overview.settled,overview.split,overview.value,paging,productPage.value,productPage.error]);
useEffect(()=>{if(!active||!applied)return;const bytes=new TextEncoder().encode(JSON.stringify({query:queryForLocation(applied,initial.releaseId)}));const encoded=btoa(Array.from(bytes,b=>String.fromCharCode(b)).join('')).replace(/=/g,'').replace(/\+/g,'-').replace(/\//g,'_');const url=`?page=${stage==='black'?'blackProduct':'brightProduct'}&context=${encoded}`;window.history.replaceState({},'',`${location.pathname}${url}`);onLocationChange(url);},[active,applied,stage]); useEffect(()=>{if(!active||!applied)return;const bytes=new TextEncoder().encode(JSON.stringify({query:queryForLocation(applied,initial.releaseId)}));const encoded=btoa(Array.from(bytes,b=>String.fromCharCode(b)).join('')).replace(/=/g,'').replace(/\+/g,'-').replace(/\//g,'_');const url=`?page=${stage==='black'?'blackProduct':'brightProduct'}&context=${encoded}`;window.history.replaceState({},'',`${location.pathname}${url}`);onLocationChange(url);},[active,applied,stage]);
const trendQuery=useMemo(()=>applied?{...applied,month:undefined,quarter:undefined,...(trendGranularity==='year'?{year:undefined}:{}),trendGranularity}:null,[applied,trendGranularity]); const trendQuery=useMemo(()=>applied?{...applied,month:undefined,quarter:undefined,...(trendGranularity==='year'?{year:undefined}:{}),trendGranularity}:null,[applied,trendGranularity]);
const trendKey=JSON.stringify({query:trendQuery,revision}); const trendKey=JSON.stringify({query:trendQuery,revision,refreshKey});
useEffect(()=>{if(!active||!trendQuery||supportError)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,supportError]); const trendReady=!!trendQuery&&queryReady(trendQuery);
const reuseOverviewTrend=!!applied&&!!trendQuery&&sameTrendScope(applied,trendQuery);
const trendGate=!overview.split&&!reuseOverviewTrend||overview.settled;
const sharedTrend=reuseOverviewTrend?overview.value:null;
useEffect(()=>{if(!active||!trendQuery||!trendReady||!trendGate)return;if(sharedTrend){setTrendResponse({key:trendKey,value:overviewTrend(sharedTrend),query:trendQuery});setTrendLoading(false);setTrendError('');return;}const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,trendReady,trendGate,sharedTrend]);
const visibleTrend=useMemo(()=>fullYearTrend(trendResponse?.key===trendKey?trendResponse.value.trend:[],applied?.year||year,trendGranularity),[trendResponse,trendKey,applied?.year,year,trendGranularity]); const visibleTrend=useMemo(()=>fullYearTrend(trendResponse?.key===trendKey?trendResponse.value.trend:[],applied?.year||year,trendGranularity),[trendResponse,trendKey,applied?.year,year,trendGranularity]);
const displayedTrend=useRetainedResult(trendResponse?.key===trendKey?visibleTrend:null)||visibleTrend; const displayedTrend=useRetainedResult(trendResponse?.key===trendKey?visibleTrend:null)||visibleTrend;
const selectedTrendPeriod=trendGranularity==='month'&&applied?.month const selectedTrendPeriod=trendGranularity==='month'&&applied?.month
@@ -80,12 +102,14 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
: trendGranularity==='quarter'&&applied?.quarter ? `${applied.year}-Q${applied.quarter}` : undefined; : trendGranularity==='quarter'&&applied?.quarter ? `${applied.year}-Q${applied.quarter}` : undefined;
const selectTrendPeriod=(value:number|string)=>{automaticPeriod.current=false;const next=nextTrendPeriod(value,{period,year,month,quarter});setYear(next.year);setMonth(next.month);setQuarter(next.quarter);setPeriod(next.period);}; const selectTrendPeriod=(value:number|string)=>{automaticPeriod.current=false;const next=nextTrendPeriod(value,{period,year,month,quarter});setYear(next.year);setMonth(next.month);setQuarter(next.quarter);setPeriod(next.period);};
const dirty=!!applied&&JSON.stringify(draft)!==JSON.stringify(applied); const dirty=!!applied&&JSON.stringify(draft)!==JSON.stringify(applied);
const queryBusy=supportLoading||loading||(!error&&!supportError&&(dirty||!current)); const metadataPending=!draftReady;
const queryFailed=!!(error||supportError); const metadataFailed=!draftReady&&!!supportError;
const queryBusy=!metadataFailed&&(metadataPending||loading||(!error&&(dirty||!current)));
const queryFailed=!!error||metadataFailed;
const submit=()=>{automaticPeriod.current=false;setAppliedSearch(search.trim());setDrilledProduct('');setRevision(v=>v+1);}; const submit=()=>{automaticPeriod.current=false;setAppliedSearch(search.trim());setDrilledProduct('');setRevision(v=>v+1);};
const setTrendObject=(value?:string)=>{const selected=value||'';setDrilledProduct('');setSearch(selected);setAppliedSearch(selected);}; const setTrendObject=(value?:string)=>{const selected=value||'';setDrilledProduct('');setSearch(selected);setAppliedSearch(selected);};
const returnToProducts=()=>{setDrilledProduct('');setSearch(drillParentSearch.current);setAppliedSearch(drillParentSearch.current);if(trendMode!=='defects')setTrendMode('production');}; const returnToProducts=()=>{setDrilledProduct('');setSearch(drillParentSearch.current);setAppliedSearch(drillParentSearch.current);if(trendMode!=='defects')setTrendMode('production');};
const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:catalog?.releaseId},stage,[]));setRevision(v=>v+1);}; const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,[]));setRevision(v=>v+1);};
const groups=(options?.groups||[]).filter(name=>!department.length||department.includes(options?.groupDepartments?.[name]||'')); const groups=(options?.groups||[]).filter(name=>!department.length||department.includes(options?.groupDepartments?.[name]||''));
const paths=options?.organizationPaths||[];const products=department.length||group.length?[...new Set(paths.filter(path=>(!department.length||department.includes(path.department||''))&&(!group.length||group.includes(path.group||''))).map(path=>path.product).filter((v):v is string=>!!v))]:options?.products||[]; const paths=options?.organizationPaths||[];const products=department.length||group.length?[...new Set(paths.filter(path=>(!department.length||department.includes(path.department||''))&&(!group.length||group.includes(path.group||''))).map(path=>path.product).filter((v):v is string=>!!v))]:options?.products||[];
const trendOptions=useMemo(()=>[...products.map(value=>({label:value,value})),...(drilledProduct==='__UNASSIGNED__'?[{label:'未分类',value:drilledProduct}]:[])],[products,drilledProduct]); const trendOptions=useMemo(()=>[...products.map(value=>({label:value,value})),...(drilledProduct==='__UNASSIGNED__'?[{label:'未分类',value:drilledProduct}]:[])],[products,drilledProduct]);
@@ -96,16 +120,17 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
const salesSelection=salesOrderTypeSelection(extraFilters); const salesSelection=salesOrderTypeSelection(extraFilters);
const moreFilterSummary=[salesSelection.values.length?`销售订单类型${salesSelection.exclude?'(排除)':''}:`+salesSelection.values.map(value=>value==='__EMPTY__'?'未匹配':value).join('、'):'',projectGroup.length?`项目组:${projectGroup.map(v=>v==='__UNASSIGNED__'?'未分类':v).join('、')}`:'',...sourceFilters.map(rule=>fields.find(field=>field.key===rule.field)?.label||rule.field)].filter(Boolean).join(';'); const moreFilterSummary=[salesSelection.values.length?`销售订单类型${salesSelection.exclude?'(排除)':''}:`+salesSelection.values.map(value=>value==='__EMPTY__'?'未匹配':value).join('、'):'',projectGroup.length?`项目组:${projectGroup.map(v=>v==='__UNASSIGNED__'?'未分类':v).join('、')}`:'',...sourceFilters.map(rule=>fields.find(field=>field.key===rule.field)?.label||rule.field)].filter(Boolean).join(';');
const displayed = useRetainedResult(current && applied ? { data: current, query: applied } : null); const displayed = useRetainedResult(current && applied ? { data: current, query: applied } : null);
const sortedRows=useMemo(()=>sortStageRowsByTotalWeight(displayed?.data.rows||[],stage),[displayed?.data,stage]); const sortedRows=useMemo(()=>paging?displayed?.data.rows||[]:sortStageRowsByTotalWeight(displayed?.data.rows||[],stage),[displayed?.data,stage,paging]);
const trendRefreshing=!trendError&&(trendLoading||trendResponse?.key!==trendKey); const trendRefreshing=!trendError&&(trendLoading||trendResponse?.key!==trendKey);
const showTrendLoading=useDelayedIndicator(trendRefreshing); const showTrendLoading=useDelayedIndicator(trendRefreshing);
const tableReady=!!current&&(!drilledProduct||(trendResponse?.key===trendKey&&!trendLoading&&!trendError)); const tableReady=!!current&&(!drilledProduct||(trendResponse?.key===trendKey&&!trendLoading&&!trendError));
const table=useRetainedResult(tableReady&&applied?{rows:drilledProduct?visibleTrend:sortedRows,query:applied,drilledProduct,granularity:trendGranularity,period:drilledProduct?'year' as const:applied.month?'month' as const:applied.quarter?'quarter' as const:'year' as const}:null); const table=useRetainedResult(tableReady&&applied?{rows:drilledProduct?visibleTrend:sortedRows,query:applied,drilledProduct,granularity:trendGranularity,period:drilledProduct?'year' as const:applied.month?'month' as const:applied.quarter?'quarter' as const:'year' as const}:null);
const defectDrill=useDefectDrilldown({active,mode:trendMode,stage,query:{...(applied||draft),releaseId:current?.release?.id||applied?.releaseId},trendQuery:trendQuery||draft, const defectDrill=useDefectDrilldown({active,mode:trendMode,stage,query:{...(applied||draft),releaseId:current?.release?.id||applied?.releaseId},trendQuery:trendQuery||draft,
tableQuery:drilledProduct?(trendQuery||draft):(applied||draft),tableDimension:drilledProduct?'period':'product',tableGranularity:trendGranularity, productKeys:paging?productPage.value?.rows.map(row=>row.key):undefined,tableEnabled:!paging||!!productPage.value,tableQuery:drilledProduct?(trendQuery||draft):(applied||draft),tableDimension:drilledProduct?'period':'product',tableGranularity:trendGranularity,
refreshKey:JSON.stringify([refreshKey,revision]),onUnauthorized,display:defectDisplay,onDisplay:setDefectDisplay}); refreshKey:JSON.stringify([refreshKey,revision]),onUnauthorized,display:defectDisplay,onDisplay:setDefectDisplay});
const summary=displayed?.data.summary[stage];const stageRate=stage==='bright'&&rateBasis==='withoutHeadquartersScrap'?summary?.rateWithoutHeadquartersScrap:summary?.rate; const displayedSummary=useRetainedResult(currentSummary);
const openDetail=(row:MetricRow|null,_stage:Stage,selectedMonth?:number)=>{if(!applied||!current?.release)return;setRecordVisible(true);setRecords({row,query:freezeSourceRecordsQuery(applied,current.release.id,drilledProduct?'overall':'product',row,selectedMonth)});}; const summary=displayedSummary?.summary[stage];const stageRate=stage==='bright'&&rateBasis==='withoutHeadquartersScrap'?summary?.rateWithoutHeadquartersScrap:summary?.rate;
const openDetail=(row:MetricRow|null,_stage:Stage,selectedMonth?:number)=>{if(!applied||!currentSummary?.release)return;setRecordVisible(true);setRecords({row,query:freezeSourceRecordsQuery(applied,currentSummary.release.id,drilledProduct?'overall':'product',row,selectedMonth)});};
const runExport=async()=>{setExporting(true);try{await exporter.current?.exportExcel();}catch(e){message.error(e instanceof Error?e.message:'导出失败');}finally{setExporting(false);}}; const runExport=async()=>{setExporting(true);try{await exporter.current?.exportExcel();}catch(e){message.error(e instanceof Error?e.message:'导出失败');}finally{setExporting(false);}};
const formulaKey=stage==='black'?'blackRate':rateBasis==='original'?'brightRate':'brightRateWithoutHeadquartersScrap'; const formulaKey=stage==='black'?'blackRate':rateBasis==='original'?'brightRate':'brightRateWithoutHeadquartersScrap';
return <div className="query-page stage-product-page"> return <div className="query-page stage-product-page">
@@ -124,13 +149,13 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
</div> </div>
<div className="query-toolbar-tail"><span className={`query-status ${queryBusy?'busy':queryFailed?'failed':''}`}><i/>{queryBusy?'正在查询':queryFailed?'查询失败':'查询完成'}</span><QuerySnapshotButton title={`${label}数据`} queryKey={JSON.stringify([draft,trendMode,trendObject,trendGranularity,rateBasis,fullFields,defectDisplay,revision])} disabled={queryBusy || !current} onUnauthorized={onUnauthorized} /><Button type="default" onClick={reset}>重置</Button></div> <div className="query-toolbar-tail"><span className={`query-status ${queryBusy?'busy':queryFailed?'failed':''}`}><i/>{queryBusy?'正在查询':queryFailed?'查询失败':'查询完成'}</span><QuerySnapshotButton title={`${label}数据`} queryKey={JSON.stringify([draft,trendMode,trendObject,trendGranularity,rateBasis,fullFields,defectDisplay,revision])} disabled={queryBusy || !current} onUnauthorized={onUnauthorized} /><Button type="default" onClick={reset}>重置</Button></div>
{moreFiltersOpen&&<div className="query-extra-fields" id={`stage-more-filters-${stage}`}> {moreFiltersOpen&&<div className="query-extra-fields" id={`stage-more-filters-${stage}`}>
{extraFields.map(field=><SalesOrderTypeSelect key={field.key} field={field} filters={extraFilters} releaseId={draft.releaseId} stage={stage} onUnauthorized={onUnauthorized} onChange={setExtraFilters}/>)} {draft.releaseId&&extraFields.map(field=><SalesOrderTypeSelect key={field.key} field={field} filters={extraFilters} releaseId={draft.releaseId} stage={stage} onUnauthorized={onUnauthorized} onChange={setExtraFilters}/>)}
{options?.projectGroupStatus==='READY'&&select('项目组',projectGroup,options.projectGroups||[],setProjectGroup,true)} {options?.projectGroupStatus==='READY'&&select('项目组',projectGroup,options.projectGroups||[],setProjectGroup,true)}
{catalog&&<StageSourceFilters stage={stage} fields={fields} query={draft} value={sourceFilters} onChange={setSourceFilters} onUnauthorized={onUnauthorized}/>} {catalogMatchesDraft&&<StageSourceFilters stage={stage} fields={fields} query={draft} value={sourceFilters} onChange={setSourceFilters} onUnauthorized={onUnauthorized}/>}
</div>} </div>}
</section> </section>
{error&&<Alert showIcon type="error" message={response?"更新失败,当前显示上次结果":"查询失败"} description={error} action={<Button onClick={()=>setRevision(v=>v+1)}>重试</Button>}/>} {error&&<Alert showIcon type="error" message={overview.value?"表格读取失败,指标已更新":response?"更新失败,当前显示上次结果":"查询失败"} description={error} action={<Button onClick={()=>setRevision(v=>v+1)}>重试</Button>}/>}
<section className="metric-strip stage-metric-strip" data-snapshot-module="metrics" aria-label="质量指标" aria-busy={loading} inert={!current}> <section className="metric-strip stage-metric-strip" data-snapshot-module="metrics" aria-label="质量指标" aria-busy={!currentSummary&&!error} inert={!currentSummary}>
{stage==='bright'&&<div className="rate-basis-control"><Segmented size="small" aria-label="合格率口径" value={rateBasis} options={[{label:'原口径',value:'original'},{label:'去本部报废',value:'withoutHeadquartersScrap'}]} onChange={v=>setRateBasis(v as typeof rateBasis)}/></div>} {stage==='bright'&&<div className="rate-basis-control"><Segmented size="small" aria-label="合格率口径" value={rateBasis} options={[{label:'原口径',value:'original'},{label:'去本部报废',value:'withoutHeadquartersScrap'}]} onChange={v=>setRateBasis(v as typeof rateBasis)}/></div>}
<MetricCard primary stage={stage} label={`${label}合格率`} value={rate(stageRate)} onClick={()=>setBasisOpen(true)}/> <MetricCard primary stage={stage} label={`${label}合格率`} value={rate(stageRate)} onClick={()=>setBasisOpen(true)}/>
<MetricCard label="合格重量" value={`${weight(summary?.qualifiedWeight)} kg`} onClick={()=>openDetail(null,stage)}/> <MetricCard label="合格重量" value={`${weight(summary?.qualifiedWeight)} kg`} onClick={()=>openDetail(null,stage)}/>
@@ -150,7 +175,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
</div> </div>
</QueryTrendPanel> </QueryTrendPanel>
<QueryTablePanel heading={<SectionHeading title={table?.drilledProduct?`产品${table.granularity==='quarter'?'季度':table.granularity==='year'?'年度':'月度'}质量数据`:`${label}产品质量汇总`} subtitle={table?.drilledProduct?`${table.query.year} 年 · ${table.drilledProduct==='__UNASSIGNED__'?'未分类':table.drilledProduct} · 重量 / kg 合格率 / %`:'重量 / kg 合格率 / %'}/>} <QueryTablePanel heading={<SectionHeading title={table?.drilledProduct?`产品${table.granularity==='quarter'?'季度':table.granularity==='year'?'年度':'月度'}质量数据`:`${label}产品质量汇总`} subtitle={table?.drilledProduct?`${table.query.year} 年 · ${table.drilledProduct==='__UNASSIGNED__'?'未分类':table.drilledProduct} · 重量 / kg 合格率 / %`:'重量 / kg 合格率 / %'}/>}
busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!catalog||!!supportError} busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!options}
fullFields={fullFields} onFullFields={setFullFields} defectDisplay={defectDisplay} onDefectDisplay={setDefectDisplay} fullFields={fullFields} onFullFields={setFullFields} defectDisplay={defectDisplay} onDefectDisplay={setDefectDisplay}
returnControls={drilledProduct&&<Button className="query-return-button" icon={<ArrowLeftOutlined/>} onClick={returnToProducts}>返回上一级</Button>} returnControls={drilledProduct&&<Button className="query-return-button" icon={<ArrowLeftOutlined/>} onClick={returnToProducts}>返回上一级</Button>}
exporting={exporting} exportDisabled={!!defectDrill.category&&(!defectDrill.table||defectDrill.pending||!!defectDrill.error)||!current||loading||!tableReady||!table?.rows.length} onExport={()=>void runExport()}> exporting={exporting} exportDisabled={!!defectDrill.category&&(!defectDrill.table||defectDrill.pending||!!defectDrill.error)||!current||loading||!tableReady||!table?.rows.length} onExport={()=>void runExport()}>
@@ -158,7 +183,9 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
<SmoothTableFrame busy={queryBusy||!tableReady||defectDrill.tablePending} blocked={queryBusy||!tableReady||defectDrill.tablePending}> <SmoothTableFrame busy={queryBusy||!tableReady||defectDrill.tablePending} blocked={queryBusy||!tableReady||defectDrill.tablePending}>
<MetricTable defectDrilldown={tableReady&&!queryBusy?defectDrill.table:undefined} ref={exporter} exportFilename={`${table?.drilledProduct||label}产品汇总-${table?.query.year||year}-${fullFields?'完整字段':'关键指标'}`} <MetricTable defectDrilldown={tableReady&&!queryBusy?defectDrill.table:undefined} ref={exporter} exportFilename={`${table?.drilledProduct||label}产品汇总-${table?.query.year||year}-${fullFields?'完整字段':'关键指标'}`}
paginationScope={JSON.stringify({query:table?.query,drilledProduct:table?.drilledProduct})} paginationScope={JSON.stringify({query:table?.query,drilledProduct:table?.drilledProduct})}
rows={table?.rows||[]} view={table?.drilledProduct?'overall':'product'} layoutView="product" pageSizeOverride={table?.drilledProduct?12:undefined} rows={table?.rows||[]} columnRows={paging&&currentSummary?[currentSummary.summary]:undefined} remotePage={paging?productPage.pagination:undefined}
loadExportData={paging?async()=>{const [rows,defectDrilldown]=await Promise.all([productPage.loadAll(),defectDrill.exportTable()]);return {rows,defectDrilldown};}:undefined}
view={table?.drilledProduct?'overall':'product'} layoutView="product" pageSizeOverride={table?.drilledProduct?12:undefined}
period={table?.period||period} month={table?.query.month||month} quarter={table?.query.quarter||quarter} period={table?.period||period} month={table?.query.month||month} quarter={table?.query.quarter||quarter}
fullFields={fullFields} defectDisplay={defectDisplay} rateBasis={rateBasis} onlyStage={stage} onDetail={openDetail} onBasis={()=>setBasisOpen(true)} fullFields={fullFields} defectDisplay={defectDisplay} rateBasis={rateBasis} onlyStage={stage} onDetail={openDetail} onBasis={()=>setBasisOpen(true)}
emptyText={queryBusy?'正在更新数据…':error?'查询失败,暂无可展示结果':'当前条件下暂无数据'} emptyText={queryBusy?'正在更新数据…':error?'查询失败,暂无可展示结果':'当前条件下暂无数据'}
@@ -175,7 +202,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
{displayed&&<DefectParetoPanel releaseId={displayed.data.release?.id} query={displayed.query} pending={!current||loading} refreshKey={revision} onlyStage={stage} externalScrapWeight={stage==='black'?displayed.data.defects.black.find(item=>item.name==='外废')?.weight:undefined} onUnauthorized={onUnauthorized}/>} {displayed&&<DefectParetoPanel releaseId={displayed.data.release?.id} query={displayed.query} pending={!current||loading} refreshKey={revision} onlyStage={stage} externalScrapWeight={stage==='black'?displayed.data.defects.black.find(item=>item.name==='外废')?.weight:undefined} onUnauthorized={onUnauthorized}/>}
</div> </div>
</section></div> </section></div>
<Modal open={basisOpen} title={`${label}计算口径`} onCancel={()=>setBasisOpen(false)} footer={<Button onClick={()=>setBasisOpen(false)}>关闭</Button>}><p>{readableMetricFormula((current?.context.metricFormulaDescriptions as Record<string,unknown>|undefined)?.[formulaKey])}</p></Modal> <Modal open={basisOpen} title={`${label}计算口径`} onCancel={()=>setBasisOpen(false)} footer={<Button onClick={()=>setBasisOpen(false)}>关闭</Button>}><p>{readableMetricFormula((currentSummary?.context.metricFormulaDescriptions as Record<string,unknown>|undefined)?.[formulaKey])}</p></Modal>
{records&&<RecordsDrawer key={JSON.stringify(records)} open={recordVisible} row={records.row} query={records.query} stage={stage} lockedStage onStage={()=>{}} onClose={()=>setRecordVisible(false)} onUnauthorized={onUnauthorized}/>} {records&&<RecordsDrawer key={JSON.stringify(records)} open={recordVisible} row={records.row} query={records.query} stage={stage} lockedStage onStage={()=>{}} onClose={()=>setRecordVisible(false)} onUnauthorized={onUnauthorized}/>}
</div>; </div>;
} }
+12 -4
View File
@@ -11,13 +11,15 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S
const [search,setSearch]=useState('');const [choices,setChoices]=useState<string[]>([]);const [busy,setBusy]=useState(false);const [error,setError]=useState('');const [more,setMore]=useState(false);const [revision,setRevision]=useState(0); const [search,setSearch]=useState('');const [choices,setChoices]=useState<string[]>([]);const [busy,setBusy]=useState(false);const [error,setError]=useState('');const [more,setMore]=useState(false);const [revision,setRevision]=useState(0);
const {active}=usePageCache(); const {active}=usePageCache();
const showBusy=useDelayedIndicator(busy); const showBusy=useDelayedIndicator(busy);
const contextKey=JSON.stringify(query);const current=useRef(0); const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query;
const period={year,...(month?{month}:{}),...(quarter?{quarter}:{})};
const filters={...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)};
const contextKey=JSON.stringify({releaseId,period,filters,field:field.key,stage});const current=useRef(0);
useEffect(()=>{ useEffect(()=>{
if(!active)return; if(!active)return;
const generation=++current.current,controller=new AbortController();setBusy(true);setError('');setChoices([]); const generation=++current.current,controller=new AbortController();setBusy(true);setError('');setChoices([]);
const timer=window.setTimeout(()=>{ const timer=window.setTimeout(()=>{
const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query; void api.sourceColumnValues({releaseId,period,filters,field:field.key,stages:[stage],search,limit:50,offset:0,candidateOnly:true},controller.signal)
void api.sourceColumnValues({releaseId,period:{year,...(month?{month}:{}),...(quarter?{quarter}:{})},filters:{...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)},field:field.key,stages:[stage],search,limit:50,offset:0},controller.signal)
.then(result=>{if(current.current!==generation)return;setChoices(result.values.filter((v):v is string=>typeof v==='string'));setMore(result.hasMore===true||(result.total??0)>50);}) .then(result=>{if(current.current!==generation)return;setChoices(result.values.filter((v):v is string=>typeof v==='string'));setMore(result.hasMore===true||(result.total??0)>50);})
.catch(failure=>{if(controller.signal.aborted||current.current!==generation)return;if(!onUnauthorized(failure))setError(failure instanceof Error?failure.message:'选项读取失败');}) .catch(failure=>{if(controller.signal.aborted||current.current!==generation)return;if(!onUnauthorized(failure))setError(failure instanceof Error?failure.message:'选项读取失败');})
.finally(()=>{if(current.current===generation)setBusy(false);}); .finally(()=>{if(current.current===generation)setBusy(false);});
@@ -26,10 +28,16 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S
},[active,field.key,stage,contextKey,search,revision,onUnauthorized]); },[active,field.key,stage,contextKey,search,revision,onUnauthorized]);
return <Select {...readableSelectPopup} mode="multiple" allowClear showSearch aria-label={`${field.label}筛选值`} placeholder="选择筛选值" value={value} searchValue={search} filterOption={false} onSearch={setSearch} loading={showBusy} maxTagCount="responsive" onChange={values=>{onChange(values);setSearch('');}} options={[...new Set([...choices,...value])].map(v=>({label:v,value:v}))} notFoundContent={showBusy?<Spin size="small"/>:error?<Button size="small" onClick={()=>setRevision(v=>v+1)}>读取失败,重试</Button>:'暂无匹配值'} popupRender={menu=><>{menu}{more&&<div className="query-options-hint">输入关键词搜索更多值</div>}{error&&<div role="alert" className="query-options-hint">{error}</div>}</>}/>; return <Select {...readableSelectPopup} mode="multiple" allowClear showSearch aria-label={`${field.label}筛选值`} placeholder="选择筛选值" value={value} searchValue={search} filterOption={false} onSearch={setSearch} loading={showBusy} maxTagCount="responsive" onChange={values=>{onChange(values);setSearch('');}} options={[...new Set([...choices,...value])].map(v=>({label:v,value:v}))} notFoundContent={showBusy?<Spin size="small"/>:error?<Button size="small" onClick={()=>setRevision(v=>v+1)}>读取失败,重试</Button>:'暂无匹配值'} popupRender={menu=><>{menu}{more&&<div className="query-options-hint">输入关键词搜索更多值</div>}{error&&<div role="alert" className="query-options-hint">{error}</div>}</>}/>;
} }
function ContainsValues({field,value,onChange}:{field:SourceColumnField;value:string[];onChange:(value:string[])=>void}) {
const [search,setSearch]=useState('');
return <Select {...readableSelectPopup} mode="tags" maxCount={50} maxTagCount="responsive" allowClear aria-label={`${field.label}包含文字`} placeholder="输入关键词后按回车" value={value} searchValue={search}
onSearch={text=>setSearch(text.slice(0,256).replace(/[\uD800-\uDBFF]$/u,''))} onBlur={()=>setSearch('')}
onChange={values=>{onChange(values.filter(value=>value.trim().length>0&&value.length<=256));setSearch('');}} options={[]}/>;
}
export function StageSourceFilters({stage,fields,query,value,onChange,onUnauthorized,part='both'}:{part?:'both'|'picker'|'rows';stage:Stage;fields:SourceColumnField[];query:QueryParams;value:SourceColumnFilter[];onChange:(value:SourceColumnFilter[])=>void;onUnauthorized:(error:unknown)=>boolean}) { export function StageSourceFilters({stage,fields,query,value,onChange,onUnauthorized,part='both'}:{part?:'both'|'picker'|'rows';stage:Stage;fields:SourceColumnField[];query:QueryParams;value:SourceColumnFilter[];onChange:(value:SourceColumnFilter[])=>void;onUnauthorized:(error:unknown)=>boolean}) {
const selected=value.map(rule=>rule.field);const patch=(key:string,change:Partial<SourceColumnFilter>)=>onChange(value.map(rule=>rule.field===key?{...rule,...change}:rule)); const selected=value.map(rule=>rule.field);const patch=(key:string,change:Partial<SourceColumnFilter>)=>onChange(value.map(rule=>rule.field===key?{...rule,...change}:rule));
return <div className="stage-source-filters"> return <div className="stage-source-filters">
{part!=='rows'&&<Select {...readableSelectPopup} mode="multiple" maxCount={20} maxTagCount="responsive" showSearch optionFilterProp="label" allowClear aria-label="选择筛选字段" placeholder="添加筛选字段" value={selected} options={fields.map(field=>({label:field.label,value:field.key}))} onChange={keys=>onChange(keys.map(key=>value.find(rule=>rule.field===key)||{field:key,stages:[stage],operator:'in',values:[]}))}/>} {part!=='rows'&&<Select {...readableSelectPopup} mode="multiple" maxCount={20} maxTagCount="responsive" showSearch optionFilterProp="label" allowClear aria-label="选择筛选字段" placeholder="添加筛选字段" value={selected} options={fields.map(field=>({label:field.label,value:field.key}))} onChange={keys=>onChange(keys.map(key=>value.find(rule=>rule.field===key)||{field:key,stages:[stage],operator:'in',values:[]}))}/>}
{part!=='picker'&&value.map(rule=>{const field=fields.find(item=>item.key===rule.field);return field?<div className="stage-source-filter-row" key={rule.field}><strong>{field.label}</strong><Select {...readableSelectPopup} aria-label={`${field.label}筛选方式`} value={rule.operator} options={[{label:'等于任一值',value:'in'},{label:'不等于这些值',value:'not_in'},{label:'为空',value:'is_empty'},{label:'不为空',value:'not_empty'}]} onChange={operator=>patch(rule.field,{operator,values:[]})}/>{['in','not_in'].includes(rule.operator)?<SourceValues field={field} stage={stage} query={query} value={rule.values||[]} onChange={values=>patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:<span/>}<Button type="text" aria-label={`移除${field.label}筛选`} icon={<CloseOutlined/>} onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}/></div>:<Alert key={rule.field} type="warning" message="所选来源字段当前不可用,请移除后重新选择" action={<Button onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}>移除</Button>}/>;})} {part!=='picker'&&value.map(rule=>{const field=fields.find(item=>item.key===rule.field);return field?<div className="stage-source-filter-row" key={rule.field}><strong>{field.label}</strong><Select {...readableSelectPopup} aria-label={`${field.label}筛选方式`} value={rule.operator} options={[{label:'等于任一值',value:'in'},{label:'包含',value:'contains'},{label:'不等于这些值',value:'not_in'},{label:'为空',value:'is_empty'},{label:'不为空',value:'not_empty'}]} onChange={operator=>patch(rule.field,{operator,values:[]})}/>{rule.operator==='contains'?<ContainsValues field={field} value={rule.values||[]} onChange={values=>patch(rule.field,{values})}/>:['in','not_in'].includes(rule.operator)?<SourceValues field={field} stage={stage} query={query} value={rule.values||[]} onChange={values=>patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:<span/>}<Button type="text" aria-label={`移除${field.label}筛选`} icon={<CloseOutlined/>} onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}/></div>:<Alert key={rule.field} type="warning" message="所选来源字段当前不可用,请移除后重新选择" action={<Button onClick={()=>onChange(value.filter(item=>item.field!==rule.field))}>移除</Button>}/>;})}
</div>; </div>;
} }
+4 -4
View File
@@ -7,7 +7,7 @@ export type AnalysisVisualSectionKey = 'overview' | 'stage' | 'department' | 'gr
export type AnalysisVisualDimension = 'department' | 'group' | 'product' | 'category' | 'defect'; export type AnalysisVisualDimension = 'department' | 'group' | 'product' | 'category' | 'defect';
export type AnalysisExclusionDimension = 'department' | 'group' | 'product' | 'projectGroup'; export type AnalysisExclusionDimension = 'department' | 'group' | 'product' | 'projectGroup';
export type AnalysisExclusions = Partial<Record<AnalysisExclusionDimension, string[]>>; export type AnalysisExclusions = Partial<Record<AnalysisExclusionDimension, string[]>>;
export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] } export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export type AnalysisVisualContextValue = string | number | boolean | Array<string | number | boolean> | Record<string, string[]> | Record<string, string> | SourceColumnFilter[] | AnalysisExclusions; export type AnalysisVisualContextValue = string | number | boolean | Array<string | number | boolean> | Record<string, string[]> | Record<string, string> | SourceColumnFilter[] | AnalysisExclusions;
export type AnalysisVisualContext = Record<string, AnalysisVisualContextValue>; export type AnalysisVisualContext = Record<string, AnalysisVisualContextValue>;
@@ -253,8 +253,8 @@ function safeContext(value: unknown): AnalysisVisualContext | null {
|| typeof rule.field !== 'string' || !/^[A-Za-z0-9_]{1,64}$/.test(rule.field) || typeof rule.field !== 'string' || !/^[A-Za-z0-9_]{1,64}$/.test(rule.field)
|| !Array.isArray(rule.stages) || !rule.stages.length || rule.stages.length > 2 || new Set(rule.stages).size !== rule.stages.length || !Array.isArray(rule.stages) || !rule.stages.length || rule.stages.length > 2 || new Set(rule.stages).size !== rule.stages.length
|| rule.stages.some(stage => stage !== 'black' && stage !== 'bright') || rule.stages.some(stage => stage !== 'black' && stage !== 'bright')
|| !['in', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null; || !['in', 'contains', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null;
const needsValues = rule.operator === 'in' || rule.operator === 'not_in'; const needsValues = rule.operator === 'in' || rule.operator === 'not_in' || rule.operator === 'contains';
if (needsValues && (!Array.isArray(rule.values) || !rule.values.length || rule.values.length > 50 || rule.values.some(value => typeof value !== 'string' || !value.trim() || value.length > 256))) return null; if (needsValues && (!Array.isArray(rule.values) || !rule.values.length || rule.values.length > 50 || rule.values.some(value => typeof value !== 'string' || !value.trim() || value.length > 256))) return null;
if (!needsValues && rule.values !== undefined) return null; if (!needsValues && rule.values !== undefined) return null;
filters.push({ field: rule.field, stages: [...rule.stages] as SourceColumnFilter['stages'], operator: rule.operator as SourceColumnFilter['operator'], ...(needsValues ? { values: [...rule.values as string[]] } : {}) }); filters.push({ field: rule.field, stages: [...rule.stages] as SourceColumnFilter['stages'], operator: rule.operator as SourceColumnFilter['operator'], ...(needsValues ? { values: [...rule.values as string[]] } : {}) });
@@ -1397,7 +1397,7 @@ export function columnFilterSummary(value: unknown, labelValue: unknown): string
if (!isRecord(rule) || typeof rule.field !== 'string' || !Array.isArray(rule.stages)) return []; if (!isRecord(rule) || typeof rule.field !== 'string' || !Array.isArray(rule.stages)) return [];
const stage = rule.stages.map(item => item === 'black' ? '黑片' : item === 'bright' ? '亮片' : '').filter(Boolean).join('、'); const stage = rule.stages.map(item => item === 'black' ? '黑片' : item === 'bright' ? '亮片' : '').filter(Boolean).join('、');
const label = typeof labels[rule.field] === 'string' ? labels[rule.field] : '来源字段'; const label = typeof labels[rule.field] === 'string' ? labels[rule.field] : '来源字段';
const operation = ({ in: '为', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record<string, string>)[String(rule.operator)]; const operation = ({ in: '为', contains: '包含', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record<string, string>)[String(rule.operator)];
if (!operation) return []; if (!operation) return [];
const values = Array.isArray(rule.values) ? rule.values.filter(item => typeof item === 'string').join('、') : ''; const values = Array.isArray(rule.values) ? rule.values.filter(item => typeof item === 'string').join('、') : '';
return [`${stage} ${label}${operation}${values}`.trim()]; return [`${stage} ${label}${operation}${values}`.trim()];
+17 -3
View File
@@ -1,3 +1,5 @@
import type { ProductOrder } from './productPagination';
import type { ProductPageResponse } from './types';
import { WORKSPACE_INVALIDATED } from './pageCache'; import { WORKSPACE_INVALIDATED } from './pageCache';
import { consumeEventStream, type StreamEvent } from './eventStream'; import { consumeEventStream, type StreamEvent } from './eventStream';
import { publicErrorMessage } from './publicDataDisplay'; import { publicErrorMessage } from './publicDataDisplay';
@@ -110,7 +112,7 @@ export async function request<T>(path: string, init: RequestInit = {}, timeoutMs
method, method,
signal: init.signal ?? undefined, signal: init.signal ?? undefined,
timeoutMs: timeoutMs ?? requestTimeoutFor(path), timeoutMs: timeoutMs ?? requestTimeoutFor(path),
cache: method === 'GET' && isImmutableRead(path), cache: method === 'GET' && init.cache !== 'reload' && isImmutableRead(path),
tags: requestResourceTags(path), tags: requestResourceTags(path),
category: requestCategory(path), category: requestCategory(path),
priority: requestPriority(path), priority: requestPriority(path),
@@ -122,7 +124,10 @@ export async function request<T>(path: string, init: RequestInit = {}, timeoutMs
} }
catch (error) { catch (error) {
handleAccessFailure(error); handleAccessFailure(error);
if (error instanceof Error && error.name === 'RequestTimeoutError') throw new ApiError(408, '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。'); if (error instanceof Error && error.name === 'RequestTimeoutError') {
const query = /^\/api\/(?:metrics(?:\/|$)|records(?:\/|$)|analysis\/(?:source\/|defects(?:\/|$)))/.test(requestPath(path));
throw new ApiError(408, query ? '查询处理超时,请重试或调整筛选范围。' : '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。');
}
throw error; throw error;
} }
} }
@@ -236,7 +241,7 @@ export const api = {
}, },
activateSalesOrderTypes(versionId:string,expectedVersion:string) { return mutate<{versionId:string}>('/api/sales-order-types/activate',{versionId,expectedVersion}); }, activateSalesOrderTypes(versionId:string,expectedVersion:string) { return mutate<{versionId:string}>('/api/sales-order-types/activate',{versionId,expectedVersion}); },
sourceColumnCatalog(releaseId?: string, signal?: AbortSignal) { return request<{releaseId:string;sourceFields:SourceColumnField[]}>(`/api/analysis/source/catalog?${toQuery({releaseId})}`, { signal }); }, sourceColumnCatalog(releaseId?: string, signal?: AbortSignal) { return request<{releaseId:string;sourceFields:SourceColumnField[]}>(`/api/analysis/source/catalog?${toQuery({releaseId})}`, { signal }); },
sourceColumnValues(body: Record<string,unknown>, signal?: AbortSignal) { return queryPost<{values:Array<string|null>;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); }, sourceColumnValues(body: Record<string,unknown> & {candidateOnly?:true}, signal?: AbortSignal) { return queryPost<{values:Array<string|null>;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); },
workspaceState() { return request<{ authRevision: string; dataRevision: string; currentReleaseId: string | null }>('/api/workspace/state', {}, 8000); }, workspaceState() { return request<{ authRevision: string; dataRevision: string; currentReleaseId: string | null }>('/api/workspace/state', {}, 8000); },
filterFields(manage = false, signal?: AbortSignal) { return request<FilterField[]>(`/api/filter-fields?manage=${manage}`, { signal }); }, filterFields(manage = false, signal?: AbortSignal) { return request<FilterField[]>(`/api/filter-fields?manage=${manage}`, { signal }); },
filterFieldOptions(key: string, search = '', signal?: AbortSignal,releaseId?:string,stage?:string) { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search,releaseId,stage})}`, { signal }); }, filterFieldOptions(key: string, search = '', signal?: AbortSignal,releaseId?:string,stage?:string) { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search,releaseId,stage})}`, { signal }); },
@@ -268,6 +273,15 @@ export const api = {
metrics(params: QueryParams, signal?: AbortSignal) { metrics(params: QueryParams, signal?: AbortSignal) {
return request<MetricsResponse>(`/api/metrics?${toQuery(params)}`, { signal }); return request<MetricsResponse>(`/api/metrics?${toQuery(params)}`, { signal });
}, },
productPage(params:QueryParams,page:number,pageSize:number,order:ProductOrder,signal?:AbortSignal) {
return request<ProductPageResponse>(`/api/metrics/products?${toQuery({...params,page,pageSize,order})}`,{signal});
},
exportProductRows(params:QueryParams,rowOrder:ProductOrder,signal?:AbortSignal) {
return request<MetricsResponse>(`/api/metrics?${toQuery({...params,part:'table',rowOrder})}`,{signal,cache:'reload'});
},
metricsTable(params: QueryParams, signal?: AbortSignal) {
return request<MetricsResponse>(`/api/metrics?${toQuery({ ...params, part: 'table' })}`, { signal });
},
trend(params: QueryParams, signal?: AbortSignal) { trend(params: QueryParams, signal?: AbortSignal) {
return request<TrendResponse>(`/api/metrics/trend?${toQuery(params)}`, { signal }); return request<TrendResponse>(`/api/metrics/trend?${toQuery(params)}`, { signal });
}, },
+29
View File
@@ -0,0 +1,29 @@
import type { MetricsResponse, QueryParams, TrendResponse } from './types';
/** 仅调整分组视图,保留全部来源、期间和权限筛选。 */
export function overviewQuery(query: QueryParams): QueryParams {
return { ...query, view: 'overall' };
}
/** 宽范围产品表先取小型汇总;窄范围复用完整结果,避免额外往返。 */
export function splitMetricsLoad(query: QueryParams): boolean {
return query.view === 'product' && !query.month && !query.quarter && !query.product
&& !query.productSearch && !query.monthStart && !query.monthEnd;
}
function canonical(value: unknown): unknown {
if (Array.isArray(value)) return value.map(canonical);
if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value)
.filter(([, item]) => item !== undefined).sort(([a], [b]) => a.localeCompare(b))
.map(([key, item]) => [key, canonical(item)]));
return value;
}
/** 筛选范围和粒度全部一致时,才复用服务端已计算的期间结果。 */
export function sameTrendScope(a: QueryParams, b: QueryParams): boolean {
return JSON.stringify(canonical(overviewQuery(a))) === JSON.stringify(canonical(overviewQuery(b)));
}
export function overviewTrend(value: MetricsResponse): TrendResponse {
return { release: value.release, context: value.context, trend: value.trend, coverage: value.coverage };
}
+19
View File
@@ -0,0 +1,19 @@
import { splitMetricsLoad } from './priorityMetrics.ts';
import type { QueryParams } from './types';
export type ProductOrder = 'brightQualifiedDesc' | 'blackTotalDesc' | 'brightTotalDesc' | 'productAsc';
export const PRODUCT_PAGE_SIZE = 15;
/** 已限定产品的少量结果保留原路径;宽产品表按数据库页读取。 */
export function shouldPageProducts(query: QueryParams): boolean {
return query.view === 'product' && !!query.releaseId && !query.product;
}
export function pageSelection(scope: string, selected: { scope: string; page: number; size: number }) {
return { page: selected.scope === scope ? selected.page : 1, size: selected.size };
}
/** 宽年度先显示摘要;月/季/搜索并行读取小页,避免窄筛选多一次串行等待。 */
export function canReadProductPage(query: QueryParams, overviewSettled: boolean): boolean {
return overviewSettled || !splitMetricsLoad(query);
}
+6 -1
View File
@@ -3,7 +3,7 @@ export type ViewKind = 'overall' | 'department' | 'group' | 'product' | 'project
export type PeriodKind = 'year' | 'quarter' | 'month'; export type PeriodKind = 'year' | 'quarter' | 'month';
export interface SourceColumnField { key: string; label: string; type: string; stages: Stage[]; columns: Partial<Record<Stage,string>> } export interface SourceColumnField { key: string; label: string; type: string; stages: Stage[]; columns: Partial<Record<Stage,string>> }
export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] } export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] }
export interface QueryParams { export interface QueryParams {
year: number; year: number;
@@ -25,6 +25,11 @@ export interface QueryParams {
trendGranularity?: 'month' | 'quarter' | 'year'; trendGranularity?: 'month' | 'quarter' | 'year';
} }
export interface ProductPageResponse {
release: {id:string}; context:Record<string,unknown>; rows:MetricRow[];
total:number; page:number; pageSize:number; order:string;
}
export interface RecordsParams extends QueryParams { export interface RecordsParams extends QueryParams {
stage: Stage; stage: Stage;
page: number; page: number;
+5 -4
View File
@@ -4,8 +4,8 @@ import { api } from './api';
import { defectScope } from './defectDrilldown'; import { defectScope } from './defectDrilldown';
import type { DefectDrilldownResponse, FineDefectCatalogResponse, QueryParams, Stage } from './types'; import type { DefectDrilldownResponse, FineDefectCatalogResponse, QueryParams, Stage } from './types';
export function useDefectDrilldown({active,mode,stage,query,trendQuery,tableQuery,tableDimension,tableGranularity,onUnauthorized,display,onDisplay,refreshKey=0}:{ export function useDefectDrilldown({active,mode,stage,query,trendQuery,tableQuery,tableDimension,tableGranularity,onUnauthorized,display,onDisplay,refreshKey=0,productKeys,tableEnabled=true}:{
refreshKey?:string|number;active:boolean;mode:string;stage:Stage;query:QueryParams;trendQuery:QueryParams;tableQuery:QueryParams; productKeys?:string[];tableEnabled?:boolean;refreshKey?:string|number;active:boolean;mode:string;stage:Stage;query:QueryParams;trendQuery:QueryParams;tableQuery:QueryParams;
tableDimension:string;tableGranularity:string;onUnauthorized:(error:unknown)=>boolean;display:'weight'|'rate';onDisplay:(display:'weight'|'rate')=>void; tableDimension:string;tableGranularity:string;onUnauthorized:(error:unknown)=>boolean;display:'weight'|'rate';onDisplay:(display:'weight'|'rate')=>void;
}) { }) {
const [selection,setSelection]=useState<{stage:Stage;category:string}|null>(null); const [selection,setSelection]=useState<{stage:Stage;category:string}|null>(null);
@@ -18,7 +18,8 @@ export function useDefectDrilldown({active,mode,stage,query,trendQuery,tableQuer
const catalogKey=JSON.stringify([enabled,stage,defectScope(query),revision,refreshKey]); const catalogKey=JSON.stringify([enabled,stage,defectScope(query),revision,refreshKey]);
const common={stage,category,releaseId:query.releaseId}; const common={stage,category,releaseId:query.releaseId};
const trend=useDefectPart(enabled&&!!category&&!!query.releaseId,{...defectScope(trendQuery),...common,dimension:'period',trendGranularity:trendQuery.trendGranularity||'month'},[revision,refreshKey],onUnauthorized); const trend=useDefectPart(enabled&&!!category&&!!query.releaseId,{...defectScope(trendQuery),...common,dimension:'period',trendGranularity:trendQuery.trendGranularity||'month'},[revision,refreshKey],onUnauthorized);
const table=useDefectPart(enabled&&!!category&&!!query.releaseId,{...defectScope(tableQuery),...common,dimension:tableDimension,trendGranularity:tableGranularity},[revision,refreshKey],onUnauthorized); const exportParams={...defectScope(tableQuery),...common,dimension:tableDimension,trendGranularity:tableGranularity};
const table=useDefectPart(enabled&&!!category&&!!query.releaseId&&tableEnabled,{...exportParams,...(productKeys?{productKeys}:{})},[revision,refreshKey],onUnauthorized);
useEffect(()=>{ useEffect(()=>{
if(!enabled||!query.releaseId)return; if(!enabled||!query.releaseId)return;
const controller=new AbortController();setCatalog(null);setCatalogFailure(''); const controller=new AbortController();setCatalog(null);setCatalogFailure('');
@@ -39,7 +40,7 @@ export function useDefectDrilldown({active,mode,stage,query,trendQuery,tableQuer
const retry=<Button size="small" onClick={()=>setRevision(v=>v+1)}>重试</Button>; const retry=<Button size="small" onClick={()=>setRevision(v=>v+1)}>重试</Button>;
const controls=category?<div className="defect-drilldown-controls"><Button type="link" onClick={clear}>全部分类</Button><span>› {category}</span>{trend.pending&&<span><Spin size="small"/> 正在读取细分</span>}{trend.error&&<Alert type="error" showIcon message={trend.error} action={retry}/>} {incomplete&&<span className="defect-drilldown-notice">细分来源未完全覆盖分类总量</span>}</div>:catalogFailure&&enabled?<Alert type="error" showIcon message="细分目录读取失败" description={catalogFailure} action={<Button size="small" onClick={()=>setRevision(v=>v+1)}>重试</Button>}/>:null; const controls=category?<div className="defect-drilldown-controls"><Button type="link" onClick={clear}>全部分类</Button><span>› {category}</span>{trend.pending&&<span><Spin size="small"/> 正在读取细分</span>}{trend.error&&<Alert type="error" showIcon message={trend.error} action={retry}/>} {incomplete&&<span className="defect-drilldown-notice">细分来源未完全覆盖分类总量</span>}</div>:catalogFailure&&enabled?<Alert type="error" showIcon message="细分目录读取失败" description={catalogFailure} action={<Button size="small" onClick={()=>setRevision(v=>v+1)}>重试</Button>}/>:null;
const tableControls=table.error?<Alert type="error" showIcon message={table.error} action={retry}/>:null; const tableControls=table.error?<Alert type="error" showIcon message={table.error} action={retry}/>:null;
return {category,categories,choose,clear,controls,tableControls,pending,error,trend:trend.data,table:table.data,trendPending:trend.pending,tablePending:table.pending,trendError:trend.error,tableError:table.error}; return {category,categories,choose,clear,controls,tableControls,pending,error,exportTable:()=>category?api.defectDrilldown(exportParams):Promise.resolve(undefined),trend:trend.data,table:table.data,trendPending:trend.pending,tablePending:table.pending,trendError:trend.error,tableError:table.error};
} }
+24
View File
@@ -0,0 +1,24 @@
import { useEffect, useState } from 'react';
import { api } from './api';
import { overviewQuery, splitMetricsLoad } from './priorityMetrics';
import type { MetricsResponse, QueryParams } from './types';
/** 优先读取正式小型汇总;失败时允许后续正式查询独立回退。 */
export function usePriorityMetrics(query: QueryParams | null, enabled: boolean, refresh: string | number,
onUnauthorized: (error: unknown) => boolean, forceOverview = false) {
const key = JSON.stringify([query, refresh, forceOverview]);
const split = !!query && (forceOverview || splitMetricsLoad(query));
const [result, setResult] = useState<{ key: string; value?: MetricsResponse; failed?: boolean } | null>(null);
useEffect(() => {
if (!enabled || !query) return;
const controller = new AbortController();
api.metrics(split ? overviewQuery(query) : query, controller.signal)
.then(value => { if (!controller.signal.aborted) setResult({ key, value }); })
.catch(error => {
if (!controller.signal.aborted && !onUnauthorized(error)) setResult({ key, failed: true });
});
return () => controller.abort();
}, [enabled, key, onUnauthorized]);
const current = enabled && result?.key === key ? result : null;
return { value: current?.value || null, settled: !!current, failed: !!current?.failed, split };
}
+40
View File
@@ -0,0 +1,40 @@
import { useEffect, useState } from 'react';
import { api } from './api';
import { pageSelection, PRODUCT_PAGE_SIZE, type ProductOrder } from './productPagination';
import type { ProductPageResponse, QueryParams } from './types';
/** 查询范围变化立即使用第一页,避免先发出旧页码请求。 */
export function useProductPage(query: QueryParams, enabled: boolean, refresh: string | number,
order: ProductOrder, onUnauthorized: (error: unknown) => boolean) {
const scope = JSON.stringify([query, refresh, order]);
const [selected, setSelected] = useState({ scope, page: 1, size: PRODUCT_PAGE_SIZE });
const selection = pageSelection(scope, selected);
const key = JSON.stringify([scope, selection.page, selection.size]);
const [result, setResult] = useState<{ key: string; scope: string; value?: ProductPageResponse; error?: string } | null>(null);
useEffect(() => {
if (!enabled) return;
const controller = new AbortController();
api.productPage(query, selection.page, selection.size, order, controller.signal).then(value => {
if (controller.signal.aborted) return;
if (value.release.id !== query.releaseId || value.page !== selection.page || value.pageSize !== selection.size || value.order !== order)
throw Error('分页查询范围已变化,请重试。');
const last = Math.max(1, Math.ceil(value.total / selection.size));
if (selection.page > last) { setSelected({ scope, page: last, size: selection.size }); return; }
setResult({ key, scope, value });
}).catch(error => {
if (!controller.signal.aborted && !onUnauthorized(error)) setResult({ key, scope, error: error instanceof Error ? error.message : '表格读取失败' });
});
return () => controller.abort();
}, [enabled, key, onUnauthorized]);
const current = enabled && result?.key === key ? result : null;
return {
key, value: current?.value || null, error: current?.error || '',
pagination: { current: selection.page, pageSize: selection.size, total: result?.scope === scope ? result.value?.total || 0 : 0,
onChange: (page: number, size: number) => setSelected({ scope, page: size === selection.size ? page : 1, size }) },
loadAll: async () => {
const value = await api.exportProductRows(query, order);
if (value.release?.id !== query.releaseId) throw Error('导出数据版本已变化,请重试。');
return value.rows;
},
};
}
@@ -4,7 +4,7 @@ import fs from 'node:fs';
import ts from 'typescript'; import ts from 'typescript';
import React from 'react'; import React from 'react';
import { renderToStaticMarkup } from 'react-dom/server'; import { renderToStaticMarkup } from 'react-dom/server';
import { buildAgentReportLayout, agentReportPrimary, agentReportSupplementary } from '../src/agent/agentReportLayout.ts'; import { buildAgentReportLayout, agentReportPrimary, agentReportSupplementary, partitionAgentReportVisuals } from '../src/agent/agentReportLayout.ts';
import { stripDuplicateFirstMarkdownH1 } from '../src/analysisReportTitle.ts'; import { stripDuplicateFirstMarkdownH1 } from '../src/analysisReportTitle.ts';
import { tableTextAlignment } from '../src/tableTextAlignment.ts'; import { tableTextAlignment } from '../src/tableTextAlignment.ts';
import { sanitizeAnalysisDisplayText } from '../src/analysisDisplayText.ts'; import { sanitizeAnalysisDisplayText } from '../src/analysisDisplayText.ts';
@@ -26,7 +26,7 @@ const Document=load('../src/agent/AgentDeliverable.tsx','AgentDeliverable',{
ReportVisualFigure:({visual,image})=>image?.svg?React.createElement('figure',{'data-chart':visual.id},visual.title):React.createElement('table',{'data-fallback':visual.id},React.createElement('tbody',null,React.createElement('tr',null,React.createElement('td',null,visual.title)))), ReportVisualFigure:({visual,image})=>image?.svg?React.createElement('figure',{'data-chart':visual.id},visual.title):React.createElement('table',{'data-fallback':visual.id},React.createElement('tbody',null,React.createElement('tr',null,React.createElement('td',null,visual.title)))),
reportVisualImageSource:image=>image?.svg||'',reportVisuals:()=>figures, reportVisualImageSource:image=>image?.svg||'',reportVisuals:()=>figures,
useReportVisualImages:()=>({authorized:true,loading:false,images}),analysisSnapshotResult:turn=>turn.result, useReportVisualImages:()=>({authorized:true,loading:false,images}),analysisSnapshotResult:turn=>turn.result,
reportSnapshotIdentity:()=>'',analysisScopeText:()=>'',stripDuplicateFirstMarkdownH1,buildAgentReportLayout,agentReportPrimary,agentReportSupplementary, reportSnapshotIdentity:()=>'',analysisScopeText:()=>'',stripDuplicateFirstMarkdownH1,buildAgentReportLayout,agentReportPrimary,agentReportSupplementary,partitionAgentReportVisuals,
captureReportLongImage:()=>{},downloadAuthorizedReport:()=>{},documentCss:'', captureReportLongImage:()=>{},downloadAuthorizedReport:()=>{},documentCss:'',
}); });
const answer='# 原始报告\n\n原始导读。\n\n## 月度趋势\n\n原始分析。\n\n| 月份 | 值 |\n|---|---|\n| 10月 | 95.46% |\n\n## 报废大类\n\n原始说明。'; const answer='# 原始报告\n\n原始导读。\n\n## 月度趋势\n\n原始分析。\n\n| 月份 | 值 |\n|---|---|\n| 10月 | 95.46% |\n\n## 报废大类\n\n原始说明。';
@@ -5,6 +5,8 @@ import ts from 'typescript';
import React from 'react'; import React from 'react';
import { renderToStaticMarkup } from 'react-dom/server'; import { renderToStaticMarkup } from 'react-dom/server';
import { emptyAnalysisNotice } from '../src/analysisConversationState.ts'; import { emptyAnalysisNotice } from '../src/analysisConversationState.ts';
import { analysisChartReferences } from '../src/analysisChartReferences.ts';
import { parseAnalysisVisuals } from '../src/analysisVisualsModel.ts';
import { sanitizeAnalysisDisplayText } from '../src/analysisDisplayText.ts'; import { sanitizeAnalysisDisplayText } from '../src/analysisDisplayText.ts';
import { import {
analysisMemberListFullyRead, analysisMemberListFullyRead,
@@ -116,7 +118,7 @@ test('member-list display is opt-in and historical plain answers do not acquire
const start = source.indexOf('export const AgentConversationTurn'); const start = source.indexOf('export const AgentConversationTurn');
const code = source.slice(start).replace('export const AgentConversationTurn', 'const AgentConversationTurn'); const code = source.slice(start).replace('export const AgentConversationTurn', 'const AgentConversationTurn');
const view = { const view = {
React, memo: value => value, useMemo: callback => callback(), useState: value => [value, () => {}], useEffect: () => {}, React, memo: value => value, useCallback: callback => callback, analysisChartReferences, parseAnalysisVisuals, useMemo: callback => callback(), useState: value => [value, () => {}], useEffect: () => {},
useAnalysisVisualRecovery: turn => ({ turn, available: false, recovering: false, error: turn.recoveryError || '' }), useAnalysisVisualRecovery: turn => ({ turn, available: false, recovering: false, error: turn.recoveryError || '' }),
analysisSnapshotResult: turn => turn.result || turn, partialAnswerIsOnlyProgress: () => false, analysisIsRunning: () => false, analysisSnapshotResult: turn => turn.result || turn, partialAnswerIsOnlyProgress: () => false, analysisIsRunning: () => false,
executionEvents: () => [], RunStatus: () => null, authoredReport: () => null, analysisScopeText: () => '', emptyAnalysisNotice, executionEvents: () => [], RunStatus: () => null, authoredReport: () => null, analysisScopeText: () => '', emptyAnalysisNotice,
@@ -140,6 +142,7 @@ return AgentConversationTurn;`, {
assert.match(empty, /<button>重新分析<\/button>/); assert.match(empty, /<button>重新分析<\/button>/);
assert.match(render({id:'failed-turn',status:'FAILED',answer:'',completion:{reason:'任务理解未完成,请重试'}}), /任务理解未完成,请重试/); assert.match(render({id:'failed-turn',status:'FAILED',answer:'',completion:{reason:'任务理解未完成,请重试'}}), /任务理解未完成,请重试/);
const recovery = render({ id: 'old-turn', status: 'PARTIAL', answer: '原答案', recoveryError: '图表恢复暂不可用' }); const recovery = render({ id: 'old-turn', status: 'PARTIAL', answer: '原答案', recoveryError: '图表恢复暂不可用' });
assert.match(recovery, /role="alert">图表恢复暂不可用/); // 图表恢复失败按正式约定静默省略,保留原回答。
assert.doesNotMatch(recovery, /role="alert"|图表恢复暂不可用/);
assert.match(recovery, /原答案/); assert.match(recovery, /原答案/);
}); });
@@ -1042,3 +1042,12 @@ test('adaptive trends cover every series, preserve missing values, and keep non-
assert.equal(formatVisualValue(.9639601707533932, 'ratio'), '96.40%'); assert.equal(formatVisualValue(.9639601707533932, 'ratio'), '96.40%');
assert.equal(formatVisualValue(3, 'kg'), '3.00 kg'); assert.equal(formatVisualValue(3, 'kg'), '3.00 kg');
}); });
test('contains keywords survive analysis scope and display as substring filters',()=>{
const rule={field:'src_any',stages:['bright'],operator:'contains',values:['日立','%_']};
const parsed=parseAnalysisVisuals([visual({scope:{columnFilters:[rule],columnLabels:{src_any:'客户'}}})]);
assert.equal(parsed.visuals.length,1);
assert.deepEqual(parsed.visuals[0].scope.columnFilters,[rule]);
assert.match(columnFilterSummary([rule],{src_any:'客户'})[0],/包含.*日立/);
assert.equal(parseAnalysisVisuals([visual({scope:{columnFilters:[{...rule,values:[]}]}})]).visuals.length,0);
});
@@ -90,3 +90,12 @@ test('changing only the trend object does not recalculate the unchanged product
assert.equal(f.requests.length,3);assert.equal(f.requests[2].params.dimension,'period');assert.equal(ui.table?.dimension,'product'); assert.equal(f.requests.length,3);assert.equal(f.requests[2].params.dimension,'period');assert.equal(ui.table?.dimension,'product');
assert.equal(ui.trend,undefined);assert.equal(ui.trendPending,true);f.unmount(); assert.equal(ui.trend,undefined);assert.equal(ui.trendPending,true);f.unmount();
}); });
test('product page changes reload only fine table rows and export remains the complete scope',async()=>{
const f=fixture();f.props.productKeys=['P1'];f.render();f.catalog[0].resolve(catalogData);await tick();f.render().choose('加工废');f.render();
assert.deepEqual(f.requests[1].params.productKeys,['P1']);assert.equal(f.requests[0].params.productKeys,undefined);
f.requests[0].resolve({...response,dimension:'period'});f.requests[1].resolve({...response,dimension:'product'});await tick();f.render();
f.props.productKeys=['P2'];f.render();assert.equal(f.requests.length,3);assert.deepEqual(f.requests[2].params.productKeys,['P2']);assert.equal(f.render().trend?.dimension,'period');
const promise=f.render().exportTable();assert.equal(f.requests[3].params.productKeys,undefined);assert.equal(f.requests[3].params.dimension,'product');f.requests[3].resolve(response);await promise;f.unmount();
});
@@ -158,7 +158,7 @@ test('single-stage detail callback freezes the clicked quarterly/yearly row inst
for(const stage of ['black','bright']) { for(const stage of ['black','bright']) {
const ui=new Function('freezeSourceRecordsQuery','stage',transpile(` const ui=new Function('freezeSourceRecordsQuery','stage',transpile(`
const applied={year:2026,month:10,product:'产品A',stage,columnFilters:[{field:'customer',stages:[stage],operator:'in',values:['001']}]}; const applied={year:2026,month:10,product:'产品A',stage,columnFilters:[{field:'customer',stages:[stage],operator:'in',values:['001']}]};
const current={release:{id:'release-A'}},drilledProduct='产品A';let records; const currentSummary={release:{id:'release-A'}},drilledProduct='产品A';let records;
const setRecordVisible=()=>{},setRecords=value=>records=value; const setRecordVisible=()=>{},setRecords=value=>records=value;
${callback} ${callback}
return {openDetail,get:()=>records}; return {openDetail,get:()=>records};
+10
View File
@@ -248,3 +248,13 @@ test('fine defect expansion and Excel share authorized parent/child values, pres
assert.equal(fields.some(f=>f.key.startsWith('black-defect-')||f.key.startsWith('black-fine-')),false); assert.equal(fields.some(f=>f.key.startsWith('black-defect-')||f.key.startsWith('black-fine-')),false);
assert.equal(data.groups[0].children[0].rate,0); assert.equal(data.groups[0].children[0].rate,0);
}); });
test('remote page two is not sliced again and full export includes every row',async()=>{
const all=Array.from({length:35},(_,i)=>({...row,key:String(i),label:'产品'+i}));let changed;
const ui=fixture({rows:all.slice(15,30),view:'product',remotePage:{current:2,pageSize:15,total:35,onChange:(...args)=>changed=args},loadExportData:async()=>({rows:all})});
assert.equal(find(ui.tree,n=>n.type==='Table').props.dataSource.length,15);
const pagination=find(ui.tree,n=>n.type==='Pagination');assert.equal(pagination.props.total,35);assert.equal(pagination.props.current,2);
pagination.props.onChange(3,15);assert.deepEqual(changed,[3,15]);
await ui.ref.exportExcel();assert.equal(ui.exported.workbook.getWorksheet('质量汇总').rowCount,37);assert.equal(ui.exported.rows[0].key,'0');assert.equal(ui.exported.rows.at(-1).key,'34');
});
+113
View File
@@ -0,0 +1,113 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
import { overviewQuery, sameTrendScope, overviewTrend, splitMetricsLoad } from '../src/priorityMetrics.ts';
const query = { releaseId: 'r1', year: 2026, view: 'product', trendGranularity: 'month', stage: 'bright',
columnFilters: [{ field: 'customer', operator: 'contains', values: ['Acme'], stages: ['bright'] }] };
const tick = () => new Promise(resolve => setImmediate(resolve));
function fixture() {
const slots = [], effects = [], requests = []; let index = 0, dirty = false, pending = [];
const props = { query, enabled: true, refresh: 0 }; const unauthorized = [];
const hooks = {
useState(initial) { const n = index++; if (!(n in slots)) slots[n] = initial;
return [slots[n], value => { slots[n] = value; dirty = true; }]; },
useEffect(fn, deps) { const n = index++, old = effects[n];
if (!old || deps.some((d, i) => !Object.is(d, old.deps[i]))) pending.push(() => {
old?.cleanup?.(); effects[n] = { deps, cleanup: fn() };
}); },
};
const api = { metrics(params, signal) { let resolve, reject;
const promise = new Promise((r, j) => { resolve = r; reject = j; });
requests.push({ params, signal, resolve, reject }); return promise;
} };
const source = fs.readFileSync(new URL('../src/usePriorityMetrics.ts', import.meta.url), 'utf8');
const code = ts.transpileModule(source.slice(source.indexOf('export function')).replace('export function', 'function'),
{ compilerOptions: { target: ts.ScriptTarget.ES2022 } }).outputText;
const hook = new Function('hooks', 'api', 'overviewQuery', 'splitMetricsLoad', `const {useState,useEffect}=hooks;${code};return usePriorityMetrics;`)(hooks, api, overviewQuery, splitMetricsLoad);
const onUnauthorized = error => { unauthorized.push(error); return error.status === 403; };
return { props, requests, unauthorized, render() { let result, count = 0; do {
dirty = false; index = 0; pending = []; result = hook(props.query, props.enabled, props.refresh, onUnauthorized, props.forceOverview);
pending.forEach(run => run()); assert.ok(++count < 10);
} while (dirty); return result; }, unmount() { effects.forEach(e => e?.cleanup?.()); } };
}
test('overview retains every source predicate and does not mutate the table query', () => {
const small = overviewQuery(query);
assert.deepEqual(small, { ...query, view: 'overall' }); assert.equal(query.view, 'product');
assert.equal(small.columnFilters, query.columnFilters);
});
test('trend reuse requires the same release, time range, grain, stage and all filters', () => {
assert.equal(sameTrendScope(query, { ...query, view: 'overall', month: undefined }), true);
for (const changed of [{ month: 1 }, { quarter: 2 }, { monthStart: 2, monthEnd: 5 }, { year: undefined },
{ releaseId: 'r2' }, { trendGranularity: 'quarter' }, { stage: 'black' }, { includeUnassigned: false },
{ product: 'P1' }, { extraFilters: { order: ['X'] } }, { columnFilters: [] }]) {
assert.equal(sameTrendScope(query, { ...query, ...changed }), false, JSON.stringify(changed));
}
assert.equal(sameTrendScope(query, Object.fromEntries(Object.entries(query).reverse())), true);
});
test('trend adapter uses the exact formal values and performs no rate arithmetic', () => {
const value = { release: { id: 'r1' }, context: {}, trend: [{ compositeRate: .123456789 }], coverage: {} };
assert.equal(overviewTrend(value).trend, value.trend);
});
test('priority gate opens immediately after the small server aggregate and stays closed while loading', async () => {
const f = fixture(); assert.equal(f.render().settled, false);
assert.equal(f.requests.length, 1); assert.equal(f.requests[0].params.view, 'overall');
const value = { release: { id: 'r1' }, summary: { compositeRate: .9 }, trend: [] };
f.requests[0].resolve(value); await tick(); const result = f.render();
assert.equal(result.settled, true); assert.equal(result.value, value);
f.render(); assert.equal(f.requests.length, 1); f.unmount();
});
test('year switches and refresh invalidate the gate and late old replies cannot replace the new result', async () => {
const f = fixture(); f.render(); f.props.query = { ...query, year: 2025 };
assert.equal(f.render().settled, false); assert.equal(f.requests[0].signal.aborted, true);
f.requests[0].resolve({ summary: 'old' }); await tick(); assert.equal(f.render().value, null);
f.requests[1].resolve({ summary: 'new' }); await tick(); assert.equal(f.render().value.summary, 'new');
f.props.refresh++; assert.equal(f.render().settled, false); assert.equal(f.requests.length, 3); f.unmount();
});
test('overview errors allow the regular query to retry independently', async () => {
const f = fixture(); f.render(); f.requests[0].reject(Error('temporary failure')); await tick();
assert.deepEqual(f.render(), { value: null, settled: true, failed: true, split: true }); f.unmount();
});
test('authorization failures do not unlock further requests', async () => {
const f = fixture(); f.render(); f.requests[0].reject({ status: 403 }); await tick();
assert.equal(f.render().settled, false); assert.equal(f.unauthorized.length, 1); f.unmount();
});
test('inactive pages neither start requests nor accept late results', async () => {
const f = fixture(); f.props.enabled = false; f.render(); assert.equal(f.requests.length, 0);
f.props.enabled = true; f.render(); f.props.enabled = false; f.render();
assert.equal(f.requests[0].signal.aborted, true); f.requests[0].resolve({ summary: 'late' });
await tick(); assert.equal(f.render().value, null); f.unmount();
});
test('narrow queries reuse a complete response rather than adding a summary round trip', async () => {
for (const change of [{month:2},{quarter:1},{product:'P'},{productSearch:'P'},{view:'department'},{view:'overall'}]) {
const f=fixture();f.props.query={...query,...change};const first=f.render();
assert.equal(first.split,false);assert.deepEqual(f.requests[0].params,f.props.query);
f.requests[0].resolve({summary:'full',rows:[1],trend:[2]});await tick();
assert.equal(f.render().value.rows.length,1);assert.equal(f.requests.length,1);f.unmount();
}
});
test('refresh generations cannot qualify retained old table and trend results as current', () => {
const source=name=>fs.readFileSync(new URL(`../src/${name}`,import.meta.url),'utf8');
const general=source('QueryPage.tsx');
const metricExpression=general.match(/const metricKey = useMemo\(\(\) => ([^;]+), \[metricParams, queryRevision[^\]]*\]\);/)[1];
const key=new Function('metricParams','queryRevision',`const productPagingScope=true,productPage={key:'page2'};return ${metricExpression}`);
assert.notEqual(key(query,0),key(query,1));
const stage=source('StageProductPage.tsx');
const expression=stage.match(/const queryKey=([^;]+);/)[1];
const stageKey=new Function('applied','revision','refreshKey',`const productPagingScope=true,productPage={key:'page2'};return ${expression}`);
assert.notEqual(stageKey(query,0,0),stageKey(query,0,1));
assert.notEqual(stageKey(query,0,0),stageKey(query,1,0));
});
test('paged month and search tables keep their first response small',async()=>{
for(const change of [{month:2},{quarter:1},{productSearch:'P'}]) {
const f=fixture();f.props.query={...query,...change};f.props.forceOverview=true;
assert.equal(f.render().split,true);assert.equal(f.requests[0].params.view,'overall');
for(const key of Object.keys(change))assert.equal(f.requests[0].params[key],change[key]);f.unmount();
}
});
@@ -0,0 +1,67 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
import {shouldPageProducts,pageSelection,PRODUCT_PAGE_SIZE,canReadProductPage} from '../src/productPagination.ts';
const query={releaseId:'r1',year:2026,view:'product'};
const tick = () => new Promise(resolve => setImmediate(resolve));
function fixture() {
const slots = [], effects = [], requests = []; let index = 0, dirty = false, pending = [];
const props = { query, enabled: true, refresh: 0 }; const unauthorized = [];
const hooks = {
useState(initial) { const n = index++; if (!(n in slots)) slots[n] = initial;
return [slots[n], value => { slots[n] = value; dirty = true; }]; },
useEffect(fn, deps) { const n = index++, old = effects[n];
if (!old || deps.some((d, i) => !Object.is(d, old.deps[i]))) pending.push(() => {
old?.cleanup?.(); effects[n] = { deps, cleanup: fn() };
}); },
};
const api = { productPage(params,page,pageSize,order,signal) { let resolve, reject;
const promise = new Promise((r, j) => { resolve = r; reject = j; });
requests.push({ params, page, pageSize, order, signal, resolve, reject }); return promise;
} };
const source = fs.readFileSync(new URL('../src/useProductPage.ts', import.meta.url), 'utf8');
const code = ts.transpileModule(source.slice(source.indexOf('export function')).replace('export function', 'function'),
{ compilerOptions: { target: ts.ScriptTarget.ES2022 } }).outputText;
const hook = new Function('hooks', 'api', 'pageSelection', 'PRODUCT_PAGE_SIZE', `const {useState,useEffect}=hooks;${code};return useProductPage;`)(hooks, api, pageSelection, PRODUCT_PAGE_SIZE);
const onUnauthorized = error => { unauthorized.push(error); return error.status === 403; };
return { props, requests, unauthorized, render() { let result, count = 0; do {
dirty = false; index = 0; pending = []; result = hook(props.query, props.enabled, props.refresh, 'brightQualifiedDesc', onUnauthorized);
pending.forEach(run => run()); assert.ok(++count < 10);
} while (dirty); return result; }, unmount() { effects.forEach(e => e?.cleanup?.()); } };
}
const response=(request,total=80)=>({release:{id:request.params.releaseId},rows:[{key:'P'}],total,page:request.page,pageSize:request.pageSize,order:request.order});
test('only unpinned product tables use server pagination',()=>{
assert.equal(shouldPageProducts(query),true);
for(const extra of [{view:'overall'},{product:'P'},{releaseId:undefined}])assert.equal(shouldPageProducts({...query,...extra}),false);
assert.equal(shouldPageProducts({...query,month:2,productSearch:'x'}),true);
});
test('page changes fetch only the selected page; year and size changes reset immediately',async()=>{
const f=fixture();let value=f.render();assert.equal(f.requests[0].page,1);
f.requests[0].resolve(response(f.requests[0]));await tick();value=f.render();assert.equal(value.pagination.total,80);
value.pagination.onChange(3,15);f.render();assert.equal(f.requests[1].page,3);assert.equal(f.render().value,null);
f.requests[1].resolve(response(f.requests[1]));await tick();value=f.render();
value.pagination.onChange(3,50);f.render();assert.equal(f.requests[2].page,1);assert.equal(f.requests[2].pageSize,50);
f.props.query={...query,year:2025};f.render();assert.equal(f.requests[3].page,1);assert.equal(f.requests[3].pageSize,50);
assert.equal(f.requests[2].signal.aborted,true);f.requests[2].resolve(response(f.requests[2]));await tick();assert.equal(f.render().value,null);f.unmount();
});
test('empty out-of-range pages recover to the valid last page',async()=>{
const f=fixture();f.render().pagination.onChange(9,15);f.render();f.requests[1].resolve(response(f.requests[1],20));await tick();f.render();
assert.equal(f.requests[2].page,2);f.requests[2].resolve(response(f.requests[2],20));await tick();assert.equal(f.render().pagination.current,2);f.unmount();
});
test('refresh, inactive pages, errors and authorization cannot qualify stale rows',async()=>{
const f=fixture();f.render();f.requests[0].resolve(response(f.requests[0]));await tick();assert.ok(f.render().value);
f.props.refresh++;assert.equal(f.render().value,null);f.requests[1].reject(Error('retry'));await tick();assert.equal(f.render().error,'retry');
f.props.refresh++;f.render();f.requests[2].reject({status:403});await tick();assert.equal(f.unauthorized.filter(e=>e.status===403).length,1);assert.equal(f.render().value,null);
f.props.enabled=false;f.render();assert.equal(f.requests[2].signal.aborted,true);f.unmount();
});
test('wrong release replies are rejected instead of displayed',async()=>{
const f=fixture();f.render();f.requests[0].resolve({...response(f.requests[0]),release:{id:'old'}});await tick();assert.match(f.render().error,/范围已变化/);f.unmount();
});
test('annual aggregate keeps priority while narrow pages avoid a serial round trip',()=>{
assert.equal(canReadProductPage(query,false),false);assert.equal(canReadProductPage(query,true),true);
for(const narrow of [{month:2},{quarter:1},{productSearch:'P'},{monthStart:1,monthEnd:5}])assert.equal(canReadProductPage({...query,...narrow},false),true);
});
+36
View File
@@ -0,0 +1,36 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
const source=fs.readFileSync(new URL('../src/StageSourceFilters.tsx',import.meta.url),'utf8');
const output=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX,target:ts.ScriptTarget.ES2022}}).outputText;
let lastSearch;
const jsx=(type,props)=>({type,props});const exports={};
new Function('require','exports',output)(name=>{
if(name==='react/jsx-runtime')return {jsx,jsxs:jsx};
if(name==='react')return {useState:initial=>[initial,value=>{lastSearch=value;} ]};
if(name==='antd')return {Select:'Select',Button:'Button',Alert:'Alert',Spin:'Spin'};
if(name==='@ant-design/icons')return {CloseOutlined:'CloseOutlined'};
if(name==='./readableSelectPopup')return {readableSelectPopup:{}};
return {};
},exports);
const nodes=value=>!value||typeof value!=='object'?[]:[value,...[value.props?.children].flat(Infinity).flatMap(nodes)];
const field={key:'customer',label:'客户名称',type:'source_value',stages:['bright'],columns:{bright:'客户名称'}};
test('contains is selectable and accepts bounded free keywords without a value-candidate lookup',()=>{
let changed;
const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立']};
const root=exports.StageSourceFilters({stage:'bright',fields:[field],query:{year:2026},value:[rule],onChange:value=>{changed=value;},onUnauthorized:()=>false});
const all=nodes(root);const operation=all.find(x=>x.type==='Select'&&x.props['aria-label']==='客户名称筛选方式');
assert.ok(operation.props.options.some(x=>x.value==='contains'&&x.label==='包含'));
const free=all.find(x=>typeof x.type==='function'&&x.type.name==='ContainsValues');
const keywords=free.type(free.props);
assert.equal(keywords.props.mode,'tags');assert.equal(keywords.props.maxCount,50);assert.deepEqual(keywords.props.options,[]);
keywords.props.onSearch('x'.repeat(257));assert.equal(lastSearch.length,256);
keywords.props.onSearch('x'.repeat(255)+'😀');assert.equal(lastSearch,'x'.repeat(255));
keywords.props.onSearch('日立');keywords.props.onBlur();assert.equal(lastSearch,'');
keywords.props.onChange(['日立','%_',' ']);
assert.deepEqual(changed,[{...rule,values:['日立','%_']}]);
operation.props.onChange('in');assert.deepEqual(changed,[{...rule,operator:'in',values:[]}]);
});
@@ -0,0 +1,53 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
const read=name=>fs.readFileSync(new URL(`../src/${name}`,import.meta.url),'utf8');
test('stage product common queries use a resolved release and do not wait for source catalog when unfiltered',()=>{
const source=read('StageProductPage.tsx');
const releaseExpression=source.match(/const queryReleaseId=([^;]+);/)[1];
const resolvedRelease=new Function('initial','currentReleaseId','catalog',`return ${releaseExpression}`);
assert.equal(resolvedRelease({releaseId:'historical'},'current',{releaseId:'catalog'}),'historical');
assert.equal(resolvedRelease({},'current',{releaseId:'catalog'}),'current');
assert.equal(resolvedRelease({},null,{releaseId:'stale'}),undefined);
assert.equal(resolvedRelease({},undefined,{releaseId:'resolved'}),'resolved');
const expression=source.match(/const queryReady=\(query:QueryParams\)=>((?:[^;]|\n)+);/)[1];
const ready=new Function('query','queryReleaseId','catalog','options',`return ${expression}`);
assert.equal(ready({releaseId:'r1',columnFilters:[]},'r1',null,null),true);
assert.equal(ready({releaseId:'r1',columnFilters:[]},undefined,null,null),false);
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',null,null),false);
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',{releaseId:'r1'},null),true);
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r2',{releaseId:'r1'},null),false);
assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,null),false);
assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,{}),true);
assert.match(source,/if\(active&&draftReady\)setApplied/);
assert.doesNotMatch(source,/if\(!active\|\|!applied\|\|supportError\)/);
});
test('source option request context excludes its own selected values and tracks other filters',()=>{
const source=read('StageSourceFilters.tsx');
const declarations=source.match(/ const \{year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters\}=query;\n const period=.*;\n const filters=.*;\n const contextKey=[^;]+;/)[0];
const run=new Function('query','field','stage',`${ts.transpileModule(declarations,{compilerOptions:{target:ts.ScriptTarget.ES2022}}).outputText};return {filters,contextKey};`);
const base={year:2026,month:4,releaseId:'r1',department:'D1',columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['North']}]};
const first=run(base,{key:'customer'},'black');
const ownValueChanged=run({...base,columnFilters:[{field:'customer',values:['OTHER']},{field:'region',values:['North']}]},{key:'customer'},'black');
const otherValueChanged=run({...base,columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['South']}]},{key:'customer'},'black');
assert.deepEqual(first.filters.columnFilters,[{field:'region',values:['North']}]);
assert.equal(first.contextKey,ownValueChanged.contextKey);
assert.notEqual(first.contextKey,otherValueChanged.contextKey);
});
test('source value dropdowns request candidate-only results while preserving search and debounce behavior',()=>{
const source=read('StageSourceFilters.tsx');
assert.match(source,/sourceColumnValues\(\{releaseId,period,filters,field:field\.key,stages:\[stage\],search,limit:50,offset:0,candidateOnly:true\}/);
assert.match(source,/search\?300:0/);
assert.match(source,/result\.values\.filter\(\(v\):v is string=>typeof v==='string'\)/);
assert.match(read('api.ts'),/sourceColumnValues\(body: Record<string,unknown> & \{candidateOnly\?:true\}/);
});
test('ordinary product search only depends on product options', () => {
assert.match(read('StageProductPage.tsx'), /searchUnavailable=\{!options\}/);
});
@@ -30,3 +30,9 @@ test('stage query includes unclassified source rows even when an old link exclud
assert.equal(base.includeUnassigned,false); assert.equal(base.includeUnassigned,false);
} }
}); });
test('contains keywords are retained while an unfilled contains rule is inactive',()=>{
const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立','%_']};
assert.deepEqual(activeStageFilters([rule],'bright'),[rule]);
assert.deepEqual(activeStageFilters([{...rule,values:[]}],'bright'),[]);
});
+10 -1
View File
@@ -27,7 +27,16 @@ for(const type of ['line','area'])test(`900px report ${type} keeps a positive pl
assert.equal(points.length,18); assert.equal(points.length,18);
const ys=points.filter((_,i)=>i%2===1); const ys=points.filter((_,i)=>i%2===1);
assert.ok(Math.max(...ys)-Math.min(...ys)>8,'different rates must not become a flat line'); assert.ok(Math.max(...ys)-Math.min(...ys)>8,'different rates must not become a flat line');
ys.forEach((y,i)=>assert.ok(Math.abs(y-(rect.y+rect.height*(1-v.series[0].values[i])))<.001)); const series=chart.getModel().getSeriesByIndex(0).getData();
const [low,high]=chart.getModel().getComponent('yAxis').axis.scale.getExtent();
assert.ok(low>0 && high-low<20,'rate trends retain their focused percentage axis');
ys.forEach((y,i)=>{
const expected=v.series[0].values[i]*100;
assert.ok(Math.abs(series.get('y',i)-expected)<1e-10,'the plotted value preserves the formal rate');
assert.ok(expected>=low&&expected<=high,'the value must fit inside the actual axis');
const pixel=rect.y+rect.height*(high-expected)/(high-low);
assert.ok(Math.abs(y-pixel)<.001,'SVG coordinates match the formal rate on the actual axis');
});
assert.match(chart.renderToSVGString(),/<svg/); assert.match(chart.renderToSVGString(),/<svg/);
} finally {chart.dispose();} } finally {chart.dispose();}
}); });