diff --git a/README.md b/README.md index 145eebc..27f5e66 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,14 @@ DSH CLI 固定为 `0.2.2-rc.1`,依赖与官方 Python SDK 校验记录在 `run 底部表格同步展开当前分类的细项,重量/废率切换与 Excel 导出跟随当前显示;产品和期间行维度保持不变。细项使用正式阶段同期总重量作分母,月、季、年均先汇总分量再计算;没有来源或分母时显示缺失,真实零显示零。细项与正式分类总量不一致时提示覆盖不足,不补造分类。 +## 筛选索引与查询加载 + +来源字段的等于、排除和空值筛选共用按构建冻结的值字典及记录索引;汇总、趋势、来源明细和导出沿用当前授权与正式指标计算。源字段值保留空值、大小写及前导零语义,旧配置筛选的字符串 `null` 约定不变。下拉框使用只读候选模式,只取一页及是否还有;需要精确总数的原接口继续保留。 + +启动时在单线程后台补建当前完整正式版本的索引,导入及归类在事务提交后进入同一去重队列。完整构建标记提交后才启用;未完成、失败或未补建的历史构建沿用原始查询,不阻止启动和正式数据发布。索引增加持久存储及后台构建成本,规模取决于非空字段数和不同值数量;高基数宽表应先测容量。保留清理随无引用构建回收索引;升级后原维护兼容保护继续生效。 + +黑片/亮片页的普通查询在数据版本已解析后独立加载,来源字段条件仍等待对应版本的字段目录;产品搜索不因无关元数据失败而禁用。 + ## 旧版本时间迁移 连接与 MySQL 会话现在统一使用 UTC,界面仍按本地时区显示。V25 只校正旧代码按 UTC 墙上时间写入的截止/有效期、报告调度及报告租约列(前者由 Java 写入,租约使用 SQL UTC_TIMESTAMP),不改 SQL CURRENT_TIMESTAMP 自动生成的创建时间,也不修改业务日期或指标。若旧库存在此类值,迁移要求先备份并核实旧会话相对 UTC 的偏移,然后设置 `MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS`(旧会话 UTC 为 `0`,旧会话中国标准时间为 `28800`)。本机启动可在私有 local-config 中记录 `legacyTimestampOffsetSeconds`;不能仅凭电脑当前时区猜测。全新空库不需要该参数;迁移标记和历史值修改在同一事务提交,防止提交确认丢失后重复平移。还原旧备份时应重新核实其原运行时区。 diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisExplorationTools.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisExplorationTools.java index 378c182..9c7c77f 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisExplorationTools.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisExplorationTools.java @@ -369,7 +369,7 @@ public class AnalysisExplorationTools { SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value); SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value); appendDetailExclusions(sql, args, query); - new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor); + new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor,releaseId); DataScope.append(sql, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); boolean includeUnassigned = query.includeUnassigned() diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisFineDefectsService.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisFineDefectsService.java index 40d1c8e..a1087b2 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisFineDefectsService.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisFineDefectsService.java @@ -276,7 +276,7 @@ public class AnalysisFineDefectsService { SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value); if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__")) predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')"); - new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor); + new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor,releaseId); DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); } diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisInvestigationService.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisInvestigationService.java index 3774df4..62c0bff 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisInvestigationService.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisInvestigationService.java @@ -90,7 +90,7 @@ public class AnalysisInvestigationService { SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x); SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x); if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')"); - new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor); + new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor,release); DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); } if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());} diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryController.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryController.java index 3b973c7..7d49ff3 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryController.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryController.java @@ -43,12 +43,16 @@ public class AnalysisSourceQueryController { @PostMapping("/values") public Map values(@RequestBody Map input,@AuthenticationPrincipal AppUserPrincipal actor) { String scope=captureScope(actor); - if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset").containsAll(input.keySet()))throw bad("候选值请求包含未知字段"); + if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset","candidateOnly").containsAll(input.keySet()))throw bad("候选值请求包含未知字段"); String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null)); Map context=context(input); if(!(input.get("field") instanceof String field)||field.isBlank()||input.get("search")!=null&&!(input.get("search") instanceof String))throw bad("候选字段或搜索值无效"); - Map result=new LinkedHashMap<>(columns.values(release,context,field,strings(input.get("stages"),2),Objects.toString(input.get("search"),null), - integer(input.getOrDefault("limit",50),1,200),integer(input.getOrDefault("offset",0),0,100000),actor)); + if(input.containsKey("candidateOnly")&&!(input.get("candidateOnly") instanceof Boolean))throw bad("候选分页参数无效"); + List stages=strings(input.get("stages"),2);String search=Objects.toString(input.get("search"),null); + int limit=integer(input.getOrDefault("limit",50),1,200),offset=integer(input.getOrDefault("offset",0),0,100000); + Map result=new LinkedHashMap<>(Boolean.TRUE.equals(input.get("candidateOnly")) + ? columns.values(release,context,field,stages,search,limit,offset,true,actor) + : columns.values(release,context,field,stages,search,limit,offset,actor)); result.put("period",input.get("period"));result.put("filters",input.get("filters")); result.put("values",AnalysisProtocolService.list(result.get("rows")).stream().map(row->AnalysisProtocolService.map(row).get("value")).toList()); result.put("candidateOnly",true);reauthorize(actor,scope,false);return result; diff --git a/backend/src/main/java/cn/yunsheng/quality/analytics/AnalyticsController.java b/backend/src/main/java/cn/yunsheng/quality/analytics/AnalyticsController.java index 56130b1..35e314b 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analytics/AnalyticsController.java +++ b/backend/src/main/java/cn/yunsheng/quality/analytics/AnalyticsController.java @@ -111,7 +111,7 @@ public class AnalyticsController { throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前字段授权不包含所选报废类别"); List args = new ArrayList<>(extra.isEmpty()?List.of(selected, selected):List.of(selected)); StringBuilder where = baseWhere(args, year, month, quarter, department, group, product, projectGroup, includeUnassigned, actor); - new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor); + new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor,selected); List aggregated; if (dimension.equals("category")) { List selectedComponents = selectedCategory == null ? visibleComponents : List.of(selectedCategory); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/FilterFieldsController.java b/backend/src/main/java/cn/yunsheng/quality/filters/FilterFieldsController.java index 073a6cc..95f24b9 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/FilterFieldsController.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/FilterFieldsController.java @@ -36,9 +36,11 @@ public class FilterFieldsController { governance.requireUserActivePolicy(actor); Map> result=new LinkedHashMap<>(); String release=releases.resolvePublishedRelease(null); + SourceFilterIndex index=new SourceFilterIndex(jdbc); for(String stage:List.of("black","bright")) { // Raw source headers are stable within each build. Read a representative row of every current build. - List raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2 WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release); + String hint=index.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":""; + List raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2"+hint+" WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release); Set columns=new TreeSet<>(); var json=tools.jackson.databind.json.JsonMapper.builder().build(); Set allowed=governance.allowedFields(actor); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/QueryFilterFields.java b/backend/src/main/java/cn/yunsheng/quality/filters/QueryFilterFields.java index c297342..42bf310 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/QueryFilterFields.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/QueryFilterFields.java @@ -48,6 +48,9 @@ public class QueryFilterFields { return Collections.unmodifiableMap(result); } public void append(StringBuilder where,List args,String stage,Map> filters,AppUserPrincipal actor) { + append(where,args,stage,filters,actor,null); + } + public void append(StringBuilder where,List args,String stage,Map> filters,AppUserPrincipal actor,String releaseId) { if(filters==null || filters.isEmpty())return; Map fields=new HashMap<>(); visible(actor).forEach(f->fields.put(f.key(),f)); for(var entry:filters.entrySet()) { @@ -61,6 +64,7 @@ public class QueryFilterFields { boolean empty=entry.getValue().contains(EMPTY); if(empty && !f.allowEmpty())throw bad(f.label()+"不允许筛选空值"); String column=stage.equals("black")?f.blackColumn():f.brightColumn(); + if(releaseId!=null&&new SourceFilterIndex(jdbc).appendLegacy(where,args,stage,releaseId,column,entry.getValue()))continue; // Paths and selected values are bound parameters; never interpolate configured identifiers. String expression="NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs"; List values=entry.getValue().stream().filter(v->!v.equals(EMPTY)).toList(); @@ -90,10 +94,15 @@ public class QueryFilterFields { StringBuilder scope=new StringBuilder(" WHERE rm.release_id=?"); List scoped=new ArrayList<>();scoped.add(releaseId); DataScope.append(scope,scoped,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); + var index=new SourceFilterIndex(jdbc); + if(index.ready(releaseId)) { + selects.add(index.candidateSelect(stage,column,scope.toString(),scoped,args).replace("AS candidate_value","AS value")); + continue; + } selects.add("SELECT NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs AS value FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+scope); args.add(path(column));args.addAll(scoped); } - String predicate=" WHERE value IS NOT NULL"; + String predicate=" WHERE value IS NOT NULL AND CAST(value AS BINARY)<>CAST('null' AS BINARY)"; if(search!=null && !search.isBlank()) { predicate+=" AND LOCATE(LOWER(?),LOWER(value))>0";args.add(search); } List found=jdbc.query("SELECT DISTINCT value FROM ("+String.join(" UNION ALL ",selects)+") choices"+predicate+" ORDER BY value LIMIT 201",(rs,n)->rs.getString(1),args.toArray()); return Map.of("values",found.stream().limit(200).toList(),"hasMore",found.size()>200); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java index 2f07d49..cde99bc 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java @@ -80,11 +80,17 @@ public class SourceColumnQueries { private final JdbcTemplate jdbc; private final GovernanceAccessService governance; private final JsonMapper json; + private SourceFilterIndex filterIndex; + @org.springframework.beans.factory.annotation.Autowired + public void setFilterIndex(SourceFilterIndex index) { this.filterIndex=index; } + public boolean indexed(String release) { return filterIndex.ready(release); } + public SourceColumnQueries(JdbcTemplate jdbc, GovernanceAccessService governance, JsonMapper json) { this.jdbc = jdbc; this.governance = governance; this.json = json; + this.filterIndex = new SourceFilterIndex(jdbc); } /** @@ -110,7 +116,7 @@ public class SourceColumnQueries { "f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs"); String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) " + "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id " - + "FROM release_member rm JOIN fact_" + stage + " f2" + factIndexHint(stage) + " " + + "FROM release_member rm JOIN fact_" + stage + " f2" + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage)) + " " + "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id" + representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative " + "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id"; @@ -249,6 +255,8 @@ public class SourceColumnQueries { String operator = (String) rule.get("operator"); String expression; Object column = ((Map) field.get("columns")).get(stage); + @SuppressWarnings("unchecked") List selected = (List) rule.getOrDefault("values",List.of()); + if("source_value".equals(field.get("type")) && filterIndex.append(where,args,stage,releaseId,String.valueOf(column),operator,selected))continue; if ("source_value".equals(field.get("type"))) { expression = SOURCE_VALUE; } else { @@ -383,12 +391,16 @@ public class SourceColumnQueries { public Map values(String releaseId, Map context, String fieldKey, List stages, String search, int limit, int offset, AppUserPrincipal actor) { + return values(releaseId,context,fieldKey,stages,search,limit,offset,false,actor); + } + public Map values(String releaseId,Map context,String fieldKey,List stages, + String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) { requireQueryAccess(actor); - return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,actor)); + return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,candidateOnly,actor)); } private Map valuesSnapshot(String releaseId,Map context,String fieldKey, - List stages,String search,int limit,int offset,AppUserPrincipal actor) { + List stages,String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) { if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效"); if (fieldKey == null || fieldKey.isBlank()) throw bad("来源列标识无效"); if (stages == null || stages.isEmpty() || stages.size() > STAGES.size() @@ -419,6 +431,10 @@ public class SourceColumnQueries { StringBuilder where = new StringBuilder(" WHERE rm.release_id=?"); List whereArgs = new ArrayList<>(List.of(releaseId)); appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor); + if("source_value".equals(field.get("type")) && filterIndex.ready(releaseId)) { + selects.add(filterIndex.candidateSelect(stage,String.valueOf(columns.get(stage)),where.toString(),whereArgs,unionArgs)); + continue; + } String fact = stage.equals("black") ? "fact_black" : "fact_bright"; selects.add("SELECT " + expression + " AS candidate_value " + "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id " @@ -437,6 +453,13 @@ public class SourceColumnQueries { } String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition; String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value"; + if(candidateOnly) { + String sql="SELECT candidate_value FROM ("+candidates+") matches GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?"; + List parameters=combine(unionArgs,searchArgs);parameters.add(limit+1);parameters.add(offset); + List> found=pages.get(new QueryKey(sql,parameters,"candidates:"+fieldKey),()->List.copyOf(jdbc.query(sql,(rs,row)->Map.of("value",rs.getString(1)),parameters.toArray()))); + boolean more=found.size()>limit; + return Map.of("rows",List.copyOf(found.subList(0,Math.min(limit,found.size()))),"hasMore",more,"limit",limit,"offset",offset,"pageComplete",true,"complete",offset==0&&!more,"releaseId",releaseId,"field",fieldKey,"selectedStages",List.copyOf(stages)); + } long total = total("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", combine(unionArgs,searchArgs)); List> rows = page("SELECT candidate_value FROM (" + candidates + ") matches " + "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?", @@ -563,7 +586,7 @@ public class SourceColumnQueries { List groups = parseSelections(context.dimensions().get("group"), "group"); if (!context.includeUnassigned() && !groups.contains("__UNASSIGNED__")) where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')"); - new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor); + new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor,releaseId); if (!context.columnFilters().isEmpty()) append(where, args, stage, context.columnFilters(), releaseId, actor); DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java new file mode 100644 index 0000000..72628f2 --- /dev/null +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java @@ -0,0 +1,122 @@ +package cn.yunsheng.quality.filters; + +import java.util.List; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; + +/** Sparse, build-owned source values. Every consumer still authorizes and scopes the actual facts. */ +@Service +public class SourceFilterIndex { + public record BuildAvailable(String buildId) {} + private static final int FORMAT = 1; + private final JdbcTemplate jdbc; + private volatile Boolean available; + public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; } + + public boolean available() { + if (available == null) { + Integer count=jdbc.queryForObject("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('source_filter_build','source_filter_field','source_filter_value','source_filter_member')",Integer.class); + available=count!=null&&count==4; + } + return available; + } + + public boolean ready(String release) { + if(release==null||!available())return false; + // Only successful immutable builds are retained; a pending backfill is checked again later. + Integer missing=jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm LEFT JOIN source_filter_build sb ON sb.build_id=rm.transform_build_id AND sb.format_version=? WHERE rm.release_id=? AND sb.build_id IS NULL",Integer.class,FORMAT,release); + if(missing==null||missing!=0)return false; + return true; + } + + /** Idempotent and atomic with fact construction, including callers which instantiate this helper. */ + public void ensureBuild(String build) { + if(!available())return; + var tx=new TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())); + tx.executeWithoutResult(status->{ + jdbc.queryForObject("SELECT id FROM transform_build WHERE id=? FOR UPDATE",String.class,build); + Integer done=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build WHERE build_id=? AND format_version=?",Integer.class,build,FORMAT); + if(done!=null&&done>0)return; + jdbc.update("DELETE m FROM source_filter_member m JOIN source_filter_field sf ON sf.id=m.field_id WHERE sf.build_id=?",build); + jdbc.update("DELETE v FROM source_filter_value v JOIN source_filter_field sf ON sf.id=v.field_id WHERE sf.build_id=?",build); + jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build); + jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build); + // Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source. + jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash)) ENGINE=InnoDB"); + try { + for(String stage:List.of("black","bright"))buildStage(build,stage); + jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT); + } finally { jdbc.execute("DROP TEMPORARY TABLE source_filter_staging"); } + }); + } + + private void buildStage(String build,String stage) { + var bounds=jdbc.queryForMap("SELECT MIN(id) AS first_id,MAX(id) AS last_id FROM fact_"+stage+" FORCE INDEX(uq_fact_"+stage+"_origin) WHERE build_id=?",build); + if(bounds.get("first_id")==null)return; + long after=((Number)bounds.get("first_id")).longValue()-1; + long last=((Number)bounds.get("last_id")).longValue(); + while(after ids=jdbc.queryForList("SELECT id FROM fact_"+stage+" FORCE INDEX(PRIMARY) WHERE build_id=? AND id>? AND id<=? ORDER BY id LIMIT 500",Long.class,build,after,last); + if(ids.isEmpty())return; + long end=ids.getLast(); + String rows=" FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE WHERE f.build_id=? AND f.id>? AND f.id<=?"; + jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end); + String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')"; + String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?"; + jdbc.update("DELETE FROM source_filter_staging"); + jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end); + jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash"); + Integer collisions=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)",Integer.class); + if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布"); + jdbc.update("INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash"); + after=end; + } + } + + /** Same-row membership is shared by metrics, trends, details and exports; empty values are sparse. */ + public boolean append(StringBuilder where,List args,String stage,String release,String column, + String operator,List values) { + if(!ready(release))return false; + where.append(" AND "); + membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"), + operator.equals("in")||operator.equals("not_in")?values:null,false); + return true; + } + + /** Legacy configured filters classify only the byte-exact literal string "null" as empty. */ + public boolean appendLegacy(StringBuilder where,List args,String stage,String release,String column,List selected) { + if(!ready(release))return false; + var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList(); + where.append(" AND ("); + if(values.isEmpty())where.append("FALSE"); + else membership(where,args,stage,release,column,false,values,true); + if(selected.contains(QueryFilterFields.EMPTY)) { + where.append(" OR ");membership(where,args,stage,release,column,true,null,true); + } + where.append(')');return true; + } + + private void membership(StringBuilder where,List args,String stage,String release,String column, + boolean negative,List values,boolean legacy) { + where.append("f.id ").append(negative?"NOT IN (":"IN (") + .append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin"); + args.add(release);args.add(stage);args.add(column); + if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)"); + if(values!=null) { + where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')'); + args.addAll(values); + } + where.append(')'); + } + + /** Dictionary-first candidates short-circuit after one authorized matching member per value. */ + public String candidateSelect(String stage,String column,String scope,List scopeArgs,List args) { + args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs); + return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id " + +"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id " + +scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)"; + } +} diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfill.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfill.java new file mode 100644 index 0000000..a7af05c --- /dev/null +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfill.java @@ -0,0 +1,194 @@ +package cn.yunsheng.quality.filters; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.context.event.EventListener; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +/** Asynchronously builds current and newly completed snapshots; unindexed queries retain their raw-value fallback. */ +@Component +public class SourceFilterIndexBackfill implements ApplicationRunner { + private static final Logger LOG = LoggerFactory.getLogger(SourceFilterIndexBackfill.class); + private static final int MAX_PENDING_BUILDS = 1024; + + private final JdbcTemplate jdbc; + private final SourceFilterIndex index; + private final ExecutorService worker; + private final Object pendingLock = new Object(); + private final LinkedHashSet pending = new LinkedHashSet<>(); + private boolean drainScheduled; + private String inFlight; + + @Autowired + public SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index) { + this(jdbc, index, createWorker()); + } + + SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index, ExecutorService worker) { + this.jdbc = jdbc; + this.index = index; + this.worker = worker; + } + + @Override + public void run(ApplicationArguments args) { + List currentBuilds; + try { + if (!index.available()) return; + currentBuilds = jdbc.query(""" + SELECT DISTINCT b.id + FROM current_release c + JOIN data_release r ON r.id=c.release_id + JOIN release_member rm ON rm.release_id=r.id + JOIN transform_build b ON b.id=rm.transform_build_id + WHERE c.dataset_id='quality' + AND r.status='PUBLISHED' + AND r.complete=TRUE + AND b.status='COMPLETED' + ORDER BY b.id + """, (rs, row) -> rs.getString(1)); + } catch (RuntimeException failure) { + LOG.error("Source filter index backfill discovery failed; raw-value fallback remains available", failure); + return; + } + if (currentBuilds.isEmpty()) return; + int accepted = enqueue(currentBuilds); + LOG.info("Source filter index backfill queued: discovered={}, accepted={}", currentBuilds.size(), accepted); + } + + @EventListener + public void onBuildAvailable(SourceFilterIndex.BuildAvailable event) { + if (event == null || event.buildId() == null || event.buildId().isBlank()) return; + int accepted = enqueue(List.of(event.buildId())); + if (accepted > 0) LOG.info("Source filter index build queued after commit: build={}", event.buildId()); + } + + private int enqueue(List builds) { + boolean schedule = false; + int accepted = 0; + int deferred = 0; + synchronized (pendingLock) { + for (String build : builds) { + if (build == null || build.isBlank() || build.equals(inFlight) || pending.contains(build)) continue; + if (pending.size() >= MAX_PENDING_BUILDS) { + deferred++; + continue; + } + pending.add(build); + accepted++; + } + if (!pending.isEmpty() && !drainScheduled) { + drainScheduled = true; + schedule = true; + } + } + if (deferred > 0) { + LOG.warn("Source filter index queue is full; {} builds keep raw-value fallback", deferred); + } + if (schedule) scheduleDrain(); + return accepted; + } + + private void scheduleDrain() { + try { + worker.execute(this::drainPending); + } catch (RejectedExecutionException failure) { + int queued; + synchronized (pendingLock) { + drainScheduled = false; + queued = pending.size(); + } + LOG.error("Source filter index drain could not be scheduled; {} builds keep raw-value fallback", queued, failure); + } + } + + private void drainPending() { + long started = System.nanoTime(); + int succeeded = 0; + int failed = 0; + while (true) { + String buildId; + synchronized (pendingLock) { + if (pending.isEmpty()) { + drainScheduled = false; + break; + } + var iterator = pending.iterator(); + buildId = iterator.next(); + iterator.remove(); + inFlight = buildId; + } + if (Thread.currentThread().isInterrupted()) { + synchronized (pendingLock) { + pending.add(buildId); + inFlight = null; + drainScheduled = false; + } + LOG.info("Source filter index drain interrupted; {} builds remain queued", pendingCount()); + return; + } + + long buildStarted = System.nanoTime(); + try { + index.ensureBuild(buildId); + succeeded++; + LOG.info("Source filter index build completed: build={}, queuedRemaining={}, elapsedMs={}", buildId, + pendingCount(), (System.nanoTime() - buildStarted) / 1_000_000L); + } catch (RuntimeException failure) { + if (Thread.currentThread().isInterrupted()) { + synchronized (pendingLock) { + pending.add(buildId); + inFlight = null; + drainScheduled = false; + } + LOG.info("Source filter index drain interrupted while building {}; {} builds remain queued", buildId, + pendingCount()); + return; + } + failed++; + LOG.error("Source filter index build failed: build={}; raw-value fallback remains available, continuing", + buildId, failure); + } finally { + synchronized (pendingLock) { inFlight = null; } + } + } + LOG.info("Source filter index drain finished: succeeded={}, failed={}, elapsedMs={}", succeeded, failed, + (System.nanoTime() - started) / 1_000_000L); + } + + private int pendingCount() { + synchronized (pendingLock) { return pending.size(); } + } + + @PreDestroy + public void stopWorker() { + worker.shutdownNow(); + try { + if (!worker.awaitTermination(2, TimeUnit.SECONDS)) { + LOG.warn("Source filter index backfill worker did not stop within two seconds"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + + private static ExecutorService createWorker() { + return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1), runnable -> { + Thread thread = new Thread(runnable, "source-filter-index-backfill"); + thread.setDaemon(true); + return thread; + }, new ThreadPoolExecutor.AbortPolicy()); + } +} diff --git a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java index f29284b..6c6b7c7 100644 --- a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java +++ b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java @@ -1,13 +1,21 @@ package cn.yunsheng.quality.metrics; +import cn.yunsheng.quality.filters.SourceFilterIndex; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; /** Builds immutable, per-transform-build source-component rollups for read-only metric queries. */ @Service public class MetricsRollupService { + private static final Logger LOG = LoggerFactory.getLogger(MetricsRollupService.class); static final String DIMENSION_COLLATION = "utf8mb4_0900_as_cs"; private static final List FACT_TABLES = List.of("fact_black", "fact_bright"); private static final String COMPONENTS = "JSON_OBJECT(" + @@ -22,10 +30,18 @@ public class MetricsRollupService { private final JdbcTemplate jdbc; private final cn.yunsheng.quality.analysis.FineDefectRollups fineDefects; + private final SourceFilterIndex sourceFilterIndex; + private ApplicationEventPublisher eventPublisher; public MetricsRollupService(JdbcTemplate jdbc) { this.jdbc = jdbc; this.fineDefects = new cn.yunsheng.quality.analysis.FineDefectRollups(jdbc, tools.jackson.databind.json.JsonMapper.builder().build()); + this.sourceFilterIndex = new SourceFilterIndex(jdbc); + } + + @Autowired + public void setApplicationEventPublisher(ApplicationEventPublisher eventPublisher) { + this.eventPublisher = eventPublisher; } public record GroupKey(String stage,String month,String department,String group,String project,String product) {} @@ -36,6 +52,7 @@ public class MetricsRollupService { jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent); if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总"); fineDefects.copyBuild(parent, child); + announceSourceFilterBuild(child); } @Transactional @@ -74,6 +91,30 @@ public class MetricsRollupService { """, buildId, buildId, buildId); fineDefects.refreshBuild(buildId); new SalesOrderMetricRollups(jdbc).refreshCurrentBuild(buildId); + announceSourceFilterBuild(buildId); + } + + void announceSourceFilterBuild(String buildId) { + if (eventPublisher == null) { + // Directly constructed fixtures have no application event bus; retain + // their explicit synchronous behavior (legacy schemas no-op safely). + sourceFilterIndex.ensureBuild(buildId); + return; + } + Runnable publish = () -> { + try { + eventPublisher.publishEvent(new SourceFilterIndex.BuildAvailable(buildId)); + } catch (RuntimeException failure) { + LOG.error("Source filter index notification failed for {}; raw-value fallback remains available", buildId, failure); + } + }; + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override public void afterCommit() { publish.run(); } + }); + } else { + publish.run(); + } } static String rollupInsertSql(String table) { diff --git a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java index f5102f7..647e524 100644 --- a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java +++ b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java @@ -39,7 +39,10 @@ public class MetricsService { private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128; private final JdbcTemplate jdbc; private final GovernanceAccessService governance; - private final SourceColumnQueries sourceColumns; + private SourceColumnQueries sourceColumns; + @Autowired + public void setSourceColumns(SourceColumnQueries sourceColumns) { this.sourceColumns = sourceColumns; } + private final TransactionTemplate readSnapshot; private final TransactionTemplate governanceSnapshot; @@ -160,7 +163,7 @@ public class MetricsService { appendDimensions(where, args, query); appendExclusions(where, args, query); if (factFallback) { - new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor); + new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor,releaseId); if (!query.columnFilters().isEmpty()) sourceColumns .append(where, args, stage, query.columnFilters(), releaseId, actor); @@ -396,7 +399,7 @@ public class MetricsService { appendDimensions(where, args, query); appendExclusions(where, args, query); if(factFallback) { - new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor); + new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId); if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); @@ -599,7 +602,8 @@ public class MetricsService { "sourceMonths", sourceMonths); } - private static String factFilterHint(String stage,boolean factFallback,MetricsQuery query) { + private String factFilterHint(String stage,boolean factFallback,MetricsQuery query) { + if(sourceColumns.indexed(query.releaseId()))return ""; if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return ""; // 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。 if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null @@ -622,7 +626,7 @@ public class MetricsService { appendDimensions(where, args, query); appendExclusions(where, args, query); if(factFallback) { - new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor); + new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId); if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); diff --git a/backend/src/main/java/cn/yunsheng/quality/records/RecordController.java b/backend/src/main/java/cn/yunsheng/quality/records/RecordController.java index e589804..c526ad6 100644 --- a/backend/src/main/java/cn/yunsheng/quality/records/RecordController.java +++ b/backend/src/main/java/cn/yunsheng/quality/records/RecordController.java @@ -53,6 +53,10 @@ public class RecordController { private final JsonMapper mapper; private final GovernanceAccessService governance; private final RecordsWorkbookExportService workbookExport; + private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns; + @org.springframework.beans.factory.annotation.Autowired + public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; } + public RecordController(JdbcTemplate jdbc, ReleaseService releases, JsonMapper mapper, GovernanceAccessService governance, RecordsWorkbookExportService workbookExport) { @@ -61,6 +65,7 @@ public class RecordController { this.mapper = mapper; this.governance = governance; this.workbookExport = workbookExport; + this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper); } @@ -156,10 +161,10 @@ public class RecordController { args.add(search); args.add(jsonPath(selectedField)); } - new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor); + new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor,selectedRelease); var sourceFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters); for(var rule:sourceFilters)if(!List.of(stage).containsAll((List)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段"); - if(!sourceFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,stage,sourceFilters,selectedRelease,actor); + if(!sourceFilters.isEmpty())sourceColumns.append(predicate,args,stage,sourceFilters,selectedRelease,actor); DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); String countSql = """ diff --git a/backend/src/main/java/cn/yunsheng/quality/records/RecordsWorkbookExportService.java b/backend/src/main/java/cn/yunsheng/quality/records/RecordsWorkbookExportService.java index 2a742c3..6df156f 100644 --- a/backend/src/main/java/cn/yunsheng/quality/records/RecordsWorkbookExportService.java +++ b/backend/src/main/java/cn/yunsheng/quality/records/RecordsWorkbookExportService.java @@ -52,6 +52,10 @@ public class RecordsWorkbookExportService { private final ReleaseService releases; private final GovernanceAccessService governance; private final JsonMapper mapper; + private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns; + @org.springframework.beans.factory.annotation.Autowired + public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; } + public RecordsWorkbookExportService(JdbcTemplate jdbc, ReleaseService releases, GovernanceAccessService governance, JsonMapper mapper) { @@ -59,6 +63,7 @@ public class RecordsWorkbookExportService { this.releases = releases; this.governance = governance; this.mapper = mapper; + this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper); } public Path export(Integer year, Integer month, Integer quarter, String department, String group, @@ -120,8 +125,11 @@ public class RecordsWorkbookExportService { LinkedHashSet rawFields = new LinkedHashSet<>(); if (releaseId != null) { QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product, - projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd); - scan(rawScope, rs -> rawFields.addAll(visibleRaw(readJson(rs.getString(1)), allowedFields).keySet())); + projectGroup, includeUnassigned, "CAST(JSON_KEYS(f.raw_values) AS CHAR)", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd); + scan(rawScope, rs -> { + Object decoded=mapper.readValue(rs.getString(1),Object.class); + if(decoded instanceof List keys)for(Object key:keys)if(key instanceof String name&&(allowedFields.contains("*")||allowedFields.contains(name)))rawFields.add(name); + }); } boolean showOrderType=SalesOrderTypes.allowed(allowedFields); if (rawFields.size() + HEADER_COUNT + (showOrderType?1:0) > MAX_COLUMNS) @@ -167,8 +175,8 @@ public class RecordsWorkbookExportService { SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value); if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__")) predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')"); - new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor); - if(!columnFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor); + new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor,releaseId); + if(!columnFilters.isEmpty())sourceColumns.append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor); DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); String columns = select == null ? "f.raw_values" : select; diff --git a/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java b/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java index 6121127..99f230a 100644 --- a/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java +++ b/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java @@ -15,6 +15,7 @@ import cn.yunsheng.quality.ingestion.ImportWorker; import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild; import cn.yunsheng.quality.ingestion.MappingValueCodec; import cn.yunsheng.quality.metrics.MetricsService; +import cn.yunsheng.quality.metrics.MetricsRollupService; import cn.yunsheng.quality.release.ReleaseService; import cn.yunsheng.quality.source.SourceCatalogService; import cn.yunsheng.quality.source.SourceDeletionService; @@ -66,6 +67,7 @@ public class DataWorkflowService { private final TaskExecutor executor; private final TransactionTemplate transaction; private final MetricsService metrics; + private MetricsRollupService rollups; private final AtomicReference runningOperation = new AtomicReference<>(); private final AtomicReference runningLease = new AtomicReference<>(); @@ -108,6 +110,12 @@ public class DataWorkflowService { this.executor = executor; this.transaction = new TransactionTemplate(manager); this.metrics=metrics; + this.rollups=new MetricsRollupService(jdbc); + } + + @Autowired + public void setMetricsRollupService(MetricsRollupService rollups) { + this.rollups = rollups; } public Map status(AppUserPrincipal actor) { @@ -376,7 +384,7 @@ public class DataWorkflowService { sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor); updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED"); var selectedRules=unclassifiedRules; - var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor)); + var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,rollups).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor)); jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token); publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false); jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token); diff --git a/backend/src/main/resources/db/migration/V43__source_filter_indexes.sql b/backend/src/main/resources/db/migration/V43__source_filter_indexes.sql new file mode 100644 index 0000000..5eeea39 --- /dev/null +++ b/backend/src/main/resources/db/migration/V43__source_filter_indexes.sql @@ -0,0 +1,39 @@ +CREATE TABLE source_filter_build ( + build_id CHAR(36) NOT NULL PRIMARY KEY, + format_version INT NOT NULL, + CONSTRAINT fk_source_filter_build FOREIGN KEY (build_id) REFERENCES transform_build(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE source_filter_field ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + build_id CHAR(36) NOT NULL, + stage VARCHAR(16) NOT NULL, + column_name LONGTEXT COLLATE utf8mb4_0900_bin NOT NULL, + name_hash BINARY(32) NOT NULL, + UNIQUE KEY uq_source_filter_field (build_id,stage,name_hash), + CONSTRAINT fk_source_filter_field_build FOREIGN KEY (build_id) REFERENCES transform_build(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE source_filter_value ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + field_id BIGINT NOT NULL, + value_hash BINARY(32) NOT NULL, + value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL, + UNIQUE KEY uq_source_filter_value_hash (field_id,value_hash), + KEY ix_source_filter_value_text (field_id,value(128)), + CONSTRAINT fk_source_filter_value_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE source_filter_member ( + field_id BIGINT NOT NULL, + fact_id BIGINT NOT NULL, + value_id BIGINT NOT NULL, + PRIMARY KEY (field_id,fact_id), + KEY ix_source_filter_member_value (value_id,fact_id), + CONSTRAINT fk_source_filter_member_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id), + CONSTRAINT fk_source_filter_member_value FOREIGN KEY (value_id) REFERENCES source_filter_value(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- Cover representative-row selection without reading every row's large source JSON. +ALTER TABLE fact_black ADD KEY ix_black_source_catalog (build_id,source_revision_id,department,group_name,id); +ALTER TABLE fact_bright ADD KEY ix_bright_source_catalog (build_id,source_revision_id,department,group_name,id); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 3457798..6ced49e 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -265,7 +265,9 @@ class SourceColumnQueriesMySqlTest { return fields.stream().filter(item -> item.get("label").equals(label)).findFirst().orElse(null); } - @Test void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters() throws Exception { + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(booleans={false,true}) + void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed) throws Exception { jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))"); jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE); jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))"); @@ -275,6 +277,7 @@ class SourceColumnQueriesMySqlTest { } jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black"); jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT"); + if(indexed){installFilterIndex();var index=new SourceFilterIndex(jdbc);for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);} var headerReads=new java.util.concurrent.atomic.AtomicInteger(); var capturing=new JdbcTemplate(jdbc.getDataSource()) { @Override public List query(String sql,org.springframework.jdbc.core.RowMapper mapper,Object... args) { @@ -417,6 +420,132 @@ class SourceColumnQueriesMySqlTest { assertThrows(ResponseStatusException.class,()->controller.query(input,actor)); } + @Test void indexedFiltersCandidatesAndTuplesMatchRawSemanticsAndStayAuthorized() throws Exception { + when(governance.allowedFields(actor)).thenReturn(Set.of("*")); + List samples=List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[1,2]}","{\"委托单\":{\"a\":1}}","{\"委托单\":\"é\"}","{\"委托单\":\"é\"}","{\"委托单\":\"A\"}","{\"委托单\":\"a\"}","{\"委托单\":\" padded \"}","{\"委托单\":\"null\"}"); + for(String raw:samples)insert("black","build-black",BLACK_REVISION,"部门A","组A",raw); + String longKey="来源".repeat(300); + insert("black","build-black",BLACK_REVISION,"部门A","组A",JsonMapper.builder().build().writeValueAsString(Map.of(longKey,"long-key-value"))); + insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}"); + String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key"); + List> rules=new java.util.ArrayList<>(); + for(String op:List.of("in","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}")) + rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value))); + var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList(); + var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor); + var rawMembers=queries.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor); + jdbc.update("INSERT INTO query_filter_field VALUES ('plain','原始委托单','委托单','委托单',TRUE,TRUE,1)"); + List> legacySelections=List.of(List.of("null"),List.of("0007"),List.of(QueryFilterFields.EMPTY),List.of("0007",QueryFilterFields.EMPTY)); + var rawLegacy=legacySelections.stream().map(this::legacyCount).toList(); + installFilterIndex(); + var index=new SourceFilterIndex(jdbc); + for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build); + var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); + assertTrue(indexed.indexed(RELEASE)); + assertEquals(rawLegacy,legacySelections.stream().map(this::legacyCount).toList()); + assertEquals("long-key-value",jdbc.queryForObject("SELECT sv.value FROM source_filter_field sf JOIN source_filter_value sv ON sv.field_id=sf.id WHERE sf.column_name=? AND sf.build_id='build-black'",String.class,longKey)); + var combined=List.of(Map.of("field",key,"stages",List.of("black"),"operator","in","values",List.of("0007")),Map.of("field",field(indexed.catalog(RELEASE,actor),"加工单位").get("key"),"stages",List.of("black"),"operator","in","values",List.of("机二"))); + assertEquals(0L,indexed.memberPage(RELEASE,Map.of("year",2026,"columnFilters",combined),List.of(key),List.of("black"),50,0,actor).get("totalCount")); + assertEquals(List.of(),indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),"hidden-only-value",50,0,true,actor).get("rows")); + jdbc.update("INSERT INTO transform_build VALUES ('build-child')"); + jdbc.update("INSERT INTO fact_black(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT 'build-child',source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no FROM fact_black WHERE build_id='build-black'"); + index.ensureBuild("build-child"); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member sm JOIN source_filter_field sf ON sf.id=sm.field_id JOIN fact_black f ON f.id=sm.fact_id WHERE sf.build_id='build-child' AND f.build_id<>'build-child'",Long.class)); + assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList()); + assertEquals(rawValues,indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor)); + assertEquals(rawMembers,indexed.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor)); + var page=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,2,0,true,actor); + assertEquals(2,((List)page.get("rows")).size());assertEquals(true,page.get("hasMore"));assertFalse(page.containsKey("totalCount")); + long members=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class); + index.ensureBuild("build-black");assertEquals(members,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class)); + when(governance.allowedFields(actor)).thenReturn(Set.of("加工单位")); + assertThrows(ResponseStatusException.class,()->indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),null,50,0,true,actor)); + when(governance.allowedFields(actor)).thenReturn(Set.of("*")); + jdbc.update("DELETE FROM source_filter_build WHERE build_id='build-black'"); + assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList()); + } + + @Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception { + installFilterIndex(); + var fail=new java.util.concurrent.atomic.AtomicBoolean(true); + var failing=new JdbcTemplate(jdbc.getDataSource()) { + @Override public int update(String sql) { + if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure"); + return super.update(sql); + } + @Override public int update(String sql,Object... args) { + if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure"); + return super.update(sql,args); + } + }; + var index=new SourceFilterIndex(failing); + assertThrows(IllegalStateException.class,()->index.ensureBuild("build-black")); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build",Long.class)); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_field",Long.class)); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_value",Long.class)); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class)); + assertFalse(index.ready(RELEASE)); + for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build); + assertTrue(index.ready(RELEASE)); + } + + @Test + @EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_FILTER_BENCHMARK",matches="true") + void syntheticWideSourceIndexBenchmark() throws Exception { + when(governance.allowedFields(actor)).thenReturn(Set.of("*")); + int size=Integer.getInteger("quality.benchmarkRows",10000); + jdbc.execute("CREATE TABLE benchmark_digit(n INT PRIMARY KEY)");for(int n=0;n<10;n++)jdbc.update("INSERT INTO benchmark_digit VALUES (?)",n); + for(String stage:List.of("black","bright")) { + jdbc.execute("DELETE FROM fact_"+stage); + StringBuilder raw=new StringBuilder("JSON_OBJECT('客户名称',CONCAT('客户',MOD(n,300))"); + for(int f=0;f<120;f++)raw.append(",'字段").append(f).append("',CONCAT('value-").append(f).append("-',").append(stage.equals("black")?"MOD(n,50)":"n").append(")"); + raw.append(')'); + String numbers="SELECT a.n+10*b.n+100*c.n+1000*d.n+10000*e.n AS n FROM benchmark_digit a CROSS JOIN benchmark_digit b CROSS JOIN benchmark_digit c CROSS JOIN benchmark_digit d CROSS JOIN benchmark_digit e LIMIT "+size; + jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT ?,?,'2026-01','部门A','组A','P-1','项目A',"+raw+",'data',n+1 FROM ("+numbers+") numbers","build-"+stage,stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION); + } + var rawQuery=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); + String field=(String)field(rawQuery.catalog(RELEASE,actor),"客户名称").get("key"); + var old=new java.util.LinkedHashMap(); + for(String stage:List.of("black","bright")) { + long start=System.nanoTime();var result=rawQuery.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor); + old.put(stage,Map.of("ms",(System.nanoTime()-start)/1_000_000L,"result",result)); + } + installFilterIndex();var index=new SourceFilterIndex(jdbc); + for(String build:List.of("build-black","build-bright","build-hidden")) { + long start=System.nanoTime();index.ensureBuild(build);System.out.println("FILTER_BENCH_BUILD "+build+" rows="+size+" ms="+(System.nanoTime()-start)/1_000_000L); + } + var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); + for(String stage:List.of("black","bright")) { + long start=System.nanoTime();var result=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor); + long elapsed=(System.nanoTime()-start)/1_000_000L; + assertEquals(((Map)old.get(stage)).get("result"),result); + System.out.println("FILTER_BENCH_QUERY "+stage+" rawMs="+((Map)old.get(stage)).get("ms")+" indexMs="+elapsed); + } + for(String table:List.of("source_filter_field","source_filter_value","source_filter_member"))jdbc.execute("ANALYZE TABLE "+table); + System.out.println("FILTER_BENCH_SPACE "+jdbc.queryForList("SELECT table_name,table_rows,data_length,index_length FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name LIKE 'source_filter_%'")); + } + + private long legacyCount(List selected) { + var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList(List.of(RELEASE)); + new QueryFilterFields(jdbc,governance).append(where,args,"black",Map.of("plain",selected),actor,RELEASE); + DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); + return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray()); + } + + private long filteredCount(SourceColumnQueries source,Map rule) { + var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList(List.of(RELEASE)); + source.append(where,args,"black",List.of(rule),RELEASE,actor); + DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); + return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray()); + } + + private void installFilterIndex() throws Exception { + jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)"); + for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id); + String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/V43__source_filter_indexes.sql")); + for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement); + } + private void schema() { jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))"); jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))"); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfillTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfillTest.java new file mode 100644 index 0000000..d11499d --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfillTest.java @@ -0,0 +1,117 @@ +package cn.yunsheng.quality.filters; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.util.ArrayDeque; +import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; + +class SourceFilterIndexBackfillTest { + @Test + void skipsWhenIndexTablesAreUnavailable() { + JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class); + SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class); + ManualExecutor worker = new ManualExecutor(); + doReturn(false).when(index).available(); + var runner = new SourceFilterIndexBackfill(jdbc, index, worker); + + runner.run(null); + + verify(jdbc, never()).query(anyString(), org.mockito.ArgumentMatchers.>any()); + verify(index, never()).ensureBuild(anyString()); + assertEquals(0, worker.queued()); + runner.stopWorker(); + } + + @Test + void coalescesEventsDuringDrainAndContinuesAfterAnIndividualFailure() { + JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class); + SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class); + ManualExecutor worker = new ManualExecutor(); + doReturn(true).when(index).available(); + doReturn(List.of("failed-build", "ready-build")).when(jdbc) + .query(anyString(), org.mockito.ArgumentMatchers.>any()); + var runner = new SourceFilterIndexBackfill(jdbc, index, worker); + doAnswer(invocation -> { + runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain")); + runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain")); + runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("failed-build")); + throw new IllegalStateException("synthetic index failure"); + }).when(index).ensureBuild("failed-build"); + + runner.run(null); + + assertEquals(1, worker.queued()); + verify(index, never()).ensureBuild(anyString()); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(jdbc).query(sql.capture(), org.mockito.ArgumentMatchers.>any()); + assertTrue(sql.getValue().contains("c.dataset_id='quality'")); + assertTrue(sql.getValue().contains("r.status='PUBLISHED'")); + assertTrue(sql.getValue().contains("r.complete=TRUE")); + assertTrue(sql.getValue().contains("b.status='COMPLETED'")); + + worker.runNext(); + + verify(index).ensureBuild("failed-build"); + verify(index).ensureBuild("ready-build"); + verify(index).ensureBuild("event-during-drain"); + assertEquals(0, worker.queued()); + runner.stopWorker(); + assertTrue(worker.isShutdown()); + } + + @Test + void componentHasAnAutowiredProductionConstructorAndCleanShutdown() { + try (var context = new AnnotationConfigApplicationContext()) { + context.registerBean(JdbcTemplate.class, () -> org.mockito.Mockito.mock(JdbcTemplate.class)); + context.registerBean(SourceFilterIndex.class, () -> org.mockito.Mockito.mock(SourceFilterIndex.class)); + context.register(SourceFilterIndexBackfill.class); + context.refresh(); + + assertNotNull(context.getBean(SourceFilterIndexBackfill.class)); + } + } + + private static final class ManualExecutor extends AbstractExecutorService { + private final ArrayDeque tasks = new ArrayDeque<>(); + private boolean shutdown; + + @Override public void shutdown() { shutdown = true; } + + @Override public List shutdownNow() { + shutdown = true; + List pending = List.copyOf(tasks); + tasks.clear(); + return pending; + } + + @Override public boolean isShutdown() { return shutdown; } + + @Override public boolean isTerminated() { return shutdown && tasks.isEmpty(); } + + @Override public boolean awaitTermination(long timeout, TimeUnit unit) { return isTerminated(); } + + @Override public void execute(Runnable command) { + if (shutdown) throw new RejectedExecutionException("executor is stopped"); + tasks.add(command); + } + + int queued() { return tasks.size(); } + + void runNext() { tasks.removeFirst().run(); } + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsRollupServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsRollupServiceTest.java index afb4255..934ea69 100644 --- a/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsRollupServiceTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsRollupServiceTest.java @@ -1,8 +1,21 @@ package cn.yunsheng.quality.metrics; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import cn.yunsheng.quality.filters.SourceFilterIndex; import org.junit.jupiter.api.Test; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; class MetricsRollupServiceTest { @Test @@ -17,4 +30,65 @@ class MetricsRollupServiceTest { assertTrue(sql.contains("GROUP BY f.build_id, f.month_key " + collation)); assertTrue(sql.contains("f.group_name " + collation + ", f.project_group " + collation)); } + + @Test + void publishesFilterBuildAvailableOnlyAfterTheOuterTransactionCommits() { + var service = new MetricsRollupService(mock(JdbcTemplate.class)); + var publisher = mock(ApplicationEventPublisher.class); + service.setApplicationEventPublisher(publisher); + String build = "11111111-1111-4111-8111-111111111111"; + + TransactionSynchronizationManager.initSynchronization(); + try { + service.announceSourceFilterBuild(build); + verifyNoInteractions(publisher); + var synchronizations = TransactionSynchronizationManager.getSynchronizations(); + assertEquals(1, synchronizations.size()); + for (TransactionSynchronization synchronization : synchronizations) synchronization.afterCommit(); + verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build)); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + } + + @Test + void publishesImmediatelyWhenThereIsNoTransactionSynchronization() { + var service = new MetricsRollupService(mock(JdbcTemplate.class)); + var publisher = mock(ApplicationEventPublisher.class); + service.setApplicationEventPublisher(publisher); + String build = "22222222-2222-4222-8222-222222222222"; + + service.announceSourceFilterBuild(build); + + verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build)); + } + + @Test + void notificationFailureAfterCommitDoesNotEscapeTheTransactionSynchronization() { + var service = new MetricsRollupService(mock(JdbcTemplate.class)); + var publisher = mock(ApplicationEventPublisher.class); + service.setApplicationEventPublisher(publisher); + String build = "33333333-3333-4333-8333-333333333333"; + var event = new SourceFilterIndex.BuildAvailable(build); + doThrow(new IllegalStateException("synthetic event delivery failure")).when(publisher).publishEvent(event); + + TransactionSynchronizationManager.initSynchronization(); + try { + service.announceSourceFilterBuild(build); + var synchronizations = TransactionSynchronizationManager.getSynchronizations(); + assertDoesNotThrow(() -> synchronizations.get(0).afterCommit()); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + } + + @Test + void directFixtureConstructionKeepsTheSynchronousLegacySchemaFallback() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + var service = new MetricsRollupService(jdbc); + + service.announceSourceFilterBuild("44444444-4444-4444-8444-444444444444"); + + verify(jdbc).queryForObject(contains("information_schema.tables"), eq(Integer.class)); + } } diff --git a/deploy/macos/retention.py b/deploy/macos/retention.py index 6af57b5..0a4708f 100644 --- a/deploy/macos/retention.py +++ b/deploy/macos/retention.py @@ -26,6 +26,7 @@ SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService' TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED') BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup', 'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build', + 'source_filter_member', 'source_filter_value', 'source_filter_field', 'source_filter_build', 'fact_black', 'fact_bright') OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion', 'release_action', 'candidate_reconciliation') @@ -253,7 +254,24 @@ class Database: self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid)) for t in BUILD_CHILDREN: if t in self.columns: - count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,)) + if t == 'source_filter_member': + # Members have no build_id; drain each owned field in + # order without aliases because the exclusion uses LOCK TABLES. + count = 0 + for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)): + count = self.execute('DELETE FROM source_filter_member WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],)) + if count: + break + elif t == 'source_filter_value': + # Values are likewise owned through fields and are deleted + # only after all field memberships have been drained. + count = 0 + for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)): + count = self.execute('DELETE FROM source_filter_value WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],)) + if count: + break + else: + count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,)) if count: return count, 0 self.execute('DELETE FROM transform_build WHERE id=%s', (bid,)) diff --git a/deploy/macos/test_retention.py b/deploy/macos/test_retention.py index 3f33398..8b4f411 100644 --- a/deploy/macos/test_retention.py +++ b/deploy/macos/test_retention.py @@ -16,6 +16,7 @@ from unittest.mock import patch ROOT=Path(__file__).resolve().parents[2] sys.path.insert(0,str(ROOT/'scripts')) import backup_retention as br +import managed_test_resources spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py')) ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret) @@ -91,14 +92,18 @@ class DatabaseRetentionTests(unittest.TestCase): cfg=json.loads((ROOT/'.runtime/local-config.json').read_text()) cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True) cls.name='quality_retention_test_'+uuid.uuid4().hex[:12] + managed_test_resources.create(ROOT,cls.name,cls.execute_admin) with cls.admin.cursor() as c: - c.execute('CREATE DATABASE '+ret.ident(cls.name)) c.execute('USE '+ret.ident(cls.name)) ddl=[ 'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))', 'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))', 'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)', 'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))', + 'CREATE TABLE source_filter_build(build_id varchar(36) PRIMARY KEY,format_version int,FOREIGN KEY(build_id) REFERENCES transform_build(id))', + 'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))', + 'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))', + 'CREATE TABLE source_filter_member(field_id bigint NOT NULL,fact_id bigint NOT NULL,value_id bigint NOT NULL,PRIMARY KEY(field_id,fact_id),KEY(value_id,fact_id),FOREIGN KEY(field_id) REFERENCES source_filter_field(id),FOREIGN KEY(value_id) REFERENCES source_filter_value(id))', 'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))', 'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))', 'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)', @@ -115,14 +120,25 @@ class DatabaseRetentionTests(unittest.TestCase): @classmethod def tearDownClass(cls): - with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name)) + managed_test_resources.drop(ROOT,cls.name,cls.execute_admin) cls.admin.close() + @classmethod + def execute_admin(cls,sql,args=None): + result='' + statements=[part.strip() for part in sql.split(';') if part.strip()] + with cls.admin.cursor() as cursor: + for statement in statements: + cursor.execute(statement,args if len(statements)==1 else None) + result='\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount) + return result + def setUp(self): self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name) self.db=ret.Database(ROOT,self.name);self.db.root=self.root self.db.execute('SET FOREIGN_KEY_CHECKS=0') - for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table)) + for table in self.db.tables: + if table!='__quality_test_owner':self.db.execute('TRUNCATE TABLE '+ret.ident(table)) self.db.execute('SET FOREIGN_KEY_CHECKS=1') self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')") self.db.execute("INSERT INTO current_release VALUES('quality','current')") @@ -187,14 +203,24 @@ class DatabaseRetentionTests(unittest.TestCase): def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self): self.retire_old();state={'builds':{'obsolete':1}} + self.db.execute("INSERT INTO source_filter_build VALUES('obsolete',1)") + self.db.execute("INSERT INTO source_filter_field VALUES(9,'obsolete','black','空字段',%s),(10,'obsolete','black','客户',%s),(11,'obsolete','black','工单',%s)", (b'0'*32,b'1'*32,b'2'*32)) + self.db.execute("INSERT INTO source_filter_value VALUES(100,10,%s,'客户A'),(101,10,%s,'客户B'),(102,11,%s,'工单A')", (b'1'*32,b'2'*32,b'3'*32)) + self.db.execute("INSERT INTO source_filter_member VALUES(10,1,100),(10,2,101),(11,3,102)") with self.db.frozen('current'): self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0)) + self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_member')[0]['n'],2) + self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_value')[0]['n'],3) + self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_field')[0]['n'],3) + self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_build')[0]['n'],1) old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0] self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64) self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,)) - for _ in range(4): + for _ in range(20): with self.db.frozen('current'):self.db.build_step(state,1800,4000,1) self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")) + for table in ('source_filter_member','source_filter_value','source_filter_field','source_filter_build'): + self.assertFalse(self.db.rows('SELECT * FROM '+ret.ident(table))) self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'")) self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'")) diff --git a/deploy/macos/test_revision_retention.py b/deploy/macos/test_revision_retention.py index ee95b87..a11c258 100644 --- a/deploy/macos/test_revision_retention.py +++ b/deploy/macos/test_revision_retention.py @@ -13,6 +13,7 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'scripts')) import backup_retention as br +import managed_test_resources import revision_retention as revisions spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py')) ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret) @@ -26,12 +27,15 @@ class SourceRevisionTests(unittest.TestCase): cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root', password=cfg['rootPassword'], autocommit=True) cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12] + managed_test_resources.create(ROOT, cls.name, cls.execute_admin) ddl = [ 'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))', 'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))', 'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))', "CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)", 'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', + 'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))', + 'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))', 'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', 'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', 'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))', @@ -42,23 +46,34 @@ class SourceRevisionTests(unittest.TestCase): 'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))'] try: with cls.admin.cursor() as c: - c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name)) + c.execute('USE ' + ret.ident(cls.name)) for statement in ddl: c.execute(statement + ' ENGINE=InnoDB') except BaseException: - with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name)) + managed_test_resources.drop(ROOT, cls.name, cls.execute_admin) cls.admin.close(); raise @classmethod def tearDownClass(cls): - with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name)) + managed_test_resources.drop(ROOT, cls.name, cls.execute_admin) cls.admin.close() + @classmethod + def execute_admin(cls, sql, args=None): + result = '' + statements = [part.strip() for part in sql.split(';') if part.strip()] + with cls.admin.cursor() as cursor: + for statement in statements: + cursor.execute(statement, args if len(statements) == 1 else None) + result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount) + return result + def setUp(self): self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve() self.db = ret.Database(ROOT, self.name); self.db.root = self.root self.db.execute('SET FOREIGN_KEY_CHECKS=0') - for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table)) + for table in self.db.tables: + if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table)) self.db.execute('SET FOREIGN_KEY_CHECKS=1') self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')") self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')") @@ -121,6 +136,21 @@ class SourceRevisionTests(unittest.TestCase): self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0) self.assertTrue(self.present()); self.assertTrue(path.exists()) + def test_indexed_business_values_do_not_count_as_revision_metadata(self): + path = self.add_revision('indexed-only', b'indexed-only workbook') + self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,)) + self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,)) + self.collect(100); result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 1) + self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists()) + + def test_indexed_field_names_do_not_count_as_revision_metadata(self): + path = self.add_revision('indexed-field-only', b'field-only workbook') + self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,)) + self.collect(100); result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 1) + self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists()) + def test_new_reference_resets_continuous_grace(self): self.add_revision(); self.collect(100) self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500) diff --git a/docs/当前进度.md b/docs/当前进度.md index 89fad32..aee40fb 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -3,6 +3,14 @@ 核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。 +## 2026-11-13:统一筛选提速(发布验收中) + +实现:V43新增按构建保存的来源字段字典和稀疏记录索引,及两阶段字段目录覆盖索引;来源筛选、配置筛选、候选、指标、趋势、明细与明细导出共用。来源列查询实例统一复用,候选模式省去精确总数,产品页常规查询不再等待慢目录,本字段选中值变化不重复读取自己的候选;明细导出先只读字段名再读业务行。 + +生命周期:启动仅在单线程后台补当前完整发布版本;导入和归类通过提交后事件进入同一有界去重队列。完整标记提交后才启用,故障保留原始查询,业务事务不等待索引。旧配置 `null`、来源字符串 `null`、空值、Unicode、同一事实行和当前权限分别保持;构建保留清理识别4张新表,未解除维护兼容保护。 + +验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 + ## 2026-11-13:来源字段筛选超时修复 定位:客户等原始字段筛选时,汇总、趋势和覆盖检查重复读取字段目录,数据库对年度查询选择了原表扫描,约48秒超过页面30秒期限。复用服务级字段目录缓存,目录及无窄范围限制的年度来源筛选使用既有构建索引;JSON值每次表达式只提取一次,保留空值、字符串和大小写语义。单月、季度、产品及扩展条件保留原查询计划。查询超时提示改为重试或调整范围,不再误报连接断开。权限和正式计算规则不变,无新增表或索引。 diff --git a/scripts/revision_retention.py b/scripts/revision_retention.py index 6714c50..2596802 100644 --- a/scripts/revision_retention.py +++ b/scripts/revision_retention.py @@ -9,9 +9,11 @@ import uuid # These are measured business values, not documents that refer to source IDs. -VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'), - ('fact_bright', 'raw_values'), ('fact_bright', 'components'), - ('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')} +BUSINESS_VALUE_COLUMNS = {('fact_black', 'raw_values'), ('fact_black', 'components'), + ('fact_bright', 'raw_values'), ('fact_bright', 'components'), + ('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components'), + ('source_filter_field', 'column_name'), + ('source_filter_value', 'value')} TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'} @@ -54,7 +56,7 @@ class Cleaner: def metadata_mentions(self, needles, excluded=None): for table, columns in self.db.column_types.items(): for column, kind in columns.items(): - if kind not in TEXT_TYPES or (table, column) in VALUE_JSON: + if kind not in TEXT_TYPES or (table, column) in BUSINESS_VALUE_COLUMNS: continue conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles] args = list(needles) diff --git a/web/src/StageProductPage.tsx b/web/src/StageProductPage.tsx index 578011a..f645fa4 100644 --- a/web/src/StageProductPage.tsx +++ b/web/src/StageProductPage.tsx @@ -37,7 +37,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU const [moreFiltersOpen,setMoreFiltersOpen]=useState(false); const [extraFilters,setExtraFilters]=useState>(initial.extraFilters||{}); const [extraFields,setExtraFields]=useState([]); - const [options,setOptions]=useState(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');const [supportLoading,setSupportLoading]=useState(true); + const [options,setOptions]=useState(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState(''); const [applied,setApplied]=useState(null);const [revision,setRevision]=useState(0);const [response,setResponse]=useState<{key:string;value:MetricsResponse}|null>(null);const [loading,setLoading]=useState(false);const [failure,setFailure]=useState<{key:string;message:string}|null>(null); const [defectDisplay,setDefectDisplay]=useState<'weight'|'rate'>('weight'); const [fullFields,setFullFields]=useState(true);const [rateBasis,setRateBasis]=useState<'original'|'withoutHeadquartersScrap'>('original');const [basisOpen,setBasisOpen]=useState(false);const [trendMode,setTrendMode]=useState<'production'|'defects'|'data'>('production'); @@ -50,29 +50,39 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU const supportGeneration=useRef(0);const supportController=useRef(null); const loadSupport=useCallback(async()=>{ const generation=++supportGeneration.current;supportController.current?.abort();const controller=new AbortController();supportController.current=controller; - setSupportLoading(true);setSupportError(''); - try{const [choices,source,filterFields]=await Promise.all([api.options(controller.signal),api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal),api.filterFields(false,controller.signal)]);if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);setCatalog(source);setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));} - catch(error){if(generation===supportGeneration.current&&!onUnauthorized(error))setSupportError(error instanceof Error?error.message:'查询条件读取失败');} - finally{if(generation===supportGeneration.current)setSupportLoading(false);} + setSupportError(''); + const fail=(error:unknown)=>{if(generation===supportGeneration.current&&!controller.signal.aborted&&!onUnauthorized(error))setSupportError(previous=>previous||(error instanceof Error?error.message:'查询条件读取失败'));}; + const requests=[ + api.options(controller.signal).then(choices=>{if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);}).catch(fail), + api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal).then(source=>{if(generation===supportGeneration.current)setCatalog(source);}).catch(fail), + api.filterFields(false,controller.signal).then(filterFields=>{if(generation===supportGeneration.current)setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));}).catch(fail), + ]; + await Promise.all(requests); },[onUnauthorized,initial.releaseId,currentReleaseId]); useEffect(()=>{if(active)void loadSupport();return()=>{supportGeneration.current++;supportController.current?.abort();};},[active,refreshKey,loadSupport]); const fields=useMemo(()=>catalog?.sourceFields.filter(field=>field.type==='source_value'&&field.stages.includes(stage))||[],[catalog,stage]); const exactProductSearch=useMemo(()=>(options?.products||[]).find(name=>name.trim().toLocaleLowerCase()===appliedSearch.trim().toLocaleLowerCase())||'', [options,appliedSearch]); const queryProducts=useMemo(()=>drilledProduct?[drilledProduct]:exactProductSearch?product.length&&!product.includes(exactProductSearch)?['__codex_no_product_search_match__']:[exactProductSearch]:product,[drilledProduct,exactProductSearch,product]); useEffect(()=>{setDrilledProduct('');},[department,group,product,projectGroup,sourceFilters,extraFilters]); - const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:initial.releaseId||currentReleaseId||catalog?.releaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,catalog,stage,sourceFilters,extraFilters,initial.releaseId,currentReleaseId]); + const queryReleaseId=initial.releaseId||(currentReleaseId!==undefined?currentReleaseId||undefined:catalog?.releaseId); + const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,queryReleaseId,stage,sourceFilters,extraFilters]); + const catalogMatchesDraft=!!catalog&&!!draft.releaseId&&catalog.releaseId===draft.releaseId; + const queryReady=(query:QueryParams)=>!!queryReleaseId&&query.releaseId===queryReleaseId&&(!query.productSearch||!!options)&&(!(query.columnFilters||[]).length||!!catalog&&catalog.releaseId===query.releaseId); + const draftReady=queryReady(draft); + const appliedReady=!!applied&&queryReady(applied); // Apply organizational/source filters directly, like the other query pages. Product text search keeps its explicit submit. - useEffect(()=>{if(active&&catalog&&!supportLoading&&!supportError)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,catalog,draft,supportLoading,supportError]); + useEffect(()=>{if(active&&draftReady)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,draft,draftReady]); const queryKey=JSON.stringify({query:applied,revision});const current=response?.key===queryKey?response.value:null;const error=failure?.key===queryKey?failure.message:''; useEffect(()=>{ - if(!active||!applied||supportError)return;const controller=new AbortController();setLoading(true);setFailure(null); + if(!active||!applied||!appliedReady)return;const controller=new AbortController();setLoading(true);setFailure(null); void api.metrics(applied,controller.signal).then(value=>{if(!controller.signal.aborted)setResponse({key:queryKey,value});}).catch(e=>{if(!controller.signal.aborted&&!onUnauthorized(e))setFailure({key:queryKey,message:e instanceof Error?e.message:'查询失败'});}).finally(()=>{if(!controller.signal.aborted)setLoading(false);}); return()=>controller.abort(); - },[active,queryKey,onUnauthorized,supportError]); + },[active,queryKey,onUnauthorized,appliedReady]); useEffect(()=>{if(!active||!applied)return;const bytes=new TextEncoder().encode(JSON.stringify({query:queryForLocation(applied,initial.releaseId)}));const encoded=btoa(Array.from(bytes,b=>String.fromCharCode(b)).join('')).replace(/=/g,'').replace(/\+/g,'-').replace(/\//g,'_');const url=`?page=${stage==='black'?'blackProduct':'brightProduct'}&context=${encoded}`;window.history.replaceState({},'',`${location.pathname}${url}`);onLocationChange(url);},[active,applied,stage]); const trendQuery=useMemo(()=>applied?{...applied,month:undefined,quarter:undefined,...(trendGranularity==='year'?{year:undefined}:{}),trendGranularity}:null,[applied,trendGranularity]); const trendKey=JSON.stringify({query:trendQuery,revision}); - useEffect(()=>{if(!active||!trendQuery||supportError)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,supportError]); + const trendReady=!!trendQuery&&queryReady(trendQuery); + useEffect(()=>{if(!active||!trendQuery||!trendReady)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,trendReady]); const visibleTrend=useMemo(()=>fullYearTrend(trendResponse?.key===trendKey?trendResponse.value.trend:[],applied?.year||year,trendGranularity),[trendResponse,trendKey,applied?.year,year,trendGranularity]); const displayedTrend=useRetainedResult(trendResponse?.key===trendKey?visibleTrend:null)||visibleTrend; const selectedTrendPeriod=trendGranularity==='month'&&applied?.month @@ -80,12 +90,14 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU : trendGranularity==='quarter'&&applied?.quarter ? `${applied.year}-Q${applied.quarter}` : undefined; const selectTrendPeriod=(value:number|string)=>{automaticPeriod.current=false;const next=nextTrendPeriod(value,{period,year,month,quarter});setYear(next.year);setMonth(next.month);setQuarter(next.quarter);setPeriod(next.period);}; const dirty=!!applied&&JSON.stringify(draft)!==JSON.stringify(applied); - const queryBusy=supportLoading||loading||(!error&&!supportError&&(dirty||!current)); - const queryFailed=!!(error||supportError); + const metadataPending=!draftReady; + const metadataFailed=!draftReady&&!!supportError; + const queryBusy=!metadataFailed&&(metadataPending||loading||(!error&&(dirty||!current))); + const queryFailed=!!error||metadataFailed; const submit=()=>{automaticPeriod.current=false;setAppliedSearch(search.trim());setDrilledProduct('');setRevision(v=>v+1);}; const setTrendObject=(value?:string)=>{const selected=value||'';setDrilledProduct('');setSearch(selected);setAppliedSearch(selected);}; const returnToProducts=()=>{setDrilledProduct('');setSearch(drillParentSearch.current);setAppliedSearch(drillParentSearch.current);if(trendMode!=='defects')setTrendMode('production');}; - const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:catalog?.releaseId},stage,[]));setRevision(v=>v+1);}; + const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,[]));setRevision(v=>v+1);}; const groups=(options?.groups||[]).filter(name=>!department.length||department.includes(options?.groupDepartments?.[name]||'')); const paths=options?.organizationPaths||[];const products=department.length||group.length?[...new Set(paths.filter(path=>(!department.length||department.includes(path.department||''))&&(!group.length||group.includes(path.group||''))).map(path=>path.product).filter((v):v is string=>!!v))]:options?.products||[]; const trendOptions=useMemo(()=>[...products.map(value=>({label:value,value})),...(drilledProduct==='__UNASSIGNED__'?[{label:'未分类',value:drilledProduct}]:[])],[products,drilledProduct]); @@ -124,9 +136,9 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
{queryBusy?'正在查询':queryFailed?'查询失败':'查询完成'}
{moreFiltersOpen&&
- {extraFields.map(field=>)} + {draft.releaseId&&extraFields.map(field=>)} {options?.projectGroupStatus==='READY'&&select('项目组',projectGroup,options.projectGroups||[],setProjectGroup,true)} - {catalog&&} + {catalogMatchesDraft&&}
} {error&&setRevision(v=>v+1)}>重试}/>} @@ -150,7 +162,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU } - busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!catalog||!!supportError} + busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!options} fullFields={fullFields} onFullFields={setFullFields} defectDisplay={defectDisplay} onDefectDisplay={setDefectDisplay} returnControls={drilledProduct&&} exporting={exporting} exportDisabled={!!defectDrill.category&&(!defectDrill.table||defectDrill.pending||!!defectDrill.error)||!current||loading||!tableReady||!table?.rows.length} onExport={()=>void runExport()}> diff --git a/web/src/StageSourceFilters.tsx b/web/src/StageSourceFilters.tsx index 8653fec..303e7fd 100644 --- a/web/src/StageSourceFilters.tsx +++ b/web/src/StageSourceFilters.tsx @@ -11,13 +11,15 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S const [search,setSearch]=useState('');const [choices,setChoices]=useState([]);const [busy,setBusy]=useState(false);const [error,setError]=useState('');const [more,setMore]=useState(false);const [revision,setRevision]=useState(0); const {active}=usePageCache(); const showBusy=useDelayedIndicator(busy); - const contextKey=JSON.stringify(query);const current=useRef(0); + const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query; + const period={year,...(month?{month}:{}),...(quarter?{quarter}:{})}; + const filters={...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)}; + const contextKey=JSON.stringify({releaseId,period,filters,field:field.key,stage});const current=useRef(0); useEffect(()=>{ if(!active)return; const generation=++current.current,controller=new AbortController();setBusy(true);setError('');setChoices([]); const timer=window.setTimeout(()=>{ - const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query; - void api.sourceColumnValues({releaseId,period:{year,...(month?{month}:{}),...(quarter?{quarter}:{})},filters:{...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)},field:field.key,stages:[stage],search,limit:50,offset:0},controller.signal) + void api.sourceColumnValues({releaseId,period,filters,field:field.key,stages:[stage],search,limit:50,offset:0,candidateOnly:true},controller.signal) .then(result=>{if(current.current!==generation)return;setChoices(result.values.filter((v):v is string=>typeof v==='string'));setMore(result.hasMore===true||(result.total??0)>50);}) .catch(failure=>{if(controller.signal.aborted||current.current!==generation)return;if(!onUnauthorized(failure))setError(failure instanceof Error?failure.message:'选项读取失败');}) .finally(()=>{if(current.current===generation)setBusy(false);}); diff --git a/web/src/api.ts b/web/src/api.ts index e0a6c56..63db6e0 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -239,7 +239,7 @@ export const api = { }, activateSalesOrderTypes(versionId:string,expectedVersion:string) { return mutate<{versionId:string}>('/api/sales-order-types/activate',{versionId,expectedVersion}); }, sourceColumnCatalog(releaseId?: string, signal?: AbortSignal) { return request<{releaseId:string;sourceFields:SourceColumnField[]}>(`/api/analysis/source/catalog?${toQuery({releaseId})}`, { signal }); }, - sourceColumnValues(body: Record, signal?: AbortSignal) { return queryPost<{values:Array;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); }, + sourceColumnValues(body: Record & {candidateOnly?:true}, signal?: AbortSignal) { return queryPost<{values:Array;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); }, workspaceState() { return request<{ authRevision: string; dataRevision: string; currentReleaseId: string | null }>('/api/workspace/state', {}, 8000); }, filterFields(manage = false, signal?: AbortSignal) { return request(`/api/filter-fields?manage=${manage}`, { signal }); }, filterFieldOptions(key: string, search = '', signal?: AbortSignal,releaseId?:string,stage?:string) { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search,releaseId,stage})}`, { signal }); }, diff --git a/web/verification/stage_product_filter_speed.test.mjs b/web/verification/stage_product_filter_speed.test.mjs new file mode 100644 index 0000000..182d16e --- /dev/null +++ b/web/verification/stage_product_filter_speed.test.mjs @@ -0,0 +1,53 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import ts from 'typescript'; + +const read=name=>fs.readFileSync(new URL(`../src/${name}`,import.meta.url),'utf8'); + +test('stage product common queries use a resolved release and do not wait for source catalog when unfiltered',()=>{ + const source=read('StageProductPage.tsx'); + const releaseExpression=source.match(/const queryReleaseId=([^;]+);/)[1]; + const resolvedRelease=new Function('initial','currentReleaseId','catalog',`return ${releaseExpression}`); + assert.equal(resolvedRelease({releaseId:'historical'},'current',{releaseId:'catalog'}),'historical'); + assert.equal(resolvedRelease({},'current',{releaseId:'catalog'}),'current'); + assert.equal(resolvedRelease({},null,{releaseId:'stale'}),undefined); + assert.equal(resolvedRelease({},undefined,{releaseId:'resolved'}),'resolved'); + const expression=source.match(/const queryReady=\(query:QueryParams\)=>((?:[^;]|\n)+);/)[1]; + const ready=new Function('query','queryReleaseId','catalog','options',`return ${expression}`); + assert.equal(ready({releaseId:'r1',columnFilters:[]},'r1',null,null),true); + assert.equal(ready({releaseId:'r1',columnFilters:[]},undefined,null,null),false); + assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',null,null),false); + assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',{releaseId:'r1'},null),true); + assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r2',{releaseId:'r1'},null),false); + assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,null),false); + assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,{}),true); + assert.match(source,/if\(active&&draftReady\)setApplied/); + assert.doesNotMatch(source,/if\(!active\|\|!applied\|\|supportError\)/); +}); + +test('source option request context excludes its own selected values and tracks other filters',()=>{ + const source=read('StageSourceFilters.tsx'); + const declarations=source.match(/ const \{year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters\}=query;\n const period=.*;\n const filters=.*;\n const contextKey=[^;]+;/)[0]; + const run=new Function('query','field','stage',`${ts.transpileModule(declarations,{compilerOptions:{target:ts.ScriptTarget.ES2022}}).outputText};return {filters,contextKey};`); + const base={year:2026,month:4,releaseId:'r1',department:'D1',columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['North']}]}; + const first=run(base,{key:'customer'},'black'); + const ownValueChanged=run({...base,columnFilters:[{field:'customer',values:['OTHER']},{field:'region',values:['North']}]},{key:'customer'},'black'); + const otherValueChanged=run({...base,columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['South']}]},{key:'customer'},'black'); + assert.deepEqual(first.filters.columnFilters,[{field:'region',values:['North']}]); + assert.equal(first.contextKey,ownValueChanged.contextKey); + assert.notEqual(first.contextKey,otherValueChanged.contextKey); +}); + +test('source value dropdowns request candidate-only results while preserving search and debounce behavior',()=>{ + const source=read('StageSourceFilters.tsx'); + assert.match(source,/sourceColumnValues\(\{releaseId,period,filters,field:field\.key,stages:\[stage\],search,limit:50,offset:0,candidateOnly:true\}/); + assert.match(source,/search\?300:0/); + assert.match(source,/result\.values\.filter\(\(v\):v is string=>typeof v==='string'\)/); + assert.match(read('api.ts'),/sourceColumnValues\(body: Record & \{candidateOnly\?:true\}/); +}); + + +test('ordinary product search only depends on product options', () => { + assert.match(read('StageProductPage.tsx'), /searchUnavailable=\{!options\}/); +});