From cbe7fc17bbb79e2f6c9c935c783af94026890f1d Mon Sep 17 00:00:00 2001 From: lyun Date: Sat, 31 Oct 2026 19:18:46 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=EF=BC=9A=E5=88=86=E7=B1=BB?= =?UTF-8?q?=E8=A7=84=E5=88=99=E4=BB=85=E5=A4=84=E7=90=86=E6=9C=AA=E5=88=86?= =?UTF-8?q?=E7=B1=BB=E5=B9=B6=E4=BF=9D=E7=95=99=E5=B7=B2=E6=9C=89=E5=BD=92?= =?UTF-8?q?=E5=B1=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ClassificationPreviewService.java | 4 +- .../governance/MappingIssueService.java | 10 ++- .../quality/ingestion/BusinessRowMapper.java | 13 ++- .../ingestion/ConfigurableMatchRules.java | 23 ++++- .../quality/ingestion/ImportWorker.java | 15 +++- .../ingestion/UnclassifiedMappingRebuild.java | 88 +++++++++++++++++++ .../quality/metrics/MetricsRollupService.java | 21 +++++ .../quality/workflow/DataWorkflowService.java | 14 +++ .../ClassificationPreviewMySqlTest.java | 6 +- .../governance/ClassificationPreviewTest.java | 10 +-- .../ingestion/ConfigurableMatchRulesTest.java | 22 +++++ .../UnclassifiedMappingRebuildMySqlTest.java | 68 ++++++++++++++ .../UnclassifiedMappingScopeTest.java | 14 +++ web/src/ClassificationRuleEditor.tsx | 4 +- web/src/ManagementPages.tsx | 2 +- web/src/StructuredConfigEditors.tsx | 8 +- web/src/classificationMaintenance.ts | 7 +- web/src/configurationWorkbook.ts | 14 +-- .../classification_maintenance.test.mjs | 6 +- .../configuration_workbook.test.mjs | 7 +- 20 files changed, 320 insertions(+), 36 deletions(-) create mode 100644 backend/src/main/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuild.java create mode 100644 backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java create mode 100644 backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingScopeTest.java diff --git a/backend/src/main/java/cn/yunsheng/quality/governance/ClassificationPreviewService.java b/backend/src/main/java/cn/yunsheng/quality/governance/ClassificationPreviewService.java index a063819..dfddc9c 100644 --- a/backend/src/main/java/cn/yunsheng/quality/governance/ClassificationPreviewService.java +++ b/backend/src/main/java/cn/yunsheng/quality/governance/ClassificationPreviewService.java @@ -7,7 +7,7 @@ import java.util.*; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; -/** Counts current immutable source rows, including already classified rows, without changing mappings. */ +/** 只预览当前未分类记录的归类影响,不改动对应关系。 */ @Service public class ClassificationPreviewService { private final GovernanceSupport g; @@ -47,7 +47,7 @@ public class ClassificationPreviewService { private List> load(String release,Request request){ List queries=new ArrayList<>();List args=new ArrayList<>(); for(String stage:request.stage().equals("all")?List.of("black","bright"):List.of(request.stage())){ - StringBuilder where=new StringBuilder(" WHERE rm.release_id=?");args.add(release); + StringBuilder where=new StringBuilder(" WHERE rm.release_id=? AND (f.group_name IS NULL OR TRIM(f.group_name)='')");args.add(release); for(var condition:request.conditions()){ String expression=FIELDS.get(condition.field()); if(condition.operator().equals("IS_BLANK"))where.append(" AND (").append(expression).append(" IS NULL OR REGEXP_LIKE(").append(expression).append(",'^[[:space:]]*$'))"); diff --git a/backend/src/main/java/cn/yunsheng/quality/governance/MappingIssueService.java b/backend/src/main/java/cn/yunsheng/quality/governance/MappingIssueService.java index a01b102..ae8806e 100644 --- a/backend/src/main/java/cn/yunsheng/quality/governance/MappingIssueService.java +++ b/backend/src/main/java/cn/yunsheng/quality/governance/MappingIssueService.java @@ -20,12 +20,18 @@ public class MappingIssueService { private final MappingIssueDimensions dimensions;private final GovernanceSupport g;private final CandidateAcceptanceService acceptance;private final ImportRunService imports;private final ObjectProvider worker;private final TransactionTemplate transaction; public MappingIssueService(GovernanceSupport g,CandidateAcceptanceService acceptance,ImportRunService imports,ObjectProvider worker,PlatformTransactionManager manager){this.g=g;this.dimensions=new MappingIssueDimensions(g);this.acceptance=acceptance;this.imports=imports;this.worker=worker;this.transaction=new TransactionTemplate(manager);this.transaction.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);} /** Aggregated from immutable transformed facts, retaining their exact source/build lineage. */ - public void capture(String runId,String releaseId){ - Map groups=new TreeMap<>();for(String stage:List.of("black","bright"))for(Map f:g.rows("SELECT f.group_name,f.total_weight,f.components,f.raw_values,f.month_key,rm.source_document_id,rm.source_revision_id,rm.transform_build_id FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id WHERE rm.release_id=? AND (JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.categoryMappingStatus'))='unmapped' OR JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.projectGroupStatus'))='unmapped')",releaseId)){ + public record SourceRow(String stage,String buildId,String sheet,int row){} + public void capture(String runId,String releaseId){captureRows(runId,releaseId,null);} + public void captureRows(String runId,String releaseId,List selected){ + Map groups=new TreeMap<>();for(String stage:List.of("black","bright")){ + String restriction="";List args=new ArrayList<>(List.of(releaseId)); + if(selected!=null){var rows=selected.stream().filter(row->stage.equals(row.stage())).toList();if(rows.isEmpty())continue;restriction=" AND ("+String.join(" OR ",Collections.nCopies(rows.size(),"(f.build_id=? AND f.sheet_name=? AND f.physical_row_no=?)"))+")";for(var row:rows){args.add(row.buildId());args.add(row.sheet());args.add(row.row());}} + for(Map f:g.rows("SELECT f.group_name,f.total_weight,f.components,f.raw_values,f.month_key,rm.source_document_id,rm.source_revision_id,rm.transform_build_id FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id WHERE rm.release_id=? AND (JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.categoryMappingStatus'))='unmapped' OR JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.projectGroupStatus'))='unmapped')"+restriction,args.toArray())){ Map c=map(f.get("components")),raw=map(f.get("rawValues")); if("unmapped".equals(c.get("categoryMappingStatus")))add(groups,"CATEGORY",c.get("sourcePdtCode")+"|"+c.get("sourceSalesRegion"),f,raw,stage); if("unmapped".equals(c.get("projectGroupStatus")))add(groups,"PROJECT_GROUP",c.get("sourceCustomerCode")+"|"+f.get("groupName"),f,raw,stage); } + } for(Group group:groups.values()){ List candidates=new ArrayList<>();for(String revision:group.revisions)candidates.addAll(g.jdbc.query("SELECT DISTINCT c.id FROM source_candidate c JOIN source_revision r ON r.sha256=c.sha256 JOIN source_document d ON d.id=r.source_document_id WHERE r.id=? AND "+SourceIdentity.canonicalSlotSql("c.source_slot")+"="+SourceIdentity.canonicalSlotSql("d.source_slot"),(rs,n)->rs.getString(1),revision)); String id=GovernanceSupport.sha(runId+"\n"+group.type+"\n"+group.key);Map details=Map.of("sourceCandidateIds",candidates.stream().distinct().toList(),"sourceDocumentIds",group.documents,"sourceRevisionIds",group.revisions,"transformBuildIds",group.builds,"periods",group.periods,"originalValues",group.originals,"originalValueRows",group.originalValueRows,"releaseId",releaseId); diff --git a/backend/src/main/java/cn/yunsheng/quality/ingestion/BusinessRowMapper.java b/backend/src/main/java/cn/yunsheng/quality/ingestion/BusinessRowMapper.java index 5800bda..c94247c 100644 --- a/backend/src/main/java/cn/yunsheng/quality/ingestion/BusinessRowMapper.java +++ b/backend/src/main/java/cn/yunsheng/quality/ingestion/BusinessRowMapper.java @@ -26,6 +26,7 @@ public final class BusinessRowMapper { private final Map productLookup; private final ConfigurableRuleEngine rules; private final ConfigurableMatchRules matchRules; + private final Map savedGroups; public BusinessRowMapper(Map categoryLookup, Map projectLookup, String mappingVersion) { this(categoryLookup,projectLookup,mappingVersion,Map.of(),Map.of()); @@ -43,8 +44,18 @@ public final class BusinessRowMapper { this.departmentLookup=departmentLookup;this.productLookup=productLookup; this.rules=rules; this.matchRules=ConfigurableMatchRules.fromDefinition(mappingDefinition); + this.savedGroups=Map.of(); } + private BusinessRowMapper(BusinessRowMapper base,Map savedGroups){ + this.categoryLookup=base.categoryLookup;this.projectLookup=base.projectLookup;this.mappingVersion=base.mappingVersion; + this.departmentLookup=base.departmentLookup;this.productLookup=base.productLookup;this.rules=base.rules;this.matchRules=base.matchRules; + this.savedGroups=Map.copyOf(savedGroups); + } + public boolean hasUnclassifiedRules(){return matchRules.hasUnclassifiedRules();} + public BusinessRowMapper preserveAssignedGroups(Map groups){return new BusinessRowMapper(this,groups);} + static String sourceRowKey(String sheet,int row){return sheet+"\n"+row;} + public String mappingVersion() { return mappingVersion; } public List requiredHeaders(String stage) { return rules.requiredHeaders(stage,"black".equals(stage) ? BLACK_REQUIRED : BRIGHT_REQUIRED); } @@ -153,7 +164,7 @@ public final class BusinessRowMapper { product=matchRules.apply(stage,"product",matchSource,computed(department,null,product,null),product); String sourceKey=joinedKey(pdt,region);boolean blankCategoryKey=pdt==null||pdt.isEmpty()||region==null||region.isEmpty(); String category=blankCategoryKey?null:categoryLookup.get(sourceKey); - category=matchRules.apply(stage,"group",matchSource,computed(department,category,product,null),category); + category=matchRules.applyGroup(stage,matchSource,computed(department,category,product,null),category,savedGroups.get(sourceRowKey(input.sheet(),input.physicalRow()))).value(); boolean unmappedCategory=!blankCategoryKey&&category==null; String projectKey=joinedKey(customer,category);boolean blankProjectKey=customer==null||customer.isEmpty()||category==null||category.isEmpty(); boolean projectMappingKnown=!blankProjectKey&&projectLookup.containsKey(projectKey); diff --git a/backend/src/main/java/cn/yunsheng/quality/ingestion/ConfigurableMatchRules.java b/backend/src/main/java/cn/yunsheng/quality/ingestion/ConfigurableMatchRules.java index 7a806da..6f433f9 100644 --- a/backend/src/main/java/cn/yunsheng/quality/ingestion/ConfigurableMatchRules.java +++ b/backend/src/main/java/cn/yunsheng/quality/ingestion/ConfigurableMatchRules.java @@ -21,11 +21,13 @@ public final class ConfigurableMatchRules { private static final Pattern UNSAFE_TEXT = Pattern.compile("(?i)(<\\s*script\\b|javascript\\s*:|on[a-z]+\\s*=|\\beval\\s*\\(|document\\.(?:cookie|write)|--|/\\*|\\*/|\\b(?:DROP|ALTER|DELETE|INSERT|UPDATE|TRUNCATE|CREATE|EXEC(?:UTE)?|UNION\\s+SELECT|SELECT\\s+.+\\s+FROM)\\b)"); private final List> rules; + private final List> executionRules; private ConfigurableMatchRules(List> rules) { this.rules = rules.stream().map(ConfigurableMatchRules::copyRule) .sorted(Comparator.comparingInt(rule -> ((Number) rule.get("priority")).intValue())) .toList(); + this.executionRules=java.util.stream.Stream.concat(this.rules.stream().filter(rule->!Boolean.TRUE.equals(rule.get("onlyUnclassified"))),this.rules.stream().filter(rule->Boolean.TRUE.equals(rule.get("onlyUnclassified")))).toList(); } /** Default mapping behavior for old mapping versions that do not contain matchRules. */ @@ -73,7 +75,8 @@ public final class ConfigurableMatchRules { errors.add(path + " 必须为对象"); continue; } - for (Object key : rule.keySet()) if (!Set.of("id", "name", "enabled", "priority", "stage", "conditions", "target", "value").contains(String.valueOf(key))) errors.add(path + " 未知字段: " + key); + for (Object key : rule.keySet()) if (!Set.of("id", "name", "enabled", "priority", "stage", "conditions", "target", "value", "onlyUnclassified").contains(String.valueOf(key))) errors.add(path + " 未知字段: " + key); + if(rule.containsKey("onlyUnclassified") && (!(rule.get("onlyUnclassified") instanceof Boolean) || Boolean.TRUE.equals(rule.get("onlyUnclassified")) && !"group".equals(rule.get("target")))) errors.add(path+" onlyUnclassified仅适用于业务分组且须为布尔值"); String id = text(rule.get("id")); if (id != null) textBudget += id.length(); if (id == null || !id.matches("[A-Za-z0-9][A-Za-z0-9_-]{0,63}")) errors.add(path + " id须为1至64位字母、数字、下划线或连字符"); @@ -136,8 +139,15 @@ public final class ConfigurableMatchRules { } public Match applyMatch(String stage, String target, Map raw, Map computed, String current) { - for (Map rule : rules) { + return applyMatch(stage,target,raw,computed,current,false); + } + + private Match applyMatch(String stage,String target,Map raw,Map computed,String current,boolean ordinaryOnly){ + // 普通对应与规则先执行;补分类规则只接收仍为空的结果,不能抢占已有归属。 + for (Map rule : executionRules) { + if(ordinaryOnly&&Boolean.TRUE.equals(rule.get("onlyUnclassified")))continue; if (!Boolean.TRUE.equals(rule.get("enabled")) || !target.equals(rule.get("target"))) continue; + if(Boolean.TRUE.equals(rule.get("onlyUnclassified")) && current!=null && !current.isBlank()) continue; if (!"all".equals(rule.get("stage")) && !stage.equals(rule.get("stage"))) continue; boolean matches = true; for (Object rawCondition : (List) rule.get("conditions")) { @@ -151,6 +161,15 @@ public final class ConfigurableMatchRules { public record Match(boolean matched, String value) {} + /** 同一来源再次导入时,补分类规则仍不能覆盖其已保存的归属。普通规则和基础对应照常优先。 */ + public Match applyGroup(String stage,Map raw,Map computed,String current,String savedGroup) { + if(savedGroup==null||savedGroup.isBlank())return applyMatch(stage,"group",raw,computed,current); + var result=applyMatch(stage,"group",raw,computed,current,true); + return result.matched()||result.value()!=null&&!result.value().isBlank()?result:new Match(false,savedGroup); + } + + public boolean hasUnclassifiedRules(){return rules.stream().anyMatch(rule->Boolean.TRUE.equals(rule.get("onlyUnclassified")));} + public List> rules() { return rules.stream().map(ConfigurableMatchRules::copyRule).toList(); } public static List> fields() { diff --git a/backend/src/main/java/cn/yunsheng/quality/ingestion/ImportWorker.java b/backend/src/main/java/cn/yunsheng/quality/ingestion/ImportWorker.java index 7861cb4..ebef872 100644 --- a/backend/src/main/java/cn/yunsheng/quality/ingestion/ImportWorker.java +++ b/backend/src/main/java/cn/yunsheng/quality/ingestion/ImportWorker.java @@ -236,6 +236,8 @@ public class ImportWorker { List facts = new ArrayList<>(BATCH_SIZE); List issues = new ArrayList<>(); Set availableHeaders = new HashSet<>(); + Map savedGroups=rowMapper.hasUnclassifiedRules()?loadAssignedGroups(jdbc,candidate.stage(),runId,revisionId):Map.of(); + BusinessRowMapper effectiveMapper=rowMapper.preserveAssignedGroups(savedGroups); long[] rowCount = {0}; int[] blocking = {0}; int[] blankCategory = {0}; @@ -244,7 +246,7 @@ public class ImportWorker { String declaredScope=jdbc.queryForObject("SELECT scope_code FROM source_candidate WHERE id=?",String.class,candidate.id().toString()); XlsxStreamingReader.ReadSummary parsed = reader.read(archivePath, expectedSheet, parserConfig, row -> { availableHeaders.addAll(row.values().keySet()); - BusinessRowMapper.RowValues value = parserConfig==null?rowMapper.transform(candidate.stage(), row, candidate.year(), candidate.month()):rowMapper.transformConfigured(candidate.stage(),row,candidate.year(),candidate.month()); + BusinessRowMapper.RowValues value = parserConfig==null?effectiveMapper.transform(candidate.stage(), row, candidate.year(), candidate.month()):effectiveMapper.transformConfigured(candidate.stage(),row,candidate.year(),candidate.month()); if(declaredScope!=null&&!"ALL".equals(declaredScope)&&!java.util.Objects.equals(cn.yunsheng.quality.common.DepartmentLabels.queryValue(declaredScope),value.department()))throw new IllegalArgumentException("来源行超出已声明组织范围,须修订来源或协议"); facts.add(value); rowCount[0]++; @@ -325,6 +327,15 @@ public class ImportWorker { } } + static Map loadAssignedGroups(JdbcTemplate jdbc,String stage,String runId,String revisionId){ + if(!Set.of("black","bright").contains(stage))throw new IllegalArgumentException("来源阶段无效"); + // 仅复用该任务基线中同一不可变来源修订的归属;新文件或修订不借用旧记录。 + Map savedGroups=new HashMap<>(); + jdbc.queryForList("SELECT f.sheet_name,f.physical_row_no,f.group_name FROM fact_"+stage+" f JOIN release_member m ON m.transform_build_id=f.build_id JOIN import_run r ON r.expected_current_release_id=m.release_id WHERE r.id=? AND f.source_revision_id=? AND f.group_name IS NOT NULL AND TRIM(f.group_name)<>''",runId,revisionId) + .forEach(row->savedGroups.put(BusinessRowMapper.sourceRowKey(String.valueOf(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue()),String.valueOf(row.get("group_name")))); + return savedGroups; + } + private void persistIssues(String runId, SourceFileCandidate candidate, List issues) { for (Issue issue : issues) { jdbc.update(""" @@ -416,6 +427,8 @@ public class ImportWorker { return cn.yunsheng.quality.governance.MasterDataSupport.enabledLookup(definition,lookup,values); } + void captureMappingIssueRows(String run,String release,java.util.List rows){mappingIssues.captureRows(run,release,rows);} + String reviewedMappingFor(String runId,String activeMapping){ String frozen=jdbc.queryForObject("SELECT expected_mapping_version FROM import_run WHERE id=?",String.class,runId); if(frozen==null)return activeMapping; diff --git a/backend/src/main/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuild.java b/backend/src/main/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuild.java new file mode 100644 index 0000000..56d81c1 --- /dev/null +++ b/backend/src/main/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuild.java @@ -0,0 +1,88 @@ +package cn.yunsheng.quality.ingestion; + +import cn.yunsheng.quality.identity.AppUserPrincipal; +import cn.yunsheng.quality.metrics.MetricsRollupService; +import java.util.*; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import org.springframework.jdbc.core.JdbcTemplate; +import tools.jackson.databind.json.JsonMapper; + +/** 未分类维护只重新判断空归属行;其余事实与原汇总按不可变版本原样保留。 */ +public final class UnclassifiedMappingRebuild { + private final JdbcTemplate jdbc;private final JsonMapper json;private final ImportWorker worker;private final MetricsRollupService rollups; + public UnclassifiedMappingRebuild(JdbcTemplate jdbc,JsonMapper json,ImportWorker worker,MetricsRollupService rollups){this.jdbc=jdbc;this.json=json;this.worker=worker;this.rollups=rollups;} + @SuppressWarnings("unchecked") static Map map(Object value){return value instanceof Map?(Map)value:Map.of();} + @SuppressWarnings("unchecked") static List> rules(Map definition){return (List>)definition.getOrDefault("matchRules",ConfigurableMatchRules.defaultRules());} + static Map> indexed(List> rules){var result=new LinkedHashMap>();for(var rule:rules)result.put(String.valueOf(rule.get("id")),rule);return result;} + public static boolean unchanged(Map before,Map after){var left=new LinkedHashMap<>(before);var right=new LinkedHashMap<>(after);left.remove("matchRules");right.remove("matchRules");return left.equals(right)&&rules(before).equals(rules(after));} + public static List> changedRules(Map before,Map after){ + var old=indexed(rules(before));var next=indexed(rules(after));var changed=new LinkedHashSet();changed.addAll(old.keySet());changed.addAll(next.keySet());changed.removeIf(id->Objects.equals(old.get(id),next.get(id))); + boolean scoped=changed.stream().anyMatch(id->Boolean.TRUE.equals(map(old.get(id)).get("onlyUnclassified"))||Boolean.TRUE.equals(map(next.get(id)).get("onlyUnclassified"))); + if(!scoped)return null; + var left=new LinkedHashMap<>(before);var right=new LinkedHashMap<>(after);left.remove("matchRules");right.remove("matchRules"); + if(!left.equals(right))throw new IllegalArgumentException("未分类维护不能同时修改其他对应或名录,请分开保存"); + var result=new ArrayList>(); + for(String id:changed){var prior=old.get(id);var rule=next.get(id); + if(rule==null){if(!Boolean.TRUE.equals(prior.get("onlyUnclassified")))throw new IllegalArgumentException("不能在未分类维护中移除普通匹配规则");continue;} + if(!Boolean.TRUE.equals(rule.get("onlyUnclassified"))||!"group".equals(rule.get("target")))throw new IllegalArgumentException("未分类规则不能改为覆盖已有分类"); + for(var raw:(List)rule.get("conditions")){var condition=map(raw);if(!Set.of("@department","@product","PDT代码","销售片区").contains(condition.get("field"))||!Set.of("EQUALS","IS_BLANK").contains(condition.get("operator")))throw new IllegalArgumentException("未分类维护仅支持产业部、产品、PDT代码和销售区域条件");} + if(Boolean.TRUE.equals(rule.get("enabled")))result.add(rule); + } + return result; + } + record Change(String stage,String parent,String sheet,int row,String month,String department,String product,String oldGroup,String oldProject,String group,String project,Map components){} + public record Candidate(String releaseId,String runId,int scannedRows,int classifiedRows){} + private String text(Object value){return value==null?null:String.valueOf(value);} + private Map decode(Object value){if(value==null)return new LinkedHashMap<>();return new LinkedHashMap<>(json.readValue(String.valueOf(value),Map.class));} + private String encode(Object value){return json.writeValueAsString(value);} + private static String hash(String value){try{return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)));}catch(Exception error){throw new IllegalStateException(error);}} + + public Candidate create(String operation,String parent,String mapping,String rule,Map definition,List> changedRules,AppUserPrincipal actor){ + // 调用方事务覆盖候选、数据副本和进度;失败不改变旧版本,禁止执行 Excel 导入器。 + var matcher=ConfigurableMatchRules.fromDefinition(Map.of("matchRules",changedRules));var projectRules=ConfigurableMatchRules.fromDefinition(definition); + var changes=new ArrayList();int scanned=0;var projectLookups=new HashMap>(); + for(String stage:List.of("black","bright")){ + var rows=jdbc.queryForList("SELECT f.build_id,f.sheet_name,f.physical_row_no,f.month_key,f.department,f.group_name,f.project_group,f.product,CAST(f.components AS CHAR) components,CAST(f.raw_values AS CHAR) raw_values,CAST(b.parser_config AS CHAR) parser_config FROM release_member m JOIN fact_"+stage+" f ON f.build_id=m.transform_build_id JOIN transform_build b ON b.id=f.build_id WHERE m.release_id=? AND (f.group_name IS NULL OR TRIM(f.group_name)='')",parent); + scanned+=rows.size(); + for(var row:rows){ + var components=decode(row.get("components"));var raw=decode(row.get("raw_values"));var restored=BusinessRowMapper.restoreStoredRow(stage,text(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue(),raw,decode(row.get("parser_config"))); + restored.values().forEach((key,value)->raw.put(key,value==null?null:value.display())); + raw.put("PDT代码",components.get("sourcePdtCode"));raw.put("销售片区",components.get("sourceSalesRegion"));raw.put("客户代码",components.get("sourceCustomerCode")); + var computed=new LinkedHashMap();computed.put("department",text(row.get("department")));computed.put("group",text(row.get("group_name")));computed.put("product",text(row.get("product")));computed.put("projectGroup",text(row.get("project_group"))); + var match=matcher.applyMatch(stage,"group",raw,computed,text(row.get("group_name")));if(!match.matched())continue; + String month=text(row.get("month_key")),group=match.value(),customer=text(components.get("sourceCustomerCode")); + var lookup=projectLookups.computeIfAbsent(month,key->worker.loadMappings("PROJECT_GROUP",mapping,Integer.parseInt(key.substring(0,4)),Integer.parseInt(key.substring(5,7)))); + boolean blank=customer==null||customer.isBlank();String key=blank?null:customer+"|"+group;boolean known=!blank&&lookup.containsKey(key);String project=known?lookup.get(key):null; + computed.put("group",group);computed.put("projectGroup",project);var projectMatch=projectRules.applyMatch(stage,"projectGroup",raw,computed,project);project=projectMatch.value(); + components.put("categoryMappingStatus","mapped");components.put("projectGroup",project);components.put("projectGroupStatus",projectMatch.matched()?"mapped":blank?"source_key_blank":known?(project==null?"mapped_null":"mapped"):"unmapped"); + changes.add(new Change(stage,text(row.get("build_id")),text(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue(),month,text(row.get("department")),text(row.get("product")),text(row.get("group_name")),text(row.get("project_group")),group,project,components)); + } + } + String release=UUID.randomUUID().toString(),run=UUID.randomUUID().toString(); + int releaseInserted=jdbc.update("INSERT INTO data_release(id,name,status,parent_release_id,expected_current_release_id,rule_version,mapping_version,target_version_id,complete,validation,created_by) SELECT ?,?,'CANDIDATE',id,id,rule_version,?,target_version_id,TRUE,JSON_SET(COALESCE(validation,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0),? FROM data_release WHERE id=? AND rule_version=?",release,"未分类归类 "+operation.substring(0,8),mapping,scanned,changes.size(),actor.getId().toString(),parent,rule); + if(releaseInserted!=1)throw new IllegalStateException("当前数据版本已变化,未分类归类未提交"); + jdbc.update("INSERT INTO import_run(id,idempotency_key,status,stage,requested_sources,expected_current_release_id,created_by,processed,total,result_release_id,expected_mapping_version,expected_rule_version,workflow_operation_id,validation,started_at,completed_at) VALUES(?,?,'COMPLETED','CLASSIFICATION_READY',JSON_ARRAY(),?,?,?,?,?,?,?,?,JSON_OBJECT('recalculationScope','UNCLASSIFIED','excelFilesParsed',0),CURRENT_TIMESTAMP(6),CURRENT_TIMESTAMP(6))",run,"UNCLASSIFIED:"+operation,parent,actor.getId().toString(),changes.size(),scanned,release,mapping,rule,operation); + var members=jdbc.queryForList("SELECT m.source_document_id,m.source_revision_id,m.transform_build_id FROM release_member m WHERE m.release_id=?",parent); + var changedOrigins=new ArrayList(); + for(var member:members){ + String old=text(member.get("transform_build_id")),build=UUID.randomUUID().toString(); + int inserted=jdbc.update("INSERT INTO transform_build(id,source_revision_id,build_key,parser_version,rule_version,mapping_version,status,row_count,rejected_count,component_summary,error_summary,started_at,completed_at,parser_config,parser_config_hash) SELECT ?,source_revision_id,?,parser_version,rule_version,?,'COMPLETED',row_count,rejected_count,JSON_SET(COALESCE(component_summary,JSON_OBJECT()),'$.classificationParentBuildId',id),error_summary,CURRENT_TIMESTAMP(6),CURRENT_TIMESTAMP(6),parser_config,parser_config_hash FROM transform_build WHERE id=? AND status='COMPLETED' AND rule_version=?",build,hash("UNCLASSIFIED\n"+operation+"\n"+old+"\n"+mapping),mapping,old,rule);if(inserted!=1)throw new IllegalStateException("来源快照已变化"); + // 只复制已冻结结果,不解析文件、不重算已分类行的数值或维度。 + for(String stage:List.of("black","bright"))jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,sheet_name,physical_row_no,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,raw_values,components) SELECT ?,source_revision_id,sheet_name,physical_row_no,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,raw_values,components FROM fact_"+stage+" WHERE build_id=?",build,old); + jdbc.update("INSERT INTO defect_component(build_id,stage,sheet_name,physical_row_no,defect_name,weight,source_field) SELECT ?,stage,sheet_name,physical_row_no,defect_name,weight,source_field FROM defect_component WHERE build_id=?",build,old); + rollups.copyBuild(old,build);var keys=new LinkedHashSet(); + for(var change:changes)if(change.parent().equals(old)){ + int updated=jdbc.update("UPDATE fact_"+change.stage()+" SET group_name=?,project_group=?,components=CAST(? AS JSON) WHERE build_id=? AND sheet_name=? AND physical_row_no=? AND (group_name IS NULL OR TRIM(group_name)='')",change.group(),change.project(),encode(change.components()),build,change.sheet(),change.row());if(updated!=1)throw new IllegalStateException("未分类行已变化,归类未提交"); + changedOrigins.add(new cn.yunsheng.quality.governance.MappingIssueService.SourceRow(change.stage(),build,change.sheet(),change.row())); + keys.add(new MetricsRollupService.GroupKey(change.stage(),change.month(),change.department(),change.oldGroup(),change.oldProject(),change.product()));keys.add(new MetricsRollupService.GroupKey(change.stage(),change.month(),change.department(),change.group(),change.project(),change.product())); + } + rollups.refreshGroups(build,keys); + jdbc.update("INSERT INTO release_member(release_id,source_document_id,source_revision_id,transform_build_id) VALUES(?,?,?,?)",release,member.get("source_document_id"),member.get("source_revision_id"),build); + } + ReleaseProtocolDependencies.inheritFrozen(jdbc,release,parent); + jdbc.update("INSERT INTO release_source_exclusion(release_id,source_document_id,request_id) SELECT ?,source_document_id,request_id FROM release_source_exclusion WHERE release_id=?",release,parent); + worker.captureMappingIssueRows(run,release,changedOrigins); + return new Candidate(release,run,scanned,changes.size()); + } +} diff --git a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java index c6b860c..3f947f0 100644 --- a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java +++ b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java @@ -24,6 +24,27 @@ public class MetricsRollupService { public MetricsRollupService(JdbcTemplate jdbc) { this.jdbc = jdbc; } + public record GroupKey(String stage,String month,String department,String group,String project,String product) {} + + /** 归类维护沿用旧汇总;仅重建归属发生变化的分组,重量计算结果不变。 */ + public void copyBuild(String parent,String child) { + jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent); + if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总"); + } + + public void refreshGroups(String build,java.util.Set keys) { + for(String stage:List.of("black","bright")) { + var selected=keys.stream().filter(k->stage.equals(k.stage())).toList();if(selected.isEmpty())continue; + String predicate=String.join(" OR ",java.util.Collections.nCopies(selected.size(),"(%s.month_key<=>? AND %s.department COLLATE "+DIMENSION_COLLATION+"<=>? AND %s.group_name COLLATE "+DIMENSION_COLLATION+"<=>? AND %s.project_group COLLATE "+DIMENSION_COLLATION+"<=>? AND %s.product COLLATE "+DIMENSION_COLLATION+"<=>?)")); + var values=new java.util.ArrayList();for(var key:selected){values.add(key.month());values.add(key.department());values.add(key.group());values.add(key.project());values.add(key.product());} + String deleted=predicate.replace("%s","r"),inserted=predicate.replace("%s","f"); + var deleteArgs=new java.util.ArrayList(List.of(build,stage));deleteArgs.addAll(values); + jdbc.update("DELETE r FROM metric_rollup r WHERE r.build_id=? AND r.stage=? AND ("+deleted+")",deleteArgs.toArray()); + var insertArgs=new java.util.ArrayList(List.of(stage,build));insertArgs.addAll(values); + jdbc.update(rollupInsertSql("fact_"+stage).replace("WHERE f.build_id=?","WHERE f.build_id=? AND ("+inserted+")"),insertArgs.toArray()); + } + } + /** Idempotently refreshes a completed build in the caller's transaction. */ @Transactional public void refreshBuild(String buildId) { diff --git a/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java b/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java index c82320f..2c8b52c 100644 --- a/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java +++ b/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java @@ -12,6 +12,7 @@ import cn.yunsheng.quality.identity.JdbcAppUserDetailsService; import cn.yunsheng.quality.ingestion.ConfigurableRuleEngine; import cn.yunsheng.quality.ingestion.ImportRunService; import cn.yunsheng.quality.ingestion.ImportWorker; +import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild; import cn.yunsheng.quality.ingestion.MappingValueCodec; import cn.yunsheng.quality.metrics.MetricsService; import cn.yunsheng.quality.release.ReleaseService; @@ -346,6 +347,7 @@ public class DataWorkflowService { String kind = String.valueOf(payload.get("kind")); Map definition = asMap(payload.get("definition")); if (definition == null) throw failure("INVALID_DEFINITION", "definition 须为 JSON 对象"); + List> unclassifiedRules=null; if ("mapping".equals(kind)) { Map previous=Map.of(); String base=boundVersion("mapping_version","mapping_version",operation.expectedCurrentReleaseId()); @@ -356,6 +358,8 @@ public class DataWorkflowService { completeMappingTargets(definition,previous); List errors = validateMapping(definition); if (!errors.isEmpty()) throw failure("MAPPING_INVALID", String.join(";", errors)); + if(UnclassifiedMappingRebuild.unchanged(previous,definition)){complete(operation.id(),token,operation.expectedCurrentReleaseId(),null,Map.of("unchanged",true,"classifiedRows",0,"excelFilesParsed",0));return;} + unclassifiedRules=UnclassifiedMappingRebuild.changedRules(previous,definition); } else { List errors = validateRule(definition); if (!errors.isEmpty()) throw failure("RULE_INVALID", String.join(";", errors)); @@ -367,6 +371,16 @@ public class DataWorkflowService { String ruleId = "rule".equals(kind) ? versionId : boundVersion("rule_version", "rule_version", operation.expectedCurrentReleaseId()); String mappingId = "mapping".equals(kind) ? versionId : boundVersion("mapping_version", "mapping_version", operation.expectedCurrentReleaseId()); if (ruleId == null || mappingId == null) throw failure("ACTIVE_VERSION_MISSING", "当前数据版本缺少可执行规则或映射"); + if(unclassifiedRules!=null){ + sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor); + updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED"); + var selectedRules=unclassifiedRules; + var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor)); + jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token); + publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false); + jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token); + return; + } rebuildWithVersions(operation.id(), actor, token, null, mappingId, ruleId, true, "mapping".equals(kind) ? versionId : null, "rule".equals(kind) ? versionId : null); } diff --git a/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewMySqlTest.java index 02d28b7..01e518f 100644 --- a/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewMySqlTest.java @@ -39,10 +39,10 @@ class ClassificationPreviewMySqlTest { private ClassificationPreviewService.Condition equal(String field,String value){return new ClassificationPreviewService.Condition(field,"EQUALS",value);} @Test void removingTwoConditionsBroadensAcrossMonthsStagesAndExistingGroupsWithExactCaseMatching(){ var exact=List.of(equal("@department","A"),equal("@product","P"),equal("PDT代码","0001"),equal("销售片区","X")); - assertEquals(1L,service.preview(new ClassificationPreviewService.Request("all","G",exact),actor).get("matchedRows")); + assertEquals(0L,service.preview(new ClassificationPreviewService.Request("all","G",exact),actor).get("matchedRows")); var broad=service.preview(new ClassificationPreviewService.Request("all","G",List.of(equal("@department","A"),equal("销售片区","X"))),actor); - assertEquals(4L,broad.get("matchedRows"));assertEquals(3L,broad.get("changedRows"));assertEquals(0,new java.math.BigDecimal("100").compareTo((java.math.BigDecimal)broad.get("totalWeight"))); - assertEquals(2L,service.preview(new ClassificationPreviewService.Request("all","G",List.of(new ClassificationPreviewService.Condition("PDT代码","IS_BLANK",""))),actor).get("matchedRows")); + assertEquals(1L,broad.get("matchedRows"));assertEquals(1L,broad.get("changedRows"));assertEquals(0,new java.math.BigDecimal("20").compareTo((java.math.BigDecimal)broad.get("totalWeight"))); + assertEquals(0L,service.preview(new ClassificationPreviewService.Request("all","G",List.of(new ClassificationPreviewService.Condition("PDT代码","IS_BLANK",""))),actor).get("matchedRows")); assertEquals(0L,service.preview(new ClassificationPreviewService.Request("all","G",List.of(equal("@department","A' OR 1=1 --"))),actor).get("matchedRows")); assertEquals(3,jdbc.queryForObject("SELECT COUNT(*) FROM fact_black",Integer.class)); } diff --git a/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewTest.java b/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewTest.java index 45c1fa3..816a6bf 100644 --- a/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewTest.java @@ -10,17 +10,17 @@ import org.springframework.web.server.ResponseStatusException; import tools.jackson.databind.json.JsonMapper; class ClassificationPreviewTest { - @Test void removedConditionsBroadenAcrossBothStagesAndAllMonthsIncludingExistingGroupsWithoutWrites(){ + @Test void removedConditionsBroadenAcrossStagesAndMonthsButQueryOnlyUnclassifiedRows(){ var jdbc=mock(JdbcTemplate.class);var access=mock(CandidateAcceptanceService.class);var actor=mock(AppUserPrincipal.class); when(jdbc.queryForList(startsWith("SELECT release_id"),any(Object[].class))).thenReturn(List.of(Map.of("release_id","r"))); when(jdbc.queryForList(startsWith("SELECT * FROM ("),any(Object[].class))).thenReturn(List.of( - Map.of("stage","black","month","2026-01","current_group","已有分类","row_count",10L,"total_weight",100), - Map.of("stage","bright","month","2026-09","current_group","G","row_count",5L,"total_weight",20))); + Map.of("stage","black","month","2026-01","current_group","","row_count",10L,"total_weight",100), + Map.of("stage","bright","month","2026-09","current_group","","row_count",5L,"total_weight",20))); var service=new ClassificationPreviewService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),access); var request=new ClassificationPreviewService.Request("all","G",List.of(new ClassificationPreviewService.Condition("@department","EQUALS","A"))); - var result=service.preview(request,actor);assertEquals(15L,result.get("matchedRows"));assertEquals(10L,result.get("changedRows")); + var result=service.preview(request,actor);assertEquals(15L,result.get("matchedRows"));assertEquals(15L,result.get("changedRows")); var sql=org.mockito.ArgumentCaptor.forClass(String.class);verify(jdbc).queryForList(sql.capture(),eq("r"),eq("A"),eq("r"),eq("A")); - assertTrue(sql.getValue().contains("fact_black"));assertTrue(sql.getValue().contains("fact_bright"));assertFalse(sql.getValue().contains("group_name=''"));assertFalse(sql.getValue().contains("sourcePdtCode"));assertFalse(sql.getValue().contains("f.month_key=?")); + assertTrue(sql.getValue().contains("fact_black"));assertTrue(sql.getValue().contains("fact_bright"));assertTrue(sql.getValue().contains("TRIM(f.group_name)=''"));assertFalse(sql.getValue().contains("sourcePdtCode"));assertFalse(sql.getValue().contains("f.month_key=?")); service.preview(new ClassificationPreviewService.Request("all","其他",request.conditions()),actor); verify(jdbc,times(1)).queryForList(startsWith("SELECT * FROM ("),any(Object[].class));verify(access,times(2)).requireRead("r",actor); doThrow(new ResponseStatusException(org.springframework.http.HttpStatus.FORBIDDEN)).when(access).requireRead("r",actor);assertThrows(ResponseStatusException.class,()->service.preview(request,actor)); diff --git a/backend/src/test/java/cn/yunsheng/quality/ingestion/ConfigurableMatchRulesTest.java b/backend/src/test/java/cn/yunsheng/quality/ingestion/ConfigurableMatchRulesTest.java index c60e9a6..30b7ca2 100644 --- a/backend/src/test/java/cn/yunsheng/quality/ingestion/ConfigurableMatchRulesTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/ingestion/ConfigurableMatchRulesTest.java @@ -9,6 +9,28 @@ import org.junit.jupiter.api.Test; import tools.jackson.databind.json.JsonMapper; class ConfigurableMatchRulesTest { + @Test void unclassifiedRulesNeverReplaceLookupOrOrdinaryRuleAssignments() { + var pending = new LinkedHashMap(rule("pending", -10000, "all", "group", "补充分组", List.of(condition("销售片区", "EQUALS", "华东")))); + pending.put("onlyUnclassified", true); + var definition=Map.of("matchRules",List.of(pending,ConfigurableMatchRules.defaultRules().getFirst())); + assertTrue(ConfigurableMatchRules.validate(definition).isEmpty()); + var empty=mapper(Map.of(),Map.of(),definition); + assertEquals("补充分组",empty.transform("black",row("001","普通产品","华东","产业部"),2026,1).group()); + assertEquals("工业电机-VCM",empty.transform("black",row("001","VCM产品","华东","产业部"),2026,1).group()); + assertEquals("已有分组",mapper(Map.of("P1|华东","已有分组"),Map.of(),definition).transform("bright",row("001","普通产品","华东","产业部"),2026,1).group()); + } + @Test void reimportKeepsSavedGroupsWhenOnlyUnclassifiedRulesChange() { + var pending=new LinkedHashMap(rule("pending",-10000,"all","group","新补充分组",List.of(condition("销售片区","EQUALS","华东")))); + pending.put("onlyUnclassified",true); + var definition=Map.of("matchRules",List.of(pending,ConfigurableMatchRules.defaultRules().getFirst())); + var saved=Map.of(BusinessRowMapper.sourceRowKey("测试",2),"已保存归属"); + var mapper=mapper(Map.of(),Map.of(),definition); + assertTrue(mapper.hasUnclassifiedRules()); + assertEquals("已保存归属",mapper.preserveAssignedGroups(saved).transform("black",row("001","普通产品","华东","产业部"),2026,1).group()); + assertEquals("新补充分组",mapper.transform("black",row("001","普通产品","华东","产业部"),2026,1).group()); + assertEquals("工业电机-VCM",mapper.preserveAssignedGroups(saved).transform("black",row("001","VCM产品","华东","产业部"),2026,1).group()); + assertEquals("明确的新对应",mapper(Map.of("P1|华东","明确的新对应"),Map.of(),definition).preserveAssignedGroups(saved).transform("black",row("001","普通产品","华东","产业部"),2026,1).group()); + } @Test void missingRulesKeepLegacyVcmBehaviorAndExplicitEmptyArrayRemovesIt() { Map categories = Map.of("P1|华东", "原始分组"); var legacy = mapper(categories, Map.of(), Map.of()); diff --git a/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java new file mode 100644 index 0000000..cbb51cb --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java @@ -0,0 +1,68 @@ +package cn.yunsheng.quality.ingestion; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; +import cn.yunsheng.quality.identity.AppUserPrincipal; +import cn.yunsheng.quality.metrics.MetricsRollupService; +import java.util.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.*; +import org.springframework.transaction.support.TransactionTemplate; +import tools.jackson.databind.json.JsonMapper; + +@EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_COLUMN_TEST_CREATE_DATABASE",matches="true") +class UnclassifiedMappingRebuildMySqlTest { + JdbcTemplate admin,jdbc;String database;DriverManagerDataSource dataSource;ImportWorker worker;MetricsRollupService rollups;JsonMapper json=JsonMapper.builder().build(); + final String parent="11111111-1111-1111-1111-111111111111",black="22222222-2222-2222-2222-222222222222",bright="33333333-3333-3333-3333-333333333333"; + AppUserPrincipal actor=mock(AppUserPrincipal.class); + Map pending(){return Map.of("id","classification-test","name","区域补分类","enabled",true,"onlyUnclassified",true,"priority",-9900,"stage","all","target","group","value","G","conditions",List.of(Map.of("field","@department","operator","EQUALS","value","A"),Map.of("field","销售片区","operator","EQUALS","value","X")));} + @BeforeEach void setup()throws Exception{ + String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL"),user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?")); + admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_unclassified_delta_"+UUID.randomUUID().toString().replace("-","");admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + dataSource=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(dataSource); + jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY,source_revision_id VARCHAR(100),build_key CHAR(64),parser_version VARCHAR(100),rule_version VARCHAR(100),mapping_version VARCHAR(100),status VARCHAR(32),row_count BIGINT,rejected_count BIGINT,component_summary JSON,error_summary JSON,started_at TIMESTAMP(6) NULL,completed_at TIMESTAMP(6) NULL,parser_config JSON,parser_config_hash VARCHAR(64))"); + jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(190),status VARCHAR(32),parent_release_id CHAR(36),expected_current_release_id CHAR(36),rule_version VARCHAR(100),mapping_version VARCHAR(100),target_version_id VARCHAR(100),complete BOOLEAN,validation JSON,created_by CHAR(36))"); + jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id VARCHAR(100),source_revision_id VARCHAR(100),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))"); + for(String stage:List.of("black","bright"))jdbc.execute("CREATE TABLE fact_"+stage+"(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id VARCHAR(100),sheet_name VARCHAR(100),physical_row_no INT,month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(24,8),scrap_weight DECIMAL(24,8),total_weight DECIMAL(24,8),raw_values JSON,components JSON,UNIQUE(build_id,sheet_name,physical_row_no))"); + jdbc.execute("CREATE TABLE defect_component(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),stage VARCHAR(16),sheet_name VARCHAR(100),physical_row_no INT,defect_name VARCHAR(100),weight DECIMAL(24,8),source_field VARCHAR(100))"); + try(var stream=getClass().getResourceAsStream("/db/migration/V16__metric_rollups.sql")){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);} + jdbc.execute("CREATE TABLE import_run(id CHAR(36) PRIMARY KEY,idempotency_key VARCHAR(190),status VARCHAR(32),stage VARCHAR(80),requested_sources JSON,expected_current_release_id CHAR(36),created_by CHAR(36),processed BIGINT,total BIGINT,result_release_id CHAR(36),expected_mapping_version VARCHAR(100),expected_rule_version VARCHAR(100),workflow_operation_id CHAR(36),validation JSON,started_at TIMESTAMP(6),completed_at TIMESTAMP(6))"); + jdbc.execute("CREATE TABLE release_protocol_dependency(release_id CHAR(36),protocol_id VARCHAR(100),PRIMARY KEY(release_id,protocol_id))");jdbc.execute("CREATE TABLE release_source_exclusion(release_id CHAR(36),source_document_id VARCHAR(100),request_id VARCHAR(100))"); + jdbc.update("INSERT INTO data_release VALUES(?,'parent','PUBLISHED',NULL,NULL,'rule','old-map',NULL,TRUE,JSON_OBJECT(),?)",parent,UUID.randomUUID().toString()); + for(String build:List.of(black,bright)){jdbc.update("INSERT INTO transform_build VALUES(?,?,?,'parser','rule','old-map','COMPLETED',?,0,JSON_OBJECT(),JSON_ARRAY(),NOW(6),NOW(6),NULL,'legacy')",build,build,"x"+build,build.equals(black)?3:1);jdbc.update("INSERT INTO release_member VALUES(?,?,?,?)",parent,build,build,build);} + seed("black",black,1,"A",null,10,2);seed("black",black,2,"A","KEEP",5,1);seed("black",black,3,"B",null,2,0);seed("bright",bright,1,"A",null,20,3); + rollups=new MetricsRollupService(jdbc);rollups.refreshBuild(black);rollups.refreshBuild(bright); + worker=mock(ImportWorker.class);when(worker.loadMappings(eq("PROJECT_GROUP"),eq("new-map"),anyInt(),anyInt())).thenReturn(Map.of("001|G","PROJECT"));when(actor.getId()).thenReturn(UUID.randomUUID()); + } + void seed(String stage,String build,int row,String department,String group,int good,int scrap){ + var components=new LinkedHashMap();components.put("sourcePdtCode","P");components.put("sourceSalesRegion","X");components.put("sourceCustomerCode","001");components.put("categoryMappingStatus",group==null?"unmapped":"mapped");components.put("projectGroupStatus","source_key_blank");components.put(stage.equals("black")?"black_material_scrap":"bright_material_scrap",scrap); + jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,sheet_name,physical_row_no,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,?,'S',?,'2026-01',?,?,NULL,'P',?,?,?,CAST(? AS JSON),CAST(? AS JSON))",build,build,row,department,group,good,scrap,good+scrap,json.writeValueAsString(Map.of("客户代码","001","PDT代码","P","销售片区","X")),json.writeValueAsString(components)); + jdbc.update("INSERT INTO defect_component(build_id,stage,sheet_name,physical_row_no,defect_name,weight,source_field) VALUES(?,?,'S',?,'料废',?,'原字段')",build,stage,row,scrap); + } + UnclassifiedMappingRebuild.Candidate run(MetricsRollupService metrics){return new TransactionTemplate(new DataSourceTransactionManager(dataSource)).execute(status->new UnclassifiedMappingRebuild(jdbc,json,worker,metrics).create(UUID.randomUUID().toString(),parent,"new-map","rule",Map.of("matchRules",List.of(pending())),List.of(pending()),actor));} + @AfterEach void cleanup(){if(admin!=null&&database!=null&&database.matches("quality_unclassified_delta_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);} + @Test void onlyEmptyRowsAreMatchedAndOldFactsWeightsAndExistingClassificationsRemainUnchanged(){ + var candidate=run(rollups);assertEquals(3,candidate.scannedRows());assertEquals(2,candidate.classifiedRows()); + assertEquals(2,jdbc.queryForObject("SELECT COUNT(*) FROM release_member m JOIN fact_black f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND f.group_name IS NULL",Integer.class,parent)); + assertEquals("KEEP",jdbc.queryForObject("SELECT f.group_name FROM release_member m JOIN fact_black f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND f.physical_row_no=2",String.class,candidate.releaseId())); + assertEquals("G",jdbc.queryForObject("SELECT f.group_name FROM release_member m JOIN fact_black f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND f.physical_row_no=1",String.class,candidate.releaseId())); + assertEquals("PROJECT",jdbc.queryForObject("SELECT f.project_group FROM release_member m JOIN fact_bright f ON f.build_id=m.transform_build_id WHERE m.release_id=?",String.class,candidate.releaseId())); + var before=jdbc.queryForObject("SELECT SUM(total_weight) FROM metric_rollup r JOIN release_member m ON m.transform_build_id=r.build_id WHERE m.release_id=?",java.math.BigDecimal.class,parent);var after=jdbc.queryForObject("SELECT SUM(total_weight) FROM metric_rollup r JOIN release_member m ON m.transform_build_id=r.build_id WHERE m.release_id=?",java.math.BigDecimal.class,candidate.releaseId());assertEquals(0,before.compareTo(after)); + assertEquals(3,jdbc.queryForObject("SELECT SUM(excluded_blank_rows) FROM metric_rollup r JOIN release_member m ON m.transform_build_id=r.build_id WHERE m.release_id=?",Integer.class,parent)); + assertEquals(1,jdbc.queryForObject("SELECT SUM(excluded_blank_rows) FROM metric_rollup r JOIN release_member m ON m.transform_build_id=r.build_id WHERE m.release_id=?",Integer.class,candidate.releaseId())); + verify(worker,never()).execute(anyString());verify(worker).captureMappingIssueRows(eq(candidate.runId()),eq(candidate.releaseId()),argThat(rows->rows.size()==2)); + } + @Test void failureRollsBackDerivedSnapshotWithoutTouchingThePublishedFacts(){ + var failing=spy(rollups);doThrow(new IllegalStateException("forced rollback")).when(failing).refreshGroups(anyString(),anySet()); + assertThrows(IllegalStateException.class,()->run(failing));assertEquals(1,jdbc.queryForObject("SELECT COUNT(*) FROM data_release",Integer.class));assertEquals(2,jdbc.queryForObject("SELECT COUNT(*) FROM transform_build",Integer.class));assertEquals(0,jdbc.queryForObject("SELECT COUNT(*) FROM import_run",Integer.class));assertEquals(3,jdbc.queryForObject("SELECT COUNT(*) FROM fact_black",Integer.class)); + } + @Test void reimportUsesAssignedGroupsOnlyFromItsBoundRevisionAndBaseline(){ + String run=UUID.randomUUID().toString();jdbc.update("INSERT INTO import_run(id,expected_current_release_id) VALUES(?,?)",run,parent); + assertEquals(Map.of(BusinessRowMapper.sourceRowKey("S",2),"KEEP"),ImportWorker.loadAssignedGroups(jdbc,"black",run,black)); + assertTrue(ImportWorker.loadAssignedGroups(jdbc,"black",run,bright).isEmpty()); + assertTrue(ImportWorker.loadAssignedGroups(jdbc,"black",UUID.randomUUID().toString(),black).isEmpty()); + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingScopeTest.java b/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingScopeTest.java new file mode 100644 index 0000000..8311d59 --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingScopeTest.java @@ -0,0 +1,14 @@ +package cn.yunsheng.quality.ingestion; +import static org.junit.jupiter.api.Assertions.*; +import java.util.*; +import org.junit.jupiter.api.Test; +class UnclassifiedMappingScopeTest { + Map rule(boolean scoped){return Map.of("id","test","onlyUnclassified",scoped,"enabled",true,"target","group","conditions",List.of(Map.of("field","@department","operator","EQUALS","value","A")));} + @Test void newScopedRuleChoosesLocalProcessingAndMixedEditsCannotFallBackToFullRebuild(){ + var before=Map.of("categoryLookup",Map.of("P|X","G"),"matchRules",List.of()); + var after=Map.of("categoryLookup",Map.of("P|X","G"),"matchRules",List.of(rule(true)));assertEquals(1,UnclassifiedMappingRebuild.changedRules(before,after).size()); + var mixed=new LinkedHashMap<>(after);mixed.put("categoryLookup",Map.of("P|X","OTHER"));assertThrows(IllegalArgumentException.class,()->UnclassifiedMappingRebuild.changedRules(before,mixed)); + assertThrows(IllegalArgumentException.class,()->UnclassifiedMappingRebuild.changedRules(after,Map.of("categoryLookup",before.get("categoryLookup"),"matchRules",List.of(rule(false))))); + assertTrue(UnclassifiedMappingRebuild.unchanged(Map.of(),Map.of("matchRules",ConfigurableMatchRules.defaultRules()))); + } +} diff --git a/web/src/ClassificationRuleEditor.tsx b/web/src/ClassificationRuleEditor.tsx index a4e715e..5c594ac 100644 --- a/web/src/ClassificationRuleEditor.tsx +++ b/web/src/ClassificationRuleEditor.tsx @@ -41,7 +41,7 @@ export function ClassificationRuleEditor({ children, onUnauthorized }: { catch(value){setError(value instanceof Error?value.message:'规则保存失败');} }; return <>{children(open)}{revision.current++;setBusy(false);setSelection(null);}}> {project?

客户代码 {String(selection?.row.customerCode)} · 业务分组 {String(selection?.row.businessGroup)}

:<>
@@ -59,7 +59,7 @@ export function ClassificationRuleEditor({ children, onUnauthorized }: {
)}} - {!project&&
保留条件同时满足时匹配,覆盖全部月份和来源文件。
} + {!project&&
只处理未分类记录,覆盖全部月份和来源文件;已有分类保持不动。
} {error&&} {preview&&
命中 {Number(preview.matchedRows).toLocaleString()} 行总重量 {Number(preview.totalWeight).toLocaleString()} kg0?'orange':'default'}>当前归属不同 {Number(preview.changedRows).toLocaleString()} 行 JSON.stringify([row.stage,row.department,row.month,row.currentGroup])} dataSource={(preview.items??[]) as Record[]} diff --git a/web/src/ManagementPages.tsx b/web/src/ManagementPages.tsx index 46a94ae..3331eef 100644 --- a/web/src/ManagementPages.tsx +++ b/web/src/ManagementPages.tsx @@ -555,7 +555,7 @@ function ConfigManagementPage({ kind, user, onUnauthorized, onNavigate, extraSec }, ]; - const simpleStageLabel: Record = { QUEUED: '等待处理', VALIDATING: '检查配置与来源', VERSION_CREATED: '保存配置', REBUILDING: '重新计算报表', PUBLISHING: '更新当前报表', COMPLETED: '已完成', FAILED: '处理失败' }; + const simpleStageLabel: Record = { QUEUED: '等待处理', VALIDATING: '检查配置与来源', VERSION_CREATED: '保存配置', CLASSIFYING_UNASSIGNED:'处理未分类记录', REBUILDING: '重新计算报表', PUBLISHING: '更新当前报表', COMPLETED: '已完成', FAILED: '处理失败' }; const applyCurrent = async () => { try { await form.validateFields(); diff --git a/web/src/StructuredConfigEditors.tsx b/web/src/StructuredConfigEditors.tsx index e24b44c..6d8a7e5 100644 --- a/web/src/StructuredConfigEditors.tsx +++ b/web/src/StructuredConfigEditors.tsx @@ -198,7 +198,7 @@ export function MappingDefinitionEditor({ value, onChange, simplified = false, r } type MatchCondition = { field: string; operator: string; value?: string }; -type MatchRule = { id?: string; name: string; enabled: boolean; priority: number; stage: string; conditions: MatchCondition[]; target: string; value: string }; +type MatchRule = { onlyUnclassified?:boolean; id?: string; name: string; enabled: boolean; priority: number; stage: string; conditions: MatchCondition[]; target: string; value: string }; type MatchMetadata = { fields?: unknown[]; operators?: unknown[]; targets?: unknown[]; stages?: unknown[]; defaultRules?: unknown[]; limits?: Record }; const defaultMatchRules: MatchRule[] = [{ id: 'default-vcm-group', name: '成品规格含VCM归入工业电机-VCM', enabled: true, priority: 100, stage: 'all', conditions: [{ field: '成品规格', operator: 'CONTAINS', value: 'VCM' }], target: 'group', value: '工业电机-VCM' }]; const plainObject = (value: unknown) => value && typeof value === 'object' && !Array.isArray(value) ? value as Record : {}; @@ -229,11 +229,11 @@ export function MatchRulesDefinitionEditor({ value, onChange, metadata, readOnly const sourceRules = Array.isArray(value.matchRules) ? value.matchRules : (metadata?.defaultRules ?? defaultMatchRules); const rules = sourceRules.map((item, index) => { const row = plainObject(item); - return { id: String(row.id || `match-${index + 1}`), name: String(row.name || `条件规则 ${index + 1}`), enabled: row.enabled !== false, priority: Number(row.priority ?? (index + 1) * 100), stage: String(row.stage || 'all'), conditions: Array.isArray(row.conditions) ? row.conditions.map(condition => { const conditionRow = plainObject(condition); return { field: String(conditionRow.field || ''), operator: String(conditionRow.operator || 'EQUALS'), ...(conditionRow.value === undefined ? {} : { value: String(conditionRow.value) }) }; }) : [], target: String(row.target || 'group'), value: String(row.value || '') } as MatchRule; + return { ...(row.onlyUnclassified===undefined?{}:{onlyUnclassified:row.onlyUnclassified===true}), id: String(row.id || `match-${index + 1}`), name: String(row.name || `条件规则 ${index + 1}`), enabled: row.enabled !== false, priority: Number(row.priority ?? (index + 1) * 100), stage: String(row.stage || 'all'), conditions: Array.isArray(row.conditions) ? row.conditions.map(condition => { const conditionRow = plainObject(condition); return { field: String(conditionRow.field || ''), operator: String(conditionRow.operator || 'EQUALS'), ...(conditionRow.value === undefined ? {} : { value: String(conditionRow.value) }) }; }) : [], target: String(row.target || 'group'), value: String(row.value || '') } as MatchRule; }); const labelFor = (options: MatchOption[], id: string) => options.find(option => option.value === id)?.label || id; const conditionText = (condition: MatchCondition) => `${labelFor(fields, condition.field)} ${labelFor(operators, condition.operator)}${['IS_BLANK', 'IS_NOT_BLANK'].includes(condition.operator) ? '' : ` ${condition.value || '…'}`}`; - const readableRule = (rule: MatchRule) => `当 ${rule.conditions.map(conditionText).join(' 且 ') || '(未设置条件)'} → ${labelFor(targets, rule.target)}:${rule.value || '(未填写)'}`; + const readableRule = (rule: MatchRule) => `${rule.onlyUnclassified?'仅未分类:':''}当 ${rule.conditions.map(conditionText).join(' 且 ') || '(未设置条件)'} → ${labelFor(targets, rule.target)}:${rule.value || '(未填写)'}`; const commit = (next: MatchRule[]) => onChange({ ...value, matchRules: next.map((rule, index) => ({ ...rule, id: rule.id || `match-${index + 1}` })) }); const open = (rule?: MatchRule) => { setError(''); @@ -268,7 +268,7 @@ export function MatchRulesDefinitionEditor({ value, onChange, metadata, readOnly { title: '判断值', render: (_, condition, index) => ['IS_BLANK', 'IS_NOT_BLANK'].includes(condition.operator) ? 无需填写 : setEditing({ ...editing, conditions: editing.conditions.map((row, rowIndex) => rowIndex === index ? { ...row, value: event.target.value } : row) })} /> }, { title: '', width: 82, render: (_, __, index) => }, ]} /> -
setEditing({ ...editing, value: event.target.value })} placeholder="例如:工业电机-VCM" />
+ rule.id===editing.id && rule.onlyUnclassified===true)} checkedChildren="仅未分类" unCheckedChildren="全部匹配行" onChange={onlyUnclassified=>setEditing({...editing,onlyUnclassified})}/>
setEditing({ ...editing, value: event.target.value })} placeholder="例如:工业电机-VCM" />
{error && } } diff --git a/web/src/classificationMaintenance.ts b/web/src/classificationMaintenance.ts index f75e9bc..b05798e 100644 --- a/web/src/classificationMaintenance.ts +++ b/web/src/classificationMaintenance.ts @@ -7,7 +7,7 @@ export function classificationConditions(row: Record) { return [['@department', row.department], ['@product', row.product], ['PDT代码', row.pdtCode], ['销售片区', row.salesRegion]].map(([field,value]) => ({field: String(field), operator: text(value) ? 'EQUALS' : 'IS_BLANK', value: text(value)})); } export function classificationSignature(stage: string, conditions: ClassificationCondition[]) { - return JSON.stringify([stage, (Array.isArray(conditions) ? conditions : []).map(c => [c.field, c.operator, c.value ?? '']).sort((a,b)=>JSON.stringify(a).localeCompare(JSON.stringify(b)))]); + return JSON.stringify([stage, (Array.isArray(conditions) ? conditions : []).map(c => [c.field, c.operator, c.value ?? '']).sort((a,b)=>JSON.stringify(a)JSON.stringify(b)?1:0)]); } export function classificationRuleId(row: Record, options: ClassificationRuleOptions = {}) { const scope = classificationSignature(options.stage ?? text(row.stage), options.conditions ?? classificationConditions(row)); @@ -45,12 +45,13 @@ export function editClassification(base: Definition, selection: ClassificationSe if (!conditions.length || conditions.length > 4 || new Set(conditions.map(c=>c.field)).size !== conditions.length) throw new Error('请保留至少一项且不重复的匹配条件'); if (conditions.some(c=>!['@department','@product','PDT代码','销售片区'].includes(c.field) || !['EQUALS','IS_BLANK'].includes(c.operator) || (c.operator === 'EQUALS' && !c.value.trim()))) throw new Error('请检查匹配条件'); const rules = Array.isArray(result.matchRules) ? result.matchRules as Definition[] : structuredClone(defaultRules) as Definition[]; - const id = classificationRuleId(row, {conditions,stage}); + const same = rules.find(rule=>rule.target==='group' && rule.value===target && classificationSignature(String(rule.stage),rule.conditions as ClassificationCondition[])===classificationSignature(stage,conditions)); + const id = same?.id ? String(same.id) : classificationRuleId(row, {conditions,stage}); const prior = rules.find(rule => rule.id === id); if (prior && classificationSignature(String(prior.stage), prior.conditions as ClassificationCondition[]) !== classificationSignature(stage,conditions)) throw new Error('规则标识冲突,请在条件匹配中维护'); const duplicate = rules.find(rule=>rule.enabled !== false && rule.target === 'group' && rule.value !== target && classificationSignature(String(rule.stage), rule.conditions as ClassificationCondition[]) === classificationSignature(stage,conditions)); if (duplicate) throw new Error(`相同条件已指向“${String(duplicate.value)}”,请在条件匹配中编辑已有规则`); - result.matchRules = [{ id, name: options.name?.trim() || `${target}分类规则`, enabled: true, priority: -10000 + (4-conditions.length)*100 + (stage === 'all' ? 10 : 0), stage, conditions, target: 'group', value: target }, ...rules.filter(rule => rule.id !== id)]; + result.matchRules = [{ id, name: options.name?.trim() || `${target}分类规则`, enabled: true, onlyUnclassified:true, priority: -10000 + (4-conditions.length)*100 + (stage === 'all' ? 10 : 0), stage, conditions, target: 'group', value: target }, ...rules.filter(rule => rule.id !== id)]; if ((result.matchRules as unknown[]).length > 100) throw new Error('条件规则已满100条,请先整理规则'); } return result; diff --git a/web/src/configurationWorkbook.ts b/web/src/configurationWorkbook.ts index 154d28f..8feea69 100644 --- a/web/src/configurationWorkbook.ts +++ b/web/src/configurationWorkbook.ts @@ -45,8 +45,8 @@ export function configurationTables(value:Definition,scope:ConfigurationScope='a } if(scope==='all'||scope==='matchRules'){ const rules=Array.isArray(value.matchRules)?value.matchRules:defaults; - tables.push({name:'条件匹配',headers:['规则编号','规则名称','优先级','适用范围','目标类型','目标分类','状态','条件序号','条件字段','字段说明','判断方式','判断值'],rows:array(rules).flatMap((rule,index)=>array(rule.conditions).map((condition,n)=>[ - text(rule.id)||`match-${index+1}`,text(rule.name),text(rule.priority),STAGES[text(rule.stage)]??text(rule.stage),TARGETS[text(rule.target)]??text(rule.target),text(rule.value),state(rule.enabled),String(n+1),text(condition.field),FIELDS[text(condition.field)]??text(condition.field),OPERATORS[text(condition.operator)]??text(condition.operator),text(condition.value)]))}); + tables.push({name:'条件匹配',headers:['规则编号','规则名称','优先级','适用范围','目标类型','目标分类','状态','条件序号','条件字段','字段说明','判断方式','判断值','仅未分类'],rows:array(rules).flatMap((rule,index)=>array(rule.conditions).map((condition,n)=>[ + text(rule.id)||`match-${index+1}`,text(rule.name),text(rule.priority),STAGES[text(rule.stage)]??text(rule.stage),TARGETS[text(rule.target)]??text(rule.target),text(rule.value),state(rule.enabled),String(n+1),text(condition.field),FIELDS[text(condition.field)]??text(condition.field),OPERATORS[text(condition.operator)]??text(condition.operator),text(condition.value),yes(rule.onlyUnclassified===true)]))}); } if(scope==='all'){ tables.push({name:'组织层级',headers:['组织编码','上级组织编码'],rows:array(value.organizationHierarchy).map(row=>[text(row.code),text(row.parentCode)])}); @@ -64,8 +64,10 @@ export function configurationTables(value:Definition,scope:ConfigurationScope='a export function importConfigurationTables(base:Definition,tables:ConfigurationTable[],scope:ConfigurationScope,mode:ImportMode,defaults:unknown[]=[]){ const result=structuredClone(base);const expected=configurationTables(base,scope,defaults);const changes:{name:string;rows:number;added:number;updated:number;removed:number}[]=[]; const names=new Set(); - for(const table of tables){ + for(const incomingTable of tables){ + let table=incomingTable; const original=expected.find(item=>item.name===table.name);if(!original)throw new Error(`工作表“${table.name}”不属于当前配置类型`); + if(table.name==='条件匹配'&&equal(table.headers,original.headers.slice(0,-1))){const prior=new Map(array(Array.isArray(base.matchRules)?base.matchRules:defaults).map(rule=>[text(rule.id),rule]));table={...table,headers:original.headers,rows:table.rows.map(row=>[...row,yes(prior.get(row[0])?.onlyUnclassified===true)])};} if(names.has(table.name))throw new Error(`重复工作表:${table.name}`);names.add(table.name); if(!equal(table.headers,original.headers))throw new Error(`${table.name}表头不匹配,请使用本页面导出的表格`); if(table.rows.length>20000)throw new Error(`${table.name}超过20000行上限`); @@ -98,12 +100,12 @@ export function importConfigurationTables(base:Definition,tables:ConfigurationTa }else if(table.name==='条件匹配'){ const previous=array(Array.isArray(base.matchRules)?base.matchRules:defaults);oldIds=previous.map(row=>text(row.id));const byId=new Map(); for(const [n,cells] of table.rows.entries()){ - const [id,name,priority,stageLabel,targetLabel,value,status,position,fieldLabel,_fieldDescription,operatorLabel,conditionValue]=cells; + const [id,name,priority,stageLabel,targetLabel,value,status,position,fieldLabel,_fieldDescription,operatorLabel,conditionValue,unclassifiedOnly]=cells; const stage=reverse(STAGES,stageLabel),target=reverse(TARGETS,targetLabel),field=fieldLabel,operator=reverse(OPERATORS,operatorLabel); if(!/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test(id)||!name||!value||name.length>200||value.length>200)throw new Error(`条件匹配第${n+2}行规则编号、名称或目标分类无效`); if(!STAGES[stage]||!TARGETS[target]||!OPERATORS[operator]||!field||(field.startsWith('@')&&!Object.hasOwn(FIELDS,field))||field.length>200||(!['IS_BLANK','IS_NOT_BLANK'].includes(operator)&&!conditionValue)||conditionValue.length>200)throw new Error(`条件匹配第${n+2}行条件无效`); if(['IS_BLANK','IS_NOT_BLANK'].includes(operator)&&conditionValue)throw new Error(`条件匹配第${n+2}行空值判断不应填写判断值`); - const metadata={id,name,priority:number(priority,'优先级',-10000,10000),stage,target,value,enabled:enabled(status)}; + const prior=previous.find(rule=>rule.id===id);const scope=enabled(unclassifiedOnly,false);const metadata={id,name,priority:number(priority,'优先级',-10000,10000),stage,target,value,enabled:enabled(status),...(scope||Object.hasOwn(prior??{},'onlyUnclassified')?{onlyUnclassified:scope}:{})};if(scope&&target!=='group')throw new Error('仅未分类只适用于业务分组规则'); const row=byId.get(id);if(row&&!equal({...row,conditions:undefined},{...metadata,conditions:undefined}))throw new Error(`规则${id}各行的名称、目标、优先级或状态不一致`); const rule=row??{...metadata,conditions:[]};const conditions=rule.conditions as Definition[];const order=number(position,'条件序号',1,20); if(conditions.some(c=>c.order===order))throw new Error(`规则${id}条件序号重复`);conditions.push({order,field,operator,value:conditionValue});byId.set(id,rule); @@ -148,7 +150,7 @@ export async function createConfigurationWorkbook(value:Definition,scope:Configu table.rows.forEach(row=>sheet.addRow(row.map((value,index)=>numeric.has(table.headers[index])&&value!==''?Number(value):value)));sheet.columns.forEach((column,index)=>{column.numFmt=numeric.has(table.headers[index])?'0':'@';column.alignment={vertical:'middle',wrapText:true};}); sheet.getRow(1).font={bold:true,color:{argb:'FFFFFFFF'}};sheet.getRow(1).fill={type:'pattern',pattern:'solid',fgColor:{argb:'FF5769B8'}};sheet.getRow(1).height=30; sheet.autoFilter={from:{row:1,column:1},to:{row:Math.max(1,table.rows.length+1),column:table.headers.length}}; - sheet.eachRow((row,n)=>{if(n>1){let lines=1;row.eachCell((cell,index)=>{const width=Number(sheet.getColumn(index).width)||22;const length=Array.from(cell.text).reduce((sum,char)=>sum+(char.charCodeAt(0)>255?2:1),0);lines=Math.max(lines,Math.ceil(length/Math.max(8,width-2)));const header=table.headers[index-1];const choices=header==='状态'?['启用','停用']:['明确不设归属','允许空值','使用默认对应'].includes(header)?['是','否']:header==='适用范围'||header==='阶段'?Object.values(STAGES):header==='目标类型'?Object.values(TARGETS):header==='判断方式'&&table.name==='条件匹配'?Object.values(OPERATORS):null;if(choices)cell.dataValidation={type:'list',allowBlank:false,formulae:['"'+choices.join(',')+'"']};});row.height=Math.min(409,Math.max(24,lines*16));}}); + sheet.eachRow((row,n)=>{if(n>1){let lines=1;row.eachCell((cell,index)=>{const width=Number(sheet.getColumn(index).width)||22;const length=Array.from(cell.text).reduce((sum,char)=>sum+(char.charCodeAt(0)>255?2:1),0);lines=Math.max(lines,Math.ceil(length/Math.max(8,width-2)));const header=table.headers[index-1];const choices=header==='状态'?['启用','停用']:['明确不设归属','允许空值','使用默认对应','仅未分类'].includes(header)?['是','否']:header==='适用范围'||header==='阶段'?Object.values(STAGES):header==='目标类型'?Object.values(TARGETS):header==='判断方式'&&table.name==='条件匹配'?Object.values(OPERATORS):null;if(choices)cell.dataValidation={type:'list',allowBlank:false,formulae:['"'+choices.join(',')+'"']};});row.height=Math.min(409,Math.max(24,lines*16));}}); } return workbook; } diff --git a/web/verification/classification_maintenance.test.mjs b/web/verification/classification_maintenance.test.mjs index 017a5bc..3c63a4e 100644 --- a/web/verification/classification_maintenance.test.mjs +++ b/web/verification/classification_maintenance.test.mjs @@ -13,7 +13,7 @@ test('blank-key source creates a narrowly scoped rule and preserves inherited de {field:'PDT代码',operator:'IS_BLANK',value:''}, {field:'销售片区',operator:'EQUALS',value:'销售区A'}, ]); - assert.equal(result.matchRules[0].stage,'black');assert.deepEqual(result.matchRules[1],defaults[0]); + assert.equal(result.matchRules[0].onlyUnclassified,true);assert.equal(result.matchRules[0].stage,'black');assert.deepEqual(result.matchRules[1],defaults[0]); assert.deepEqual(base,{projectGroupLookup:{'001|G':'项目组A'},extra:['keep']}); const changed=editClassification(result,selection,'业务组A',defaults); assert.equal(changed.matchRules.length,2);assert.equal(changed.matchRules[0].value,'业务组A'); @@ -43,11 +43,11 @@ test('editing a disabled project relationship enables only that relationship and assert.throws(()=>editClassification(base,selection,'不存在的项目组',[]),/未登记/); }); -test('removable conditions apply to all matching rows without an implicit unclassified predicate',()=>{ +test('removable conditions retain an explicit unclassified-only scope',()=>{ const options={stage:'all',conditions:[{field:'@department',operator:'EQUALS',value:'产业部A'},{field:'销售片区',operator:'EQUALS',value:'销售区A'}]}; const result=editClassification({matchRules:[]},selection,'G',[],options); assert.deepEqual(result.matchRules[0].conditions,options.conditions);assert.equal(result.matchRules[0].stage,'all'); - assert.equal(result.matchRules[0].conditions.some(c=>c.field==='@group'),false); + assert.equal(result.matchRules[0].onlyUnclassified,true); assert.equal(classificationRuleId(source,options),classificationRuleId(source,{...options,conditions:[...options.conditions].reverse()})); const specific=editClassification({matchRules:[]},selection,'G',[],{stage:'all'}); assert.ok(specific.matchRules[0].priorityt.name==='指标计算公式').rows[0][1]='qualified/(qualified+scrap)';const imported=importConfigurationTables(rules,tables,'rules','MERGE').definition;assert.equal(imported.metricRules.RATE.formula,'qualified/(qualified+scrap)');assert.equal(imported.metricRules.RATE.nullPolicy,undefined); const fields={filterFields:[{key:'0001',label:'客户',blackColumn:'客户代码',brightColumn:'客户代码',enabled:false,allowEmpty:true,sortOrder:1}]};const book=await createConfigurationWorkbook(fields,'filterFields');tables=await readConfigurationWorkbook(await book.xlsx.writeBuffer());assert.deepEqual(importConfigurationTables(fields,tables,'filterFields','REPLACE').definition,fields); }); + +test('legacy twelve-column workbooks preserve the existing unclassified-only restriction',()=>{ + const tables=configurationTables(base,'matchRules');tables[0].headers.pop();tables[0].rows.forEach(row=>row.pop()); + for(const mode of ['MERGE','REPLACE'])assert.equal(importConfigurationTables(base,tables,'matchRules',mode).definition.matchRules[0].onlyUnclassified,true); +});