修复:分类规则仅处理未分类并保留已有归属
This commit is contained in:
+2
-2
@@ -7,7 +7,7 @@ import java.util.*;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/** Counts current immutable source rows, including already classified rows, without changing mappings. */
|
||||
/** 只预览当前未分类记录的归类影响,不改动对应关系。 */
|
||||
@Service
|
||||
public class ClassificationPreviewService {
|
||||
private final GovernanceSupport g;
|
||||
@@ -47,7 +47,7 @@ public class ClassificationPreviewService {
|
||||
private List<Map<String,Object>> load(String release,Request request){
|
||||
List<String> queries=new ArrayList<>();List<Object> args=new ArrayList<>();
|
||||
for(String stage:request.stage().equals("all")?List.of("black","bright"):List.of(request.stage())){
|
||||
StringBuilder where=new StringBuilder(" WHERE rm.release_id=?");args.add(release);
|
||||
StringBuilder where=new StringBuilder(" WHERE rm.release_id=? AND (f.group_name IS NULL OR TRIM(f.group_name)='')");args.add(release);
|
||||
for(var condition:request.conditions()){
|
||||
String expression=FIELDS.get(condition.field());
|
||||
if(condition.operator().equals("IS_BLANK"))where.append(" AND (").append(expression).append(" IS NULL OR REGEXP_LIKE(").append(expression).append(",'^[[:space:]]*$'))");
|
||||
|
||||
@@ -20,12 +20,18 @@ public class MappingIssueService {
|
||||
private final MappingIssueDimensions dimensions;private final GovernanceSupport g;private final CandidateAcceptanceService acceptance;private final ImportRunService imports;private final ObjectProvider<ImportWorker> worker;private final TransactionTemplate transaction;
|
||||
public MappingIssueService(GovernanceSupport g,CandidateAcceptanceService acceptance,ImportRunService imports,ObjectProvider<ImportWorker> worker,PlatformTransactionManager manager){this.g=g;this.dimensions=new MappingIssueDimensions(g);this.acceptance=acceptance;this.imports=imports;this.worker=worker;this.transaction=new TransactionTemplate(manager);this.transaction.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);}
|
||||
/** Aggregated from immutable transformed facts, retaining their exact source/build lineage. */
|
||||
public void capture(String runId,String releaseId){
|
||||
Map<String,Group> groups=new TreeMap<>();for(String stage:List.of("black","bright"))for(Map<String,Object> f:g.rows("SELECT f.group_name,f.total_weight,f.components,f.raw_values,f.month_key,rm.source_document_id,rm.source_revision_id,rm.transform_build_id FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id WHERE rm.release_id=? AND (JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.categoryMappingStatus'))='unmapped' OR JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.projectGroupStatus'))='unmapped')",releaseId)){
|
||||
public record SourceRow(String stage,String buildId,String sheet,int row){}
|
||||
public void capture(String runId,String releaseId){captureRows(runId,releaseId,null);}
|
||||
public void captureRows(String runId,String releaseId,List<SourceRow> selected){
|
||||
Map<String,Group> groups=new TreeMap<>();for(String stage:List.of("black","bright")){
|
||||
String restriction="";List<Object> args=new ArrayList<>(List.of(releaseId));
|
||||
if(selected!=null){var rows=selected.stream().filter(row->stage.equals(row.stage())).toList();if(rows.isEmpty())continue;restriction=" AND ("+String.join(" OR ",Collections.nCopies(rows.size(),"(f.build_id=? AND f.sheet_name=? AND f.physical_row_no=?)"))+")";for(var row:rows){args.add(row.buildId());args.add(row.sheet());args.add(row.row());}}
|
||||
for(Map<String,Object> f:g.rows("SELECT f.group_name,f.total_weight,f.components,f.raw_values,f.month_key,rm.source_document_id,rm.source_revision_id,rm.transform_build_id FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id WHERE rm.release_id=? AND (JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.categoryMappingStatus'))='unmapped' OR JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.projectGroupStatus'))='unmapped')"+restriction,args.toArray())){
|
||||
Map<String,Object> c=map(f.get("components")),raw=map(f.get("rawValues"));
|
||||
if("unmapped".equals(c.get("categoryMappingStatus")))add(groups,"CATEGORY",c.get("sourcePdtCode")+"|"+c.get("sourceSalesRegion"),f,raw,stage);
|
||||
if("unmapped".equals(c.get("projectGroupStatus")))add(groups,"PROJECT_GROUP",c.get("sourceCustomerCode")+"|"+f.get("groupName"),f,raw,stage);
|
||||
}
|
||||
}
|
||||
for(Group group:groups.values()){
|
||||
List<String> candidates=new ArrayList<>();for(String revision:group.revisions)candidates.addAll(g.jdbc.query("SELECT DISTINCT c.id FROM source_candidate c JOIN source_revision r ON r.sha256=c.sha256 JOIN source_document d ON d.id=r.source_document_id WHERE r.id=? AND "+SourceIdentity.canonicalSlotSql("c.source_slot")+"="+SourceIdentity.canonicalSlotSql("d.source_slot"),(rs,n)->rs.getString(1),revision));
|
||||
String id=GovernanceSupport.sha(runId+"\n"+group.type+"\n"+group.key);Map<String,Object> details=Map.of("sourceCandidateIds",candidates.stream().distinct().toList(),"sourceDocumentIds",group.documents,"sourceRevisionIds",group.revisions,"transformBuildIds",group.builds,"periods",group.periods,"originalValues",group.originals,"originalValueRows",group.originalValueRows,"releaseId",releaseId);
|
||||
|
||||
@@ -26,6 +26,7 @@ public final class BusinessRowMapper {
|
||||
private final Map<String,String> productLookup;
|
||||
private final ConfigurableRuleEngine rules;
|
||||
private final ConfigurableMatchRules matchRules;
|
||||
private final Map<String,String> savedGroups;
|
||||
|
||||
public BusinessRowMapper(Map<String, String> categoryLookup, Map<String, String> projectLookup, String mappingVersion) {
|
||||
this(categoryLookup,projectLookup,mappingVersion,Map.of(),Map.of());
|
||||
@@ -43,8 +44,18 @@ public final class BusinessRowMapper {
|
||||
this.departmentLookup=departmentLookup;this.productLookup=productLookup;
|
||||
this.rules=rules;
|
||||
this.matchRules=ConfigurableMatchRules.fromDefinition(mappingDefinition);
|
||||
this.savedGroups=Map.of();
|
||||
}
|
||||
|
||||
private BusinessRowMapper(BusinessRowMapper base,Map<String,String> savedGroups){
|
||||
this.categoryLookup=base.categoryLookup;this.projectLookup=base.projectLookup;this.mappingVersion=base.mappingVersion;
|
||||
this.departmentLookup=base.departmentLookup;this.productLookup=base.productLookup;this.rules=base.rules;this.matchRules=base.matchRules;
|
||||
this.savedGroups=Map.copyOf(savedGroups);
|
||||
}
|
||||
public boolean hasUnclassifiedRules(){return matchRules.hasUnclassifiedRules();}
|
||||
public BusinessRowMapper preserveAssignedGroups(Map<String,String> groups){return new BusinessRowMapper(this,groups);}
|
||||
static String sourceRowKey(String sheet,int row){return sheet+"\n"+row;}
|
||||
|
||||
public String mappingVersion() { return mappingVersion; }
|
||||
|
||||
public List<String> requiredHeaders(String stage) { return rules.requiredHeaders(stage,"black".equals(stage) ? BLACK_REQUIRED : BRIGHT_REQUIRED); }
|
||||
@@ -153,7 +164,7 @@ public final class BusinessRowMapper {
|
||||
product=matchRules.apply(stage,"product",matchSource,computed(department,null,product,null),product);
|
||||
String sourceKey=joinedKey(pdt,region);boolean blankCategoryKey=pdt==null||pdt.isEmpty()||region==null||region.isEmpty();
|
||||
String category=blankCategoryKey?null:categoryLookup.get(sourceKey);
|
||||
category=matchRules.apply(stage,"group",matchSource,computed(department,category,product,null),category);
|
||||
category=matchRules.applyGroup(stage,matchSource,computed(department,category,product,null),category,savedGroups.get(sourceRowKey(input.sheet(),input.physicalRow()))).value();
|
||||
boolean unmappedCategory=!blankCategoryKey&&category==null;
|
||||
String projectKey=joinedKey(customer,category);boolean blankProjectKey=customer==null||customer.isEmpty()||category==null||category.isEmpty();
|
||||
boolean projectMappingKnown=!blankProjectKey&&projectLookup.containsKey(projectKey);
|
||||
|
||||
@@ -21,11 +21,13 @@ public final class ConfigurableMatchRules {
|
||||
private static final Pattern UNSAFE_TEXT = Pattern.compile("(?i)(<\\s*script\\b|javascript\\s*:|on[a-z]+\\s*=|\\beval\\s*\\(|document\\.(?:cookie|write)|--|/\\*|\\*/|\\b(?:DROP|ALTER|DELETE|INSERT|UPDATE|TRUNCATE|CREATE|EXEC(?:UTE)?|UNION\\s+SELECT|SELECT\\s+.+\\s+FROM)\\b)");
|
||||
|
||||
private final List<Map<String, Object>> rules;
|
||||
private final List<Map<String,Object>> executionRules;
|
||||
|
||||
private ConfigurableMatchRules(List<Map<String, Object>> rules) {
|
||||
this.rules = rules.stream().map(ConfigurableMatchRules::copyRule)
|
||||
.sorted(Comparator.comparingInt(rule -> ((Number) rule.get("priority")).intValue()))
|
||||
.toList();
|
||||
this.executionRules=java.util.stream.Stream.concat(this.rules.stream().filter(rule->!Boolean.TRUE.equals(rule.get("onlyUnclassified"))),this.rules.stream().filter(rule->Boolean.TRUE.equals(rule.get("onlyUnclassified")))).toList();
|
||||
}
|
||||
|
||||
/** Default mapping behavior for old mapping versions that do not contain matchRules. */
|
||||
@@ -73,7 +75,8 @@ public final class ConfigurableMatchRules {
|
||||
errors.add(path + " 必须为对象");
|
||||
continue;
|
||||
}
|
||||
for (Object key : rule.keySet()) if (!Set.of("id", "name", "enabled", "priority", "stage", "conditions", "target", "value").contains(String.valueOf(key))) errors.add(path + " 未知字段: " + key);
|
||||
for (Object key : rule.keySet()) if (!Set.of("id", "name", "enabled", "priority", "stage", "conditions", "target", "value", "onlyUnclassified").contains(String.valueOf(key))) errors.add(path + " 未知字段: " + key);
|
||||
if(rule.containsKey("onlyUnclassified") && (!(rule.get("onlyUnclassified") instanceof Boolean) || Boolean.TRUE.equals(rule.get("onlyUnclassified")) && !"group".equals(rule.get("target")))) errors.add(path+" onlyUnclassified仅适用于业务分组且须为布尔值");
|
||||
String id = text(rule.get("id"));
|
||||
if (id != null) textBudget += id.length();
|
||||
if (id == null || !id.matches("[A-Za-z0-9][A-Za-z0-9_-]{0,63}")) errors.add(path + " id须为1至64位字母、数字、下划线或连字符");
|
||||
@@ -136,8 +139,15 @@ public final class ConfigurableMatchRules {
|
||||
}
|
||||
|
||||
public Match applyMatch(String stage, String target, Map<String, ?> raw, Map<String, String> computed, String current) {
|
||||
for (Map<String, Object> rule : rules) {
|
||||
return applyMatch(stage,target,raw,computed,current,false);
|
||||
}
|
||||
|
||||
private Match applyMatch(String stage,String target,Map<String,?> raw,Map<String,String> computed,String current,boolean ordinaryOnly){
|
||||
// 普通对应与规则先执行;补分类规则只接收仍为空的结果,不能抢占已有归属。
|
||||
for (Map<String, Object> rule : executionRules) {
|
||||
if(ordinaryOnly&&Boolean.TRUE.equals(rule.get("onlyUnclassified")))continue;
|
||||
if (!Boolean.TRUE.equals(rule.get("enabled")) || !target.equals(rule.get("target"))) continue;
|
||||
if(Boolean.TRUE.equals(rule.get("onlyUnclassified")) && current!=null && !current.isBlank()) continue;
|
||||
if (!"all".equals(rule.get("stage")) && !stage.equals(rule.get("stage"))) continue;
|
||||
boolean matches = true;
|
||||
for (Object rawCondition : (List<?>) rule.get("conditions")) {
|
||||
@@ -151,6 +161,15 @@ public final class ConfigurableMatchRules {
|
||||
|
||||
public record Match(boolean matched, String value) {}
|
||||
|
||||
/** 同一来源再次导入时,补分类规则仍不能覆盖其已保存的归属。普通规则和基础对应照常优先。 */
|
||||
public Match applyGroup(String stage,Map<String,?> raw,Map<String,String> computed,String current,String savedGroup) {
|
||||
if(savedGroup==null||savedGroup.isBlank())return applyMatch(stage,"group",raw,computed,current);
|
||||
var result=applyMatch(stage,"group",raw,computed,current,true);
|
||||
return result.matched()||result.value()!=null&&!result.value().isBlank()?result:new Match(false,savedGroup);
|
||||
}
|
||||
|
||||
public boolean hasUnclassifiedRules(){return rules.stream().anyMatch(rule->Boolean.TRUE.equals(rule.get("onlyUnclassified")));}
|
||||
|
||||
public List<Map<String, Object>> rules() { return rules.stream().map(ConfigurableMatchRules::copyRule).toList(); }
|
||||
|
||||
public static List<Map<String, Object>> fields() {
|
||||
|
||||
@@ -236,6 +236,8 @@ public class ImportWorker {
|
||||
List<BusinessRowMapper.RowValues> facts = new ArrayList<>(BATCH_SIZE);
|
||||
List<Issue> issues = new ArrayList<>();
|
||||
Set<String> availableHeaders = new HashSet<>();
|
||||
Map<String,String> savedGroups=rowMapper.hasUnclassifiedRules()?loadAssignedGroups(jdbc,candidate.stage(),runId,revisionId):Map.of();
|
||||
BusinessRowMapper effectiveMapper=rowMapper.preserveAssignedGroups(savedGroups);
|
||||
long[] rowCount = {0};
|
||||
int[] blocking = {0};
|
||||
int[] blankCategory = {0};
|
||||
@@ -244,7 +246,7 @@ public class ImportWorker {
|
||||
String declaredScope=jdbc.queryForObject("SELECT scope_code FROM source_candidate WHERE id=?",String.class,candidate.id().toString());
|
||||
XlsxStreamingReader.ReadSummary parsed = reader.read(archivePath, expectedSheet, parserConfig, row -> {
|
||||
availableHeaders.addAll(row.values().keySet());
|
||||
BusinessRowMapper.RowValues value = parserConfig==null?rowMapper.transform(candidate.stage(), row, candidate.year(), candidate.month()):rowMapper.transformConfigured(candidate.stage(),row,candidate.year(),candidate.month());
|
||||
BusinessRowMapper.RowValues value = parserConfig==null?effectiveMapper.transform(candidate.stage(), row, candidate.year(), candidate.month()):effectiveMapper.transformConfigured(candidate.stage(),row,candidate.year(),candidate.month());
|
||||
if(declaredScope!=null&&!"ALL".equals(declaredScope)&&!java.util.Objects.equals(cn.yunsheng.quality.common.DepartmentLabels.queryValue(declaredScope),value.department()))throw new IllegalArgumentException("来源行超出已声明组织范围,须修订来源或协议");
|
||||
facts.add(value);
|
||||
rowCount[0]++;
|
||||
@@ -325,6 +327,15 @@ public class ImportWorker {
|
||||
}
|
||||
}
|
||||
|
||||
static Map<String,String> loadAssignedGroups(JdbcTemplate jdbc,String stage,String runId,String revisionId){
|
||||
if(!Set.of("black","bright").contains(stage))throw new IllegalArgumentException("来源阶段无效");
|
||||
// 仅复用该任务基线中同一不可变来源修订的归属;新文件或修订不借用旧记录。
|
||||
Map<String,String> savedGroups=new HashMap<>();
|
||||
jdbc.queryForList("SELECT f.sheet_name,f.physical_row_no,f.group_name FROM fact_"+stage+" f JOIN release_member m ON m.transform_build_id=f.build_id JOIN import_run r ON r.expected_current_release_id=m.release_id WHERE r.id=? AND f.source_revision_id=? AND f.group_name IS NOT NULL AND TRIM(f.group_name)<>''",runId,revisionId)
|
||||
.forEach(row->savedGroups.put(BusinessRowMapper.sourceRowKey(String.valueOf(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue()),String.valueOf(row.get("group_name"))));
|
||||
return savedGroups;
|
||||
}
|
||||
|
||||
private void persistIssues(String runId, SourceFileCandidate candidate, List<Issue> issues) {
|
||||
for (Issue issue : issues) {
|
||||
jdbc.update("""
|
||||
@@ -416,6 +427,8 @@ public class ImportWorker {
|
||||
return cn.yunsheng.quality.governance.MasterDataSupport.enabledLookup(definition,lookup,values);
|
||||
}
|
||||
|
||||
void captureMappingIssueRows(String run,String release,java.util.List<cn.yunsheng.quality.governance.MappingIssueService.SourceRow> rows){mappingIssues.captureRows(run,release,rows);}
|
||||
|
||||
String reviewedMappingFor(String runId,String activeMapping){
|
||||
String frozen=jdbc.queryForObject("SELECT expected_mapping_version FROM import_run WHERE id=?",String.class,runId);
|
||||
if(frozen==null)return activeMapping;
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package cn.yunsheng.quality.ingestion;
|
||||
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.metrics.MetricsRollupService;
|
||||
import java.util.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/** 未分类维护只重新判断空归属行;其余事实与原汇总按不可变版本原样保留。 */
|
||||
public final class UnclassifiedMappingRebuild {
|
||||
private final JdbcTemplate jdbc;private final JsonMapper json;private final ImportWorker worker;private final MetricsRollupService rollups;
|
||||
public UnclassifiedMappingRebuild(JdbcTemplate jdbc,JsonMapper json,ImportWorker worker,MetricsRollupService rollups){this.jdbc=jdbc;this.json=json;this.worker=worker;this.rollups=rollups;}
|
||||
@SuppressWarnings("unchecked") static Map<String,Object> map(Object value){return value instanceof Map<?,?>?(Map<String,Object>)value:Map.of();}
|
||||
@SuppressWarnings("unchecked") static List<Map<String,Object>> rules(Map<String,Object> definition){return (List<Map<String,Object>>)definition.getOrDefault("matchRules",ConfigurableMatchRules.defaultRules());}
|
||||
static Map<String,Map<String,Object>> indexed(List<Map<String,Object>> rules){var result=new LinkedHashMap<String,Map<String,Object>>();for(var rule:rules)result.put(String.valueOf(rule.get("id")),rule);return result;}
|
||||
public static boolean unchanged(Map<String,Object> before,Map<String,Object> after){var left=new LinkedHashMap<>(before);var right=new LinkedHashMap<>(after);left.remove("matchRules");right.remove("matchRules");return left.equals(right)&&rules(before).equals(rules(after));}
|
||||
public static List<Map<String,Object>> changedRules(Map<String,Object> before,Map<String,Object> after){
|
||||
var old=indexed(rules(before));var next=indexed(rules(after));var changed=new LinkedHashSet<String>();changed.addAll(old.keySet());changed.addAll(next.keySet());changed.removeIf(id->Objects.equals(old.get(id),next.get(id)));
|
||||
boolean scoped=changed.stream().anyMatch(id->Boolean.TRUE.equals(map(old.get(id)).get("onlyUnclassified"))||Boolean.TRUE.equals(map(next.get(id)).get("onlyUnclassified")));
|
||||
if(!scoped)return null;
|
||||
var left=new LinkedHashMap<>(before);var right=new LinkedHashMap<>(after);left.remove("matchRules");right.remove("matchRules");
|
||||
if(!left.equals(right))throw new IllegalArgumentException("未分类维护不能同时修改其他对应或名录,请分开保存");
|
||||
var result=new ArrayList<Map<String,Object>>();
|
||||
for(String id:changed){var prior=old.get(id);var rule=next.get(id);
|
||||
if(rule==null){if(!Boolean.TRUE.equals(prior.get("onlyUnclassified")))throw new IllegalArgumentException("不能在未分类维护中移除普通匹配规则");continue;}
|
||||
if(!Boolean.TRUE.equals(rule.get("onlyUnclassified"))||!"group".equals(rule.get("target")))throw new IllegalArgumentException("未分类规则不能改为覆盖已有分类");
|
||||
for(var raw:(List<?>)rule.get("conditions")){var condition=map(raw);if(!Set.of("@department","@product","PDT代码","销售片区").contains(condition.get("field"))||!Set.of("EQUALS","IS_BLANK").contains(condition.get("operator")))throw new IllegalArgumentException("未分类维护仅支持产业部、产品、PDT代码和销售区域条件");}
|
||||
if(Boolean.TRUE.equals(rule.get("enabled")))result.add(rule);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
record Change(String stage,String parent,String sheet,int row,String month,String department,String product,String oldGroup,String oldProject,String group,String project,Map<String,Object> components){}
|
||||
public record Candidate(String releaseId,String runId,int scannedRows,int classifiedRows){}
|
||||
private String text(Object value){return value==null?null:String.valueOf(value);}
|
||||
private Map<String,Object> decode(Object value){if(value==null)return new LinkedHashMap<>();return new LinkedHashMap<>(json.readValue(String.valueOf(value),Map.class));}
|
||||
private String encode(Object value){return json.writeValueAsString(value);}
|
||||
private static String hash(String value){try{return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)));}catch(Exception error){throw new IllegalStateException(error);}}
|
||||
|
||||
public Candidate create(String operation,String parent,String mapping,String rule,Map<String,Object> definition,List<Map<String,Object>> changedRules,AppUserPrincipal actor){
|
||||
// 调用方事务覆盖候选、数据副本和进度;失败不改变旧版本,禁止执行 Excel 导入器。
|
||||
var matcher=ConfigurableMatchRules.fromDefinition(Map.of("matchRules",changedRules));var projectRules=ConfigurableMatchRules.fromDefinition(definition);
|
||||
var changes=new ArrayList<Change>();int scanned=0;var projectLookups=new HashMap<String,Map<String,String>>();
|
||||
for(String stage:List.of("black","bright")){
|
||||
var rows=jdbc.queryForList("SELECT f.build_id,f.sheet_name,f.physical_row_no,f.month_key,f.department,f.group_name,f.project_group,f.product,CAST(f.components AS CHAR) components,CAST(f.raw_values AS CHAR) raw_values,CAST(b.parser_config AS CHAR) parser_config FROM release_member m JOIN fact_"+stage+" f ON f.build_id=m.transform_build_id JOIN transform_build b ON b.id=f.build_id WHERE m.release_id=? AND (f.group_name IS NULL OR TRIM(f.group_name)='')",parent);
|
||||
scanned+=rows.size();
|
||||
for(var row:rows){
|
||||
var components=decode(row.get("components"));var raw=decode(row.get("raw_values"));var restored=BusinessRowMapper.restoreStoredRow(stage,text(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue(),raw,decode(row.get("parser_config")));
|
||||
restored.values().forEach((key,value)->raw.put(key,value==null?null:value.display()));
|
||||
raw.put("PDT代码",components.get("sourcePdtCode"));raw.put("销售片区",components.get("sourceSalesRegion"));raw.put("客户代码",components.get("sourceCustomerCode"));
|
||||
var computed=new LinkedHashMap<String,String>();computed.put("department",text(row.get("department")));computed.put("group",text(row.get("group_name")));computed.put("product",text(row.get("product")));computed.put("projectGroup",text(row.get("project_group")));
|
||||
var match=matcher.applyMatch(stage,"group",raw,computed,text(row.get("group_name")));if(!match.matched())continue;
|
||||
String month=text(row.get("month_key")),group=match.value(),customer=text(components.get("sourceCustomerCode"));
|
||||
var lookup=projectLookups.computeIfAbsent(month,key->worker.loadMappings("PROJECT_GROUP",mapping,Integer.parseInt(key.substring(0,4)),Integer.parseInt(key.substring(5,7))));
|
||||
boolean blank=customer==null||customer.isBlank();String key=blank?null:customer+"|"+group;boolean known=!blank&&lookup.containsKey(key);String project=known?lookup.get(key):null;
|
||||
computed.put("group",group);computed.put("projectGroup",project);var projectMatch=projectRules.applyMatch(stage,"projectGroup",raw,computed,project);project=projectMatch.value();
|
||||
components.put("categoryMappingStatus","mapped");components.put("projectGroup",project);components.put("projectGroupStatus",projectMatch.matched()?"mapped":blank?"source_key_blank":known?(project==null?"mapped_null":"mapped"):"unmapped");
|
||||
changes.add(new Change(stage,text(row.get("build_id")),text(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue(),month,text(row.get("department")),text(row.get("product")),text(row.get("group_name")),text(row.get("project_group")),group,project,components));
|
||||
}
|
||||
}
|
||||
String release=UUID.randomUUID().toString(),run=UUID.randomUUID().toString();
|
||||
int releaseInserted=jdbc.update("INSERT INTO data_release(id,name,status,parent_release_id,expected_current_release_id,rule_version,mapping_version,target_version_id,complete,validation,created_by) SELECT ?,?,'CANDIDATE',id,id,rule_version,?,target_version_id,TRUE,JSON_SET(COALESCE(validation,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0),? FROM data_release WHERE id=? AND rule_version=?",release,"未分类归类 "+operation.substring(0,8),mapping,scanned,changes.size(),actor.getId().toString(),parent,rule);
|
||||
if(releaseInserted!=1)throw new IllegalStateException("当前数据版本已变化,未分类归类未提交");
|
||||
jdbc.update("INSERT INTO import_run(id,idempotency_key,status,stage,requested_sources,expected_current_release_id,created_by,processed,total,result_release_id,expected_mapping_version,expected_rule_version,workflow_operation_id,validation,started_at,completed_at) VALUES(?,?,'COMPLETED','CLASSIFICATION_READY',JSON_ARRAY(),?,?,?,?,?,?,?,?,JSON_OBJECT('recalculationScope','UNCLASSIFIED','excelFilesParsed',0),CURRENT_TIMESTAMP(6),CURRENT_TIMESTAMP(6))",run,"UNCLASSIFIED:"+operation,parent,actor.getId().toString(),changes.size(),scanned,release,mapping,rule,operation);
|
||||
var members=jdbc.queryForList("SELECT m.source_document_id,m.source_revision_id,m.transform_build_id FROM release_member m WHERE m.release_id=?",parent);
|
||||
var changedOrigins=new ArrayList<cn.yunsheng.quality.governance.MappingIssueService.SourceRow>();
|
||||
for(var member:members){
|
||||
String old=text(member.get("transform_build_id")),build=UUID.randomUUID().toString();
|
||||
int inserted=jdbc.update("INSERT INTO transform_build(id,source_revision_id,build_key,parser_version,rule_version,mapping_version,status,row_count,rejected_count,component_summary,error_summary,started_at,completed_at,parser_config,parser_config_hash) SELECT ?,source_revision_id,?,parser_version,rule_version,?,'COMPLETED',row_count,rejected_count,JSON_SET(COALESCE(component_summary,JSON_OBJECT()),'$.classificationParentBuildId',id),error_summary,CURRENT_TIMESTAMP(6),CURRENT_TIMESTAMP(6),parser_config,parser_config_hash FROM transform_build WHERE id=? AND status='COMPLETED' AND rule_version=?",build,hash("UNCLASSIFIED\n"+operation+"\n"+old+"\n"+mapping),mapping,old,rule);if(inserted!=1)throw new IllegalStateException("来源快照已变化");
|
||||
// 只复制已冻结结果,不解析文件、不重算已分类行的数值或维度。
|
||||
for(String stage:List.of("black","bright"))jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,sheet_name,physical_row_no,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,raw_values,components) SELECT ?,source_revision_id,sheet_name,physical_row_no,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,raw_values,components FROM fact_"+stage+" WHERE build_id=?",build,old);
|
||||
jdbc.update("INSERT INTO defect_component(build_id,stage,sheet_name,physical_row_no,defect_name,weight,source_field) SELECT ?,stage,sheet_name,physical_row_no,defect_name,weight,source_field FROM defect_component WHERE build_id=?",build,old);
|
||||
rollups.copyBuild(old,build);var keys=new LinkedHashSet<MetricsRollupService.GroupKey>();
|
||||
for(var change:changes)if(change.parent().equals(old)){
|
||||
int updated=jdbc.update("UPDATE fact_"+change.stage()+" SET group_name=?,project_group=?,components=CAST(? AS JSON) WHERE build_id=? AND sheet_name=? AND physical_row_no=? AND (group_name IS NULL OR TRIM(group_name)='')",change.group(),change.project(),encode(change.components()),build,change.sheet(),change.row());if(updated!=1)throw new IllegalStateException("未分类行已变化,归类未提交");
|
||||
changedOrigins.add(new cn.yunsheng.quality.governance.MappingIssueService.SourceRow(change.stage(),build,change.sheet(),change.row()));
|
||||
keys.add(new MetricsRollupService.GroupKey(change.stage(),change.month(),change.department(),change.oldGroup(),change.oldProject(),change.product()));keys.add(new MetricsRollupService.GroupKey(change.stage(),change.month(),change.department(),change.group(),change.project(),change.product()));
|
||||
}
|
||||
rollups.refreshGroups(build,keys);
|
||||
jdbc.update("INSERT INTO release_member(release_id,source_document_id,source_revision_id,transform_build_id) VALUES(?,?,?,?)",release,member.get("source_document_id"),member.get("source_revision_id"),build);
|
||||
}
|
||||
ReleaseProtocolDependencies.inheritFrozen(jdbc,release,parent);
|
||||
jdbc.update("INSERT INTO release_source_exclusion(release_id,source_document_id,request_id) SELECT ?,source_document_id,request_id FROM release_source_exclusion WHERE release_id=?",release,parent);
|
||||
worker.captureMappingIssueRows(run,release,changedOrigins);
|
||||
return new Candidate(release,run,scanned,changes.size());
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,27 @@ public class MetricsRollupService {
|
||||
|
||||
public MetricsRollupService(JdbcTemplate jdbc) { this.jdbc = jdbc; }
|
||||
|
||||
public record GroupKey(String stage,String month,String department,String group,String project,String product) {}
|
||||
|
||||
/** 归类维护沿用旧汇总;仅重建归属发生变化的分组,重量计算结果不变。 */
|
||||
public void copyBuild(String parent,String child) {
|
||||
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent);
|
||||
if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总");
|
||||
}
|
||||
|
||||
public void refreshGroups(String build,java.util.Set<GroupKey> keys) {
|
||||
for(String stage:List.of("black","bright")) {
|
||||
var selected=keys.stream().filter(k->stage.equals(k.stage())).toList();if(selected.isEmpty())continue;
|
||||
String predicate=String.join(" OR ",java.util.Collections.nCopies(selected.size(),"(%s.month_key<=>? AND %s.department COLLATE "+DIMENSION_COLLATION+"<=>? AND %s.group_name COLLATE "+DIMENSION_COLLATION+"<=>? AND %s.project_group COLLATE "+DIMENSION_COLLATION+"<=>? AND %s.product COLLATE "+DIMENSION_COLLATION+"<=>?)"));
|
||||
var values=new java.util.ArrayList<Object>();for(var key:selected){values.add(key.month());values.add(key.department());values.add(key.group());values.add(key.project());values.add(key.product());}
|
||||
String deleted=predicate.replace("%s","r"),inserted=predicate.replace("%s","f");
|
||||
var deleteArgs=new java.util.ArrayList<Object>(List.of(build,stage));deleteArgs.addAll(values);
|
||||
jdbc.update("DELETE r FROM metric_rollup r WHERE r.build_id=? AND r.stage=? AND ("+deleted+")",deleteArgs.toArray());
|
||||
var insertArgs=new java.util.ArrayList<Object>(List.of(stage,build));insertArgs.addAll(values);
|
||||
jdbc.update(rollupInsertSql("fact_"+stage).replace("WHERE f.build_id=?","WHERE f.build_id=? AND ("+inserted+")"),insertArgs.toArray());
|
||||
}
|
||||
}
|
||||
|
||||
/** Idempotently refreshes a completed build in the caller's transaction. */
|
||||
@Transactional
|
||||
public void refreshBuild(String buildId) {
|
||||
|
||||
@@ -12,6 +12,7 @@ import cn.yunsheng.quality.identity.JdbcAppUserDetailsService;
|
||||
import cn.yunsheng.quality.ingestion.ConfigurableRuleEngine;
|
||||
import cn.yunsheng.quality.ingestion.ImportRunService;
|
||||
import cn.yunsheng.quality.ingestion.ImportWorker;
|
||||
import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild;
|
||||
import cn.yunsheng.quality.ingestion.MappingValueCodec;
|
||||
import cn.yunsheng.quality.metrics.MetricsService;
|
||||
import cn.yunsheng.quality.release.ReleaseService;
|
||||
@@ -346,6 +347,7 @@ public class DataWorkflowService {
|
||||
String kind = String.valueOf(payload.get("kind"));
|
||||
Map<String, Object> definition = asMap(payload.get("definition"));
|
||||
if (definition == null) throw failure("INVALID_DEFINITION", "definition 须为 JSON 对象");
|
||||
List<Map<String,Object>> unclassifiedRules=null;
|
||||
if ("mapping".equals(kind)) {
|
||||
Map<String,Object> previous=Map.of();
|
||||
String base=boundVersion("mapping_version","mapping_version",operation.expectedCurrentReleaseId());
|
||||
@@ -356,6 +358,8 @@ public class DataWorkflowService {
|
||||
completeMappingTargets(definition,previous);
|
||||
List<String> errors = validateMapping(definition);
|
||||
if (!errors.isEmpty()) throw failure("MAPPING_INVALID", String.join(";", errors));
|
||||
if(UnclassifiedMappingRebuild.unchanged(previous,definition)){complete(operation.id(),token,operation.expectedCurrentReleaseId(),null,Map.of("unchanged",true,"classifiedRows",0,"excelFilesParsed",0));return;}
|
||||
unclassifiedRules=UnclassifiedMappingRebuild.changedRules(previous,definition);
|
||||
} else {
|
||||
List<String> errors = validateRule(definition);
|
||||
if (!errors.isEmpty()) throw failure("RULE_INVALID", String.join(";", errors));
|
||||
@@ -367,6 +371,16 @@ public class DataWorkflowService {
|
||||
String ruleId = "rule".equals(kind) ? versionId : boundVersion("rule_version", "rule_version", operation.expectedCurrentReleaseId());
|
||||
String mappingId = "mapping".equals(kind) ? versionId : boundVersion("mapping_version", "mapping_version", operation.expectedCurrentReleaseId());
|
||||
if (ruleId == null || mappingId == null) throw failure("ACTIVE_VERSION_MISSING", "当前数据版本缺少可执行规则或映射");
|
||||
if(unclassifiedRules!=null){
|
||||
sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor);
|
||||
updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED");
|
||||
var selectedRules=unclassifiedRules;
|
||||
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
|
||||
jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token);
|
||||
publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false);
|
||||
jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token);
|
||||
return;
|
||||
}
|
||||
rebuildWithVersions(operation.id(), actor, token, null, mappingId, ruleId, true,
|
||||
"mapping".equals(kind) ? versionId : null, "rule".equals(kind) ? versionId : null);
|
||||
}
|
||||
|
||||
+3
-3
@@ -39,10 +39,10 @@ class ClassificationPreviewMySqlTest {
|
||||
private ClassificationPreviewService.Condition equal(String field,String value){return new ClassificationPreviewService.Condition(field,"EQUALS",value);}
|
||||
@Test void removingTwoConditionsBroadensAcrossMonthsStagesAndExistingGroupsWithExactCaseMatching(){
|
||||
var exact=List.of(equal("@department","A"),equal("@product","P"),equal("PDT代码","0001"),equal("销售片区","X"));
|
||||
assertEquals(1L,service.preview(new ClassificationPreviewService.Request("all","G",exact),actor).get("matchedRows"));
|
||||
assertEquals(0L,service.preview(new ClassificationPreviewService.Request("all","G",exact),actor).get("matchedRows"));
|
||||
var broad=service.preview(new ClassificationPreviewService.Request("all","G",List.of(equal("@department","A"),equal("销售片区","X"))),actor);
|
||||
assertEquals(4L,broad.get("matchedRows"));assertEquals(3L,broad.get("changedRows"));assertEquals(0,new java.math.BigDecimal("100").compareTo((java.math.BigDecimal)broad.get("totalWeight")));
|
||||
assertEquals(2L,service.preview(new ClassificationPreviewService.Request("all","G",List.of(new ClassificationPreviewService.Condition("PDT代码","IS_BLANK",""))),actor).get("matchedRows"));
|
||||
assertEquals(1L,broad.get("matchedRows"));assertEquals(1L,broad.get("changedRows"));assertEquals(0,new java.math.BigDecimal("20").compareTo((java.math.BigDecimal)broad.get("totalWeight")));
|
||||
assertEquals(0L,service.preview(new ClassificationPreviewService.Request("all","G",List.of(new ClassificationPreviewService.Condition("PDT代码","IS_BLANK",""))),actor).get("matchedRows"));
|
||||
assertEquals(0L,service.preview(new ClassificationPreviewService.Request("all","G",List.of(equal("@department","A' OR 1=1 --"))),actor).get("matchedRows"));
|
||||
assertEquals(3,jdbc.queryForObject("SELECT COUNT(*) FROM fact_black",Integer.class));
|
||||
}
|
||||
|
||||
@@ -10,17 +10,17 @@ import org.springframework.web.server.ResponseStatusException;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
class ClassificationPreviewTest {
|
||||
@Test void removedConditionsBroadenAcrossBothStagesAndAllMonthsIncludingExistingGroupsWithoutWrites(){
|
||||
@Test void removedConditionsBroadenAcrossStagesAndMonthsButQueryOnlyUnclassifiedRows(){
|
||||
var jdbc=mock(JdbcTemplate.class);var access=mock(CandidateAcceptanceService.class);var actor=mock(AppUserPrincipal.class);
|
||||
when(jdbc.queryForList(startsWith("SELECT release_id"),any(Object[].class))).thenReturn(List.of(Map.of("release_id","r")));
|
||||
when(jdbc.queryForList(startsWith("SELECT * FROM ("),any(Object[].class))).thenReturn(List.of(
|
||||
Map.of("stage","black","month","2026-01","current_group","已有分类","row_count",10L,"total_weight",100),
|
||||
Map.of("stage","bright","month","2026-09","current_group","G","row_count",5L,"total_weight",20)));
|
||||
Map.of("stage","black","month","2026-01","current_group","","row_count",10L,"total_weight",100),
|
||||
Map.of("stage","bright","month","2026-09","current_group","","row_count",5L,"total_weight",20)));
|
||||
var service=new ClassificationPreviewService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),access);
|
||||
var request=new ClassificationPreviewService.Request("all","G",List.of(new ClassificationPreviewService.Condition("@department","EQUALS","A")));
|
||||
var result=service.preview(request,actor);assertEquals(15L,result.get("matchedRows"));assertEquals(10L,result.get("changedRows"));
|
||||
var result=service.preview(request,actor);assertEquals(15L,result.get("matchedRows"));assertEquals(15L,result.get("changedRows"));
|
||||
var sql=org.mockito.ArgumentCaptor.forClass(String.class);verify(jdbc).queryForList(sql.capture(),eq("r"),eq("A"),eq("r"),eq("A"));
|
||||
assertTrue(sql.getValue().contains("fact_black"));assertTrue(sql.getValue().contains("fact_bright"));assertFalse(sql.getValue().contains("group_name=''"));assertFalse(sql.getValue().contains("sourcePdtCode"));assertFalse(sql.getValue().contains("f.month_key=?"));
|
||||
assertTrue(sql.getValue().contains("fact_black"));assertTrue(sql.getValue().contains("fact_bright"));assertTrue(sql.getValue().contains("TRIM(f.group_name)=''"));assertFalse(sql.getValue().contains("sourcePdtCode"));assertFalse(sql.getValue().contains("f.month_key=?"));
|
||||
service.preview(new ClassificationPreviewService.Request("all","其他",request.conditions()),actor);
|
||||
verify(jdbc,times(1)).queryForList(startsWith("SELECT * FROM ("),any(Object[].class));verify(access,times(2)).requireRead("r",actor);
|
||||
doThrow(new ResponseStatusException(org.springframework.http.HttpStatus.FORBIDDEN)).when(access).requireRead("r",actor);assertThrows(ResponseStatusException.class,()->service.preview(request,actor));
|
||||
|
||||
@@ -9,6 +9,28 @@ import org.junit.jupiter.api.Test;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
class ConfigurableMatchRulesTest {
|
||||
@Test void unclassifiedRulesNeverReplaceLookupOrOrdinaryRuleAssignments() {
|
||||
var pending = new LinkedHashMap<String,Object>(rule("pending", -10000, "all", "group", "补充分组", List.of(condition("销售片区", "EQUALS", "华东"))));
|
||||
pending.put("onlyUnclassified", true);
|
||||
var definition=Map.<String,Object>of("matchRules",List.of(pending,ConfigurableMatchRules.defaultRules().getFirst()));
|
||||
assertTrue(ConfigurableMatchRules.validate(definition).isEmpty());
|
||||
var empty=mapper(Map.of(),Map.of(),definition);
|
||||
assertEquals("补充分组",empty.transform("black",row("001","普通产品","华东","产业部"),2026,1).group());
|
||||
assertEquals("工业电机-VCM",empty.transform("black",row("001","VCM产品","华东","产业部"),2026,1).group());
|
||||
assertEquals("已有分组",mapper(Map.of("P1|华东","已有分组"),Map.of(),definition).transform("bright",row("001","普通产品","华东","产业部"),2026,1).group());
|
||||
}
|
||||
@Test void reimportKeepsSavedGroupsWhenOnlyUnclassifiedRulesChange() {
|
||||
var pending=new LinkedHashMap<String,Object>(rule("pending",-10000,"all","group","新补充分组",List.of(condition("销售片区","EQUALS","华东"))));
|
||||
pending.put("onlyUnclassified",true);
|
||||
var definition=Map.<String,Object>of("matchRules",List.of(pending,ConfigurableMatchRules.defaultRules().getFirst()));
|
||||
var saved=Map.of(BusinessRowMapper.sourceRowKey("测试",2),"已保存归属");
|
||||
var mapper=mapper(Map.of(),Map.of(),definition);
|
||||
assertTrue(mapper.hasUnclassifiedRules());
|
||||
assertEquals("已保存归属",mapper.preserveAssignedGroups(saved).transform("black",row("001","普通产品","华东","产业部"),2026,1).group());
|
||||
assertEquals("新补充分组",mapper.transform("black",row("001","普通产品","华东","产业部"),2026,1).group());
|
||||
assertEquals("工业电机-VCM",mapper.preserveAssignedGroups(saved).transform("black",row("001","VCM产品","华东","产业部"),2026,1).group());
|
||||
assertEquals("明确的新对应",mapper(Map.of("P1|华东","明确的新对应"),Map.of(),definition).preserveAssignedGroups(saved).transform("black",row("001","普通产品","华东","产业部"),2026,1).group());
|
||||
}
|
||||
@Test void missingRulesKeepLegacyVcmBehaviorAndExplicitEmptyArrayRemovesIt() {
|
||||
Map<String, String> categories = Map.of("P1|华东", "原始分组");
|
||||
var legacy = mapper(categories, Map.of(), Map.of());
|
||||
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package cn.yunsheng.quality.ingestion;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.metrics.MetricsRollupService;
|
||||
import java.util.*;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.datasource.*;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
@EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_COLUMN_TEST_CREATE_DATABASE",matches="true")
|
||||
class UnclassifiedMappingRebuildMySqlTest {
|
||||
JdbcTemplate admin,jdbc;String database;DriverManagerDataSource dataSource;ImportWorker worker;MetricsRollupService rollups;JsonMapper json=JsonMapper.builder().build();
|
||||
final String parent="11111111-1111-1111-1111-111111111111",black="22222222-2222-2222-2222-222222222222",bright="33333333-3333-3333-3333-333333333333";
|
||||
AppUserPrincipal actor=mock(AppUserPrincipal.class);
|
||||
Map<String,Object> pending(){return Map.of("id","classification-test","name","区域补分类","enabled",true,"onlyUnclassified",true,"priority",-9900,"stage","all","target","group","value","G","conditions",List.of(Map.of("field","@department","operator","EQUALS","value","A"),Map.of("field","销售片区","operator","EQUALS","value","X")));}
|
||||
@BeforeEach void setup()throws Exception{
|
||||
String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL"),user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
||||
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_unclassified_delta_"+UUID.randomUUID().toString().replace("-","");admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
||||
dataSource=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(dataSource);
|
||||
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY,source_revision_id VARCHAR(100),build_key CHAR(64),parser_version VARCHAR(100),rule_version VARCHAR(100),mapping_version VARCHAR(100),status VARCHAR(32),row_count BIGINT,rejected_count BIGINT,component_summary JSON,error_summary JSON,started_at TIMESTAMP(6) NULL,completed_at TIMESTAMP(6) NULL,parser_config JSON,parser_config_hash VARCHAR(64))");
|
||||
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(190),status VARCHAR(32),parent_release_id CHAR(36),expected_current_release_id CHAR(36),rule_version VARCHAR(100),mapping_version VARCHAR(100),target_version_id VARCHAR(100),complete BOOLEAN,validation JSON,created_by CHAR(36))");
|
||||
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id VARCHAR(100),source_revision_id VARCHAR(100),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))");
|
||||
for(String stage:List.of("black","bright"))jdbc.execute("CREATE TABLE fact_"+stage+"(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id VARCHAR(100),sheet_name VARCHAR(100),physical_row_no INT,month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(24,8),scrap_weight DECIMAL(24,8),total_weight DECIMAL(24,8),raw_values JSON,components JSON,UNIQUE(build_id,sheet_name,physical_row_no))");
|
||||
jdbc.execute("CREATE TABLE defect_component(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),stage VARCHAR(16),sheet_name VARCHAR(100),physical_row_no INT,defect_name VARCHAR(100),weight DECIMAL(24,8),source_field VARCHAR(100))");
|
||||
try(var stream=getClass().getResourceAsStream("/db/migration/V16__metric_rollups.sql")){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);}
|
||||
jdbc.execute("CREATE TABLE import_run(id CHAR(36) PRIMARY KEY,idempotency_key VARCHAR(190),status VARCHAR(32),stage VARCHAR(80),requested_sources JSON,expected_current_release_id CHAR(36),created_by CHAR(36),processed BIGINT,total BIGINT,result_release_id CHAR(36),expected_mapping_version VARCHAR(100),expected_rule_version VARCHAR(100),workflow_operation_id CHAR(36),validation JSON,started_at TIMESTAMP(6),completed_at TIMESTAMP(6))");
|
||||
jdbc.execute("CREATE TABLE release_protocol_dependency(release_id CHAR(36),protocol_id VARCHAR(100),PRIMARY KEY(release_id,protocol_id))");jdbc.execute("CREATE TABLE release_source_exclusion(release_id CHAR(36),source_document_id VARCHAR(100),request_id VARCHAR(100))");
|
||||
jdbc.update("INSERT INTO data_release VALUES(?,'parent','PUBLISHED',NULL,NULL,'rule','old-map',NULL,TRUE,JSON_OBJECT(),?)",parent,UUID.randomUUID().toString());
|
||||
for(String build:List.of(black,bright)){jdbc.update("INSERT INTO transform_build VALUES(?,?,?,'parser','rule','old-map','COMPLETED',?,0,JSON_OBJECT(),JSON_ARRAY(),NOW(6),NOW(6),NULL,'legacy')",build,build,"x"+build,build.equals(black)?3:1);jdbc.update("INSERT INTO release_member VALUES(?,?,?,?)",parent,build,build,build);}
|
||||
seed("black",black,1,"A",null,10,2);seed("black",black,2,"A","KEEP",5,1);seed("black",black,3,"B",null,2,0);seed("bright",bright,1,"A",null,20,3);
|
||||
rollups=new MetricsRollupService(jdbc);rollups.refreshBuild(black);rollups.refreshBuild(bright);
|
||||
worker=mock(ImportWorker.class);when(worker.loadMappings(eq("PROJECT_GROUP"),eq("new-map"),anyInt(),anyInt())).thenReturn(Map.of("001|G","PROJECT"));when(actor.getId()).thenReturn(UUID.randomUUID());
|
||||
}
|
||||
void seed(String stage,String build,int row,String department,String group,int good,int scrap){
|
||||
var components=new LinkedHashMap<String,Object>();components.put("sourcePdtCode","P");components.put("sourceSalesRegion","X");components.put("sourceCustomerCode","001");components.put("categoryMappingStatus",group==null?"unmapped":"mapped");components.put("projectGroupStatus","source_key_blank");components.put(stage.equals("black")?"black_material_scrap":"bright_material_scrap",scrap);
|
||||
jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,sheet_name,physical_row_no,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,?,'S',?,'2026-01',?,?,NULL,'P',?,?,?,CAST(? AS JSON),CAST(? AS JSON))",build,build,row,department,group,good,scrap,good+scrap,json.writeValueAsString(Map.of("客户代码","001","PDT代码","P","销售片区","X")),json.writeValueAsString(components));
|
||||
jdbc.update("INSERT INTO defect_component(build_id,stage,sheet_name,physical_row_no,defect_name,weight,source_field) VALUES(?,?,'S',?,'料废',?,'原字段')",build,stage,row,scrap);
|
||||
}
|
||||
UnclassifiedMappingRebuild.Candidate run(MetricsRollupService metrics){return new TransactionTemplate(new DataSourceTransactionManager(dataSource)).execute(status->new UnclassifiedMappingRebuild(jdbc,json,worker,metrics).create(UUID.randomUUID().toString(),parent,"new-map","rule",Map.of("matchRules",List.of(pending())),List.of(pending()),actor));}
|
||||
@AfterEach void cleanup(){if(admin!=null&&database!=null&&database.matches("quality_unclassified_delta_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);}
|
||||
@Test void onlyEmptyRowsAreMatchedAndOldFactsWeightsAndExistingClassificationsRemainUnchanged(){
|
||||
var candidate=run(rollups);assertEquals(3,candidate.scannedRows());assertEquals(2,candidate.classifiedRows());
|
||||
assertEquals(2,jdbc.queryForObject("SELECT COUNT(*) FROM release_member m JOIN fact_black f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND f.group_name IS NULL",Integer.class,parent));
|
||||
assertEquals("KEEP",jdbc.queryForObject("SELECT f.group_name FROM release_member m JOIN fact_black f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND f.physical_row_no=2",String.class,candidate.releaseId()));
|
||||
assertEquals("G",jdbc.queryForObject("SELECT f.group_name FROM release_member m JOIN fact_black f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND f.physical_row_no=1",String.class,candidate.releaseId()));
|
||||
assertEquals("PROJECT",jdbc.queryForObject("SELECT f.project_group FROM release_member m JOIN fact_bright f ON f.build_id=m.transform_build_id WHERE m.release_id=?",String.class,candidate.releaseId()));
|
||||
var before=jdbc.queryForObject("SELECT SUM(total_weight) FROM metric_rollup r JOIN release_member m ON m.transform_build_id=r.build_id WHERE m.release_id=?",java.math.BigDecimal.class,parent);var after=jdbc.queryForObject("SELECT SUM(total_weight) FROM metric_rollup r JOIN release_member m ON m.transform_build_id=r.build_id WHERE m.release_id=?",java.math.BigDecimal.class,candidate.releaseId());assertEquals(0,before.compareTo(after));
|
||||
assertEquals(3,jdbc.queryForObject("SELECT SUM(excluded_blank_rows) FROM metric_rollup r JOIN release_member m ON m.transform_build_id=r.build_id WHERE m.release_id=?",Integer.class,parent));
|
||||
assertEquals(1,jdbc.queryForObject("SELECT SUM(excluded_blank_rows) FROM metric_rollup r JOIN release_member m ON m.transform_build_id=r.build_id WHERE m.release_id=?",Integer.class,candidate.releaseId()));
|
||||
verify(worker,never()).execute(anyString());verify(worker).captureMappingIssueRows(eq(candidate.runId()),eq(candidate.releaseId()),argThat(rows->rows.size()==2));
|
||||
}
|
||||
@Test void failureRollsBackDerivedSnapshotWithoutTouchingThePublishedFacts(){
|
||||
var failing=spy(rollups);doThrow(new IllegalStateException("forced rollback")).when(failing).refreshGroups(anyString(),anySet());
|
||||
assertThrows(IllegalStateException.class,()->run(failing));assertEquals(1,jdbc.queryForObject("SELECT COUNT(*) FROM data_release",Integer.class));assertEquals(2,jdbc.queryForObject("SELECT COUNT(*) FROM transform_build",Integer.class));assertEquals(0,jdbc.queryForObject("SELECT COUNT(*) FROM import_run",Integer.class));assertEquals(3,jdbc.queryForObject("SELECT COUNT(*) FROM fact_black",Integer.class));
|
||||
}
|
||||
@Test void reimportUsesAssignedGroupsOnlyFromItsBoundRevisionAndBaseline(){
|
||||
String run=UUID.randomUUID().toString();jdbc.update("INSERT INTO import_run(id,expected_current_release_id) VALUES(?,?)",run,parent);
|
||||
assertEquals(Map.of(BusinessRowMapper.sourceRowKey("S",2),"KEEP"),ImportWorker.loadAssignedGroups(jdbc,"black",run,black));
|
||||
assertTrue(ImportWorker.loadAssignedGroups(jdbc,"black",run,bright).isEmpty());
|
||||
assertTrue(ImportWorker.loadAssignedGroups(jdbc,"black",UUID.randomUUID().toString(),black).isEmpty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package cn.yunsheng.quality.ingestion;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import java.util.*;
|
||||
import org.junit.jupiter.api.Test;
|
||||
class UnclassifiedMappingScopeTest {
|
||||
Map<String,Object> rule(boolean scoped){return Map.of("id","test","onlyUnclassified",scoped,"enabled",true,"target","group","conditions",List.of(Map.of("field","@department","operator","EQUALS","value","A")));}
|
||||
@Test void newScopedRuleChoosesLocalProcessingAndMixedEditsCannotFallBackToFullRebuild(){
|
||||
var before=Map.<String,Object>of("categoryLookup",Map.of("P|X","G"),"matchRules",List.of());
|
||||
var after=Map.<String,Object>of("categoryLookup",Map.of("P|X","G"),"matchRules",List.of(rule(true)));assertEquals(1,UnclassifiedMappingRebuild.changedRules(before,after).size());
|
||||
var mixed=new LinkedHashMap<>(after);mixed.put("categoryLookup",Map.of("P|X","OTHER"));assertThrows(IllegalArgumentException.class,()->UnclassifiedMappingRebuild.changedRules(before,mixed));
|
||||
assertThrows(IllegalArgumentException.class,()->UnclassifiedMappingRebuild.changedRules(after,Map.of("categoryLookup",before.get("categoryLookup"),"matchRules",List.of(rule(false)))));
|
||||
assertTrue(UnclassifiedMappingRebuild.unchanged(Map.of(),Map.of("matchRules",ConfigurableMatchRules.defaultRules())));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user