修复:分类规则仅处理未分类并保留已有归属
This commit is contained in:
+2
-2
@@ -7,7 +7,7 @@ import java.util.*;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/** Counts current immutable source rows, including already classified rows, without changing mappings. */
|
||||
/** 只预览当前未分类记录的归类影响,不改动对应关系。 */
|
||||
@Service
|
||||
public class ClassificationPreviewService {
|
||||
private final GovernanceSupport g;
|
||||
@@ -47,7 +47,7 @@ public class ClassificationPreviewService {
|
||||
private List<Map<String,Object>> load(String release,Request request){
|
||||
List<String> queries=new ArrayList<>();List<Object> args=new ArrayList<>();
|
||||
for(String stage:request.stage().equals("all")?List.of("black","bright"):List.of(request.stage())){
|
||||
StringBuilder where=new StringBuilder(" WHERE rm.release_id=?");args.add(release);
|
||||
StringBuilder where=new StringBuilder(" WHERE rm.release_id=? AND (f.group_name IS NULL OR TRIM(f.group_name)='')");args.add(release);
|
||||
for(var condition:request.conditions()){
|
||||
String expression=FIELDS.get(condition.field());
|
||||
if(condition.operator().equals("IS_BLANK"))where.append(" AND (").append(expression).append(" IS NULL OR REGEXP_LIKE(").append(expression).append(",'^[[:space:]]*$'))");
|
||||
|
||||
@@ -20,12 +20,18 @@ public class MappingIssueService {
|
||||
private final MappingIssueDimensions dimensions;private final GovernanceSupport g;private final CandidateAcceptanceService acceptance;private final ImportRunService imports;private final ObjectProvider<ImportWorker> worker;private final TransactionTemplate transaction;
|
||||
public MappingIssueService(GovernanceSupport g,CandidateAcceptanceService acceptance,ImportRunService imports,ObjectProvider<ImportWorker> worker,PlatformTransactionManager manager){this.g=g;this.dimensions=new MappingIssueDimensions(g);this.acceptance=acceptance;this.imports=imports;this.worker=worker;this.transaction=new TransactionTemplate(manager);this.transaction.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);}
|
||||
/** Aggregated from immutable transformed facts, retaining their exact source/build lineage. */
|
||||
public void capture(String runId,String releaseId){
|
||||
Map<String,Group> groups=new TreeMap<>();for(String stage:List.of("black","bright"))for(Map<String,Object> f:g.rows("SELECT f.group_name,f.total_weight,f.components,f.raw_values,f.month_key,rm.source_document_id,rm.source_revision_id,rm.transform_build_id FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id WHERE rm.release_id=? AND (JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.categoryMappingStatus'))='unmapped' OR JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.projectGroupStatus'))='unmapped')",releaseId)){
|
||||
public record SourceRow(String stage,String buildId,String sheet,int row){}
|
||||
public void capture(String runId,String releaseId){captureRows(runId,releaseId,null);}
|
||||
public void captureRows(String runId,String releaseId,List<SourceRow> selected){
|
||||
Map<String,Group> groups=new TreeMap<>();for(String stage:List.of("black","bright")){
|
||||
String restriction="";List<Object> args=new ArrayList<>(List.of(releaseId));
|
||||
if(selected!=null){var rows=selected.stream().filter(row->stage.equals(row.stage())).toList();if(rows.isEmpty())continue;restriction=" AND ("+String.join(" OR ",Collections.nCopies(rows.size(),"(f.build_id=? AND f.sheet_name=? AND f.physical_row_no=?)"))+")";for(var row:rows){args.add(row.buildId());args.add(row.sheet());args.add(row.row());}}
|
||||
for(Map<String,Object> f:g.rows("SELECT f.group_name,f.total_weight,f.components,f.raw_values,f.month_key,rm.source_document_id,rm.source_revision_id,rm.transform_build_id FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id WHERE rm.release_id=? AND (JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.categoryMappingStatus'))='unmapped' OR JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.projectGroupStatus'))='unmapped')"+restriction,args.toArray())){
|
||||
Map<String,Object> c=map(f.get("components")),raw=map(f.get("rawValues"));
|
||||
if("unmapped".equals(c.get("categoryMappingStatus")))add(groups,"CATEGORY",c.get("sourcePdtCode")+"|"+c.get("sourceSalesRegion"),f,raw,stage);
|
||||
if("unmapped".equals(c.get("projectGroupStatus")))add(groups,"PROJECT_GROUP",c.get("sourceCustomerCode")+"|"+f.get("groupName"),f,raw,stage);
|
||||
}
|
||||
}
|
||||
for(Group group:groups.values()){
|
||||
List<String> candidates=new ArrayList<>();for(String revision:group.revisions)candidates.addAll(g.jdbc.query("SELECT DISTINCT c.id FROM source_candidate c JOIN source_revision r ON r.sha256=c.sha256 JOIN source_document d ON d.id=r.source_document_id WHERE r.id=? AND "+SourceIdentity.canonicalSlotSql("c.source_slot")+"="+SourceIdentity.canonicalSlotSql("d.source_slot"),(rs,n)->rs.getString(1),revision));
|
||||
String id=GovernanceSupport.sha(runId+"\n"+group.type+"\n"+group.key);Map<String,Object> details=Map.of("sourceCandidateIds",candidates.stream().distinct().toList(),"sourceDocumentIds",group.documents,"sourceRevisionIds",group.revisions,"transformBuildIds",group.builds,"periods",group.periods,"originalValues",group.originals,"originalValueRows",group.originalValueRows,"releaseId",releaseId);
|
||||
|
||||
@@ -26,6 +26,7 @@ public final class BusinessRowMapper {
|
||||
private final Map<String,String> productLookup;
|
||||
private final ConfigurableRuleEngine rules;
|
||||
private final ConfigurableMatchRules matchRules;
|
||||
private final Map<String,String> savedGroups;
|
||||
|
||||
public BusinessRowMapper(Map<String, String> categoryLookup, Map<String, String> projectLookup, String mappingVersion) {
|
||||
this(categoryLookup,projectLookup,mappingVersion,Map.of(),Map.of());
|
||||
@@ -43,8 +44,18 @@ public final class BusinessRowMapper {
|
||||
this.departmentLookup=departmentLookup;this.productLookup=productLookup;
|
||||
this.rules=rules;
|
||||
this.matchRules=ConfigurableMatchRules.fromDefinition(mappingDefinition);
|
||||
this.savedGroups=Map.of();
|
||||
}
|
||||
|
||||
private BusinessRowMapper(BusinessRowMapper base,Map<String,String> savedGroups){
|
||||
this.categoryLookup=base.categoryLookup;this.projectLookup=base.projectLookup;this.mappingVersion=base.mappingVersion;
|
||||
this.departmentLookup=base.departmentLookup;this.productLookup=base.productLookup;this.rules=base.rules;this.matchRules=base.matchRules;
|
||||
this.savedGroups=Map.copyOf(savedGroups);
|
||||
}
|
||||
public boolean hasUnclassifiedRules(){return matchRules.hasUnclassifiedRules();}
|
||||
public BusinessRowMapper preserveAssignedGroups(Map<String,String> groups){return new BusinessRowMapper(this,groups);}
|
||||
static String sourceRowKey(String sheet,int row){return sheet+"\n"+row;}
|
||||
|
||||
public String mappingVersion() { return mappingVersion; }
|
||||
|
||||
public List<String> requiredHeaders(String stage) { return rules.requiredHeaders(stage,"black".equals(stage) ? BLACK_REQUIRED : BRIGHT_REQUIRED); }
|
||||
@@ -153,7 +164,7 @@ public final class BusinessRowMapper {
|
||||
product=matchRules.apply(stage,"product",matchSource,computed(department,null,product,null),product);
|
||||
String sourceKey=joinedKey(pdt,region);boolean blankCategoryKey=pdt==null||pdt.isEmpty()||region==null||region.isEmpty();
|
||||
String category=blankCategoryKey?null:categoryLookup.get(sourceKey);
|
||||
category=matchRules.apply(stage,"group",matchSource,computed(department,category,product,null),category);
|
||||
category=matchRules.applyGroup(stage,matchSource,computed(department,category,product,null),category,savedGroups.get(sourceRowKey(input.sheet(),input.physicalRow()))).value();
|
||||
boolean unmappedCategory=!blankCategoryKey&&category==null;
|
||||
String projectKey=joinedKey(customer,category);boolean blankProjectKey=customer==null||customer.isEmpty()||category==null||category.isEmpty();
|
||||
boolean projectMappingKnown=!blankProjectKey&&projectLookup.containsKey(projectKey);
|
||||
|
||||
@@ -21,11 +21,13 @@ public final class ConfigurableMatchRules {
|
||||
private static final Pattern UNSAFE_TEXT = Pattern.compile("(?i)(<\\s*script\\b|javascript\\s*:|on[a-z]+\\s*=|\\beval\\s*\\(|document\\.(?:cookie|write)|--|/\\*|\\*/|\\b(?:DROP|ALTER|DELETE|INSERT|UPDATE|TRUNCATE|CREATE|EXEC(?:UTE)?|UNION\\s+SELECT|SELECT\\s+.+\\s+FROM)\\b)");
|
||||
|
||||
private final List<Map<String, Object>> rules;
|
||||
private final List<Map<String,Object>> executionRules;
|
||||
|
||||
private ConfigurableMatchRules(List<Map<String, Object>> rules) {
|
||||
this.rules = rules.stream().map(ConfigurableMatchRules::copyRule)
|
||||
.sorted(Comparator.comparingInt(rule -> ((Number) rule.get("priority")).intValue()))
|
||||
.toList();
|
||||
this.executionRules=java.util.stream.Stream.concat(this.rules.stream().filter(rule->!Boolean.TRUE.equals(rule.get("onlyUnclassified"))),this.rules.stream().filter(rule->Boolean.TRUE.equals(rule.get("onlyUnclassified")))).toList();
|
||||
}
|
||||
|
||||
/** Default mapping behavior for old mapping versions that do not contain matchRules. */
|
||||
@@ -73,7 +75,8 @@ public final class ConfigurableMatchRules {
|
||||
errors.add(path + " 必须为对象");
|
||||
continue;
|
||||
}
|
||||
for (Object key : rule.keySet()) if (!Set.of("id", "name", "enabled", "priority", "stage", "conditions", "target", "value").contains(String.valueOf(key))) errors.add(path + " 未知字段: " + key);
|
||||
for (Object key : rule.keySet()) if (!Set.of("id", "name", "enabled", "priority", "stage", "conditions", "target", "value", "onlyUnclassified").contains(String.valueOf(key))) errors.add(path + " 未知字段: " + key);
|
||||
if(rule.containsKey("onlyUnclassified") && (!(rule.get("onlyUnclassified") instanceof Boolean) || Boolean.TRUE.equals(rule.get("onlyUnclassified")) && !"group".equals(rule.get("target")))) errors.add(path+" onlyUnclassified仅适用于业务分组且须为布尔值");
|
||||
String id = text(rule.get("id"));
|
||||
if (id != null) textBudget += id.length();
|
||||
if (id == null || !id.matches("[A-Za-z0-9][A-Za-z0-9_-]{0,63}")) errors.add(path + " id须为1至64位字母、数字、下划线或连字符");
|
||||
@@ -136,8 +139,15 @@ public final class ConfigurableMatchRules {
|
||||
}
|
||||
|
||||
public Match applyMatch(String stage, String target, Map<String, ?> raw, Map<String, String> computed, String current) {
|
||||
for (Map<String, Object> rule : rules) {
|
||||
return applyMatch(stage,target,raw,computed,current,false);
|
||||
}
|
||||
|
||||
private Match applyMatch(String stage,String target,Map<String,?> raw,Map<String,String> computed,String current,boolean ordinaryOnly){
|
||||
// 普通对应与规则先执行;补分类规则只接收仍为空的结果,不能抢占已有归属。
|
||||
for (Map<String, Object> rule : executionRules) {
|
||||
if(ordinaryOnly&&Boolean.TRUE.equals(rule.get("onlyUnclassified")))continue;
|
||||
if (!Boolean.TRUE.equals(rule.get("enabled")) || !target.equals(rule.get("target"))) continue;
|
||||
if(Boolean.TRUE.equals(rule.get("onlyUnclassified")) && current!=null && !current.isBlank()) continue;
|
||||
if (!"all".equals(rule.get("stage")) && !stage.equals(rule.get("stage"))) continue;
|
||||
boolean matches = true;
|
||||
for (Object rawCondition : (List<?>) rule.get("conditions")) {
|
||||
@@ -151,6 +161,15 @@ public final class ConfigurableMatchRules {
|
||||
|
||||
public record Match(boolean matched, String value) {}
|
||||
|
||||
/** 同一来源再次导入时,补分类规则仍不能覆盖其已保存的归属。普通规则和基础对应照常优先。 */
|
||||
public Match applyGroup(String stage,Map<String,?> raw,Map<String,String> computed,String current,String savedGroup) {
|
||||
if(savedGroup==null||savedGroup.isBlank())return applyMatch(stage,"group",raw,computed,current);
|
||||
var result=applyMatch(stage,"group",raw,computed,current,true);
|
||||
return result.matched()||result.value()!=null&&!result.value().isBlank()?result:new Match(false,savedGroup);
|
||||
}
|
||||
|
||||
public boolean hasUnclassifiedRules(){return rules.stream().anyMatch(rule->Boolean.TRUE.equals(rule.get("onlyUnclassified")));}
|
||||
|
||||
public List<Map<String, Object>> rules() { return rules.stream().map(ConfigurableMatchRules::copyRule).toList(); }
|
||||
|
||||
public static List<Map<String, Object>> fields() {
|
||||
|
||||
@@ -236,6 +236,8 @@ public class ImportWorker {
|
||||
List<BusinessRowMapper.RowValues> facts = new ArrayList<>(BATCH_SIZE);
|
||||
List<Issue> issues = new ArrayList<>();
|
||||
Set<String> availableHeaders = new HashSet<>();
|
||||
Map<String,String> savedGroups=rowMapper.hasUnclassifiedRules()?loadAssignedGroups(jdbc,candidate.stage(),runId,revisionId):Map.of();
|
||||
BusinessRowMapper effectiveMapper=rowMapper.preserveAssignedGroups(savedGroups);
|
||||
long[] rowCount = {0};
|
||||
int[] blocking = {0};
|
||||
int[] blankCategory = {0};
|
||||
@@ -244,7 +246,7 @@ public class ImportWorker {
|
||||
String declaredScope=jdbc.queryForObject("SELECT scope_code FROM source_candidate WHERE id=?",String.class,candidate.id().toString());
|
||||
XlsxStreamingReader.ReadSummary parsed = reader.read(archivePath, expectedSheet, parserConfig, row -> {
|
||||
availableHeaders.addAll(row.values().keySet());
|
||||
BusinessRowMapper.RowValues value = parserConfig==null?rowMapper.transform(candidate.stage(), row, candidate.year(), candidate.month()):rowMapper.transformConfigured(candidate.stage(),row,candidate.year(),candidate.month());
|
||||
BusinessRowMapper.RowValues value = parserConfig==null?effectiveMapper.transform(candidate.stage(), row, candidate.year(), candidate.month()):effectiveMapper.transformConfigured(candidate.stage(),row,candidate.year(),candidate.month());
|
||||
if(declaredScope!=null&&!"ALL".equals(declaredScope)&&!java.util.Objects.equals(cn.yunsheng.quality.common.DepartmentLabels.queryValue(declaredScope),value.department()))throw new IllegalArgumentException("来源行超出已声明组织范围,须修订来源或协议");
|
||||
facts.add(value);
|
||||
rowCount[0]++;
|
||||
@@ -325,6 +327,15 @@ public class ImportWorker {
|
||||
}
|
||||
}
|
||||
|
||||
static Map<String,String> loadAssignedGroups(JdbcTemplate jdbc,String stage,String runId,String revisionId){
|
||||
if(!Set.of("black","bright").contains(stage))throw new IllegalArgumentException("来源阶段无效");
|
||||
// 仅复用该任务基线中同一不可变来源修订的归属;新文件或修订不借用旧记录。
|
||||
Map<String,String> savedGroups=new HashMap<>();
|
||||
jdbc.queryForList("SELECT f.sheet_name,f.physical_row_no,f.group_name FROM fact_"+stage+" f JOIN release_member m ON m.transform_build_id=f.build_id JOIN import_run r ON r.expected_current_release_id=m.release_id WHERE r.id=? AND f.source_revision_id=? AND f.group_name IS NOT NULL AND TRIM(f.group_name)<>''",runId,revisionId)
|
||||
.forEach(row->savedGroups.put(BusinessRowMapper.sourceRowKey(String.valueOf(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue()),String.valueOf(row.get("group_name"))));
|
||||
return savedGroups;
|
||||
}
|
||||
|
||||
private void persistIssues(String runId, SourceFileCandidate candidate, List<Issue> issues) {
|
||||
for (Issue issue : issues) {
|
||||
jdbc.update("""
|
||||
@@ -416,6 +427,8 @@ public class ImportWorker {
|
||||
return cn.yunsheng.quality.governance.MasterDataSupport.enabledLookup(definition,lookup,values);
|
||||
}
|
||||
|
||||
void captureMappingIssueRows(String run,String release,java.util.List<cn.yunsheng.quality.governance.MappingIssueService.SourceRow> rows){mappingIssues.captureRows(run,release,rows);}
|
||||
|
||||
String reviewedMappingFor(String runId,String activeMapping){
|
||||
String frozen=jdbc.queryForObject("SELECT expected_mapping_version FROM import_run WHERE id=?",String.class,runId);
|
||||
if(frozen==null)return activeMapping;
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package cn.yunsheng.quality.ingestion;
|
||||
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.metrics.MetricsRollupService;
|
||||
import java.util.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/** 未分类维护只重新判断空归属行;其余事实与原汇总按不可变版本原样保留。 */
|
||||
public final class UnclassifiedMappingRebuild {
|
||||
private final JdbcTemplate jdbc;private final JsonMapper json;private final ImportWorker worker;private final MetricsRollupService rollups;
|
||||
public UnclassifiedMappingRebuild(JdbcTemplate jdbc,JsonMapper json,ImportWorker worker,MetricsRollupService rollups){this.jdbc=jdbc;this.json=json;this.worker=worker;this.rollups=rollups;}
|
||||
@SuppressWarnings("unchecked") static Map<String,Object> map(Object value){return value instanceof Map<?,?>?(Map<String,Object>)value:Map.of();}
|
||||
@SuppressWarnings("unchecked") static List<Map<String,Object>> rules(Map<String,Object> definition){return (List<Map<String,Object>>)definition.getOrDefault("matchRules",ConfigurableMatchRules.defaultRules());}
|
||||
static Map<String,Map<String,Object>> indexed(List<Map<String,Object>> rules){var result=new LinkedHashMap<String,Map<String,Object>>();for(var rule:rules)result.put(String.valueOf(rule.get("id")),rule);return result;}
|
||||
public static boolean unchanged(Map<String,Object> before,Map<String,Object> after){var left=new LinkedHashMap<>(before);var right=new LinkedHashMap<>(after);left.remove("matchRules");right.remove("matchRules");return left.equals(right)&&rules(before).equals(rules(after));}
|
||||
public static List<Map<String,Object>> changedRules(Map<String,Object> before,Map<String,Object> after){
|
||||
var old=indexed(rules(before));var next=indexed(rules(after));var changed=new LinkedHashSet<String>();changed.addAll(old.keySet());changed.addAll(next.keySet());changed.removeIf(id->Objects.equals(old.get(id),next.get(id)));
|
||||
boolean scoped=changed.stream().anyMatch(id->Boolean.TRUE.equals(map(old.get(id)).get("onlyUnclassified"))||Boolean.TRUE.equals(map(next.get(id)).get("onlyUnclassified")));
|
||||
if(!scoped)return null;
|
||||
var left=new LinkedHashMap<>(before);var right=new LinkedHashMap<>(after);left.remove("matchRules");right.remove("matchRules");
|
||||
if(!left.equals(right))throw new IllegalArgumentException("未分类维护不能同时修改其他对应或名录,请分开保存");
|
||||
var result=new ArrayList<Map<String,Object>>();
|
||||
for(String id:changed){var prior=old.get(id);var rule=next.get(id);
|
||||
if(rule==null){if(!Boolean.TRUE.equals(prior.get("onlyUnclassified")))throw new IllegalArgumentException("不能在未分类维护中移除普通匹配规则");continue;}
|
||||
if(!Boolean.TRUE.equals(rule.get("onlyUnclassified"))||!"group".equals(rule.get("target")))throw new IllegalArgumentException("未分类规则不能改为覆盖已有分类");
|
||||
for(var raw:(List<?>)rule.get("conditions")){var condition=map(raw);if(!Set.of("@department","@product","PDT代码","销售片区").contains(condition.get("field"))||!Set.of("EQUALS","IS_BLANK").contains(condition.get("operator")))throw new IllegalArgumentException("未分类维护仅支持产业部、产品、PDT代码和销售区域条件");}
|
||||
if(Boolean.TRUE.equals(rule.get("enabled")))result.add(rule);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
record Change(String stage,String parent,String sheet,int row,String month,String department,String product,String oldGroup,String oldProject,String group,String project,Map<String,Object> components){}
|
||||
public record Candidate(String releaseId,String runId,int scannedRows,int classifiedRows){}
|
||||
private String text(Object value){return value==null?null:String.valueOf(value);}
|
||||
private Map<String,Object> decode(Object value){if(value==null)return new LinkedHashMap<>();return new LinkedHashMap<>(json.readValue(String.valueOf(value),Map.class));}
|
||||
private String encode(Object value){return json.writeValueAsString(value);}
|
||||
private static String hash(String value){try{return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)));}catch(Exception error){throw new IllegalStateException(error);}}
|
||||
|
||||
public Candidate create(String operation,String parent,String mapping,String rule,Map<String,Object> definition,List<Map<String,Object>> changedRules,AppUserPrincipal actor){
|
||||
// 调用方事务覆盖候选、数据副本和进度;失败不改变旧版本,禁止执行 Excel 导入器。
|
||||
var matcher=ConfigurableMatchRules.fromDefinition(Map.of("matchRules",changedRules));var projectRules=ConfigurableMatchRules.fromDefinition(definition);
|
||||
var changes=new ArrayList<Change>();int scanned=0;var projectLookups=new HashMap<String,Map<String,String>>();
|
||||
for(String stage:List.of("black","bright")){
|
||||
var rows=jdbc.queryForList("SELECT f.build_id,f.sheet_name,f.physical_row_no,f.month_key,f.department,f.group_name,f.project_group,f.product,CAST(f.components AS CHAR) components,CAST(f.raw_values AS CHAR) raw_values,CAST(b.parser_config AS CHAR) parser_config FROM release_member m JOIN fact_"+stage+" f ON f.build_id=m.transform_build_id JOIN transform_build b ON b.id=f.build_id WHERE m.release_id=? AND (f.group_name IS NULL OR TRIM(f.group_name)='')",parent);
|
||||
scanned+=rows.size();
|
||||
for(var row:rows){
|
||||
var components=decode(row.get("components"));var raw=decode(row.get("raw_values"));var restored=BusinessRowMapper.restoreStoredRow(stage,text(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue(),raw,decode(row.get("parser_config")));
|
||||
restored.values().forEach((key,value)->raw.put(key,value==null?null:value.display()));
|
||||
raw.put("PDT代码",components.get("sourcePdtCode"));raw.put("销售片区",components.get("sourceSalesRegion"));raw.put("客户代码",components.get("sourceCustomerCode"));
|
||||
var computed=new LinkedHashMap<String,String>();computed.put("department",text(row.get("department")));computed.put("group",text(row.get("group_name")));computed.put("product",text(row.get("product")));computed.put("projectGroup",text(row.get("project_group")));
|
||||
var match=matcher.applyMatch(stage,"group",raw,computed,text(row.get("group_name")));if(!match.matched())continue;
|
||||
String month=text(row.get("month_key")),group=match.value(),customer=text(components.get("sourceCustomerCode"));
|
||||
var lookup=projectLookups.computeIfAbsent(month,key->worker.loadMappings("PROJECT_GROUP",mapping,Integer.parseInt(key.substring(0,4)),Integer.parseInt(key.substring(5,7))));
|
||||
boolean blank=customer==null||customer.isBlank();String key=blank?null:customer+"|"+group;boolean known=!blank&&lookup.containsKey(key);String project=known?lookup.get(key):null;
|
||||
computed.put("group",group);computed.put("projectGroup",project);var projectMatch=projectRules.applyMatch(stage,"projectGroup",raw,computed,project);project=projectMatch.value();
|
||||
components.put("categoryMappingStatus","mapped");components.put("projectGroup",project);components.put("projectGroupStatus",projectMatch.matched()?"mapped":blank?"source_key_blank":known?(project==null?"mapped_null":"mapped"):"unmapped");
|
||||
changes.add(new Change(stage,text(row.get("build_id")),text(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue(),month,text(row.get("department")),text(row.get("product")),text(row.get("group_name")),text(row.get("project_group")),group,project,components));
|
||||
}
|
||||
}
|
||||
String release=UUID.randomUUID().toString(),run=UUID.randomUUID().toString();
|
||||
int releaseInserted=jdbc.update("INSERT INTO data_release(id,name,status,parent_release_id,expected_current_release_id,rule_version,mapping_version,target_version_id,complete,validation,created_by) SELECT ?,?,'CANDIDATE',id,id,rule_version,?,target_version_id,TRUE,JSON_SET(COALESCE(validation,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0),? FROM data_release WHERE id=? AND rule_version=?",release,"未分类归类 "+operation.substring(0,8),mapping,scanned,changes.size(),actor.getId().toString(),parent,rule);
|
||||
if(releaseInserted!=1)throw new IllegalStateException("当前数据版本已变化,未分类归类未提交");
|
||||
jdbc.update("INSERT INTO import_run(id,idempotency_key,status,stage,requested_sources,expected_current_release_id,created_by,processed,total,result_release_id,expected_mapping_version,expected_rule_version,workflow_operation_id,validation,started_at,completed_at) VALUES(?,?,'COMPLETED','CLASSIFICATION_READY',JSON_ARRAY(),?,?,?,?,?,?,?,?,JSON_OBJECT('recalculationScope','UNCLASSIFIED','excelFilesParsed',0),CURRENT_TIMESTAMP(6),CURRENT_TIMESTAMP(6))",run,"UNCLASSIFIED:"+operation,parent,actor.getId().toString(),changes.size(),scanned,release,mapping,rule,operation);
|
||||
var members=jdbc.queryForList("SELECT m.source_document_id,m.source_revision_id,m.transform_build_id FROM release_member m WHERE m.release_id=?",parent);
|
||||
var changedOrigins=new ArrayList<cn.yunsheng.quality.governance.MappingIssueService.SourceRow>();
|
||||
for(var member:members){
|
||||
String old=text(member.get("transform_build_id")),build=UUID.randomUUID().toString();
|
||||
int inserted=jdbc.update("INSERT INTO transform_build(id,source_revision_id,build_key,parser_version,rule_version,mapping_version,status,row_count,rejected_count,component_summary,error_summary,started_at,completed_at,parser_config,parser_config_hash) SELECT ?,source_revision_id,?,parser_version,rule_version,?,'COMPLETED',row_count,rejected_count,JSON_SET(COALESCE(component_summary,JSON_OBJECT()),'$.classificationParentBuildId',id),error_summary,CURRENT_TIMESTAMP(6),CURRENT_TIMESTAMP(6),parser_config,parser_config_hash FROM transform_build WHERE id=? AND status='COMPLETED' AND rule_version=?",build,hash("UNCLASSIFIED\n"+operation+"\n"+old+"\n"+mapping),mapping,old,rule);if(inserted!=1)throw new IllegalStateException("来源快照已变化");
|
||||
// 只复制已冻结结果,不解析文件、不重算已分类行的数值或维度。
|
||||
for(String stage:List.of("black","bright"))jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,sheet_name,physical_row_no,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,raw_values,components) SELECT ?,source_revision_id,sheet_name,physical_row_no,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,raw_values,components FROM fact_"+stage+" WHERE build_id=?",build,old);
|
||||
jdbc.update("INSERT INTO defect_component(build_id,stage,sheet_name,physical_row_no,defect_name,weight,source_field) SELECT ?,stage,sheet_name,physical_row_no,defect_name,weight,source_field FROM defect_component WHERE build_id=?",build,old);
|
||||
rollups.copyBuild(old,build);var keys=new LinkedHashSet<MetricsRollupService.GroupKey>();
|
||||
for(var change:changes)if(change.parent().equals(old)){
|
||||
int updated=jdbc.update("UPDATE fact_"+change.stage()+" SET group_name=?,project_group=?,components=CAST(? AS JSON) WHERE build_id=? AND sheet_name=? AND physical_row_no=? AND (group_name IS NULL OR TRIM(group_name)='')",change.group(),change.project(),encode(change.components()),build,change.sheet(),change.row());if(updated!=1)throw new IllegalStateException("未分类行已变化,归类未提交");
|
||||
changedOrigins.add(new cn.yunsheng.quality.governance.MappingIssueService.SourceRow(change.stage(),build,change.sheet(),change.row()));
|
||||
keys.add(new MetricsRollupService.GroupKey(change.stage(),change.month(),change.department(),change.oldGroup(),change.oldProject(),change.product()));keys.add(new MetricsRollupService.GroupKey(change.stage(),change.month(),change.department(),change.group(),change.project(),change.product()));
|
||||
}
|
||||
rollups.refreshGroups(build,keys);
|
||||
jdbc.update("INSERT INTO release_member(release_id,source_document_id,source_revision_id,transform_build_id) VALUES(?,?,?,?)",release,member.get("source_document_id"),member.get("source_revision_id"),build);
|
||||
}
|
||||
ReleaseProtocolDependencies.inheritFrozen(jdbc,release,parent);
|
||||
jdbc.update("INSERT INTO release_source_exclusion(release_id,source_document_id,request_id) SELECT ?,source_document_id,request_id FROM release_source_exclusion WHERE release_id=?",release,parent);
|
||||
worker.captureMappingIssueRows(run,release,changedOrigins);
|
||||
return new Candidate(release,run,scanned,changes.size());
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,27 @@ public class MetricsRollupService {
|
||||
|
||||
public MetricsRollupService(JdbcTemplate jdbc) { this.jdbc = jdbc; }
|
||||
|
||||
public record GroupKey(String stage,String month,String department,String group,String project,String product) {}
|
||||
|
||||
/** 归类维护沿用旧汇总;仅重建归属发生变化的分组,重量计算结果不变。 */
|
||||
public void copyBuild(String parent,String child) {
|
||||
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent);
|
||||
if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总");
|
||||
}
|
||||
|
||||
public void refreshGroups(String build,java.util.Set<GroupKey> keys) {
|
||||
for(String stage:List.of("black","bright")) {
|
||||
var selected=keys.stream().filter(k->stage.equals(k.stage())).toList();if(selected.isEmpty())continue;
|
||||
String predicate=String.join(" OR ",java.util.Collections.nCopies(selected.size(),"(%s.month_key<=>? AND %s.department COLLATE "+DIMENSION_COLLATION+"<=>? AND %s.group_name COLLATE "+DIMENSION_COLLATION+"<=>? AND %s.project_group COLLATE "+DIMENSION_COLLATION+"<=>? AND %s.product COLLATE "+DIMENSION_COLLATION+"<=>?)"));
|
||||
var values=new java.util.ArrayList<Object>();for(var key:selected){values.add(key.month());values.add(key.department());values.add(key.group());values.add(key.project());values.add(key.product());}
|
||||
String deleted=predicate.replace("%s","r"),inserted=predicate.replace("%s","f");
|
||||
var deleteArgs=new java.util.ArrayList<Object>(List.of(build,stage));deleteArgs.addAll(values);
|
||||
jdbc.update("DELETE r FROM metric_rollup r WHERE r.build_id=? AND r.stage=? AND ("+deleted+")",deleteArgs.toArray());
|
||||
var insertArgs=new java.util.ArrayList<Object>(List.of(stage,build));insertArgs.addAll(values);
|
||||
jdbc.update(rollupInsertSql("fact_"+stage).replace("WHERE f.build_id=?","WHERE f.build_id=? AND ("+inserted+")"),insertArgs.toArray());
|
||||
}
|
||||
}
|
||||
|
||||
/** Idempotently refreshes a completed build in the caller's transaction. */
|
||||
@Transactional
|
||||
public void refreshBuild(String buildId) {
|
||||
|
||||
@@ -12,6 +12,7 @@ import cn.yunsheng.quality.identity.JdbcAppUserDetailsService;
|
||||
import cn.yunsheng.quality.ingestion.ConfigurableRuleEngine;
|
||||
import cn.yunsheng.quality.ingestion.ImportRunService;
|
||||
import cn.yunsheng.quality.ingestion.ImportWorker;
|
||||
import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild;
|
||||
import cn.yunsheng.quality.ingestion.MappingValueCodec;
|
||||
import cn.yunsheng.quality.metrics.MetricsService;
|
||||
import cn.yunsheng.quality.release.ReleaseService;
|
||||
@@ -346,6 +347,7 @@ public class DataWorkflowService {
|
||||
String kind = String.valueOf(payload.get("kind"));
|
||||
Map<String, Object> definition = asMap(payload.get("definition"));
|
||||
if (definition == null) throw failure("INVALID_DEFINITION", "definition 须为 JSON 对象");
|
||||
List<Map<String,Object>> unclassifiedRules=null;
|
||||
if ("mapping".equals(kind)) {
|
||||
Map<String,Object> previous=Map.of();
|
||||
String base=boundVersion("mapping_version","mapping_version",operation.expectedCurrentReleaseId());
|
||||
@@ -356,6 +358,8 @@ public class DataWorkflowService {
|
||||
completeMappingTargets(definition,previous);
|
||||
List<String> errors = validateMapping(definition);
|
||||
if (!errors.isEmpty()) throw failure("MAPPING_INVALID", String.join(";", errors));
|
||||
if(UnclassifiedMappingRebuild.unchanged(previous,definition)){complete(operation.id(),token,operation.expectedCurrentReleaseId(),null,Map.of("unchanged",true,"classifiedRows",0,"excelFilesParsed",0));return;}
|
||||
unclassifiedRules=UnclassifiedMappingRebuild.changedRules(previous,definition);
|
||||
} else {
|
||||
List<String> errors = validateRule(definition);
|
||||
if (!errors.isEmpty()) throw failure("RULE_INVALID", String.join(";", errors));
|
||||
@@ -367,6 +371,16 @@ public class DataWorkflowService {
|
||||
String ruleId = "rule".equals(kind) ? versionId : boundVersion("rule_version", "rule_version", operation.expectedCurrentReleaseId());
|
||||
String mappingId = "mapping".equals(kind) ? versionId : boundVersion("mapping_version", "mapping_version", operation.expectedCurrentReleaseId());
|
||||
if (ruleId == null || mappingId == null) throw failure("ACTIVE_VERSION_MISSING", "当前数据版本缺少可执行规则或映射");
|
||||
if(unclassifiedRules!=null){
|
||||
sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor);
|
||||
updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED");
|
||||
var selectedRules=unclassifiedRules;
|
||||
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
|
||||
jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token);
|
||||
publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false);
|
||||
jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token);
|
||||
return;
|
||||
}
|
||||
rebuildWithVersions(operation.id(), actor, token, null, mappingId, ruleId, true,
|
||||
"mapping".equals(kind) ? versionId : null, "rule".equals(kind) ? versionId : null);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user