修复:项目组分类仅补齐未分类并在完成后刷新列表

This commit is contained in:
2026-10-31 20:04:24 +08:00
parent cbe7fc17bb
commit 572873c6b8
21 changed files with 257 additions and 64 deletions
@@ -12,19 +12,22 @@ import org.springframework.stereotype.Service;
public class ClassificationPreviewService {
private final GovernanceSupport g;
private final CandidateAcceptanceService access;
private record Key(String release,String stage,List<Condition> conditions) { Key { conditions=List.copyOf(conditions); } }
private record Key(String release,String kind,String stage,List<Condition> conditions) { Key { conditions=List.copyOf(conditions); } }
private final ImmutableQueryCache<Key,List<Map<String,Object>>> previews=new ImmutableQueryCache<>(64);
public ClassificationPreviewService(GovernanceSupport g,CandidateAcceptanceService access){this.g=g;this.access=access;}
public record Condition(String field,String operator,String value){}
public record Request(String stage,String target,List<Condition> conditions){}
public record Request(String stage,String target,List<Condition> conditions,String kind){
public Request(String stage,String target,List<Condition> conditions){this(stage,target,conditions,"group");}
public boolean project(){return "projectGroup".equals(kind);}
}
private static final Map<String,String> FIELDS=Map.of("@department","f.department","@product","f.product",
"PDT代码","JSON_VALUE(f.components,'$.sourcePdtCode')","销售片区","JSON_VALUE(f.components,'$.sourceSalesRegion')");
"PDT代码","JSON_VALUE(f.components,'$.sourcePdtCode')","销售片区","JSON_VALUE(f.components,'$.sourceSalesRegion')","客户代码","JSON_VALUE(f.components,'$.sourceCustomerCode')","@group","f.group_name");
public Map<String,Object> preview(Request request,AppUserPrincipal actor){
validate(request);access.requireSourceDetail(actor);
var current=g.rows("SELECT release_id FROM current_release WHERE dataset_id='quality'");
if(current.isEmpty())return Map.of("items",List.of(),"matchedRows",0,"changedRows",0,"totalWeight",BigDecimal.ZERO);
String release=String.valueOf(current.getFirst().get("releaseId"));access.requireRead(release,actor);
var items=previews.get(new Key(release,request.stage(),request.conditions()),()->load(release,request));
var items=previews.get(new Key(release,request.project()?"projectGroup":"group",request.stage(),request.conditions()),()->load(release,request));
long matched=0,changed=0;BigDecimal weight=BigDecimal.ZERO;
for(var row:items){long count=((Number)row.get("rowCount")).longValue();matched+=count;
if(!Objects.equals(row.get("currentGroup"),request.target()))changed+=count;
@@ -32,12 +35,13 @@ public class ClassificationPreviewService {
return Map.of("releaseId",release,"items",items,"matchedRows",matched,"changedRows",changed,"totalWeight",weight);
}
static void validate(Request request){
if(request==null||!Set.of("all","black","bright").contains(Objects.toString(request.stage(),""))
if(request==null||(request.kind()!=null&&!Set.of("group","projectGroup").contains(request.kind()))||!Set.of("all","black","bright").contains(Objects.toString(request.stage(),""))
||request.target()==null||request.target().isBlank()||request.target().length()>160
||request.conditions()==null||request.conditions().isEmpty()||request.conditions().size()>4)
throw GovernanceSupport.error(HttpStatus.BAD_REQUEST,"分类规则条件无效");
if(request.project()&&(request.conditions().size()!=2||request.conditions().stream().anyMatch(c->c==null||!"EQUALS".equals(c.operator()))))throw GovernanceSupport.error(HttpStatus.BAD_REQUEST,"项目组归类须同时指定客户代码和业务分组");
Set<String> fields=new HashSet<>();
for(var condition:request.conditions())if(condition==null||condition.field()==null||!FIELDS.containsKey(condition.field())||!fields.add(condition.field())
for(var condition:request.conditions())if(condition==null||condition.field()==null||!Set.copyOf(request.project()?List.of("客户代码","@group"):List.of("@department","@product","PDT代码","销售片区")).contains(condition.field())||!fields.add(condition.field())
||!Set.of("EQUALS","IS_BLANK").contains(Objects.toString(condition.operator(),""))
||condition.value()==null||condition.value().length()>200
||("EQUALS".equals(condition.operator())&&condition.value().isBlank())
@@ -47,14 +51,16 @@ public class ClassificationPreviewService {
private List<Map<String,Object>> load(String release,Request request){
List<String> queries=new ArrayList<>();List<Object> args=new ArrayList<>();
for(String stage:request.stage().equals("all")?List.of("black","bright"):List.of(request.stage())){
StringBuilder where=new StringBuilder(" WHERE rm.release_id=? AND (f.group_name IS NULL OR TRIM(f.group_name)='')");args.add(release);
String column=request.project()?"f.project_group":"f.group_name";
String predicate=request.project()?cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild.projectBlankSql("f"):"(f.group_name IS NULL OR TRIM(f.group_name)='')";
StringBuilder where=new StringBuilder(" WHERE rm.release_id=? AND "+predicate);args.add(release);
for(var condition:request.conditions()){
String expression=FIELDS.get(condition.field());
if(condition.operator().equals("IS_BLANK"))where.append(" AND (").append(expression).append(" IS NULL OR REGEXP_LIKE(").append(expression).append(",'^[[:space:]]*$'))");
else{where.append(" AND CAST(").append(expression).append(" AS BINARY)=CAST(? AS BINARY)");args.add(condition.value());}
}
queries.add("SELECT '"+stage+"' AS stage,f.department,f.month_key AS month,COALESCE(f.group_name,'') AS current_group,COUNT(*) AS row_count,SUM(f.total_weight) AS total_weight "
+"FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+where+" GROUP BY f.department,f.month_key,COALESCE(f.group_name,'')");
queries.add("SELECT '"+stage+"' AS stage,f.department,f.month_key AS month,COALESCE("+column+",'') AS current_group,COUNT(*) AS row_count,SUM(f.total_weight) AS total_weight "
+"FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+where+" GROUP BY f.department,f.month_key,COALESCE("+column+",'')");
}
return g.rows("SELECT * FROM ("+String.join(" UNION ALL ",queries)+") impact ORDER BY stage,month,current_group",args.toArray());
}
@@ -14,6 +14,7 @@ public class MappingIssueController {
@PostMapping("/classification-preview") public Map<String,Object> classificationPreview(@RequestBody ClassificationPreviewService.Request request,@AuthenticationPrincipal AppUserPrincipal actor){return preview.preview(request,actor);}
@GetMapping("/classification-metadata") public Map<String,Object> classificationMetadata(){return cn.yunsheng.quality.ingestion.ConfigurableMatchRules.metadata();}
@GetMapping("/unclassified") public Map<String,Object> unclassified(@RequestParam(defaultValue="1") int page,@RequestParam(defaultValue="50") int pageSize,@AuthenticationPrincipal AppUserPrincipal actor){return unclassified.list(page,pageSize,actor);}
@GetMapping("/unclassified-projects") public Map<String,Object> unclassifiedProjects(@RequestParam(defaultValue="1") int page,@RequestParam(defaultValue="15") int pageSize,@RequestParam(required=false) String customerCode,@RequestParam(required=false) String businessGroup,@RequestParam(required=false) String department,@AuthenticationPrincipal AppUserPrincipal actor){return unclassified.listProjects(page,pageSize,customerCode,businessGroup,department,actor);}
@GetMapping public Map<String,Object> list(@RequestParam(required=false) String importRunId,@RequestParam(required=false) String status,@RequestParam(defaultValue="1") int page,@RequestParam(defaultValue="50") int pageSize,@RequestParam(required=false) String mappingType,@RequestParam(required=false) String customerCode,@RequestParam(required=false) String businessGroup,@RequestParam(required=false) String department,@AuthenticationPrincipal AppUserPrincipal actor){return service.list(importRunId,status,page,pageSize,customerCode,businessGroup,department,mappingType,actor);}
@GetMapping("/{id}") public Map<String,Object> detail(@PathVariable String id,@AuthenticationPrincipal AppUserPrincipal actor){return service.detail(id,actor);}
@PostMapping("/{id}/resolve") @PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER','RULE_MAINTAINER')")
@@ -12,6 +12,7 @@ public class UnclassifiedSourceService {
private final GovernanceSupport g;
private final CandidateAcceptanceService access;
private final ImmutableQueryCache<String, List<Map<String, Object>>> inventories = new ImmutableQueryCache<>(12);
private final ImmutableQueryCache<String,List<Map<String,Object>>> projects=new ImmutableQueryCache<>(12);
public UnclassifiedSourceService(GovernanceSupport g, CandidateAcceptanceService access) {
this.g = g;
this.access = access;
@@ -50,4 +51,35 @@ public class UnclassifiedSourceService {
GROUP BY f.department,f.product,pdt_code,sales_region,source_key
""".formatted(stage, stage);
}
/** 项目组待处理列表只展示当前版本的空归属,避免重复出现已经归类的历史问题。 */
public Map<String,Object> listProjects(int page,int size,String customer,String group,String department,AppUserPrincipal actor){
if(page<1||size<1||size>100)throw GovernanceSupport.error(HttpStatus.BAD_REQUEST,"分页无效");
access.requireSourceDetail(actor);
var current=g.rows("SELECT release_id FROM current_release WHERE dataset_id='quality'");
if(current.isEmpty())return Map.of("items",List.of(),"total",0,"page",page,"pageSize",size,"filterOptions",MappingIssueDimensions.options(List.of()));
String release=String.valueOf(current.getFirst().get("releaseId"));access.requireRead(release,actor);
var all=projects.get(release,()->{
Map<String,Project> result=new TreeMap<>();
for(String stage:List.of("black","bright"))for(var row:g.rows(projectSql(stage),release)){
String code=String.valueOf(row.get("customerCode")),business=String.valueOf(row.get("businessGroup")),key=code+"|"+business;
var item=result.computeIfAbsent(key,k->new Project(code,business));
item.count+=((Number)row.get("rowCount")).longValue();if(row.get("totalWeight") instanceof Number n)item.weight=item.weight.add(new java.math.BigDecimal(n.toString()));
if(row.get("department") instanceof String name&&!name.isBlank())item.departments.add(name);
if(row.get("month")!=null)item.periods.add(String.valueOf(row.get("month")));
}
return result.entrySet().stream().map(entry->{var item=entry.getValue();return Map.<String,Object>of("id",GovernanceSupport.sha(release+"\nPROJECT_GROUP\n"+entry.getKey()),"mappingType","PROJECT_GROUP","sourceKey",entry.getKey(),"customerCode",item.customer,"businessGroup",item.group,"departments",List.copyOf(item.departments),"rowCount",item.count,"totalWeight",item.weight,"status","OPEN","details",Map.of("periods",List.copyOf(item.periods)));}).toList();
});
var filtered=all.stream().filter(row->MappingIssueDimensions.matches(row,customer,group,department)).toList();
int offset=(int)Math.min(filtered.size(),(long)(page-1)*size);
return Map.of("items",filtered.subList(offset,Math.min(filtered.size(),offset+size)),"total",filtered.size(),"page",page,"pageSize",size,"releaseId",release,"filterOptions",MappingIssueDimensions.options(all));
}
private static class Project{
final String customer,group;final Set<String> departments=new TreeSet<>(),periods=new TreeSet<>();long count;java.math.BigDecimal weight=java.math.BigDecimal.ZERO;
Project(String customer,String group){this.customer=customer;this.group=group;}
}
static String projectSql(String stage){
if(!Set.of("black","bright").contains(stage))throw new IllegalArgumentException("来源阶段无效");
return "SELECT JSON_VALUE(f.components,'$.sourceCustomerCode') customer_code,f.group_name business_group,f.department,f.month_key month,COUNT(*) row_count,SUM(f.total_weight) total_weight FROM release_member m JOIN fact_"+stage+" f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND "+cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild.projectBlankSql("f")+" AND JSON_VALUE(f.components,'$.sourceCustomerCode') IS NOT NULL AND TRIM(JSON_VALUE(f.components,'$.sourceCustomerCode'))<>'' AND f.group_name IS NOT NULL AND TRIM(f.group_name)<>'' GROUP BY customer_code,business_group,f.department,f.month_key";
}
}
@@ -27,6 +27,7 @@ public final class BusinessRowMapper {
private final ConfigurableRuleEngine rules;
private final ConfigurableMatchRules matchRules;
private final Map<String,String> savedGroups;
private final Map<String,String> savedProjects;
public BusinessRowMapper(Map<String, String> categoryLookup, Map<String, String> projectLookup, String mappingVersion) {
this(categoryLookup,projectLookup,mappingVersion,Map.of(),Map.of());
@@ -44,16 +45,18 @@ public final class BusinessRowMapper {
this.departmentLookup=departmentLookup;this.productLookup=productLookup;
this.rules=rules;
this.matchRules=ConfigurableMatchRules.fromDefinition(mappingDefinition);
this.savedGroups=Map.of();
this.savedGroups=Map.of();this.savedProjects=Map.of();
}
private BusinessRowMapper(BusinessRowMapper base,Map<String,String> savedGroups){
private BusinessRowMapper(BusinessRowMapper base,Map<String,String> savedGroups,Map<String,String> savedProjects){
this.categoryLookup=base.categoryLookup;this.projectLookup=base.projectLookup;this.mappingVersion=base.mappingVersion;
this.departmentLookup=base.departmentLookup;this.productLookup=base.productLookup;this.rules=base.rules;this.matchRules=base.matchRules;
this.savedGroups=Map.copyOf(savedGroups);
this.savedGroups=Map.copyOf(savedGroups);this.savedProjects=Map.copyOf(savedProjects);
}
public boolean hasUnclassifiedRules(){return matchRules.hasUnclassifiedRules();}
public BusinessRowMapper preserveAssignedGroups(Map<String,String> groups){return new BusinessRowMapper(this,groups);}
public BusinessRowMapper preserveAssignedGroups(Map<String,String> groups){return new BusinessRowMapper(this,groups,savedProjects);}
public boolean hasUnclassifiedRules(String target){return matchRules.hasUnclassifiedRules(target);}
public BusinessRowMapper preserveAssignedProjects(Map<String,String> projects){return new BusinessRowMapper(this,savedGroups,projects);}
static String sourceRowKey(String sheet,int row){return sheet+"\n"+row;}
public String mappingVersion() { return mappingVersion; }
@@ -169,11 +172,11 @@ public final class BusinessRowMapper {
String projectKey=joinedKey(customer,category);boolean blankProjectKey=customer==null||customer.isEmpty()||category==null||category.isEmpty();
boolean projectMappingKnown=!blankProjectKey&&projectLookup.containsKey(projectKey);
String projectGroup=projectMappingKnown?projectLookup.get(projectKey):null;
ConfigurableMatchRules.Match projectRule=matchRules.applyMatch(stage,"projectGroup",matchSource,computed(department,category,product,projectGroup),projectGroup);
ConfigurableMatchRules.Match projectRule=matchRules.applyProject(stage,matchSource,computed(department,category,product,projectGroup),projectGroup,savedProjects.get(sourceRowKey(input.sheet(),input.physicalRow())),projectMappingKnown&&(projectGroup==null||projectGroup.isBlank()));
projectGroup=projectRule.value();boolean knownNullProjectGroup=projectMappingKnown&&(projectGroup==null||projectGroup.isEmpty());
if(knownNullProjectGroup)projectGroup=null;
boolean projectUnmapped=!projectRule.matched()&&!blankProjectKey&&!projectMappingKnown;
String projectGroupStatus=projectRule.matched()?"mapped":projectUnmapped?"unmapped":blankProjectKey?"source_key_blank":knownNullProjectGroup?"mapped_null":"mapped";
boolean projectUnmapped=projectGroup==null&&!projectRule.matched()&&!blankProjectKey&&!projectMappingKnown;
String projectGroupStatus=projectRule.matched()||projectGroup!=null?"mapped":projectUnmapped?"unmapped":blankProjectKey?"source_key_blank":knownNullProjectGroup?"mapped_null":"mapped";
String categoryStatus=unmappedCategory?"unmapped":blankCategoryKey&&category==null?"source_key_blank":"mapped";
return new PreparedSource(source,raw,new DimensionValues(department,category,product,projectGroup,categoryStatus,projectGroupStatus,
pdt,region,customer,sourceDepartment,productLookupKey,unmappedCategory,projectUnmapped));
@@ -76,7 +76,7 @@ public final class ConfigurableMatchRules {
continue;
}
for (Object key : rule.keySet()) if (!Set.of("id", "name", "enabled", "priority", "stage", "conditions", "target", "value", "onlyUnclassified").contains(String.valueOf(key))) errors.add(path + " 未知字段: " + key);
if(rule.containsKey("onlyUnclassified") && (!(rule.get("onlyUnclassified") instanceof Boolean) || Boolean.TRUE.equals(rule.get("onlyUnclassified")) && !"group".equals(rule.get("target")))) errors.add(path+" onlyUnclassified仅适用于业务分组且须为布尔值");
if(rule.containsKey("onlyUnclassified") && (!(rule.get("onlyUnclassified") instanceof Boolean) || Boolean.TRUE.equals(rule.get("onlyUnclassified")) && !Set.of("group","projectGroup").contains(rule.get("target")))) errors.add(path+" onlyUnclassified仅适用于业务分组或项目组且须为布尔值");
String id = text(rule.get("id"));
if (id != null) textBudget += id.length();
if (id == null || !id.matches("[A-Za-z0-9][A-Za-z0-9_-]{0,63}")) errors.add(path + " id须为1至64位字母、数字、下划线或连字符");
@@ -168,6 +168,14 @@ public final class ConfigurableMatchRules {
return result.matched()||result.value()!=null&&!result.value().isBlank()?result:new Match(false,savedGroup);
}
public Match applyProject(String stage,Map<String,?> raw,Map<String,String> computed,String current,String savedProject,boolean explicitNull){
if(savedProject!=null&&!savedProject.isBlank()){
var ordinary=applyMatch(stage,"projectGroup",raw,computed,current,true);
return ordinary.matched()||ordinary.value()!=null&&!ordinary.value().isBlank()?ordinary:new Match(false,savedProject);
}
return applyMatch(stage,"projectGroup",raw,computed,current,explicitNull);
}
public boolean hasUnclassifiedRules(String target){return rules.stream().anyMatch(rule->target.equals(rule.get("target"))&&Boolean.TRUE.equals(rule.get("onlyUnclassified")));}
public boolean hasUnclassifiedRules(){return rules.stream().anyMatch(rule->Boolean.TRUE.equals(rule.get("onlyUnclassified")));}
public List<Map<String, Object>> rules() { return rules.stream().map(ConfigurableMatchRules::copyRule).toList(); }
@@ -236,8 +236,8 @@ public class ImportWorker {
List<BusinessRowMapper.RowValues> facts = new ArrayList<>(BATCH_SIZE);
List<Issue> issues = new ArrayList<>();
Set<String> availableHeaders = new HashSet<>();
Map<String,String> savedGroups=rowMapper.hasUnclassifiedRules()?loadAssignedGroups(jdbc,candidate.stage(),runId,revisionId):Map.of();
BusinessRowMapper effectiveMapper=rowMapper.preserveAssignedGroups(savedGroups);
Map<String,String> savedGroups=rowMapper.hasUnclassifiedRules("group")?loadAssignedGroups(jdbc,candidate.stage(),runId,revisionId):Map.of();
BusinessRowMapper effectiveMapper=rowMapper.preserveAssignedGroups(savedGroups).preserveAssignedProjects(rowMapper.hasUnclassifiedRules("projectGroup")?loadAssignedDimension(jdbc,candidate.stage(),runId,revisionId,"project_group"):Map.of());
long[] rowCount = {0};
int[] blocking = {0};
int[] blankCategory = {0};
@@ -328,11 +328,14 @@ public class ImportWorker {
}
static Map<String,String> loadAssignedGroups(JdbcTemplate jdbc,String stage,String runId,String revisionId){
if(!Set.of("black","bright").contains(stage))throw new IllegalArgumentException("来源阶段无效");
return loadAssignedDimension(jdbc,stage,runId,revisionId,"group_name");
}
static Map<String,String> loadAssignedDimension(JdbcTemplate jdbc,String stage,String runId,String revisionId,String column){
if(!Set.of("black","bright").contains(stage)||!Set.of("group_name","project_group").contains(column))throw new IllegalArgumentException("来源维度无效");
// 仅复用该任务基线中同一不可变来源修订的归属;新文件或修订不借用旧记录。
Map<String,String> savedGroups=new HashMap<>();
jdbc.queryForList("SELECT f.sheet_name,f.physical_row_no,f.group_name FROM fact_"+stage+" f JOIN release_member m ON m.transform_build_id=f.build_id JOIN import_run r ON r.expected_current_release_id=m.release_id WHERE r.id=? AND f.source_revision_id=? AND f.group_name IS NOT NULL AND TRIM(f.group_name)<>''",runId,revisionId)
.forEach(row->savedGroups.put(BusinessRowMapper.sourceRowKey(String.valueOf(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue()),String.valueOf(row.get("group_name"))));
jdbc.queryForList("SELECT f.sheet_name,f.physical_row_no,f."+column+" assigned_value FROM fact_"+stage+" f JOIN release_member m ON m.transform_build_id=f.build_id JOIN import_run r ON r.expected_current_release_id=m.release_id WHERE r.id=? AND f.source_revision_id=? AND f."+column+" IS NOT NULL AND TRIM(f."+column+")<>''",runId,revisionId)
.forEach(row->savedGroups.put(BusinessRowMapper.sourceRowKey(String.valueOf(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue()),String.valueOf(row.get("assigned_value"))));
return savedGroups;
}
@@ -25,13 +25,15 @@ public final class UnclassifiedMappingRebuild {
var result=new ArrayList<Map<String,Object>>();
for(String id:changed){var prior=old.get(id);var rule=next.get(id);
if(rule==null){if(!Boolean.TRUE.equals(prior.get("onlyUnclassified")))throw new IllegalArgumentException("不能在未分类维护中移除普通匹配规则");continue;}
if(!Boolean.TRUE.equals(rule.get("onlyUnclassified"))||!"group".equals(rule.get("target")))throw new IllegalArgumentException("未分类规则不能改为覆盖已有分类");
for(var raw:(List<?>)rule.get("conditions")){var condition=map(raw);if(!Set.of("@department","@product","PDT代码","销售片区").contains(condition.get("field"))||!Set.of("EQUALS","IS_BLANK").contains(condition.get("operator")))throw new IllegalArgumentException("未分类维护仅支持产业部、产品、PDT代码和销售区域条件");}
if(!Boolean.TRUE.equals(rule.get("onlyUnclassified"))||!Set.of("group","projectGroup").contains(rule.get("target")))throw new IllegalArgumentException("未分类规则不能改为覆盖已有分类");
for(var raw:(List<?>)rule.get("conditions")){var condition=map(raw);var fields="projectGroup".equals(rule.get("target"))?Set.of("客户代码","@group"):Set.of("@department","@product","PDT代码","销售片区");if(!fields.contains(condition.get("field"))||!Set.of("EQUALS","IS_BLANK").contains(condition.get("operator")))throw new IllegalArgumentException("未分类维护条件与分类类型不匹配");}
if(Boolean.TRUE.equals(rule.get("enabled")))result.add(rule);
}
return result;
}
record Change(String stage,String parent,String sheet,int row,String month,String department,String product,String oldGroup,String oldProject,String group,String project,Map<String,Object> components){}
record Change(String stage,String parent,String sheet,int row,String month,String department,String product,String oldGroup,String oldProject,String group,String project,boolean groupChanged,boolean projectChanged,Map<String,Object> components){}
static boolean blank(String value){return value==null||value.isBlank();}
public static String projectBlankSql(String alias){if(!Set.of("f").contains(alias))throw new IllegalArgumentException("来源别名无效");return "("+alias+".project_group IS NULL OR TRIM("+alias+".project_group)='') AND COALESCE(JSON_VALUE("+alias+".components,'$.projectGroupStatus'),'')<>'mapped_null'";}
public record Candidate(String releaseId,String runId,int scannedRows,int classifiedRows){}
private String text(Object value){return value==null?null:String.valueOf(value);}
private Map<String,Object> decode(Object value){if(value==null)return new LinkedHashMap<>();return new LinkedHashMap<>(json.readValue(String.valueOf(value),Map.class));}
@@ -43,20 +45,48 @@ public final class UnclassifiedMappingRebuild {
var matcher=ConfigurableMatchRules.fromDefinition(Map.of("matchRules",changedRules));var projectRules=ConfigurableMatchRules.fromDefinition(definition);
var changes=new ArrayList<Change>();int scanned=0;var projectLookups=new HashMap<String,Map<String,String>>();
for(String stage:List.of("black","bright")){
var rows=jdbc.queryForList("SELECT f.build_id,f.sheet_name,f.physical_row_no,f.month_key,f.department,f.group_name,f.project_group,f.product,CAST(f.components AS CHAR) components,CAST(f.raw_values AS CHAR) raw_values,CAST(b.parser_config AS CHAR) parser_config FROM release_member m JOIN fact_"+stage+" f ON f.build_id=m.transform_build_id JOIN transform_build b ON b.id=f.build_id WHERE m.release_id=? AND (f.group_name IS NULL OR TRIM(f.group_name)='')",parent);
var candidates=new ArrayList<String>();var args=new ArrayList<Object>();args.add(parent);
for(var changed:changedRules){
if(!"all".equals(changed.get("stage"))&&!stage.equals(changed.get("stage")))continue;
if("group".equals(changed.get("target"))){candidates.add("(f.group_name IS NULL OR TRIM(f.group_name)='')");continue;}
var conditions=new ArrayList<String>();conditions.add(projectBlankSql("f"));
for(var rawCondition:(List<?>)changed.get("conditions")){
var condition=map(rawCondition);String field="@group".equals(condition.get("field"))?"f.group_name":"JSON_VALUE(f.components,'$.sourceCustomerCode')";
if("IS_BLANK".equals(condition.get("operator")))conditions.add("("+field+" IS NULL OR TRIM("+field+")='')");
else{conditions.add("CAST("+field+" AS BINARY)=CAST(? AS BINARY)");args.add(condition.get("value"));}
}
candidates.add("("+String.join(" AND ",conditions)+")");
}
if(candidates.isEmpty())continue;
var rows=jdbc.queryForList("SELECT f.build_id,f.sheet_name,f.physical_row_no,f.month_key,f.department,f.group_name,f.project_group,f.product,CAST(f.components AS CHAR) components,CAST(f.raw_values AS CHAR) raw_values,CAST(b.parser_config AS CHAR) parser_config FROM release_member m JOIN fact_"+stage+" f ON f.build_id=m.transform_build_id JOIN transform_build b ON b.id=f.build_id WHERE m.release_id=? AND ("+String.join(" OR ",candidates)+")",args.toArray());
scanned+=rows.size();
for(var row:rows){
var components=decode(row.get("components"));var raw=decode(row.get("raw_values"));var restored=BusinessRowMapper.restoreStoredRow(stage,text(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue(),raw,decode(row.get("parser_config")));
restored.values().forEach((key,value)->raw.put(key,value==null?null:value.display()));
raw.put("PDT代码",components.get("sourcePdtCode"));raw.put("销售片区",components.get("sourceSalesRegion"));raw.put("客户代码",components.get("sourceCustomerCode"));
var computed=new LinkedHashMap<String,String>();computed.put("department",text(row.get("department")));computed.put("group",text(row.get("group_name")));computed.put("product",text(row.get("product")));computed.put("projectGroup",text(row.get("project_group")));
var match=matcher.applyMatch(stage,"group",raw,computed,text(row.get("group_name")));if(!match.matched())continue;
String month=text(row.get("month_key")),group=match.value(),customer=text(components.get("sourceCustomerCode"));
var lookup=projectLookups.computeIfAbsent(month,key->worker.loadMappings("PROJECT_GROUP",mapping,Integer.parseInt(key.substring(0,4)),Integer.parseInt(key.substring(5,7))));
boolean blank=customer==null||customer.isBlank();String key=blank?null:customer+"|"+group;boolean known=!blank&&lookup.containsKey(key);String project=known?lookup.get(key):null;
computed.put("group",group);computed.put("projectGroup",project);var projectMatch=projectRules.applyMatch(stage,"projectGroup",raw,computed,project);project=projectMatch.value();
components.put("categoryMappingStatus","mapped");components.put("projectGroup",project);components.put("projectGroupStatus",projectMatch.matched()?"mapped":blank?"source_key_blank":known?(project==null?"mapped_null":"mapped"):"unmapped");
changes.add(new Change(stage,text(row.get("build_id")),text(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue(),month,text(row.get("department")),text(row.get("product")),text(row.get("group_name")),text(row.get("project_group")),group,project,components));
String oldGroup=text(row.get("group_name")),oldProject=text(row.get("project_group"));
var computed=new LinkedHashMap<String,String>();computed.put("department",text(row.get("department")));computed.put("group",oldGroup);computed.put("product",text(row.get("product")));computed.put("projectGroup",oldProject);
var groupMatch=matcher.applyMatch(stage,"group",raw,computed,oldGroup);String group=groupMatch.value(),project=oldProject;
boolean canFillProject=blank(oldProject)&&!"mapped_null".equals(components.get("projectGroupStatus"));
if(groupMatch.matched()){
components.put("categoryMappingStatus","mapped");computed.put("group",group);
if(canFillProject){
String month=text(row.get("month_key")),customer=text(components.get("sourceCustomerCode"));
var lookup=projectLookups.computeIfAbsent(month,key->worker.loadMappings("PROJECT_GROUP",mapping,Integer.parseInt(key.substring(0,4)),Integer.parseInt(key.substring(5,7))));
boolean missingKey=blank(customer)||blank(group);String key=missingKey?null:customer+"|"+group;boolean known=!missingKey&&lookup.containsKey(key);project=known?lookup.get(key):null;
boolean explicitNull=known&&blank(project);
computed.put("projectGroup",project);var match=projectRules.applyProject(stage,raw,computed,project,null,explicitNull);project=match.value();
components.put("projectGroup",project);components.put("projectGroupStatus",match.matched()||!blank(project)?"mapped":missingKey?"source_key_blank":known?"mapped_null":"unmapped");
}
}
computed.put("group",group);computed.put("projectGroup",project);
if(canFillProject&&blank(project)&&!"mapped_null".equals(components.get("projectGroupStatus"))){
var projectMatch=matcher.applyMatch(stage,"projectGroup",raw,computed,project);
if(projectMatch.matched()){project=projectMatch.value();components.put("projectGroup",project);components.put("projectGroupStatus","mapped");}
}
boolean groupChanged=!Objects.equals(group,oldGroup),projectChanged=!Objects.equals(project,oldProject);
if(!groupChanged&&!projectChanged)continue;
changes.add(new Change(stage,text(row.get("build_id")),text(row.get("sheet_name")),((Number)row.get("physical_row_no")).intValue(),text(row.get("month_key")),text(row.get("department")),text(row.get("product")),oldGroup,oldProject,group,project,groupChanged,projectChanged,components));
}
}
String release=UUID.randomUUID().toString(),run=UUID.randomUUID().toString();
@@ -73,7 +103,7 @@ public final class UnclassifiedMappingRebuild {
jdbc.update("INSERT INTO defect_component(build_id,stage,sheet_name,physical_row_no,defect_name,weight,source_field) SELECT ?,stage,sheet_name,physical_row_no,defect_name,weight,source_field FROM defect_component WHERE build_id=?",build,old);
rollups.copyBuild(old,build);var keys=new LinkedHashSet<MetricsRollupService.GroupKey>();
for(var change:changes)if(change.parent().equals(old)){
int updated=jdbc.update("UPDATE fact_"+change.stage()+" SET group_name=?,project_group=?,components=CAST(? AS JSON) WHERE build_id=? AND sheet_name=? AND physical_row_no=? AND (group_name IS NULL OR TRIM(group_name)='')",change.group(),change.project(),encode(change.components()),build,change.sheet(),change.row());if(updated!=1)throw new IllegalStateException("未分类行已变化,归类未提交");
int updated=jdbc.update("UPDATE fact_"+change.stage()+" SET group_name=?,project_group=?,components=CAST(? AS JSON) WHERE build_id=? AND sheet_name=? AND physical_row_no=? AND (group_name<=>?) AND (project_group<=>?) AND "+(change.groupChanged()?"(group_name IS NULL OR TRIM(group_name)='')":"(project_group IS NULL OR TRIM(project_group)='')"),change.group(),change.project(),encode(change.components()),build,change.sheet(),change.row(),change.oldGroup(),change.oldProject());if(updated!=1)throw new IllegalStateException("未分类行已变化,归类未提交");
changedOrigins.add(new cn.yunsheng.quality.governance.MappingIssueService.SourceRow(change.stage(),build,change.sheet(),change.row()));
keys.add(new MetricsRollupService.GroupKey(change.stage(),change.month(),change.department(),change.oldGroup(),change.oldProject(),change.product()));keys.add(new MetricsRollupService.GroupKey(change.stage(),change.month(),change.department(),change.group(),change.project(),change.product()));
}
@@ -46,4 +46,25 @@ class ClassificationPreviewMySqlTest {
assertEquals(0L,service.preview(new ClassificationPreviewService.Request("all","G",List.of(equal("@department","A' OR 1=1 --"))),actor).get("matchedRows"));
assertEquals(3,jdbc.queryForObject("SELECT COUNT(*) FROM fact_black",Integer.class));
}
private void projectFixture(){
for(String stage:List.of("black","bright")){jdbc.update("DELETE FROM fact_"+stage);jdbc.execute("ALTER TABLE fact_"+stage+" ADD project_group VARCHAR(40)");}
jdbc.update("INSERT INTO fact_black VALUES('b','A','P','G','2026-01',10,JSON_OBJECT('sourceCustomerCode','001','projectGroupStatus','unmapped'),NULL),('b','A','P','G','2026-02',20,JSON_OBJECT('sourceCustomerCode','001','projectGroupStatus','mapped'),'KEEP'),('b','B','P','G','2026-02',60,JSON_OBJECT('sourceCustomerCode','002','projectGroupStatus','unmapped'),NULL)");
jdbc.update("INSERT INTO fact_bright VALUES('l','A','P','G','2026-03',30,JSON_OBJECT('sourceCustomerCode','001','projectGroupStatus','unmapped'),NULL),('l','A','P','G','2026-04',40,JSON_OBJECT('sourceCustomerCode','001','projectGroupStatus','mapped_null'),NULL)");
}
@Test void projectPreviewRequiresBothKeysAndExcludesAssignedAndExplicitNullProjects(){
projectFixture();var request=new ClassificationPreviewService.Request("all","NEW",List.of(equal("客户代码","001"),equal("@group","G")),"projectGroup");
var result=service.preview(request,actor);assertEquals(2L,result.get("matchedRows"));assertEquals(2L,result.get("changedRows"));assertEquals(0,new java.math.BigDecimal("40").compareTo((java.math.BigDecimal)result.get("totalWeight")));
assertThrows(org.springframework.web.server.ResponseStatusException.class,()->service.preview(new ClassificationPreviewService.Request("all","NEW",List.of(equal("客户代码","001")),"projectGroup"),actor));
assertThrows(org.springframework.web.server.ResponseStatusException.class,()->service.preview(new ClassificationPreviewService.Request("all","NEW",List.of(equal("客户代码","001"),equal("@product","P")),"projectGroup"),actor));
}
@Test void projectInventoryIsCurrentUnclassifiedOnlyAndKeepsCustomerGroupDepartmentFilters(){
projectFixture();var service=new UnclassifiedSourceService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),mock(CandidateAcceptanceService.class));
var all=service.listProjects(1,15,null,null,null,actor);assertEquals(2,all.get("total"));
var filtered=service.listProjects(1,15,"001","G","A",actor);assertEquals(1,filtered.get("total"));var item=(Map<?,?>)((List<?>)filtered.get("items")).getFirst();assertEquals(2L,item.get("rowCount"));
assertEquals(0,service.listProjects(1,15,"001","G","B",actor).get("total"));
for(String stage:List.of("black","bright"))jdbc.update("UPDATE fact_"+stage+" SET project_group='NEW' WHERE JSON_VALUE(components,'$.sourceCustomerCode')='001' AND JSON_VALUE(components,'$.projectGroupStatus')='unmapped'");
jdbc.update("UPDATE current_release SET release_id='r2'");jdbc.update("INSERT INTO release_member VALUES('r2','b'),('r2','l')");
assertEquals(0,service.listProjects(1,15,"001",null,null,actor).get("total"));assertEquals(1,service.listProjects(1,15,null,null,null,actor).get("total"));
}
}
@@ -31,6 +31,18 @@ class ConfigurableMatchRulesTest {
assertEquals("工业电机-VCM",mapper.preserveAssignedGroups(saved).transform("black",row("001","VCM产品","华东","产业部"),2026,1).group());
assertEquals("明确的新对应",mapper(Map.of("P1|华东","明确的新对应"),Map.of(),definition).preserveAssignedGroups(saved).transform("black",row("001","普通产品","华东","产业部"),2026,1).group());
}
@Test void projectScopeCannotOverrideLookupSavedOrExplicitNullAssignments(){
var rule=new LinkedHashMap<String,Object>(rule("pending-project",-9900,"all","projectGroup","NEW",List.of(condition("客户代码","EQUALS","001"),condition("@group","EQUALS","G"))));rule.put("onlyUnclassified",true);
var definition=Map.<String,Object>of("matchRules",List.of(rule));assertTrue(ConfigurableMatchRules.validate(definition).isEmpty());
var row=row("001","普通产品","华东","产业部");var groups=Map.of("P1|华东","G");
var empty=mapper(groups,Map.of(),definition);assertEquals("NEW",empty.transform("black",row,2026,1).components().get("projectGroup"));
var kept=empty.preserveAssignedProjects(Map.of(BusinessRowMapper.sourceRowKey("测试",2),"SAVED" )).transform("black",row,2026,1);
assertEquals("SAVED",kept.components().get("projectGroup"));assertEquals("mapped",kept.components().get("projectGroupStatus"));assertFalse(kept.unmappedProjectGroup());
assertEquals("LOOKUP",mapper(groups,Map.of("001|G","LOOKUP"),definition).transform("black",row,2026,1).components().get("projectGroup"));
var explicit=new LinkedHashMap<String,String>();explicit.put("001|G",null);var result=mapper(groups,explicit,definition).transform("black",row,2026,1);
assertNull(result.components().get("projectGroup"));assertEquals("mapped_null",result.components().get("projectGroupStatus"));
}
@Test void missingRulesKeepLegacyVcmBehaviorAndExplicitEmptyArrayRemovesIt() {
Map<String, String> categories = Map.of("P1|华东", "原始分组");
var legacy = mapper(categories, Map.of(), Map.of());
@@ -65,4 +65,29 @@ class UnclassifiedMappingRebuildMySqlTest {
assertTrue(ImportWorker.loadAssignedGroups(jdbc,"black",run,bright).isEmpty());
assertTrue(ImportWorker.loadAssignedGroups(jdbc,"black",UUID.randomUUID().toString(),black).isEmpty());
}
@Test void projectMaintenanceOnlyFillsMatchingEmptyProjectsAcrossStages(){
for(String stage:List.of("black","bright"))jdbc.update("UPDATE fact_"+stage+" SET group_name='G',components=JSON_SET(components,'$.categoryMappingStatus','mapped','$.projectGroupStatus','unmapped')");
jdbc.update("UPDATE fact_black SET project_group='KEEP',components=JSON_SET(components,'$.projectGroup','KEEP','$.projectGroupStatus','mapped') WHERE physical_row_no=2");
jdbc.update("UPDATE fact_black SET components=JSON_SET(components,'$.sourceCustomerCode','002') WHERE physical_row_no=3");
seed("black",black,4,"A","G",3,1);
jdbc.update("UPDATE fact_black SET components=JSON_SET(components,'$.projectGroupStatus','mapped_null') WHERE physical_row_no=4");
rollups.refreshBuild(black);rollups.refreshBuild(bright);
var pending=new LinkedHashMap<String,Object>(pending());pending.put("target","projectGroup");pending.put("value","NEW_PROJECT");pending.put("conditions",List.of(Map.of("field","客户代码","operator","EQUALS","value","001"),Map.of("field","@group","operator","EQUALS","value","G")));
var candidate=new TransactionTemplate(new DataSourceTransactionManager(dataSource)).execute(status->new UnclassifiedMappingRebuild(jdbc,json,worker,rollups).create(UUID.randomUUID().toString(),parent,"new-map","rule",Map.of("matchRules",List.of(pending)),List.of(pending),actor));
assertEquals(2,candidate.scannedRows());assertEquals(2,candidate.classifiedRows());
assertEquals("KEEP",jdbc.queryForObject("SELECT f.project_group FROM fact_black f JOIN release_member m ON m.transform_build_id=f.build_id WHERE m.release_id=? AND f.physical_row_no=2",String.class,candidate.releaseId()));
assertEquals(2,jdbc.queryForObject("SELECT COUNT(*) FROM fact_black f JOIN release_member m ON m.transform_build_id=f.build_id WHERE m.release_id=? AND f.project_group IS NULL",Integer.class,candidate.releaseId()));
for(String stage:List.of("black","bright")){
assertEquals("NEW_PROJECT",jdbc.queryForObject("SELECT f.project_group FROM fact_"+stage+" f JOIN release_member m ON m.transform_build_id=f.build_id WHERE m.release_id=? AND f.physical_row_no=1",String.class,candidate.releaseId()));
assertEquals(0,jdbc.queryForObject("SELECT COUNT(*) FROM fact_"+stage+" n JOIN release_member m ON m.transform_build_id=n.build_id JOIN fact_"+stage+" o ON o.build_id=? AND o.sheet_name=n.sheet_name AND o.physical_row_no=n.physical_row_no WHERE m.release_id=? AND NOT(n.group_name<=>o.group_name AND n.raw_values<=>o.raw_values AND n.total_weight<=>o.total_weight AND n.scrap_weight<=>o.scrap_weight AND n.qualified_weight<=>o.qualified_weight)",Integer.class,stage.equals("black")?black:bright,candidate.releaseId()));
}
verify(worker,never()).execute(anyString());verify(worker,never()).loadMappings(anyString(),anyString(),anyInt(),anyInt());
assertEquals(0,jdbc.queryForObject("SELECT COUNT(*) FROM metric_rollup r JOIN release_member m ON m.transform_build_id=r.build_id WHERE m.release_id=? AND r.project_group='NEW_PROJECT' AND r.source_row_count<>1",Integer.class,candidate.releaseId()));
}
@Test void assigningBusinessGroupDoesNotReplaceAnAlreadyAssignedProject(){
jdbc.update("UPDATE fact_black SET project_group='KEEP',components=JSON_SET(components,'$.projectGroup','KEEP','$.projectGroupStatus','mapped') WHERE physical_row_no=1");rollups.refreshBuild(black);
var candidate=run(rollups);
assertEquals("KEEP",jdbc.queryForObject("SELECT f.project_group FROM fact_black f JOIN release_member m ON m.transform_build_id=f.build_id WHERE m.release_id=? AND f.physical_row_no=1",String.class,candidate.releaseId()));
}
}
@@ -11,4 +11,11 @@ class UnclassifiedMappingScopeTest {
assertThrows(IllegalArgumentException.class,()->UnclassifiedMappingRebuild.changedRules(after,Map.of("categoryLookup",before.get("categoryLookup"),"matchRules",List.of(rule(false)))));
assertTrue(UnclassifiedMappingRebuild.unchanged(Map.of(),Map.of("matchRules",ConfigurableMatchRules.defaultRules())));
}
@Test void projectRulesUseIncrementalPathAndCannotBeDowngraded(){
var project=new LinkedHashMap<String,Object>(rule(true));project.put("target","projectGroup");project.put("conditions",List.of(Map.of("field","客户代码","operator","EQUALS","value","001"),Map.of("field","@group","operator","EQUALS","value","G")));
var before=Map.<String,Object>of("matchRules",List.of());var after=Map.<String,Object>of("matchRules",List.of(project));
assertEquals(List.of(project),UnclassifiedMappingRebuild.changedRules(before,after));
var ordinary=new LinkedHashMap<>(project);ordinary.put("onlyUnclassified",false);assertThrows(IllegalArgumentException.class,()->UnclassifiedMappingRebuild.changedRules(after,Map.of("matchRules",List.of(ordinary))));
}
}