优化:完善分类规则与Excel配置维护并加速查询翻页
This commit is contained in:
@@ -215,7 +215,7 @@ public class AnalysisFineDefectsService {
|
||||
&& !categories(stage).stream().anyMatch(item -> item.name().equals(category))) throw bad("category 不属于所选 stage");
|
||||
String productSearch = optionalText(input, "productSearch");
|
||||
if (productSearch != null && productSearch.length() > SelectionValues.MAX_VALUE_LENGTH) throw bad("产品搜索文本过长");
|
||||
boolean includeUnassigned = bool(input, "includeUnassigned", false);
|
||||
boolean includeUnassigned = bool(input, "includeUnassigned", true);
|
||||
int limit = integer(input, "limit") == null ? 20 : integer(input, "limit");
|
||||
if (limit < 1 || limit > 100) throw bad("limit 必须在 1 到 100 之间");
|
||||
return new Request(releaseId, mode, year, month, quarter, optionalText(input, "department"),
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package cn.yunsheng.quality.common;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/** Bounded process-local cache for immutable release data. Authorization must precede every lookup. */
|
||||
public final class ImmutableQueryCache<K, V> {
|
||||
private final int capacity;
|
||||
private final Map<K, V> values = new LinkedHashMap<>(16, .75f, true);
|
||||
private final Map<K, FutureTask<V>> loading = new HashMap<>();
|
||||
|
||||
public ImmutableQueryCache(int capacity) {
|
||||
if (capacity < 1) throw new IllegalArgumentException("capacity");
|
||||
this.capacity = capacity;
|
||||
}
|
||||
|
||||
public V get(K key, Supplier<V> loader) {
|
||||
FutureTask<V> task;
|
||||
boolean owner = false;
|
||||
synchronized (this) {
|
||||
if (values.containsKey(key)) return values.get(key);
|
||||
task = loading.get(key);
|
||||
if (task == null) {
|
||||
task = new FutureTask<>(() -> Objects.requireNonNull(loader.get()));
|
||||
loading.put(key, task);
|
||||
owner = true;
|
||||
}
|
||||
}
|
||||
if (owner) task.run(); // Different releases/queries can load concurrently.
|
||||
try {
|
||||
V value = task.get();
|
||||
if (owner) synchronized (this) {
|
||||
values.put(key, value);
|
||||
while (values.size() > capacity) values.remove(values.keySet().iterator().next());
|
||||
}
|
||||
return value;
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("查询等待被中断", interrupted);
|
||||
} catch (ExecutionException failure) {
|
||||
if (failure.getCause() instanceof RuntimeException cause) throw cause;
|
||||
if (failure.getCause() instanceof Error cause) throw cause;
|
||||
throw new IllegalStateException(failure.getCause());
|
||||
} finally {
|
||||
if (owner) synchronized (this) { loading.remove(key, task); }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package cn.yunsheng.quality.filters;
|
||||
|
||||
import cn.yunsheng.quality.common.DataScope;
|
||||
import cn.yunsheng.quality.common.ImmutableQueryCache;
|
||||
import cn.yunsheng.quality.common.DepartmentLabels;
|
||||
import cn.yunsheng.quality.common.SelectionValues;
|
||||
import cn.yunsheng.quality.governance.GovernanceAccessService;
|
||||
@@ -20,6 +21,7 @@ import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Isolation;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
@@ -30,6 +32,34 @@ import tools.jackson.databind.json.JsonMapper;
|
||||
/** Actor-scoped catalog and parameterized filters for published source columns. */
|
||||
@Service
|
||||
public class SourceColumnQueries {
|
||||
private record QueryKey(String sql, List<Object> args, String projection) {
|
||||
QueryKey { args = List.copyOf(args); }
|
||||
}
|
||||
private final ImmutableQueryCache<QueryKey, Long> totals = new ImmutableQueryCache<>(128);
|
||||
private final ImmutableQueryCache<QueryKey, List<String>> rawHeaders = new ImmutableQueryCache<>(32);
|
||||
private final ImmutableQueryCache<QueryKey, List<Map<String, Object>>> pages = new ImmutableQueryCache<>(64);
|
||||
|
||||
private long total(String sql, List<Object> args) {
|
||||
return totals.get(new QueryKey(sql,args,"count"), () -> {
|
||||
Long value=jdbc.queryForObject(sql,Long.class,args.toArray());return value==null?0L:value;
|
||||
});
|
||||
}
|
||||
|
||||
/** Reuse a bounded window; authorization/catalog validation still runs before this method. */
|
||||
private List<Map<String,Object>> page(String sql,List<Object> args,String projection,int offset,int limit,
|
||||
RowMapper<Map<String,Object>> mapper) {
|
||||
List<Map<String,Object>> result=new ArrayList<>();
|
||||
for(long start=offset,end=(long)offset+limit;start<end;) {
|
||||
long windowStart=start/240*240;List<Object> parameters=new ArrayList<>(args);
|
||||
parameters.add(240);parameters.add(windowStart);
|
||||
List<Map<String,Object>> window=pages.get(new QueryKey(sql,parameters,projection),
|
||||
()->List.copyOf(jdbc.query(sql,mapper,parameters.toArray())));
|
||||
int first=(int)(start-windowStart),last=(int)Math.min(window.size(),end-windowStart);
|
||||
if(first>=last)break;
|
||||
result.addAll(window.subList(first,last));start=windowStart+last;
|
||||
}
|
||||
return List.copyOf(result);
|
||||
}
|
||||
private static final List<String> STAGES = List.of("black", "bright");
|
||||
private static final List<String> OPERATORS = List.of("in", "not_in", "is_empty", "not_empty");
|
||||
private static final Map<String, String> CANONICAL_COLUMNS = Map.of(
|
||||
@@ -74,7 +104,8 @@ public class SourceColumnQueries {
|
||||
+ "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"
|
||||
+ representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative "
|
||||
+ "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id";
|
||||
List<String> keyLists = jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray());
|
||||
List<String> keyLists = rawHeaders.get(new QueryKey(sql,args,"raw-headers"),
|
||||
() -> Collections.unmodifiableList(new ArrayList<>(jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray()))));
|
||||
stagePresent.put(stage, !keyLists.isEmpty());
|
||||
for (String rawKeys : keyLists) {
|
||||
if (rawKeys == null) continue;
|
||||
@@ -308,13 +339,10 @@ public class SourceColumnQueries {
|
||||
String unionSql = String.join(" UNION ALL ", selects);
|
||||
String tupleColumns = tupleColumns(memberFields.size());
|
||||
String grouped = "SELECT " + tupleColumns + " FROM (" + unionSql + ") member_rows GROUP BY " + tupleColumns;
|
||||
Long total = jdbc.queryForObject("SELECT COUNT(*) FROM (" + grouped + ") member_tuples", Long.class, queryArgs.toArray());
|
||||
List<Object> pageArgs = new ArrayList<>(queryArgs);
|
||||
pageArgs.add(limit);
|
||||
pageArgs.add(offset);
|
||||
long total = total("SELECT COUNT(*) FROM (" + grouped + ") member_tuples", queryArgs);
|
||||
String pageSql = "SELECT " + tupleColumns + ", GROUP_CONCAT(DISTINCT member_stage ORDER BY member_stage SEPARATOR ',') AS member_stages "
|
||||
+ "FROM (" + unionSql + ") member_rows GROUP BY " + tupleColumns + " ORDER BY " + tupleColumns + " LIMIT ? OFFSET ?";
|
||||
List<Map<String, Object>> rows = jdbc.query(pageSql, (rs, row) -> {
|
||||
List<Map<String, Object>> rows = page(pageSql,queryArgs,memberFields.toString(),offset,limit,(rs, row) -> {
|
||||
Map<String, Object> values = new LinkedHashMap<>();
|
||||
for (int index = 0; index < memberFields.size(); index++) values.put(memberFields.get(index), rs.getString("c" + index));
|
||||
String joinedStages = rs.getString("member_stages");
|
||||
@@ -324,8 +352,8 @@ public class SourceColumnQueries {
|
||||
result.put("values", Collections.unmodifiableMap(values));
|
||||
result.put("stages", rowStages);
|
||||
return Collections.unmodifiableMap(result);
|
||||
}, pageArgs.toArray());
|
||||
long totalCount = total == null ? 0 : total;
|
||||
});
|
||||
long totalCount = total;
|
||||
Map<String, Object> filters = queryContext.filtersResponse();
|
||||
Map<String, Object> response = new LinkedHashMap<>();
|
||||
response.put("rows", List.copyOf(rows));
|
||||
@@ -402,16 +430,11 @@ public class SourceColumnQueries {
|
||||
}
|
||||
String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition;
|
||||
String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value";
|
||||
Long total = jdbc.queryForObject("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", Long.class,
|
||||
combine(unionArgs, searchArgs).toArray());
|
||||
List<Object> pageArgs = combine(unionArgs, searchArgs);
|
||||
pageArgs.add(limit);
|
||||
pageArgs.add(offset);
|
||||
List<Map<String, Object>> rows = jdbc.query("SELECT candidate_value FROM (" + candidates + ") matches "
|
||||
long total = total("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", combine(unionArgs,searchArgs));
|
||||
List<Map<String, Object>> rows = page("SELECT candidate_value FROM (" + candidates + ") matches "
|
||||
+ "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?",
|
||||
(rs, row) -> Map.of("value", rs.getString("candidate_value")),
|
||||
pageArgs.toArray());
|
||||
long totalCount = total == null ? 0 : total;
|
||||
combine(unionArgs,searchArgs),fieldKey,offset,limit,(rs, row) -> Map.of("value", rs.getString("candidate_value")));
|
||||
long totalCount = total;
|
||||
Map<String, Object> response = new LinkedHashMap<>();
|
||||
response.put("rows", List.copyOf(rows));
|
||||
response.put("totalCount", totalCount);
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package cn.yunsheng.quality.governance;
|
||||
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.common.ImmutableQueryCache;
|
||||
import java.math.BigDecimal;
|
||||
import java.util.*;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/** Counts current immutable source rows, including already classified rows, without changing mappings. */
|
||||
@Service
|
||||
public class ClassificationPreviewService {
|
||||
private final GovernanceSupport g;
|
||||
private final CandidateAcceptanceService access;
|
||||
private record Key(String release,String stage,List<Condition> conditions) { Key { conditions=List.copyOf(conditions); } }
|
||||
private final ImmutableQueryCache<Key,List<Map<String,Object>>> previews=new ImmutableQueryCache<>(64);
|
||||
public ClassificationPreviewService(GovernanceSupport g,CandidateAcceptanceService access){this.g=g;this.access=access;}
|
||||
public record Condition(String field,String operator,String value){}
|
||||
public record Request(String stage,String target,List<Condition> conditions){}
|
||||
private static final Map<String,String> FIELDS=Map.of("@department","f.department","@product","f.product",
|
||||
"PDT代码","JSON_VALUE(f.components,'$.sourcePdtCode')","销售片区","JSON_VALUE(f.components,'$.sourceSalesRegion')");
|
||||
public Map<String,Object> preview(Request request,AppUserPrincipal actor){
|
||||
validate(request);access.requireSourceDetail(actor);
|
||||
var current=g.rows("SELECT release_id FROM current_release WHERE dataset_id='quality'");
|
||||
if(current.isEmpty())return Map.of("items",List.of(),"matchedRows",0,"changedRows",0,"totalWeight",BigDecimal.ZERO);
|
||||
String release=String.valueOf(current.getFirst().get("releaseId"));access.requireRead(release,actor);
|
||||
var items=previews.get(new Key(release,request.stage(),request.conditions()),()->load(release,request));
|
||||
long matched=0,changed=0;BigDecimal weight=BigDecimal.ZERO;
|
||||
for(var row:items){long count=((Number)row.get("rowCount")).longValue();matched+=count;
|
||||
if(!Objects.equals(row.get("currentGroup"),request.target()))changed+=count;
|
||||
if(row.get("totalWeight") instanceof Number n)weight=weight.add(new BigDecimal(n.toString()));}
|
||||
return Map.of("releaseId",release,"items",items,"matchedRows",matched,"changedRows",changed,"totalWeight",weight);
|
||||
}
|
||||
static void validate(Request request){
|
||||
if(request==null||!Set.of("all","black","bright").contains(Objects.toString(request.stage(),""))
|
||||
||request.target()==null||request.target().isBlank()||request.target().length()>160
|
||||
||request.conditions()==null||request.conditions().isEmpty()||request.conditions().size()>4)
|
||||
throw GovernanceSupport.error(HttpStatus.BAD_REQUEST,"分类规则条件无效");
|
||||
Set<String> fields=new HashSet<>();
|
||||
for(var condition:request.conditions())if(condition==null||condition.field()==null||!FIELDS.containsKey(condition.field())||!fields.add(condition.field())
|
||||
||!Set.of("EQUALS","IS_BLANK").contains(Objects.toString(condition.operator(),""))
|
||||
||condition.value()==null||condition.value().length()>200
|
||||
||("EQUALS".equals(condition.operator())&&condition.value().isBlank())
|
||||
||("IS_BLANK".equals(condition.operator())&&!condition.value().isEmpty()))
|
||||
throw GovernanceSupport.error(HttpStatus.BAD_REQUEST,"分类条件字段、判断方式或判断值无效");
|
||||
}
|
||||
private List<Map<String,Object>> load(String release,Request request){
|
||||
List<String> queries=new ArrayList<>();List<Object> args=new ArrayList<>();
|
||||
for(String stage:request.stage().equals("all")?List.of("black","bright"):List.of(request.stage())){
|
||||
StringBuilder where=new StringBuilder(" WHERE rm.release_id=?");args.add(release);
|
||||
for(var condition:request.conditions()){
|
||||
String expression=FIELDS.get(condition.field());
|
||||
if(condition.operator().equals("IS_BLANK"))where.append(" AND (").append(expression).append(" IS NULL OR REGEXP_LIKE(").append(expression).append(",'^[[:space:]]*$'))");
|
||||
else{where.append(" AND CAST(").append(expression).append(" AS BINARY)=CAST(? AS BINARY)");args.add(condition.value());}
|
||||
}
|
||||
queries.add("SELECT '"+stage+"' AS stage,f.department,f.month_key AS month,COALESCE(f.group_name,'') AS current_group,COUNT(*) AS row_count,SUM(f.total_weight) AS total_weight "
|
||||
+"FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+where+" GROUP BY f.department,f.month_key,COALESCE(f.group_name,'')");
|
||||
}
|
||||
return g.rows("SELECT * FROM ("+String.join(" UNION ALL ",queries)+") impact ORDER BY stage,month,current_group",args.toArray());
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package cn.yunsheng.quality.governance;
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import java.time.YearMonth;
|
||||
import java.util.*;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
import org.springframework.security.core.annotation.AuthenticationPrincipal;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import static cn.yunsheng.quality.governance.GovernanceSupport.error;
|
||||
|
||||
/** Registers inputs and readiness only. No amount, conversion or savings calculation is exposed. */
|
||||
@RestController @RequestMapping("/api/governance/cost-configurations")
|
||||
public class CostConfigurationController {
|
||||
private final GovernanceSupport g;public CostConfigurationController(GovernanceSupport g){this.g=g;}
|
||||
@GetMapping @PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER','RULE_MAINTAINER','AUDIT_VIEWER')")
|
||||
public Map<String,Object> list(){return Map.of("items",g.rows("SELECT * FROM cost_configuration_version ORDER BY created_at DESC"),"calculationEnabled",false);}
|
||||
@PostMapping @PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER','RULE_MAINTAINER')") @Transactional
|
||||
public Map<String,Object> create(@RequestBody Draft r,@AuthenticationPrincipal AppUserPrincipal actor){g.required(r.name(),"名称");g.required(r.reason(),"理由");if(r.definition()==null)throw error(HttpStatus.BAD_REQUEST,"配置定义不能为空");String id=UUID.randomUUID().toString();g.jdbc.update("INSERT INTO cost_configuration_version(id,name,definition,created_by)VALUES(?,?,CAST(? AS JSON),?)",id,r.name(),g.encode(r.definition()),actor.getId().toString());g.audit(actor.getId().toString(),"COST_CONFIGURATION_REGISTERED","cost_configuration_version",id,Map.of("reason",r.reason(),"calculationEnabled",false));return Map.of("configuration",get(id));}
|
||||
@PutMapping("/{id}") @PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER','RULE_MAINTAINER')") @Transactional
|
||||
public Map<String,Object> update(@PathVariable String id,@RequestBody Draft r,@AuthenticationPrincipal AppUserPrincipal actor){Map<String,Object> before=get(id);g.required(r.name(),"名称");g.required(r.reason(),"理由");if(r.definition()==null||r.expectedRowVersion()==null||((Number)before.get("rowVersion")).longValue()!=r.expectedRowVersion()||!"DRAFT".equals(before.get("status")))throw error(HttpStatus.CONFLICT,"仅可修改未验证的同版本草稿");int changed=g.jdbc.update("UPDATE cost_configuration_version SET name=?,definition=CAST(? AS JSON),validation=NULL,row_version=row_version+1 WHERE id=? AND row_version=? AND status='DRAFT'",r.name(),g.encode(r.definition()),id,r.expectedRowVersion());if(changed==0)throw error(HttpStatus.CONFLICT,"配置版本已变化");g.audit(actor.getId().toString(),"COST_CONFIGURATION_UPDATED","cost_configuration_version",id,Map.of("reason",r.reason()));return Map.of("configuration",get(id));}
|
||||
@SuppressWarnings("unchecked") @PostMapping("/{id}/validate") @PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER','RULE_MAINTAINER')") @Transactional
|
||||
public Map<String,Object> validate(@PathVariable String id,@RequestBody Map<String,Object> r,@AuthenticationPrincipal AppUserPrincipal actor){Map<String,Object> c=g.one("SELECT * FROM cost_configuration_version WHERE id=? FOR UPDATE",id);if(!(r.get("expectedRowVersion") instanceof Number expected)||expected.longValue()!=((Number)c.get("rowVersion")).longValue()||!"DRAFT".equals(c.get("status")))throw error(HttpStatus.CONFLICT,"配置状态或版本已变化");
|
||||
Map<String,Object> def=(Map<String,Object>)c.get("definition");List<String> errors=validateDefinition(def);String readiness=errors.isEmpty()?"READY_FOR_FUTURE_INTEGRATION":"MATERIALS_REQUIRED";Map<String,Object> v=Map.of("valid",errors.isEmpty(),"readiness",readiness,"errors",errors,"calculationEnabled",false,"numericalAcceptanceCompleted",false);
|
||||
g.jdbc.update("UPDATE cost_configuration_version SET status=?,validation=CAST(? AS JSON),row_version=row_version+1 WHERE id=?",errors.isEmpty()?"VERIFIED_INPUTS":"DRAFT",g.encode(v),id);g.audit(actor.getId().toString(),"COST_CONFIGURATION_VALIDATED","cost_configuration_version",id,v);return Map.of("configuration",get(id),"validation",v);}
|
||||
static List<String> validateDefinition(Map<String,Object> definition){List<String> errors=new ArrayList<>();Object raw=definition.get("sources");if(!(raw instanceof List<?> sources)||sources.isEmpty())return List.of("至少登记一个成本来源");Map<String,List<Period>> byObject=new HashMap<>();for(Object item:sources){if(!(item instanceof Map<?,?> source)){errors.add("来源须为对象");continue;}for(String key:List.of("sourceReference","objectCode","costField","unit","currency","approvalReference","originalFormula","baselineReference"))if(source.get(key)==null||String.valueOf(source.get(key)).isBlank())errors.add("来源缺少"+key);try{String object=String.valueOf(source.get("objectCode"));YearMonth from=YearMonth.parse(String.valueOf(source.get("fromMonth"))),to=YearMonth.parse(String.valueOf(source.get("toMonth")));if(from.isAfter(to)){errors.add("成本来源有效期倒置");continue;}List<Period> periods=byObject.computeIfAbsent(object,k->new ArrayList<>());if(periods.stream().anyMatch(p->!from.isAfter(p.to)&&!to.isBefore(p.from)))errors.add("成本来源配置重复或有效期重叠:"+object);periods.add(new Period(from,to));}catch(Exception e){errors.add("成本来源须有有效的fromMonth/toMonth");}}return errors.stream().distinct().toList();}
|
||||
private Map<String,Object> get(String id){return g.one("SELECT * FROM cost_configuration_version WHERE id=?",id);}
|
||||
public record Draft(String name,Map<String,Object> definition,Long expectedRowVersion,String reason){}
|
||||
private record Period(YearMonth from,YearMonth to){}
|
||||
}
|
||||
+1
-1
@@ -26,7 +26,7 @@ public class GovernanceDependencyService {
|
||||
List<Map<String,Object>> references=g.rows("SELECT id,name,status,created_at FROM data_release WHERE "+column+"=? ORDER BY created_at DESC",id);
|
||||
List<Map<String,Object>> affected=g.rows("SELECT DISTINCT d.id AS source_document_id,d.source_slot,d.stage,d.source_year,d.source_month,r.id AS source_revision_id,b.id AS build_id FROM release_member rm JOIN source_document d ON d.id=rm.source_document_id JOIN source_revision r ON r.id=rm.source_revision_id JOIN transform_build b ON b.id=rm.transform_build_id WHERE rm.release_id=(SELECT release_id FROM current_release WHERE dataset_id='quality') ORDER BY d.source_year,d.source_month,d.stage");
|
||||
List<Map<String,Object>> frozen=g.rows("SELECT snapshot,snapshot_sha256,frozen_by,frozen_at FROM governance_dependency WHERE owner_kind=? AND owner_id=?",kind,id);
|
||||
Map<String,Object> result=new LinkedHashMap<>(Map.of("kind",kind,"version",version,"references",references,"affectedSources",affected,"frozen",frozen,"requiresNewRelease",true,"currentReleaseUnchanged",true,"costCalculationEnabled",false));if(kind.equals("mapping")){Map<String,Object> preview=mappingImpact.preview(id,actor);result.putAll(preview);Set<Object> documents=new HashSet<>();if(preview.get("hits") instanceof List<?> hits)for(Object hit:hits)if(hit instanceof Map<?,?> h)documents.add(h.get("sourceDocumentId"));result.put("affectedSources",affected.stream().filter(source->documents.contains(source.get("sourceDocumentId"))).toList());}return result;
|
||||
Map<String,Object> result=new LinkedHashMap<>(Map.of("kind",kind,"version",version,"references",references,"affectedSources",affected,"frozen",frozen,"requiresNewRelease",true,"currentReleaseUnchanged",true));if(kind.equals("mapping")){Map<String,Object> preview=mappingImpact.preview(id,actor);result.putAll(preview);Set<Object> documents=new HashSet<>();if(preview.get("hits") instanceof List<?> hits)for(Object hit:hits)if(hit instanceof Map<?,?> h)documents.add(h.get("sourceDocumentId"));result.put("affectedSources",affected.stream().filter(source->documents.contains(source.get("sourceDocumentId"))).toList());}return result;
|
||||
}
|
||||
public Map<String,Object> inspect(String id){
|
||||
Map<String,Object> release=g.one("SELECT id,name,status,parent_release_id,expected_current_release_id,rule_version,mapping_version,target_version_id,acceptance_id,complete,validation,row_version,created_at,published_at FROM data_release WHERE id=?",id);
|
||||
|
||||
@@ -7,8 +7,14 @@ import org.springframework.web.bind.annotation.*;
|
||||
@RestController @RequestMapping("/api/governance/mapping-issues")
|
||||
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_PUBLISHER','DATA_MAINTAINER','RULE_MAINTAINER','AUDIT_VIEWER')")
|
||||
public class MappingIssueController {
|
||||
private final MappingIssueService service;public MappingIssueController(MappingIssueService service){this.service=service;}
|
||||
@GetMapping public Map<String,Object> list(@RequestParam(required=false) String importRunId,@RequestParam(required=false) String status,@RequestParam(defaultValue="1") int page,@RequestParam(defaultValue="50") int pageSize,@AuthenticationPrincipal AppUserPrincipal actor){return service.list(importRunId,status,page,pageSize,actor);}
|
||||
private final MappingIssueService service;
|
||||
private final UnclassifiedSourceService unclassified;
|
||||
private final ClassificationPreviewService preview;
|
||||
public MappingIssueController(MappingIssueService service, UnclassifiedSourceService unclassified,ClassificationPreviewService preview){this.service=service;this.unclassified=unclassified;this.preview=preview;}
|
||||
@PostMapping("/classification-preview") public Map<String,Object> classificationPreview(@RequestBody ClassificationPreviewService.Request request,@AuthenticationPrincipal AppUserPrincipal actor){return preview.preview(request,actor);}
|
||||
@GetMapping("/classification-metadata") public Map<String,Object> classificationMetadata(){return cn.yunsheng.quality.ingestion.ConfigurableMatchRules.metadata();}
|
||||
@GetMapping("/unclassified") public Map<String,Object> unclassified(@RequestParam(defaultValue="1") int page,@RequestParam(defaultValue="50") int pageSize,@AuthenticationPrincipal AppUserPrincipal actor){return unclassified.list(page,pageSize,actor);}
|
||||
@GetMapping public Map<String,Object> list(@RequestParam(required=false) String importRunId,@RequestParam(required=false) String status,@RequestParam(defaultValue="1") int page,@RequestParam(defaultValue="50") int pageSize,@RequestParam(required=false) String mappingType,@RequestParam(required=false) String customerCode,@RequestParam(required=false) String businessGroup,@RequestParam(required=false) String department,@AuthenticationPrincipal AppUserPrincipal actor){return service.list(importRunId,status,page,pageSize,customerCode,businessGroup,department,mappingType,actor);}
|
||||
@GetMapping("/{id}") public Map<String,Object> detail(@PathVariable String id,@AuthenticationPrincipal AppUserPrincipal actor){return service.detail(id,actor);}
|
||||
@PostMapping("/{id}/resolve") @PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER','RULE_MAINTAINER')")
|
||||
public Map<String,Object> resolve(@PathVariable String id,@RequestBody MappingIssueService.Resolution request,@AuthenticationPrincipal AppUserPrincipal actor){return service.resolve(id,request,actor);}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package cn.yunsheng.quality.governance;
|
||||
|
||||
import java.util.*;
|
||||
import cn.yunsheng.quality.common.ImmutableQueryCache;
|
||||
|
||||
/** Derive maintenance filters from immutable source facts, never from name prefixes. */
|
||||
final class MappingIssueDimensions {
|
||||
private final GovernanceSupport g;
|
||||
private final ImmutableQueryCache<String, Map<String, Set<String>>> indexes = new ImmutableQueryCache<>(12);
|
||||
MappingIssueDimensions(GovernanceSupport g) { this.g = g; }
|
||||
|
||||
List<Map<String,Object>> decorate(List<Map<String,Object>> issues) {
|
||||
List<Map<String,Object>> result = new ArrayList<>();
|
||||
for (var issue : issues) {
|
||||
Map<String,Object> row = new LinkedHashMap<>(issue);
|
||||
String type = Objects.toString(row.get("mappingType"), "");
|
||||
String sourceKey = Objects.toString(row.get("sourceKey"), "");
|
||||
if (type.equals("PROJECT_GROUP")) {
|
||||
int separator = sourceKey.indexOf('|');
|
||||
row.put("customerCode", separator < 0 ? sourceKey : sourceKey.substring(0, separator));
|
||||
row.put("businessGroup", separator < 0 ? "" : sourceKey.substring(separator + 1));
|
||||
}
|
||||
Object details = row.get("details");
|
||||
String release = details instanceof Map<?,?> values ? Objects.toString(values.get("releaseId"), "") : "";
|
||||
Map<String,Set<String>> index = release.isBlank() ? Map.of() : indexes.get(release, () -> {
|
||||
Map<String,Set<String>> values = new HashMap<>();
|
||||
for (var source : g.rows(sourceSql("black") + " UNION " + sourceSql("bright"), release, release)) {
|
||||
String department = Objects.toString(source.get("department"), "");
|
||||
if (department.isBlank()) continue;
|
||||
index(values, "CATEGORY", key(source.get("pdtCode"), source.get("salesRegion")), department);
|
||||
index(values, "PROJECT_GROUP", key(source.get("customerCode"), source.get("groupName")), department);
|
||||
}
|
||||
return values;
|
||||
});
|
||||
row.put("departments", List.copyOf(index.getOrDefault(type + "\n" + sourceKey, Set.of())));
|
||||
result.add(row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void index(Map<String,Set<String>> index, String type, String key, String department) {
|
||||
index.computeIfAbsent(type + "\n" + key, ignored -> new TreeSet<>()).add(department);
|
||||
}
|
||||
private static String key(Object first, Object second) {
|
||||
return Objects.toString(first, "") + "|" + Objects.toString(second, "");
|
||||
}
|
||||
static String sourceSql(String stage) {
|
||||
if (!Set.of("black", "bright").contains(stage)) throw new IllegalArgumentException("stage");
|
||||
return "SELECT DISTINCT f.department,f.group_name,JSON_VALUE(f.components,'$.sourcePdtCode') AS pdt_code,"
|
||||
+ "JSON_VALUE(f.components,'$.sourceSalesRegion') AS sales_region,"
|
||||
+ "JSON_VALUE(f.components,'$.sourceCustomerCode') AS customer_code "
|
||||
+ "FROM release_member rm JOIN fact_" + stage + " f ON f.build_id=rm.transform_build_id "
|
||||
+ "WHERE rm.release_id=? AND (JSON_VALUE(f.components,'$.categoryMappingStatus')='unmapped' "
|
||||
+ "OR JSON_VALUE(f.components,'$.projectGroupStatus')='unmapped')";
|
||||
}
|
||||
static boolean matches(Map<String,Object> row, String customer, String group, String department) {
|
||||
return (customer == null || customer.isBlank() || customer.equals(row.get("customerCode")))
|
||||
&& (group == null || group.isBlank() || group.equals(row.get("businessGroup")))
|
||||
&& (department == null || department.isBlank() || ((List<?>) row.getOrDefault("departments", List.of())).contains(department));
|
||||
}
|
||||
static Map<String,Object> options(List<Map<String,Object>> rows) {
|
||||
Set<String> customers = new TreeSet<>(), groups = new TreeSet<>(), departments = new TreeSet<>();
|
||||
for (var row : rows) {
|
||||
String customer = Objects.toString(row.get("customerCode"), "");
|
||||
String group = Objects.toString(row.get("businessGroup"), "");
|
||||
if (!customer.isBlank()) customers.add(customer);
|
||||
if (!group.isBlank()) groups.add(group);
|
||||
for (Object value : (List<?>) row.getOrDefault("departments", List.of())) departments.add(String.valueOf(value));
|
||||
}
|
||||
return Map.of("customerCodes", customers, "businessGroups", groups, "departments", departments);
|
||||
}
|
||||
}
|
||||
@@ -17,8 +17,8 @@ import static cn.yunsheng.quality.governance.GovernanceSupport.error;
|
||||
|
||||
@Service
|
||||
public class MappingIssueService {
|
||||
private final GovernanceSupport g;private final CandidateAcceptanceService acceptance;private final ImportRunService imports;private final ObjectProvider<ImportWorker> worker;private final TransactionTemplate transaction;
|
||||
public MappingIssueService(GovernanceSupport g,CandidateAcceptanceService acceptance,ImportRunService imports,ObjectProvider<ImportWorker> worker,PlatformTransactionManager manager){this.g=g;this.acceptance=acceptance;this.imports=imports;this.worker=worker;this.transaction=new TransactionTemplate(manager);this.transaction.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);}
|
||||
private final MappingIssueDimensions dimensions;private final GovernanceSupport g;private final CandidateAcceptanceService acceptance;private final ImportRunService imports;private final ObjectProvider<ImportWorker> worker;private final TransactionTemplate transaction;
|
||||
public MappingIssueService(GovernanceSupport g,CandidateAcceptanceService acceptance,ImportRunService imports,ObjectProvider<ImportWorker> worker,PlatformTransactionManager manager){this.g=g;this.dimensions=new MappingIssueDimensions(g);this.acceptance=acceptance;this.imports=imports;this.worker=worker;this.transaction=new TransactionTemplate(manager);this.transaction.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);}
|
||||
/** Aggregated from immutable transformed facts, retaining their exact source/build lineage. */
|
||||
public void capture(String runId,String releaseId){
|
||||
Map<String,Group> groups=new TreeMap<>();for(String stage:List.of("black","bright"))for(Map<String,Object> f:g.rows("SELECT f.group_name,f.total_weight,f.components,f.raw_values,f.month_key,rm.source_document_id,rm.source_revision_id,rm.transform_build_id FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id WHERE rm.release_id=? AND (JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.categoryMappingStatus'))='unmapped' OR JSON_UNQUOTE(JSON_EXTRACT(f.components,'$.projectGroupStatus'))='unmapped')",releaseId)){
|
||||
@@ -34,7 +34,9 @@ public class MappingIssueService {
|
||||
}
|
||||
private void add(Map<String,Group> groups,String type,String key,Map<String,Object> f,Map<String,Object> raw,String stage){Group group=groups.computeIfAbsent(type+"\n"+key,k->new Group(type,key));group.count++;if(f.get("totalWeight") instanceof Number n)group.weight=group.weight.add(new BigDecimal(n.toString()));group.documents.add(String.valueOf(f.get("sourceDocumentId")));group.revisions.add(String.valueOf(f.get("sourceRevisionId")));group.builds.add(String.valueOf(f.get("transformBuildId")));group.periods.add(String.valueOf(f.get("monthKey")));
|
||||
boolean hasOriginal=false;for(String field:type.equals("CATEGORY")?List.of("产业部",stage.equals("black")?"黑片产业部":"亮片产业部"):List.of("项目组","项目组项目组")){Object value=raw.get(field);if(value instanceof String s&&!s.isBlank()){group.originals.add(s);hasOriginal=true;}}if(hasOriginal)group.originalValueRows++;}
|
||||
public Map<String,Object> list(String runId,String status,int page,int size,AppUserPrincipal actor){if(runId!=null){String release=g.jdbc.queryForObject("SELECT result_release_id FROM import_run WHERE id=?",String.class,runId);if(release!=null){acceptance.requireRead(release,actor);acceptance.requireSourceDetail(actor);capture(runId,release);}}if(page<1||size<1||size>100)throw error(HttpStatus.BAD_REQUEST,"分页无效");List<Object> args=new ArrayList<>();StringBuilder sql=new StringBuilder("SELECT * FROM mapping_issue_group WHERE 1=1");if(runId!=null){sql.append(" AND import_run_id=?");args.add(runId);}if(status!=null){sql.append(" AND status=?");args.add(status);}sql.append(" ORDER BY created_at DESC,id");List<Map<String,Object>> allowed=new ArrayList<>();for(var row:g.rows(sql.toString(),args.toArray())){try{authorize(row,actor);allowed.add(row);}catch(org.springframework.web.server.ResponseStatusException e){if(e.getStatusCode().value()!=403)throw e;}}int offset=(int)Math.min(allowed.size(),(long)(page-1)*size);return Map.of("items",allowed.subList(offset,Math.min(allowed.size(),offset+size)),"total",allowed.size(),"page",page,"pageSize",size);}
|
||||
public Map<String,Object> list(String runId,String status,int page,int size,AppUserPrincipal actor){return list(runId,status,page,size,null,null,null,actor);}
|
||||
public Map<String,Object> list(String runId,String status,int page,int size,String customerCode,String businessGroup,String department,AppUserPrincipal actor){return list(runId,status,page,size,customerCode,businessGroup,department,null,actor);}
|
||||
public Map<String,Object> list(String runId,String status,int page,int size,String customerCode,String businessGroup,String department,String mappingType,AppUserPrincipal actor){if(runId!=null){String release=g.jdbc.queryForObject("SELECT result_release_id FROM import_run WHERE id=?",String.class,runId);if(release!=null){acceptance.requireRead(release,actor);acceptance.requireSourceDetail(actor);capture(runId,release);}}if(page<1||size<1||size>100)throw error(HttpStatus.BAD_REQUEST,"分页无效");List<Object> args=new ArrayList<>();StringBuilder sql=new StringBuilder("SELECT * FROM mapping_issue_group WHERE 1=1");if(runId!=null){sql.append(" AND import_run_id=?");args.add(runId);}if(status!=null){sql.append(" AND status=?");args.add(status);}if(mappingType!=null){if(!Set.of("CATEGORY","PROJECT_GROUP","DEPARTMENT","PRODUCT").contains(mappingType))throw error(HttpStatus.BAD_REQUEST,"映射类型无效");sql.append(" AND mapping_type=?");args.add(mappingType);}sql.append(" ORDER BY created_at DESC,id");List<Map<String,Object>> allowed=new ArrayList<>();Map<String,Boolean> releaseAccess=new HashMap<>();for(var row:g.rows(sql.toString(),args.toArray())){String releaseId=Objects.toString(map(row.get("details")).get("releaseId"),"");try{if(!releaseAccess.containsKey(releaseId)){authorize(row,actor);releaseAccess.put(releaseId,true);}if(Boolean.TRUE.equals(releaseAccess.get(releaseId)))allowed.add(row);}catch(org.springframework.web.server.ResponseStatusException e){if(e.getStatusCode().value()!=403)throw e;releaseAccess.put(releaseId,false);}}List<Map<String,Object>> decorated=dimensions.decorate(allowed);List<Map<String,Object>> filtered=decorated.stream().filter(row->MappingIssueDimensions.matches(row,customerCode,businessGroup,department)).toList();int offset=(int)Math.min(filtered.size(),(long)(page-1)*size);return Map.of("items",filtered.subList(offset,Math.min(filtered.size(),offset+size)),"total",filtered.size(),"page",page,"pageSize",size,"filterOptions",MappingIssueDimensions.options(decorated));}
|
||||
public Map<String,Object> detail(String id,AppUserPrincipal actor){Map<String,Object> issue=g.one("SELECT * FROM mapping_issue_group WHERE id=?",id);authorize(issue,actor);return Map.of("issue",issue,"decisions",g.rows("SELECT * FROM mapping_issue_decision WHERE issue_key=? ORDER BY created_at DESC",id),"buildAttempts",g.rows("SELECT a.* FROM mapping_issue_build_attempt a JOIN mapping_issue_decision d ON d.id=a.decision_id WHERE d.issue_key=? ORDER BY a.created_at DESC",id));}
|
||||
@Transactional public Map<String,Object> resolve(String id,Resolution request,AppUserPrincipal actor){
|
||||
Map<String,Object> issue=g.one("SELECT * FROM mapping_issue_group WHERE id=? FOR UPDATE",id);authorize(issue,actor);g.required(request.reason(),"裁决理由");
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package cn.yunsheng.quality.governance;
|
||||
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.common.ImmutableQueryCache;
|
||||
import java.util.*;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/** Read-only inventory of blank business groups in the current published release. */
|
||||
@Service
|
||||
public class UnclassifiedSourceService {
|
||||
private final GovernanceSupport g;
|
||||
private final CandidateAcceptanceService access;
|
||||
private final ImmutableQueryCache<String, List<Map<String, Object>>> inventories = new ImmutableQueryCache<>(12);
|
||||
public UnclassifiedSourceService(GovernanceSupport g, CandidateAcceptanceService access) {
|
||||
this.g = g;
|
||||
this.access = access;
|
||||
}
|
||||
|
||||
public Map<String, Object> list(int page, int size, AppUserPrincipal actor) {
|
||||
if (page < 1 || size < 1 || size > 100)
|
||||
throw GovernanceSupport.error(HttpStatus.BAD_REQUEST, "分页无效");
|
||||
access.requireSourceDetail(actor);
|
||||
List<Map<String, Object>> current = g.rows("SELECT release_id FROM current_release WHERE dataset_id='quality'");
|
||||
if (current.isEmpty()) return Map.of("items", List.of(), "total", 0, "page", page, "pageSize", size);
|
||||
String release = (String) current.getFirst().get("releaseId");
|
||||
access.requireRead(release, actor);
|
||||
List<Map<String, Object>> rows = inventories.get(release, () -> {
|
||||
String sql = sourceSql("black") + " UNION ALL " + sourceSql("bright");
|
||||
return g.rows("SELECT * FROM (" + sql + ") sources ORDER BY stage,department,product,source_key",
|
||||
release, release).stream().map(row -> Collections.unmodifiableMap(new LinkedHashMap<>(row))).toList();
|
||||
});
|
||||
int offset = (int) Math.min(rows.size(), (long) (page - 1) * size);
|
||||
return Map.of("items", rows.subList(offset, Math.min(rows.size(), offset + size)), "total", (long) rows.size(),
|
||||
"page", page, "pageSize", size, "releaseId", release);
|
||||
}
|
||||
|
||||
static String sourceSql(String stage) {
|
||||
if (!Set.of("black", "bright").contains(stage)) throw new IllegalArgumentException("stage");
|
||||
return """
|
||||
SELECT '%s' AS stage, f.department, f.product,
|
||||
COALESCE(JSON_VALUE(f.components,'$.sourcePdtCode'),'') AS pdt_code,
|
||||
COALESCE(JSON_VALUE(f.components,'$.sourceSalesRegion'),'') AS sales_region,
|
||||
CONCAT(COALESCE(JSON_VALUE(f.components,'$.sourcePdtCode'),''),'|',
|
||||
COALESCE(JSON_VALUE(f.components,'$.sourceSalesRegion'),'')) AS source_key,
|
||||
COUNT(*) AS row_count, SUM(f.total_weight) AS total_weight,
|
||||
GROUP_CONCAT(DISTINCT f.month_key ORDER BY f.month_key SEPARATOR '、') AS periods
|
||||
FROM release_member rm JOIN fact_%s f ON f.build_id=rm.transform_build_id
|
||||
WHERE rm.release_id=? AND (f.group_name IS NULL OR f.group_name='')
|
||||
GROUP BY f.department,f.product,pdt_code,sales_region,source_key
|
||||
""".formatted(stage, stage);
|
||||
}
|
||||
}
|
||||
@@ -60,7 +60,7 @@ public class MetricsController {
|
||||
@RequestParam(required = false) String productSearch,
|
||||
@RequestParam(required = false) String projectGroup,
|
||||
@RequestParam(defaultValue = "overall") String view,
|
||||
@RequestParam(defaultValue = "false") boolean includeUnassigned,
|
||||
@RequestParam(defaultValue = "true") boolean includeUnassigned,
|
||||
@RequestParam(required = false) String releaseId,
|
||||
@RequestParam(defaultValue = "month") String trendGranularity,
|
||||
@RequestParam(required = false) Integer rowLimit,
|
||||
|
||||
@@ -378,7 +378,9 @@ public class MetricsService {
|
||||
if (view.equals("overall")) return List.of(aggregates.summary().toMetricRow("overall", "总体", null, null, null));
|
||||
List<MetricRow> rows = new ArrayList<>();
|
||||
aggregates.dimensions().forEach((key, stats) -> {
|
||||
boolean unassigned = key.equals("__UNASSIGNED__");
|
||||
boolean unassigned = key == null || key.isBlank() || key.equals("__UNASSIGNED__");
|
||||
// Display filtering only: summary, coverage and trend retain every matching source row.
|
||||
if (view.equals("group") && unassigned) return;
|
||||
String label = unassigned ? "未映射/未填写" : key;
|
||||
String department = view.equals("department") && !unassigned ? DepartmentLabels.display(key) : null;
|
||||
String group = view.equals("group") && !unassigned ? key : null;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package cn.yunsheng.quality.records;
|
||||
|
||||
import cn.yunsheng.quality.common.DataScope;
|
||||
import cn.yunsheng.quality.common.ImmutableQueryCache;
|
||||
import cn.yunsheng.quality.filters.QueryFilterFields;
|
||||
import cn.yunsheng.quality.common.DepartmentLabels;
|
||||
import cn.yunsheng.quality.common.SelectionValues;
|
||||
@@ -38,6 +39,12 @@ import org.springframework.web.server.ResponseStatusException;
|
||||
@RestController
|
||||
@RequestMapping("/api/records")
|
||||
public class RecordController {
|
||||
private record QueryKey(String sql, List<Object> args) {
|
||||
QueryKey { args = List.copyOf(args); }
|
||||
}
|
||||
private static final int PAGE_WINDOW = 240;
|
||||
private final ImmutableQueryCache<QueryKey, Long> totals = new ImmutableQueryCache<>(128);
|
||||
private final ImmutableQueryCache<QueryKey, List<String>> pageIndexes = new ImmutableQueryCache<>(128);
|
||||
private static final Set<String> COMPUTED_WEIGHT_FIELDS = Set.of(
|
||||
"qualifiedWeight", "scrapWeight", "totalWeight", "qualified", "scrap", "total");
|
||||
private final JdbcTemplate jdbc;
|
||||
@@ -95,7 +102,7 @@ public class RecordController {
|
||||
@RequestParam(required = false) String product,
|
||||
@RequestParam(required = false) String productSearch,
|
||||
@RequestParam(required = false) String projectGroup,
|
||||
@RequestParam(defaultValue = "false") boolean includeUnassigned,
|
||||
@RequestParam(defaultValue = "true") boolean includeUnassigned,
|
||||
@RequestParam(required = false) String releaseId,
|
||||
@RequestParam(defaultValue = "1") int page,
|
||||
@RequestParam(defaultValue = "50") int pageSize,
|
||||
@@ -149,17 +156,39 @@ public class RecordController {
|
||||
if(!sourceFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,stage,sourceFilters,selectedRelease,actor);
|
||||
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
|
||||
Long total = jdbc.queryForObject("""
|
||||
String countSql = """
|
||||
SELECT COUNT(*) FROM release_member rm
|
||||
JOIN source_revision sr ON sr.id = rm.source_revision_id
|
||||
JOIN %s f ON f.build_id = rm.transform_build_id
|
||||
%s
|
||||
""".formatted(fact, predicate), Long.class, args.toArray());
|
||||
List<Object> pageArgs = new ArrayList<>(args);
|
||||
String orderBy = orderBy(sort, sortDirection, selectedField, actor, pageArgs);
|
||||
pageArgs.add(pageSize);
|
||||
pageArgs.add((long) (page - 1) * pageSize);
|
||||
List<Map<String, Object>> rows = jdbc.query("""
|
||||
""".formatted(fact, predicate);
|
||||
Long total = totals.get(new QueryKey(countSql, args), () -> {
|
||||
Long count = jdbc.queryForObject(countSql, Long.class, args.toArray());
|
||||
return count == null ? 0L : count;
|
||||
});
|
||||
List<Object> indexArgs = new ArrayList<>(args);
|
||||
String orderBy = orderBy(sort, sortDirection, selectedField, actor, indexArgs) + ", f.id";
|
||||
long offset = (long) (page - 1) * pageSize;
|
||||
List<String> ids = new ArrayList<>();
|
||||
long end = Math.min(total, offset + pageSize);
|
||||
for (long start = offset; start < end;) {
|
||||
long windowStart = start / PAGE_WINDOW * PAGE_WINDOW;
|
||||
List<Object> windowArgs = new ArrayList<>(indexArgs);
|
||||
windowArgs.add(PAGE_WINDOW); windowArgs.add(windowStart);
|
||||
String indexSql = "SELECT f.id FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id "
|
||||
+ "JOIN " + fact + " f ON f.build_id=rm.transform_build_id " + predicate
|
||||
+ " ORDER BY " + orderBy + " LIMIT ? OFFSET ?";
|
||||
List<String> window = pageIndexes.get(new QueryKey(indexSql, windowArgs),
|
||||
() -> List.copyOf(jdbc.queryForList(indexSql, String.class, windowArgs.toArray())));
|
||||
int first = (int) (start - windowStart);
|
||||
int last = (int) Math.min(window.size(), end - windowStart);
|
||||
if (first >= last) break;
|
||||
ids.addAll(window.subList(first, last)); start = windowStart + last;
|
||||
}
|
||||
// Field permissions remain current on every request; no authorized row JSON is cached.
|
||||
Set<String> allowedFields = governance.allowedFields(actor);
|
||||
List<Object> rowArgs = new ArrayList<>(args); rowArgs.addAll(ids);
|
||||
List<Map<String, Object>> fetched = ids.isEmpty() ? List.of() : jdbc.query("""
|
||||
SELECT f.id, sr.id AS source_revision_id, sr.source_file, f.sheet_name, f.physical_row_no, f.month_key,
|
||||
f.department, f.group_name, f.project_group, f.product, f.qualified_weight, f.scrap_weight,
|
||||
f.total_weight, f.raw_values, f.components
|
||||
@@ -167,9 +196,12 @@ public class RecordController {
|
||||
JOIN source_revision sr ON sr.id = rm.source_revision_id
|
||||
JOIN %s f ON f.build_id = rm.transform_build_id
|
||||
%s
|
||||
ORDER BY %s
|
||||
LIMIT ? OFFSET ?
|
||||
""".formatted(fact, predicate, orderBy), (rs, row) -> mapRow(rs, stage, actor, selectedColumns), pageArgs.toArray());
|
||||
AND f.id IN (%s)
|
||||
""".formatted(fact, predicate, String.join(",", java.util.Collections.nCopies(ids.size(), "?"))),
|
||||
(rs, row) -> mapRow(rs, stage, allowedFields, selectedColumns), rowArgs.toArray());
|
||||
Map<String, Map<String, Object>> byId = new LinkedHashMap<>();
|
||||
fetched.forEach(row -> byId.put(String.valueOf(row.get("id")), row));
|
||||
List<Map<String, Object>> rows = ids.stream().map(byId::get).filter(java.util.Objects::nonNull).toList();
|
||||
return Map.of("rows", rows, "total", total == null ? 0 : total, "page", page,
|
||||
"pageSize", pageSize, "releaseId", selectedRelease);
|
||||
}
|
||||
@@ -187,7 +219,7 @@ public class RecordController {
|
||||
@RequestParam(required = false) String product,
|
||||
@RequestParam(required = false) String productSearch,
|
||||
@RequestParam(required = false) String projectGroup,
|
||||
@RequestParam(defaultValue = "false") boolean includeUnassigned,
|
||||
@RequestParam(defaultValue = "true") boolean includeUnassigned,
|
||||
@RequestParam(required = false) String releaseId,
|
||||
@RequestParam(required=false) String extraFilters,
|
||||
@RequestParam(required=false) String stage,
|
||||
@@ -232,7 +264,7 @@ public class RecordController {
|
||||
if (month != null && quarter != null) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "月份与季度筛选不可同时指定");
|
||||
}
|
||||
|
||||
private Map<String, Object> mapRow(ResultSet rs, String stage, AppUserPrincipal actor, List<String> selectedColumns) throws SQLException {
|
||||
private Map<String, Object> mapRow(ResultSet rs, String stage, Set<String> allowedFields, List<String> selectedColumns) throws SQLException {
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
String sourceFile = rs.getString("source_file");
|
||||
String sheet = rs.getString("sheet_name");
|
||||
@@ -248,13 +280,13 @@ public class RecordController {
|
||||
row.put("projectGroup", rs.getString("project_group"));
|
||||
row.put("group", rs.getString("group_name"));
|
||||
row.put("product", rs.getString("product"));
|
||||
Map<String, Object> visibleRaw = governance.filterFields(actor, readJson(rs.getString("raw_values")));
|
||||
Map<String, Object> visibleRaw = filterFields(readJson(rs.getString("raw_values")), allowedFields);
|
||||
if (selectedColumns != null) {
|
||||
Map<String, Object> selectedRaw = new LinkedHashMap<>();
|
||||
for (String column : selectedColumns) if (visibleRaw.containsKey(column)) selectedRaw.put(column, visibleRaw.get(column));
|
||||
visibleRaw = selectedRaw;
|
||||
}
|
||||
Map<String, Object> visibleComponents = governance.filterFields(actor, readJson(rs.getString("components")));
|
||||
Map<String, Object> visibleComponents = filterFields(readJson(rs.getString("components")), allowedFields);
|
||||
row.put("raw", visibleRaw);
|
||||
row.put("components", visibleComponents);
|
||||
Map<String, Object> normalized = new LinkedHashMap<>();
|
||||
@@ -266,13 +298,18 @@ public class RecordController {
|
||||
normalized.put("qualifiedWeight", rs.getBigDecimal("qualified_weight"));
|
||||
normalized.put("scrapWeight", rs.getBigDecimal("scrap_weight"));
|
||||
normalized.put("totalWeight", rs.getBigDecimal("total_weight"));
|
||||
Set<String> allowedFields = governance.allowedFields(actor);
|
||||
row.put("normalized", restrictComputedWeightFields(normalized, allowedFields));
|
||||
hideLegacyTopLevelWeightAliases(row, allowedFields);
|
||||
row.put("transformNotes", transformNotes(stage, visibleRaw, visibleComponents));
|
||||
return row;
|
||||
}
|
||||
|
||||
private static Map<String, Object> filterFields(Map<String, Object> fields, Set<String> allowed) {
|
||||
Map<String, Object> visible = new LinkedHashMap<>(fields);
|
||||
if (!allowed.contains("*")) visible.keySet().retainAll(allowed);
|
||||
return visible;
|
||||
}
|
||||
|
||||
static Map<String, Object> restrictComputedWeightFields(Map<String, Object> values, Set<String> allowedFields) {
|
||||
Map<String, Object> visible = new LinkedHashMap<>(values);
|
||||
if (allowedFields.contains("*")) return visible;
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package cn.yunsheng.quality.common;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class ImmutableQueryCacheTest {
|
||||
@Test void sharesOneConcurrentLoadAndEvictsLeastRecentlyUsedEntries() throws Exception {
|
||||
var cache=new ImmutableQueryCache<String,Integer>(2);var calls=new AtomicInteger();
|
||||
var entered=new CountDownLatch(1);var proceed=new CountDownLatch(1);
|
||||
try(var pool=Executors.newFixedThreadPool(2)) {
|
||||
var first=pool.submit(()->cache.get("a",()->{calls.incrementAndGet();entered.countDown();try{proceed.await();}catch(InterruptedException e){throw new IllegalStateException(e);}return 1;}));
|
||||
assertTrue(entered.await(2,TimeUnit.SECONDS));
|
||||
var second=pool.submit(()->cache.get("a",()->{calls.incrementAndGet();return 2;}));
|
||||
proceed.countDown();assertEquals(1,first.get());assertEquals(1,second.get());assertEquals(1,calls.get());
|
||||
}
|
||||
cache.get("b",()->2);cache.get("a",()->99);cache.get("c",()->3);
|
||||
assertEquals(1,cache.get("a",()->99));assertEquals(4,cache.get("b",()->4));
|
||||
}
|
||||
@Test void failedLoadIsNeverRetained() {
|
||||
var cache=new ImmutableQueryCache<String,Integer>(1);
|
||||
assertThrows(IllegalStateException.class,()->cache.get("a",()->{throw new IllegalStateException();}));
|
||||
assertEquals(1,cache.get("a",()->1));
|
||||
}
|
||||
}
|
||||
@@ -197,6 +197,27 @@ class SourceColumnQueriesTest {
|
||||
return sourceField(SOURCE, "black", "委托单");
|
||||
}
|
||||
|
||||
@Test void valuePagesReuseCountAndWindowButRevalidateFieldsAndSeparateSearches() {
|
||||
JdbcTemplate jdbc=mock(JdbcTemplate.class);
|
||||
AppUserPrincipal admin=new AppUserPrincipal(UUID.randomUUID(),"admin","","admin",true,false,
|
||||
List.of("SYSTEM_ADMIN"),List.of(),List.of(),List.of());
|
||||
SourceColumnQueries queries=spy(new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()));
|
||||
doReturn(List.of(sourceField())).when(queries).scopedCatalog("release",admin);
|
||||
when(jdbc.queryForObject(startsWith("SELECT COUNT(*)"),eq(Long.class),any(Object[].class))).thenReturn(40L);
|
||||
List<Map<String,Object>> values=java.util.stream.IntStream.range(0,40)
|
||||
.mapToObj(n->Map.<String,Object>of("value",String.format("%04d",n))).toList();
|
||||
when(jdbc.query(startsWith("SELECT candidate_value"),any(org.springframework.jdbc.core.RowMapper.class),any(Object[].class))).thenReturn(values);
|
||||
queries.values("release",Map.of(),SOURCE,List.of("black"),null,15,0,admin);
|
||||
var second=queries.values("release",Map.of(),SOURCE,List.of("black"),null,15,15,admin);
|
||||
assertEquals("0015",((List<Map<String,Object>>)second.get("rows")).getFirst().get("value"));
|
||||
verify(jdbc,times(1)).queryForObject(startsWith("SELECT COUNT(*)"),eq(Long.class),any(Object[].class));
|
||||
verify(jdbc,times(1)).query(startsWith("SELECT candidate_value"),any(org.springframework.jdbc.core.RowMapper.class),any(Object[].class));
|
||||
queries.values("release",Map.of(),SOURCE,List.of("black"),"001",15,0,admin);
|
||||
verify(jdbc,times(2)).queryForObject(startsWith("SELECT COUNT(*)"),eq(Long.class),any(Object[].class));
|
||||
doReturn(List.of()).when(queries).scopedCatalog("release",admin);
|
||||
assertThrows(ResponseStatusException.class,()->queries.values("release",Map.of(),SOURCE,List.of("black"),null,15,15,admin));
|
||||
}
|
||||
|
||||
private Map<String, Object> sourceField(String key, String stage, String header) {
|
||||
Map<String, Object> columns = new LinkedHashMap<>();
|
||||
columns.put("black", stage.equals("black") ? header : null);
|
||||
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package cn.yunsheng.quality.governance;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import java.util.*;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/** Opt-in disposable database; no application database or source file is modified. */
|
||||
@EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_COLUMN_TEST_CREATE_DATABASE",matches="true")
|
||||
class ClassificationPreviewMySqlTest {
|
||||
private JdbcTemplate admin,jdbc;private String database;private ClassificationPreviewService service;
|
||||
private final AppUserPrincipal actor=mock(AppUserPrincipal.class);
|
||||
@BeforeEach void create(){
|
||||
String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
||||
String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
||||
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_classification_preview_"+UUID.randomUUID().toString().replace("-","");admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
||||
jdbc=new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password));
|
||||
jdbc.execute("CREATE TABLE current_release(dataset_id VARCHAR(20),release_id VARCHAR(20))");jdbc.update("INSERT INTO current_release VALUES('quality','r')");
|
||||
jdbc.execute("CREATE TABLE release_member(release_id VARCHAR(20),transform_build_id VARCHAR(20))");jdbc.update("INSERT INTO release_member VALUES('r','b'),('r','l')");
|
||||
for(String stage:List.of("black","bright"))jdbc.execute("CREATE TABLE fact_"+stage+"(build_id VARCHAR(20),department VARCHAR(40),product VARCHAR(40),group_name VARCHAR(40),month_key VARCHAR(7),total_weight DECIMAL(20,4),components JSON)");
|
||||
seed("black","b","A","P","已有分类","2026-01","0001","X",10);
|
||||
seed("black","b","A","Q","","2026-02","0002","X",20);
|
||||
seed("black","b","B","P","其他","2026-01","0001","X",100);
|
||||
seed("bright","l","A","R","G","2026-03",null,"X",30);
|
||||
seed("bright","l","A","R","旧组","2026-04"," ","X",40);
|
||||
seed("bright","l","A","R","小写区域","2026-04","0001","x",200);
|
||||
service=new ClassificationPreviewService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),mock(CandidateAcceptanceService.class));
|
||||
}
|
||||
@AfterEach void cleanup(){if(database!=null&&database.matches("quality_classification_preview_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);}
|
||||
private void seed(String stage,String build,String department,String product,String group,String month,String code,String region,int weight){
|
||||
Map<String,Object> components=new HashMap<>();components.put("sourcePdtCode",code);components.put("sourceSalesRegion",region);
|
||||
jdbc.update("INSERT INTO fact_"+stage+" VALUES(?,?,?,?,?,?,CAST(? AS JSON))",build,department,product,group,month,weight,JsonMapper.builder().build().writeValueAsString(components));
|
||||
}
|
||||
private ClassificationPreviewService.Condition equal(String field,String value){return new ClassificationPreviewService.Condition(field,"EQUALS",value);}
|
||||
@Test void removingTwoConditionsBroadensAcrossMonthsStagesAndExistingGroupsWithExactCaseMatching(){
|
||||
var exact=List.of(equal("@department","A"),equal("@product","P"),equal("PDT代码","0001"),equal("销售片区","X"));
|
||||
assertEquals(1L,service.preview(new ClassificationPreviewService.Request("all","G",exact),actor).get("matchedRows"));
|
||||
var broad=service.preview(new ClassificationPreviewService.Request("all","G",List.of(equal("@department","A"),equal("销售片区","X"))),actor);
|
||||
assertEquals(4L,broad.get("matchedRows"));assertEquals(3L,broad.get("changedRows"));assertEquals(0,new java.math.BigDecimal("100").compareTo((java.math.BigDecimal)broad.get("totalWeight")));
|
||||
assertEquals(2L,service.preview(new ClassificationPreviewService.Request("all","G",List.of(new ClassificationPreviewService.Condition("PDT代码","IS_BLANK",""))),actor).get("matchedRows"));
|
||||
assertEquals(0L,service.preview(new ClassificationPreviewService.Request("all","G",List.of(equal("@department","A' OR 1=1 --"))),actor).get("matchedRows"));
|
||||
assertEquals(3,jdbc.queryForObject("SELECT COUNT(*) FROM fact_black",Integer.class));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package cn.yunsheng.quality.governance;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import java.util.*;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
class ClassificationPreviewTest {
|
||||
@Test void removedConditionsBroadenAcrossBothStagesAndAllMonthsIncludingExistingGroupsWithoutWrites(){
|
||||
var jdbc=mock(JdbcTemplate.class);var access=mock(CandidateAcceptanceService.class);var actor=mock(AppUserPrincipal.class);
|
||||
when(jdbc.queryForList(startsWith("SELECT release_id"),any(Object[].class))).thenReturn(List.of(Map.of("release_id","r")));
|
||||
when(jdbc.queryForList(startsWith("SELECT * FROM ("),any(Object[].class))).thenReturn(List.of(
|
||||
Map.of("stage","black","month","2026-01","current_group","已有分类","row_count",10L,"total_weight",100),
|
||||
Map.of("stage","bright","month","2026-09","current_group","G","row_count",5L,"total_weight",20)));
|
||||
var service=new ClassificationPreviewService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),access);
|
||||
var request=new ClassificationPreviewService.Request("all","G",List.of(new ClassificationPreviewService.Condition("@department","EQUALS","A")));
|
||||
var result=service.preview(request,actor);assertEquals(15L,result.get("matchedRows"));assertEquals(10L,result.get("changedRows"));
|
||||
var sql=org.mockito.ArgumentCaptor.forClass(String.class);verify(jdbc).queryForList(sql.capture(),eq("r"),eq("A"),eq("r"),eq("A"));
|
||||
assertTrue(sql.getValue().contains("fact_black"));assertTrue(sql.getValue().contains("fact_bright"));assertFalse(sql.getValue().contains("group_name=''"));assertFalse(sql.getValue().contains("sourcePdtCode"));assertFalse(sql.getValue().contains("f.month_key=?"));
|
||||
service.preview(new ClassificationPreviewService.Request("all","其他",request.conditions()),actor);
|
||||
verify(jdbc,times(1)).queryForList(startsWith("SELECT * FROM ("),any(Object[].class));verify(access,times(2)).requireRead("r",actor);
|
||||
doThrow(new ResponseStatusException(org.springframework.http.HttpStatus.FORBIDDEN)).when(access).requireRead("r",actor);assertThrows(ResponseStatusException.class,()->service.preview(request,actor));
|
||||
verify(jdbc,never()).update(anyString(),any(Object[].class));
|
||||
}
|
||||
@Test void rejectsEmptyDuplicateAndUnboundConditions(){
|
||||
assertThrows(ResponseStatusException.class,()->ClassificationPreviewService.validate(new ClassificationPreviewService.Request("all","G",List.of())));
|
||||
var condition=new ClassificationPreviewService.Condition("@department","EQUALS","A");
|
||||
assertThrows(ResponseStatusException.class,()->ClassificationPreviewService.validate(new ClassificationPreviewService.Request("all","G",List.of(condition,condition))));
|
||||
assertThrows(ResponseStatusException.class,()->ClassificationPreviewService.validate(new ClassificationPreviewService.Request("all","G",List.of(new ClassificationPreviewService.Condition("sql","EQUALS","A")))));
|
||||
}
|
||||
}
|
||||
@@ -19,12 +19,6 @@ class GovernanceValidationTest {
|
||||
assertFalse(MappingGovernanceValidator.applyRanges(Map.of("001","原组"),good,"CATEGORY",java.time.YearMonth.of(2026,5)).containsKey("001"));
|
||||
assertEquals("原组",MappingGovernanceValidator.applyRanges(Map.of("001","原组"),good,"CATEGORY",java.time.YearMonth.of(2026,2)).get("001"));
|
||||
}
|
||||
@Test void costReadinessRequiresSourceFieldUnitsApprovalAndNonoverlappingPeriod(){
|
||||
Map<String,Object> source=new LinkedHashMap<>(Map.of("sourceReference","library-cost-sheet","objectCode","0001","costField","材料成本","unit","元/kg","currency","CNY","approvalReference","APP-1","originalFormula","源工作表既有公式","fromMonth","2026-01","toMonth","2026-12"));source.put("baselineReference","原基准参数工作表/字段");
|
||||
assertTrue(CostConfigurationController.validateDefinition(Map.of("sources",List.of(source))).isEmpty());
|
||||
assertFalse(CostConfigurationController.validateDefinition(Map.of("sources",List.of(source,source))).isEmpty());
|
||||
assertFalse(CostConfigurationController.validateDefinition(Map.of("sources",List.of(Map.of("objectCode","0001")))).isEmpty());
|
||||
}
|
||||
@Test void credentialManagementAcceptsReferencesAndRejectsSecretsAndTraversal(){
|
||||
assertDoesNotThrow(()->GovernanceIdentityController.reference("env:QUALITY_MODEL_KEY","READY"));
|
||||
assertDoesNotThrow(()->GovernanceIdentityController.reference("file:apikey.txt","READY"));
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package cn.yunsheng.quality.governance;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.ingestion.ImportRunService;
|
||||
import java.util.*;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
class MappingIssueFilterTest {
|
||||
@Test void filtersApplyBeforePaginationAndPreserveLeadingZeroCustomerCodes() {
|
||||
JdbcTemplate jdbc=mock(JdbcTemplate.class);var access=mock(CandidateAcceptanceService.class);var actor=mock(AppUserPrincipal.class);
|
||||
when(jdbc.queryForList(startsWith("SELECT * FROM mapping_issue_group"),any(Object[].class))).thenReturn(List.of(issue("1","001|G1","r"),issue("2","002|G2","r"),issue("3","001|G1","r")));
|
||||
when(jdbc.queryForList(startsWith("SELECT DISTINCT f.department"),eq("r"),eq("r"))).thenReturn(List.of(
|
||||
Map.of("department","产业部A","group_name","G1","customer_code","001"),
|
||||
Map.of("department","产业部B","group_name","G2","customer_code","002")));
|
||||
var service=service(jdbc,access);
|
||||
var result=service.list(null,"OPEN",2,1,"001","G1","产业部A",actor);
|
||||
assertEquals(2,result.get("total"));
|
||||
verify(access,times(1)).requireRead("r",actor);
|
||||
verify(access,times(1)).requireSourceDetail(actor);
|
||||
var items=(List<Map<String,Object>>)result.get("items");assertEquals("3",items.getFirst().get("id"));
|
||||
assertEquals("001",items.getFirst().get("customerCode"));assertEquals("G1",items.getFirst().get("businessGroup"));
|
||||
assertEquals(List.of("产业部A"),items.getFirst().get("departments"));
|
||||
assertEquals(0,service.list(null,"OPEN",1,20,"001",null,"产业部B",actor).get("total"));
|
||||
verify(jdbc,times(1)).queryForList(startsWith("SELECT DISTINCT f.department"),eq("r"),eq("r"));
|
||||
when(jdbc.queryForList(startsWith("SELECT * FROM mapping_issue_group"),any(Object[].class)))
|
||||
.thenReturn(List.of(issue("1","001|G1","r")));
|
||||
assertEquals(1,service.list(null,"OPEN",1,20,"001",null,null,actor).get("total"));
|
||||
doThrow(new ResponseStatusException(org.springframework.http.HttpStatus.FORBIDDEN)).when(access).requireRead("r",actor);
|
||||
var denied=service.list(null,"OPEN",1,20,null,null,null,actor);
|
||||
assertEquals(0,denied.get("total"));
|
||||
assertEquals(Set.of(),((Map<?,?>)denied.get("filterOptions")).get("customerCodes"));
|
||||
verify(jdbc,times(1)).queryForList(startsWith("SELECT DISTINCT f.department"),eq("r"),eq("r"));
|
||||
verify(jdbc,never()).update(anyString(),any(Object[].class));
|
||||
}
|
||||
@Test void unauthorizedIssuesCannotLeakThroughFilterOptionsOrSourceLookup() {
|
||||
JdbcTemplate jdbc=mock(JdbcTemplate.class);var access=mock(CandidateAcceptanceService.class);var actor=mock(AppUserPrincipal.class);
|
||||
when(jdbc.queryForList(startsWith("SELECT * FROM mapping_issue_group"),any(Object[].class))).thenReturn(List.of(issue("private","secret|G","denied")));
|
||||
doThrow(new ResponseStatusException(org.springframework.http.HttpStatus.FORBIDDEN)).when(access).requireRead("denied",actor);
|
||||
var result=service(jdbc,access).list(null,"OPEN",1,20,null,null,null,actor);
|
||||
assertEquals(0,result.get("total"));
|
||||
assertEquals(Set.of(),((Map<?,?>)result.get("filterOptions")).get("customerCodes"));
|
||||
verify(jdbc,never()).queryForList(startsWith("SELECT DISTINCT f.department"),any(Object[].class));
|
||||
}
|
||||
private Map<String,Object> issue(String id,String key,String release) {
|
||||
return Map.of("id",id,"mapping_type","PROJECT_GROUP","source_key",key,"details",Map.of("releaseId",release));
|
||||
}
|
||||
private MappingIssueService service(JdbcTemplate jdbc,CandidateAcceptanceService access) {
|
||||
return new MappingIssueService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),access,mock(ImportRunService.class),mock(ObjectProvider.class),mock(PlatformTransactionManager.class));
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package cn.yunsheng.quality.governance;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import java.util.*;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
class UnclassifiedSourceServiceTest {
|
||||
@Test void paginationReusesImmutableInventoryButRechecksAccessAndPublishedRelease() {
|
||||
JdbcTemplate jdbc=mock(JdbcTemplate.class);
|
||||
var access=mock(CandidateAcceptanceService.class); var actor=mock(AppUserPrincipal.class);
|
||||
when(jdbc.queryForList(contains("current_release"),any(Object[].class))).thenReturn(List.of(Map.of("release_id","release")));
|
||||
when(jdbc.queryForList(startsWith("SELECT * FROM ("),any(Object[].class))).thenReturn(List.of(
|
||||
Map.of("stage","black","source_key","A","row_count",2L),Map.of("stage","bright","source_key","B","row_count",3L)));
|
||||
var service=new UnclassifiedSourceService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),access);
|
||||
var first=service.list(1,1,actor);var second=service.list(2,1,actor);
|
||||
assertEquals(2L,first.get("total"));assertEquals(2L,second.get("total"));
|
||||
assertEquals("B",((List<Map<String,Object>>)second.get("items")).getFirst().get("sourceKey"));
|
||||
verify(jdbc,times(1)).queryForList(startsWith("SELECT * FROM ("),eq("release"),eq("release"));
|
||||
verify(jdbc,never()).queryForObject(startsWith("SELECT COUNT(*)"),eq(Long.class),any(Object[].class));
|
||||
verify(access,times(2)).requireRead("release",actor);
|
||||
doThrow(new ResponseStatusException(org.springframework.http.HttpStatus.FORBIDDEN)).when(access).requireRead("release",actor);
|
||||
assertThrows(ResponseStatusException.class,()->service.list(2,1,actor));
|
||||
reset(access);
|
||||
when(jdbc.queryForList(contains("current_release"),any(Object[].class))).thenReturn(List.of(Map.of("release_id","new-release")));
|
||||
service.list(1,1,actor);
|
||||
verify(jdbc).queryForList(startsWith("SELECT * FROM ("),eq("new-release"),eq("new-release"));
|
||||
verify(jdbc,never()).update(anyString(),any(Object[].class));
|
||||
}
|
||||
@Test void deniedInventoryCannotReadFacts() {
|
||||
JdbcTemplate jdbc=mock(JdbcTemplate.class);var access=mock(CandidateAcceptanceService.class);var actor=mock(AppUserPrincipal.class);
|
||||
doThrow(new ResponseStatusException(org.springframework.http.HttpStatus.FORBIDDEN)).when(access).requireSourceDetail(actor);
|
||||
assertThrows(ResponseStatusException.class,()->new UnclassifiedSourceService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),access).list(1,50,actor));
|
||||
verifyNoInteractions(jdbc);
|
||||
assertTrue(UnclassifiedSourceService.sourceSql("black").contains("f.group_name IS NULL OR f.group_name=''"));
|
||||
assertThrows(IllegalArgumentException.class,()->UnclassifiedSourceService.sourceSql("wrong"));
|
||||
}
|
||||
}
|
||||
@@ -163,6 +163,48 @@ class ConfigurableMatchRulesTest {
|
||||
rule("blank-no-value", 1, "all", "group", "空备注", List.of(blankWithoutValue))))).isEmpty());
|
||||
}
|
||||
|
||||
@Test void maintenanceDraftClassifiesBlankKeyRowsAndFeedsProjectLookupWithoutChangingOtherScopes() throws Exception {
|
||||
var json = JsonMapper.builder().build();
|
||||
Map<String,Object> definition;
|
||||
try (var stream = getClass().getResourceAsStream("/classification-maintenance.json")) {
|
||||
definition = json.readValue(stream, Map.class);
|
||||
}
|
||||
var mapper = mapper((Map<String,String>) definition.get("categoryLookup"), (Map<String,String>) definition.get("projectGroupLookup"), definition);
|
||||
var source = row("001", "产品A", "销售区A", "产业部A");
|
||||
var values = new LinkedHashMap<>(source.values()); values.put("PDT代码", cell(""));
|
||||
var raw = new LinkedHashMap<>(source.rawView()); raw.put("PDT代码", "");
|
||||
var missing = new XlsxStreamingReader.RawRow(source.sheet(), source.physicalRow(), values, raw);
|
||||
var classified = mapper.transform("black", missing, 2026, 3);
|
||||
assertEquals("业务组A", classified.group());
|
||||
assertEquals("项目组A", classified.components().get("projectGroup"));
|
||||
assertFalse(classified.unmappedProjectGroup());
|
||||
assertNull(mapper.transform("bright", missing, 2026, 3).group());
|
||||
assertEquals("已有分组", mapper.transform("black", source, 2026, 3).group());
|
||||
values.put("成品规格", cell("其他产品")); raw.put("成品规格", "其他产品");
|
||||
assertNull(mapper.transform("black", new XlsxStreamingReader.RawRow(source.sheet(),source.physicalRow(),values,raw),2026,3).group());
|
||||
}
|
||||
|
||||
@Test void frontendExcelRoundTripRulesApplyAcrossFilesMonthsAndStagesWithSpecificityAndLegacyDefaults() throws Exception {
|
||||
Map<String,Object> definition;
|
||||
try (var stream = getClass().getResourceAsStream("/classification-workbook-roundtrip.json")) {
|
||||
definition = JsonMapper.builder().build().readValue(stream, Map.class);
|
||||
}
|
||||
assertTrue(ConfigurableMatchRules.validate(definition).isEmpty());
|
||||
var mapper = mapper((Map<String,String>) definition.get("categoryLookup"), (Map<String,String>) definition.get("projectGroupLookup"), definition);
|
||||
for (int month : List.of(1, 9)) {
|
||||
var source = row("001", "其他产品", "销售区A", "产业部A");
|
||||
for (String stage : List.of("black", "bright")) {
|
||||
var result = mapper.transform(stage, source, 2026, month);
|
||||
assertEquals("区域组", result.group()); // Replaces the existing lookup classification too.
|
||||
assertEquals("区域项目", result.components().get("projectGroup"));
|
||||
}
|
||||
assertEquals("特定产品组", mapper.transform("black", row("001", "产品A", "销售区A", "产业部A"), 2026, month).group());
|
||||
assertEquals("区域组", mapper.transform("bright", row("001", "产品A", "销售区A", "产业部A"), 2026, month).group());
|
||||
}
|
||||
assertEquals("工业电机-VCM", mapper.transform("black", row("001", "VCM-42", "其他销售区", "产业部A"), 2026, 3).group());
|
||||
assertEquals("原查表分组", mapper.transform("black", row("001", "其他产品", "销售区A", "其他产业部"), 2026, 3).group());
|
||||
}
|
||||
|
||||
private static BusinessRowMapper mapper(Map<String, String> groups, Map<String, String> projects, Map<String, Object> mappingDefinition) {
|
||||
return new BusinessRowMapper(groups, projects, "test-mapping", Map.of(), Map.of(), ConfigurableRuleEngine.baseline(), mappingDefinition);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package cn.yunsheng.quality.metrics;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import cn.yunsheng.quality.governance.GovernanceAccessService;
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.ingestion.BusinessRowMapper;
|
||||
import java.math.BigDecimal;
|
||||
import java.sql.ResultSet;
|
||||
import java.util.*;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.jdbc.core.*;
|
||||
|
||||
class MetricsUnclassifiedDisplayTest {
|
||||
@Test void blankGroupsAreHiddenWithoutRemovingTheirWeightFromSummaryOrTrend() throws Exception {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
MetricsService service = new MetricsService(jdbc, mock(GovernanceAccessService.class));
|
||||
var actor = new AppUserPrincipal(UUID.randomUUID(),"admin","","Admin",true,false,List.of("SYSTEM_ADMIN"),List.of("*"),List.of("*"),List.of());
|
||||
when(jdbc.query(anyString(), any(RowMapper.class), any(Object[].class))).thenAnswer(call -> {
|
||||
RowMapper<?> mapper = call.getArgument(1);
|
||||
ResultSet rs = mock(ResultSet.class);
|
||||
when(rs.getString(1)).thenReturn("release"); when(rs.getString(2)).thenReturn("Published"); when(rs.getString(3)).thenReturn("PUBLISHED");
|
||||
return List.of(mapper.mapRow(rs,0));
|
||||
});
|
||||
when(jdbc.queryForObject(contains("metric_rollup_build"),eq(Integer.class),any(Object[].class))).thenReturn(0);
|
||||
when(jdbc.queryForObject(eq("SELECT rule_version FROM data_release WHERE id=?"),eq(String.class),eq("release"))).thenReturn(BusinessRowMapper.RULE_VERSION);
|
||||
doAnswer(call -> {
|
||||
String sql = call.getArgument(0);
|
||||
RowCallbackHandler callback = call.getArgument(1);
|
||||
assertFalse(sql.contains("AND (f.group_name IS NOT NULL AND f.group_name<>'')"));
|
||||
if(sql.contains("AS bucket_key")) {
|
||||
for(String key:List.of("G1","","__UNASSIGNED__")) {
|
||||
ResultSet rs=mock(ResultSet.class);
|
||||
when(rs.getString("trend_period")).thenReturn("2026-09"); when(rs.getString("bucket_key")).thenReturn(key);
|
||||
when(rs.getLong("row_count")).thenReturn(1L);
|
||||
when(rs.getBigDecimal("qualified_weight")).thenReturn(BigDecimal.valueOf(90));
|
||||
when(rs.getBigDecimal("scrap_weight")).thenReturn(BigDecimal.TEN);
|
||||
when(rs.getBigDecimal("total_weight")).thenReturn(BigDecimal.valueOf(100));
|
||||
callback.processRow(rs);
|
||||
}
|
||||
} else if(sql.contains("SELECT f.month_key,")) {
|
||||
ResultSet rs=mock(ResultSet.class); when(rs.getString("month_key")).thenReturn("2026-09");
|
||||
when(rs.getLong("row_count")).thenReturn(3L); when(rs.getLong("excluded_blank_rows")).thenReturn(2L); callback.processRow(rs);
|
||||
} else throw new AssertionError(sql);
|
||||
return null;
|
||||
}).when(jdbc).query(anyString(),any(RowCallbackHandler.class),any(Object[].class));
|
||||
var query=new MetricsService.MetricsQuery(Integer.valueOf(2026),null,null,null,null,null,null,"group",true,"release");
|
||||
var result=service.calculate(query,actor);
|
||||
assertEquals(List.of("G1"),result.rows().stream().map(MetricsService.MetricRow::key).toList());
|
||||
assertEquals(0,BigDecimal.valueOf(300).compareTo(result.summary().black().totalWeight()));
|
||||
assertEquals(0,BigDecimal.valueOf(300).compareTo(result.trend().getFirst().black().totalWeight()));
|
||||
assertEquals(6L,result.coverage().get("includedRows"));
|
||||
assertEquals(4L,result.coverage().get("excludedBlankRows"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package cn.yunsheng.quality.records;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
import cn.yunsheng.quality.governance.GovernanceAccessService;
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.release.ReleaseService;
|
||||
import java.util.*;
|
||||
import java.util.stream.IntStream;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
class RecordPaginationTest {
|
||||
@Test void adjacentPagesReuseCountAndNarrowIndexWhileAuthorizationRemainsLive() {
|
||||
JdbcTemplate jdbc=mock(JdbcTemplate.class);var releases=mock(ReleaseService.class);
|
||||
var access=mock(GovernanceAccessService.class);var actor=mock(AppUserPrincipal.class);
|
||||
when(actor.getRoleCodes()).thenReturn(List.of("SYSTEM_ADMIN"));when(access.allowedFields(actor)).thenReturn(Set.of("*"));
|
||||
when(releases.resolvePublishedRelease(null)).thenReturn("release");
|
||||
when(jdbc.queryForObject(startsWith("SELECT COUNT(*)"),eq(Long.class),any(Object[].class))).thenReturn(40L);
|
||||
List<String> ids=IntStream.range(0,40).mapToObj(n->"id-"+n).toList();
|
||||
when(jdbc.queryForList(startsWith("SELECT f.id"),eq(String.class),any(Object[].class))).thenReturn(ids);
|
||||
when(jdbc.query(startsWith("SELECT f.id, sr.id"),any(RowMapper.class),any(Object[].class))).thenAnswer(call->{
|
||||
Object[] args=java.util.Arrays.copyOfRange(call.getArguments(),2,call.getArguments().length);List<Map<String,Object>> rows=new ArrayList<>();
|
||||
for(Object arg:args)if(arg instanceof String id&&id.startsWith("id-"))rows.add(Map.of("id",id));
|
||||
Collections.reverse(rows);return rows;
|
||||
});
|
||||
var controller=new RecordController(jdbc,releases,JsonMapper.builder().build(),access,mock(RecordsWorkbookExportService.class));
|
||||
var first=page(controller,actor,1,null);var second=page(controller,actor,2,null);
|
||||
assertEquals(40L,second.get("total"));
|
||||
assertEquals("id-0",((List<Map<String,Object>>)first.get("rows")).getFirst().get("id"));
|
||||
assertEquals("id-15",((List<Map<String,Object>>)second.get("rows")).getFirst().get("id"));
|
||||
verify(jdbc,times(1)).queryForObject(startsWith("SELECT COUNT(*)"),eq(Long.class),any(Object[].class));
|
||||
verify(jdbc,times(1)).queryForList(startsWith("SELECT f.id"),eq(String.class),any(Object[].class));
|
||||
verify(access,times(2)).requireDetail(actor);
|
||||
page(controller,actor,2,"规格A");
|
||||
verify(jdbc,times(2)).queryForObject(startsWith("SELECT COUNT(*)"),eq(Long.class),any(Object[].class));
|
||||
when(releases.resolvePublishedRelease(null)).thenReturn("new-release");page(controller,actor,1,null);
|
||||
verify(jdbc,times(3)).queryForObject(startsWith("SELECT COUNT(*)"),eq(Long.class),any(Object[].class));
|
||||
doThrow(new ResponseStatusException(org.springframework.http.HttpStatus.FORBIDDEN)).when(access).requireDetail(actor);
|
||||
assertThrows(ResponseStatusException.class,()->page(controller,actor,1,null));
|
||||
}
|
||||
|
||||
private Map<String,Object> page(RecordController controller,AppUserPrincipal actor,int page,String product) {
|
||||
return controller.records("black",2026,null,null,null,null,product,null,true,null,page,15,null,null,null,"asc",null,actor);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"categoryLookup": {
|
||||
"P1|销售区A": "已有分组"
|
||||
},
|
||||
"projectGroupLookup": {
|
||||
"001|业务组A": "项目组A"
|
||||
},
|
||||
"matchRules": [
|
||||
{
|
||||
"id": "classification-c94c311d",
|
||||
"name": "业务分组未分类维护",
|
||||
"enabled": true,
|
||||
"priority": -10000,
|
||||
"stage": "black",
|
||||
"conditions": [
|
||||
{
|
||||
"field": "@department",
|
||||
"operator": "EQUALS",
|
||||
"value": "产业部A"
|
||||
},
|
||||
{
|
||||
"field": "@product",
|
||||
"operator": "EQUALS",
|
||||
"value": "产品A"
|
||||
},
|
||||
{
|
||||
"field": "PDT代码",
|
||||
"operator": "IS_BLANK",
|
||||
"value": ""
|
||||
},
|
||||
{
|
||||
"field": "销售片区",
|
||||
"operator": "EQUALS",
|
||||
"value": "销售区A"
|
||||
},
|
||||
{
|
||||
"field": "@group",
|
||||
"operator": "IS_BLANK",
|
||||
"value": ""
|
||||
}
|
||||
],
|
||||
"target": "group",
|
||||
"value": "业务组A"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"categoryLookup": {
|
||||
"P1|销售区A": "原查表分组"
|
||||
},
|
||||
"projectGroupLookup": {
|
||||
"001|区域组": "区域项目"
|
||||
},
|
||||
"matchRules": [
|
||||
{
|
||||
"id": "classification-782ed979",
|
||||
"name": "特定产品组分类规则",
|
||||
"priority": -10000,
|
||||
"stage": "black",
|
||||
"target": "group",
|
||||
"value": "特定产品组",
|
||||
"enabled": true,
|
||||
"conditions": [
|
||||
{
|
||||
"field": "@department",
|
||||
"operator": "EQUALS",
|
||||
"value": "产业部A"
|
||||
},
|
||||
{
|
||||
"field": "@product",
|
||||
"operator": "EQUALS",
|
||||
"value": "产品A"
|
||||
},
|
||||
{
|
||||
"field": "PDT代码",
|
||||
"operator": "EQUALS",
|
||||
"value": "P1"
|
||||
},
|
||||
{
|
||||
"field": "销售片区",
|
||||
"operator": "EQUALS",
|
||||
"value": "销售区A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "classification-94305728",
|
||||
"name": "区域组分类规则",
|
||||
"priority": -9790,
|
||||
"stage": "all",
|
||||
"target": "group",
|
||||
"value": "区域组",
|
||||
"enabled": true,
|
||||
"conditions": [
|
||||
{
|
||||
"field": "@department",
|
||||
"operator": "EQUALS",
|
||||
"value": "产业部A"
|
||||
},
|
||||
{
|
||||
"field": "销售片区",
|
||||
"operator": "EQUALS",
|
||||
"value": "销售区A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "default-vcm-group",
|
||||
"name": "VCM默认",
|
||||
"priority": 100,
|
||||
"stage": "all",
|
||||
"target": "group",
|
||||
"value": "工业电机-VCM",
|
||||
"enabled": true,
|
||||
"conditions": [
|
||||
{
|
||||
"field": "成品规格",
|
||||
"operator": "CONTAINS",
|
||||
"value": "VCM"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user