优化:产品查询数据库分页并修复报告测试契约
This commit is contained in:
@@ -158,6 +158,12 @@ public class AnalysisFineDefectsService {
|
||||
/** One authorized scan (or rollup query) for every period/object and all visible children. */
|
||||
@Transactional(readOnly = true, isolation = Isolation.REPEATABLE_READ)
|
||||
public Map<String, Object> grouped(Map<String, Object> input, AppUserPrincipal actor, String dimension, String granularity) {
|
||||
return grouped(input,actor,dimension,granularity,null);
|
||||
}
|
||||
|
||||
public Map<String,Object> grouped(Map<String,Object> input,AppUserPrincipal actor,String dimension,String granularity,List<String> productKeys) {
|
||||
productKeys=cn.yunsheng.quality.metrics.ProductBuckets.parse(productKeys);
|
||||
if(productKeys!=null&&!dimension.equals("product"))throw bad("产品桶只能用于产品细分表");
|
||||
Set<String> allowed = governance.allowedFields(actor);
|
||||
Request request = parse(input, allowed, true);
|
||||
String bucket = switch (dimension) {
|
||||
@@ -184,6 +190,7 @@ public class AnalysisFineDefectsService {
|
||||
if (request.productSearch() != null) {
|
||||
predicate.append(" AND LOCATE(LOWER(?),LOWER(COALESCE(f.product,'')))>0"); args.add(request.productSearch());
|
||||
}
|
||||
cn.yunsheng.quality.metrics.ProductBuckets.append(predicate,args,productKeys);
|
||||
List<DefectField> fields = applicableFields(request,allowed);
|
||||
List<Category> categories = categories(request.stage());
|
||||
Set<String> components = new LinkedHashSet<>();
|
||||
|
||||
@@ -32,13 +32,17 @@ public class DefectDrilldownController {
|
||||
String granularity=String.valueOf(input.getOrDefault("trendGranularity","month"));
|
||||
if(!Set.of("period","overall","department","group","product","projectGroup").contains(dimension))throw bad("细分维度无效");
|
||||
if(!Set.of("month","quarter","year").contains(granularity))throw bad("趋势粒度无效");
|
||||
Map<String,Object> scope=new LinkedHashMap<>(input);scope.remove("trendGranularity");scope.remove("view");scope.put("dimension","defect");scope.put("mode","top");scope.put("limit",100);
|
||||
List<String> productKeys=cn.yunsheng.quality.metrics.ProductBuckets.parse(input.get("productKeys"));
|
||||
if(productKeys!=null&&!dimension.equals("product"))throw bad("产品桶只能用于产品细分表");
|
||||
Map<String,Object> scope=new LinkedHashMap<>(input);scope.remove("productKeys");scope.remove("trendGranularity");scope.remove("view");scope.put("dimension","defect");scope.put("mode","top");scope.put("limit",100);
|
||||
// Reuse the source parser to reject unknown filters and validate the selected category/stage.
|
||||
AnalysisFineDefectsService.Request request=AnalysisFineDefectsService.parse(scope,governance.allowedFields(actor),true);
|
||||
if(request.category()==null)throw bad("必须指定不良分类");
|
||||
var query=new MetricsService.MetricsQuery(number(scope,"year"),number(scope,"month"),number(scope,"quarter"),request.department(),request.group(),request.product(),request.projectGroup(),dimension.equals("period")?"overall":dimension,request.includeUnassigned(),request.releaseId(),granularity,AnalysisExplorationTools.exclusions(scope.get("exclusions")))
|
||||
.withProductSearch(request.productSearch()).withExtraFilters(scope.get("extraFilters")).withColumnFilters(scope.get("columnFilters")).withMonthRange(number(scope,"monthStart"),number(scope,"monthEnd")).withStage(request.stage());
|
||||
var paired=metrics.calculatePairedSnapshot(query,actor,dimension.equals("period"),() -> fine.grouped(scope,actor,dimension,granularity));
|
||||
var paired=productKeys==null
|
||||
? metrics.calculatePairedSnapshot(query,actor,dimension.equals("period"),() -> fine.grouped(scope,actor,dimension,granularity))
|
||||
: metrics.calculatePairedProductSnapshot(query,actor,productKeys,() -> fine.grouped(scope,actor,dimension,granularity,productKeys));
|
||||
// Refresh the visible fields after the snapshot; a concurrent policy revocation cannot release old fields.
|
||||
governance.requireDetail(actor);governance.requireBoundAction("QUERY","query.detail");
|
||||
Set<String> currentAllowed=governance.allowedFields(actor);
|
||||
|
||||
@@ -109,6 +109,40 @@ public class MetricsController {
|
||||
return responseAccess.visibleTrend(metrics.calculateTrend(query, actor), actor);
|
||||
}
|
||||
|
||||
/** 产品分页接口:范围、授权和正式规则与主查询一致。 */
|
||||
@GetMapping("/products")
|
||||
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN', 'DATA_ADMIN', 'USER')")
|
||||
public MetricsService.ProductPage productsHttp(@RequestParam(required = false) Integer year,
|
||||
@RequestParam(required = false) Integer month,
|
||||
@RequestParam(required = false) Integer monthStart,
|
||||
@RequestParam(required = false) Integer monthEnd,
|
||||
@RequestParam(required = false) String quarter,
|
||||
@RequestParam(required = false) String department,
|
||||
@RequestParam(required = false) String group,
|
||||
@RequestParam(required = false) String product,
|
||||
@RequestParam(required = false) String productSearch,
|
||||
@RequestParam(required = false) String projectGroup,
|
||||
@RequestParam(defaultValue = "product") String view,
|
||||
@RequestParam(defaultValue = "true") boolean includeUnassigned,
|
||||
@RequestParam(required = false) String releaseId,
|
||||
@RequestParam(defaultValue = "month") String trendGranularity,
|
||||
@RequestParam(required = false) String extraFilters,
|
||||
@RequestParam(required = false) String exclusions,
|
||||
@RequestParam(required = false) String columnFilters,
|
||||
@RequestParam(required = false) String stage,
|
||||
@RequestParam(defaultValue="1") int page,
|
||||
@RequestParam(defaultValue="15") int pageSize,
|
||||
@RequestParam(defaultValue="brightQualifiedDesc") String order,
|
||||
@AuthenticationPrincipal AppUserPrincipal actor) {
|
||||
Integer quarterNumber = parseQuarter(quarter);
|
||||
MetricsService.MetricsQuery query = new MetricsService.MetricsQuery(year, month, quarterNumber, department,
|
||||
group, product, projectGroup, view, includeUnassigned, releaseId, trendGranularity,
|
||||
parseExclusions(exclusions));
|
||||
query = query.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters)
|
||||
.withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage);
|
||||
return responseAccess.visibleProductPage(metrics.calculateProductPage(query,actor,page,pageSize,order),actor);
|
||||
}
|
||||
|
||||
/** Existing direct callers retain the pre-range controller signature. */
|
||||
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter,
|
||||
String department, String group, String product, String productSearch,
|
||||
@@ -142,15 +176,16 @@ public class MetricsController {
|
||||
product, projectGroup, view, includeUnassigned, releaseId, trendGranularity, parseExclusions(exclusions))
|
||||
.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters).withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage);
|
||||
MetricsService.MetricSnapshot visible = responseAccess.visible(tableOnly ? metrics.calculateTable(query,actor) : metrics.calculate(query,actor), actor);
|
||||
if (rowLimit == null) return visible;
|
||||
if (rowLimit < 1 || rowLimit > 500 || rowOffset < 0) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
|
||||
if (rowOrder != null && !rowOrder.equals("scrapDesc")) throw new IllegalArgumentException("rowOrder 无效");
|
||||
if (rowLimit == null && rowOrder == null) return visible;
|
||||
if (rowLimit != null && (rowLimit < 1 || rowLimit > 500 || rowOffset < 0)) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
|
||||
ProductTableOrder productOrder=rowOrder==null||rowOrder.equals("scrapDesc")?null:ProductTableOrder.parse(rowOrder);
|
||||
List<MetricsService.MetricRow> ordered = new ArrayList<>(visible.rows());
|
||||
if ("scrapDesc".equals(rowOrder)) ordered.sort(Comparator
|
||||
.comparing(this::totalScrap).reversed()
|
||||
.thenComparing(row -> row.key() == null ? "" : row.key()));
|
||||
int start = Math.min(rowOffset, ordered.size());
|
||||
int end = (int) Math.min((long) start + rowLimit, ordered.size());
|
||||
if(productOrder!=null)ordered.sort(productOrder.comparator());
|
||||
int start = rowLimit==null?0:Math.min(rowOffset, ordered.size());
|
||||
int end = rowLimit==null?ordered.size():(int) Math.min((long) start + rowLimit, ordered.size());
|
||||
Map<String, Object> context = new LinkedHashMap<>(visible.context());
|
||||
context.put("rowCount", ordered.size());
|
||||
return new MetricsService.MetricSnapshot(visible.release(), context, visible.summary(), ordered.subList(start, end),
|
||||
|
||||
@@ -31,6 +31,17 @@ public class MetricsResponseAccess {
|
||||
return new MetricSnapshot(filtered.release(), filtered.context(), hideDefects(filtered.summary()), List.of(), List.of(), defects, filtered.coverage());
|
||||
}
|
||||
|
||||
/** 分页结果返回前重新核验当前用户、字段和明细服务绑定。 */
|
||||
public MetricsService.ProductPage visibleProductPage(MetricsService.ProductPage page,AppUserPrincipal actor) {
|
||||
governance.requireUserActivePolicy(actor);
|
||||
if(!Boolean.TRUE.equals(governance.policy(actor.getId().toString()).get("detailAllowed")))
|
||||
return new MetricsService.ProductPage(page.release(),page.context(),List.of(),0,page.page(),page.pageSize(),page.order());
|
||||
governance.requireBoundAction("QUERY","query.detail");
|
||||
Set<String> allowed=governance.allowedFields(actor);
|
||||
var rows=allowed.contains("*")?page.rows():page.rows().stream().map(row->restrictDefects(row,allowed)).toList();
|
||||
return new MetricsService.ProductPage(page.release(),page.context(),rows,page.total(),page.page(),page.pageSize(),page.order());
|
||||
}
|
||||
|
||||
/** 趋势查询沿用明细授权和指标字段可见性边界。 */
|
||||
public MetricTrendSnapshot visibleTrend(MetricTrendSnapshot snapshot, AppUserPrincipal actor) {
|
||||
governance.requireUserActivePolicy(actor);
|
||||
|
||||
@@ -87,6 +87,54 @@ public class MetricsService {
|
||||
return withConsistentSnapshot(() -> calculateForReleaseSnapshot(query, actor, release, tableOnly));
|
||||
}
|
||||
|
||||
public record ProductPage(Map<String,Object> release,Map<String,Object> context,List<MetricRow> rows,long total,int page,int pageSize,String order) {}
|
||||
|
||||
/** 数据库选出当前页的产品桶,再按正式规则计算这些桶的完整分量。 */
|
||||
public ProductPage calculateProductPage(MetricsQuery query,AppUserPrincipal actor,int page,int pageSize,String orderName) {
|
||||
if(actor==null)throw new ResponseStatusException(HttpStatus.FORBIDDEN,"指标查询必须绑定当前有效用户");
|
||||
governance.requireUserActivePolicy(actor);governance.requireBoundAction("QUERY","query.aggregate");validate(query);
|
||||
if(!"product".equals(query.view()))throw new IllegalArgumentException("产品分页只支持product视图");
|
||||
if(page<1||page>1_000_000||pageSize<1||pageSize>100)throw new IllegalArgumentException("页码或每页数量无效");
|
||||
ProductTableOrder requestedOrder=ProductTableOrder.parse(orderName);
|
||||
ProductTableOrder order=requestedOrder.visibleTo(governance.allowedFields(actor));
|
||||
String releaseId=query.releaseId()==null||query.releaseId().isBlank()?currentReleaseId():query.releaseId();
|
||||
if(releaseId==null){var empty=emptySnapshot(query);return new ProductPage(empty.release(),empty.context(),List.of(),0,page,pageSize,requestedOrder.wire());}
|
||||
ReleaseInfo release=release(releaseId,actor);
|
||||
return withConsistentSnapshot(()->{
|
||||
boolean unrolled=hasUnrolledBuilds(releaseId),sales=salesOrderRollupReady(releaseId,query,actor);
|
||||
boolean source=!query.extraFilters().isEmpty()||!query.columnFilters().isEmpty();
|
||||
var branches=new ArrayList<String>();var args=new ArrayList<Object>();
|
||||
for(String stage:List.of("black","bright")) {
|
||||
if(query.stage()!=null&&!query.stage().equals(stage))continue;
|
||||
if(!source||sales)addProductRankBranch(branches,args,stageScope(stage,releaseId,query,actor,false,sales));
|
||||
if(!sales&&(source||unrolled))addProductRankBranch(branches,args,stageScope(stage,releaseId,query,actor,true,false));
|
||||
}
|
||||
String candidates="WITH candidates AS ("+String.join(" UNION ALL ",branches)+") ";
|
||||
String ranked="SELECT ANY_VALUE(bucket_key) AS bucket_key,"+order.rankExpression()+" AS sort_weight FROM candidates GROUP BY CAST(bucket_key AS BINARY)";
|
||||
String sql=candidates+", ranked AS ("+ranked+") SELECT bucket_key,COUNT(*) OVER() AS row_count FROM ranked ORDER BY "+order.sqlOrder()+" LIMIT ? OFFSET ?";
|
||||
var pageArgs=new ArrayList<Object>(args);pageArgs.add(pageSize);pageArgs.add((long)(page-1)*pageSize);
|
||||
var keys=new ArrayList<String>();long[] count={0};
|
||||
jdbc.query(sql,(RowCallbackHandler)rs->{keys.add(rs.getString("bucket_key"));count[0]=rs.getLong("row_count");},pageArgs.toArray());
|
||||
if(keys.isEmpty()) {
|
||||
Long total=jdbc.queryForObject(candidates+"SELECT COUNT(*) FROM (SELECT CAST(bucket_key AS BINARY) FROM candidates GROUP BY CAST(bucket_key AS BINARY)) all_products",Long.class,args.toArray());
|
||||
count[0]=total==null?0:total;
|
||||
}
|
||||
var snapshot=calculateForReleaseSnapshot(query,actor,release,true,keys);
|
||||
var byKey=new LinkedHashMap<String,MetricRow>();snapshot.rows().forEach(row->byKey.put(row.key(),row));
|
||||
var rows=keys.stream().map(byKey::get).filter(Objects::nonNull).toList();
|
||||
return new ProductPage(snapshot.release(),snapshot.context(),rows,count[0],page,pageSize,requestedOrder.wire());
|
||||
});
|
||||
}
|
||||
|
||||
private void addProductRankBranch(List<String> branches,List<Object> args,StageScope scope) {
|
||||
boolean black=scope.stage().equals("black");
|
||||
String nil="CAST(NULL AS DECIMAL(38,8))";
|
||||
branches.add("SELECT "+ProductBuckets.EXPRESSION+" AS bucket_key,"+(black?"SUM(f.total_weight)":nil)+" AS black_total,"+
|
||||
(black?nil:"SUM(f.qualified_weight)")+" AS bright_good,"+(black?nil:"SUM(f.scrap_weight)")+" AS bright_scrap "+
|
||||
"FROM release_member rm JOIN "+scope.table()+" f"+scope.hint()+" ON f.build_id=rm.transform_build_id"+scope.where()+" GROUP BY bucket_key");
|
||||
args.addAll(scope.args());
|
||||
}
|
||||
|
||||
/** 只按时间桶生成趋势,不构造总体摘要或产品/分组维度行。 */
|
||||
public MetricTrendSnapshot calculateTrend(MetricsQuery query, AppUserPrincipal actor) {
|
||||
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "指标查询必须绑定当前有效用户");
|
||||
@@ -105,6 +153,15 @@ public class MetricsService {
|
||||
/** Authorize before opening the read snapshot, so service-binding locks do not span source scans. */
|
||||
public <T> PairedSnapshot<T> calculatePairedSnapshot(MetricsQuery query, AppUserPrincipal actor,
|
||||
boolean trendOnly, Supplier<T> source) {
|
||||
return calculatePairedSnapshot(query,actor,trendOnly,source,null);
|
||||
}
|
||||
|
||||
public <T> PairedSnapshot<T> calculatePairedProductSnapshot(MetricsQuery query,AppUserPrincipal actor,List<String> productKeys,Supplier<T> source) {
|
||||
if(!"product".equals(query.view()))throw new IllegalArgumentException("产品桶范围无效");
|
||||
return calculatePairedSnapshot(query,actor,false,source,ProductBuckets.parse(productKeys));
|
||||
}
|
||||
|
||||
private <T> PairedSnapshot<T> calculatePairedSnapshot(MetricsQuery query,AppUserPrincipal actor,boolean trendOnly,Supplier<T> source,List<String> productKeys) {
|
||||
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN,"指标查询必须绑定当前有效用户");
|
||||
governance.requireUserActivePolicy(actor);
|
||||
governance.requireBoundAction("QUERY","query.aggregate");
|
||||
@@ -114,7 +171,7 @@ public class MetricsService {
|
||||
ReleaseInfo release=release(releaseId,actor);
|
||||
return withConsistentSnapshot(() -> trendOnly
|
||||
? new PairedSnapshot<T>(null,calculateTrendForReleaseSnapshot(query,actor,release),source.get())
|
||||
: new PairedSnapshot<T>(calculateForReleaseSnapshot(query,actor,release),null,source.get()));
|
||||
: new PairedSnapshot<T>(calculateForReleaseSnapshot(query,actor,release,productKeys!=null,productKeys),null,source.get()));
|
||||
}
|
||||
|
||||
private MetricTrendSnapshot calculateTrendForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor,
|
||||
@@ -257,9 +314,13 @@ public class MetricsService {
|
||||
}
|
||||
|
||||
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release, boolean tableOnly) {
|
||||
return calculateForReleaseSnapshot(query,actor,release,tableOnly,null);
|
||||
}
|
||||
|
||||
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release, boolean tableOnly,List<String> productKeys) {
|
||||
String releaseId = release.id();
|
||||
boolean hasUnrolledBuilds = hasUnrolledBuilds(releaseId);
|
||||
AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds, null, !tableOnly);
|
||||
AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds, null, !tableOnly,productKeys);
|
||||
String ruleVersion=jdbc.queryForObject("SELECT rule_version FROM data_release WHERE id=?",String.class,releaseId);
|
||||
ConfigurableRuleEngine rules=rulesForVersion(ruleVersion);
|
||||
MetricRow summary = rules.applyMetrics(aggregates.summary().toMetricRow("overall", "总体", null, null, null));
|
||||
@@ -268,7 +329,7 @@ public class MetricsService {
|
||||
MetricRow adjusted=rules.applyMetrics(new MetricRow(row.key(),row.label(),row.department(),row.group(),row.product(),row.projectGroup(),row.compositeRate(),row.black(),row.bright(),row.coverage(),row.compositeRateWithoutHeadquartersScrap()));
|
||||
return new TrendMetricRow(row.month(),adjusted.key(),adjusted.label(),adjusted.department(),adjusted.group(),adjusted.product(),adjusted.projectGroup(),adjusted.compositeRate(),adjusted.black(),adjusted.bright(),adjusted.coverage(),adjusted.compositeRateWithoutHeadquartersScrap(),row.period(),row.periodLabel(),row.granularity());
|
||||
}).toList();
|
||||
Map<String, Object> coverage = queryCoverage(releaseId, query, actor, hasUnrolledBuilds);
|
||||
Map<String, Object> coverage = productKeys==null?queryCoverage(releaseId, query, actor, hasUnrolledBuilds):Map.of();
|
||||
Map<String, Object> context = context(query, query.includeUnassigned());
|
||||
context.put("ruleVersionId",ruleVersion==null?BusinessRowMapper.RULE_VERSION:ruleVersion);
|
||||
context.put("calculationRules",rules.calculationRules());context.put("metricFormulaDescriptions",rules.formulaDescriptions());
|
||||
@@ -376,24 +437,29 @@ public class MetricsService {
|
||||
|
||||
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
|
||||
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments, boolean includePeriods) {
|
||||
return queryAggregates(releaseId,query,actor,hasUnrolledBuilds,canonicalGroupDepartments,includePeriods,null);
|
||||
}
|
||||
|
||||
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
|
||||
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments, boolean includePeriods,List<String> productKeys) {
|
||||
AggregateSet result = new AggregateSet();
|
||||
boolean salesRollup = salesOrderRollupReady(releaseId, query, actor);
|
||||
boolean hasSourceFilters = !query.extraFilters().isEmpty() || !query.columnFilters().isEmpty();
|
||||
if (!hasSourceFilters || salesRollup) {
|
||||
aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods);
|
||||
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods);
|
||||
aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods,productKeys);
|
||||
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods,productKeys);
|
||||
}
|
||||
if (!salesRollup && (hasUnrolledBuilds || hasSourceFilters)) {
|
||||
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods);
|
||||
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods);
|
||||
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods,productKeys);
|
||||
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods,productKeys);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private void aggregateStage(AggregateSet result, String stage, String view, String releaseId,
|
||||
MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup,
|
||||
Map<String, String> canonicalGroupDepartments, boolean includePeriods) {
|
||||
if(query.stage()!=null&&!query.stage().equals(stage))return;
|
||||
Map<String, String> canonicalGroupDepartments, boolean includePeriods,List<String> productKeys) {
|
||||
if(query.stage()!=null&&!query.stage().equals(stage)||productKeys!=null&&productKeys.isEmpty())return;
|
||||
String table = factFallback ? (stage.equals("black") ? "fact_black" : "fact_bright") : (salesRollup ? "sales_order_metric_rollup" : "metric_rollup");
|
||||
String dimensionColumn = view.equals("overall") ? null : switch (view) {
|
||||
case "department" -> "f.department";
|
||||
@@ -407,22 +473,11 @@ public class MetricsService {
|
||||
boolean canonicalGroupRows = "group".equals(view) && canonicalGroupDepartments != null;
|
||||
String ownerDepartment = canonicalGroupRows ? ", f.department COLLATE utf8mb4_0900_as_cs AS owner_department" : "";
|
||||
String aggregateKeys = canonicalGroupRows ? "trend_period, bucket_key, owner_department" : "trend_period, bucket_key";
|
||||
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
|
||||
List<Object> args = new ArrayList<>();
|
||||
args.add(releaseId);
|
||||
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
|
||||
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
|
||||
if (salesRollup) new SalesOrderMetricRollups(jdbc).append(where, args, query.extraFilters());
|
||||
appendTime(where, args, query);
|
||||
appendDimensions(where, args, query);
|
||||
appendExclusions(where, args, query);
|
||||
if(factFallback) {
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
|
||||
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
|
||||
}
|
||||
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
if (!effectiveIncludeUnassigned(query))
|
||||
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
|
||||
StageScope scope=stageScope(stage,releaseId,query,actor,factFallback,salesRollup);
|
||||
List<Object> args=new ArrayList<>(scope.args());
|
||||
StringBuilder limitedWhere=new StringBuilder(scope.where());
|
||||
ProductBuckets.append(limitedWhere,args,productKeys);
|
||||
String where=limitedWhere.toString();
|
||||
String trendBucket = includePeriods ? trendBucketExpression(query.trendGranularity()) : "'all'";
|
||||
String sql = """
|
||||
SELECT %s AS trend_period, %s AS bucket_key%s,
|
||||
@@ -463,6 +518,29 @@ public class MetricsService {
|
||||
}, args.toArray());
|
||||
}
|
||||
|
||||
private record StageScope(String stage,String table,String hint,String where,List<Object> args) {}
|
||||
|
||||
private StageScope stageScope(String stage,String releaseId,MetricsQuery query,AppUserPrincipal actor,boolean factFallback,boolean salesRollup) {
|
||||
String table=factFallback?(stage.equals("black")?"fact_black":"fact_bright"):(salesRollup?"sales_order_metric_rollup":"metric_rollup");
|
||||
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
|
||||
List<Object> args = new ArrayList<>();
|
||||
args.add(releaseId);
|
||||
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
|
||||
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
|
||||
if (salesRollup) new SalesOrderMetricRollups(jdbc).append(where, args, query.extraFilters());
|
||||
appendTime(where, args, query);
|
||||
appendDimensions(where, args, query);
|
||||
appendExclusions(where, args, query);
|
||||
if(factFallback) {
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
|
||||
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
|
||||
}
|
||||
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
if (!effectiveIncludeUnassigned(query))
|
||||
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
|
||||
return new StageScope(stage,table,factFilterHint(stage,factFallback,query),where.toString(),List.copyOf(args));
|
||||
}
|
||||
|
||||
private String componentSum(String key) {
|
||||
String path = "$." + key;
|
||||
String extracted = "JSON_EXTRACT(f.components, '" + path + "')";
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package cn.yunsheng.quality.metrics;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/** 已授权范围内的产品桶限制;空字符串与NULL桶保留各自语义。 */
|
||||
public final class ProductBuckets {
|
||||
public static final String EXPRESSION="COALESCE(f.product COLLATE utf8mb4_0900_as_cs, '__UNASSIGNED__') COLLATE utf8mb4_0900_as_cs";
|
||||
private ProductBuckets() {}
|
||||
public static List<String> parse(Object input) {
|
||||
if(input==null)return null;
|
||||
if(!(input instanceof List<?> values)||values.size()>100)throw new IllegalArgumentException("产品桶必须为不超过100项的列表");
|
||||
var result=new LinkedHashSet<String>();
|
||||
for(Object value:values) {
|
||||
if(!(value instanceof String key)||key.length()>512)throw new IllegalArgumentException("产品桶名称无效");
|
||||
result.add(key);
|
||||
}
|
||||
return List.copyOf(result);
|
||||
}
|
||||
public static void append(StringBuilder where,List<Object> args,List<String> keys) {
|
||||
if(keys==null)return;
|
||||
if(keys.isEmpty()){where.append(" AND 1=0");return;}
|
||||
// STRCMP avoids hash-join comparisons dropping canonically equivalent Unicode text.
|
||||
where.append(" AND (").append(String.join(" OR ",Collections.nCopies(keys.size(),
|
||||
"STRCMP("+EXPRESSION+",CONVERT(? USING utf8mb4) COLLATE utf8mb4_0900_as_cs)=0"))).append(')');
|
||||
args.addAll(keys);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package cn.yunsheng.quality.metrics;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.Comparator;
|
||||
|
||||
/** SQL分页与全量导出共用的重量排序约定;比率仍由正式规则计算。 */
|
||||
public enum ProductTableOrder {
|
||||
BRIGHT_QUALIFIED("brightQualifiedDesc", "SUM(bright_good)"),
|
||||
BLACK_TOTAL("blackTotalDesc", "SUM(black_total)"),
|
||||
BRIGHT_TOTAL("brightTotalDesc", "CASE WHEN SUM(bright_good) IS NULL AND SUM(bright_scrap) IS NULL THEN NULL ELSE COALESCE(SUM(bright_good),0)+COALESCE(SUM(bright_scrap),0) END"),
|
||||
PRODUCT("productAsc", null);
|
||||
|
||||
private final String wire;
|
||||
private final String expression;
|
||||
ProductTableOrder(String wire, String expression) { this.wire=wire; this.expression=expression; }
|
||||
public String wire() { return wire; }
|
||||
public static ProductTableOrder parse(String value) {
|
||||
for (var order:values()) if (order.wire.equals(value)) return order;
|
||||
throw new IllegalArgumentException("产品排序无效");
|
||||
}
|
||||
public String rankExpression() { return expression==null?"NULL":expression; }
|
||||
public String sqlOrder() {
|
||||
// UTF-16字节序与既有Java对象键次序一致,同重量时固定顺序。
|
||||
return (expression==null?"":"sort_weight DESC, ")+"CAST(CONVERT(bucket_key USING utf16) AS BINARY) ASC";
|
||||
}
|
||||
public ProductTableOrder visibleTo(java.util.Set<String> fields) {
|
||||
String field=switch(this) {
|
||||
case BRIGHT_QUALIFIED -> "bright_qualified_weight";
|
||||
case BLACK_TOTAL -> "black_total_weight";
|
||||
case BRIGHT_TOTAL -> "bright_total_weight";
|
||||
case PRODUCT -> "*";
|
||||
};
|
||||
return fields.contains("*")||fields.contains(field)?this:PRODUCT;
|
||||
}
|
||||
private BigDecimal weight(MetricsService.MetricRow row) {
|
||||
return switch(this) {
|
||||
case BRIGHT_QUALIFIED -> row.bright().qualifiedWeight();
|
||||
case BLACK_TOTAL -> row.black().totalWeight();
|
||||
case BRIGHT_TOTAL -> row.bright().totalWeight();
|
||||
case PRODUCT -> null;
|
||||
};
|
||||
}
|
||||
public Comparator<MetricsService.MetricRow> comparator() {
|
||||
Comparator<MetricsService.MetricRow> keys=Comparator.comparing(row->row.key()==null?"":row.key());
|
||||
return this==PRODUCT?keys:Comparator.comparing(this::weight,Comparator.nullsLast(Comparator.reverseOrder())).thenComparing(keys);
|
||||
}
|
||||
}
|
||||
@@ -84,4 +84,19 @@ class MetricsResponseAccessTest {
|
||||
assertTrue(aggregateOnly.trend().isEmpty());
|
||||
verify(governance, never()).requireBoundAction("QUERY", "query.detail");
|
||||
}
|
||||
@Test void productPageRechecksCurrentDetailAndFieldPolicyAndHidesCountsWithoutDetailAccess() {
|
||||
var page=new ProductPage(Map.of("id","r"),Map.of(),List.of(row),120,2,15,"productAsc");
|
||||
var access=new MetricsResponseAccess(governance);
|
||||
when(governance.policy(actor.getId().toString())).thenReturn(Map.of("detailAllowed",true));
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("black_total_weight"));
|
||||
var visible=access.visibleProductPage(page,actor);
|
||||
assertEquals(120,visible.total());assertEquals(2,visible.page());
|
||||
assertEquals(stage.totalWeight(),visible.rows().getFirst().black().totalWeight());
|
||||
assertNull(visible.rows().getFirst().bright().totalWeight());
|
||||
verify(governance).requireUserActivePolicy(actor);
|
||||
when(governance.policy(actor.getId().toString())).thenReturn(Map.of("detailAllowed",false));
|
||||
var revoked=access.visibleProductPage(page,actor);
|
||||
assertTrue(revoked.rows().isEmpty());assertEquals(0,revoked.total());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+55
@@ -93,4 +93,59 @@ class SalesOrderMetricRollupsMySqlTest {
|
||||
rollups.refreshCurrentBuild(build);assertFalse(rollups.ready(release,query.extraFilters(),true,Set.of("*")));
|
||||
assertEquals(0,jdbc.queryForObject("SELECT COUNT(*) FROM sales_order_metric_rollup_build WHERE version_id=?",Integer.class,version));
|
||||
}
|
||||
@Test void databaseProductPagesMatchFormalRowsAcrossRawRollupsFiltersAndStableNullTies() {
|
||||
// Include blanks, a missing stage, equal weights and UTF-16 ordering edges.
|
||||
int n=0;
|
||||
for(String product:List.of("", " ", "edge-😀", "edge-\uE000", "edge-é", "edge-e\u0301", "edge-A", "edge-a")) {
|
||||
jdbc.update("INSERT INTO fact_bright(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',?,100,1,999,JSON_OBJECT('销售订单号','order-0'),JSON_OBJECT('bright_processing_scrap',1))",build,product);
|
||||
n++;
|
||||
}
|
||||
jdbc.update("INSERT INTO fact_black(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',NULL,NULL,0,0,JSON_OBJECT(),JSON_OBJECT())",build);
|
||||
// Stage-specific representatives remain distinct exactly as in the existing formal table.
|
||||
jdbc.update("INSERT INTO fact_black(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',?,50,1,51,JSON_OBJECT(),JSON_OBJECT())",build,"edge-e\u0301");
|
||||
var base=query("product",true,Map.of());
|
||||
var scopes=List.of(base,base.withStage("bright"),base.withStage("black"),base.withMonthRange(1,1),base.withProductSearch("edge"),
|
||||
query("product",false,Map.of(SalesOrderTypes.KEY,List.of("标准销售订单"))),
|
||||
query("product",true,Map.of(SalesOrderTypes.EXCLUDE_KEY,List.of("样品销售订单"))));
|
||||
// No rollups -> facts; regular rollups; then sales-order rollups.
|
||||
for(int path=0;path<3;path++) {
|
||||
if(path==1) for(String stage:List.of("black","bright")) {
|
||||
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT build_id,?,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,1,0,0,0 FROM fact_"+stage,stage);
|
||||
}
|
||||
if(path==1)jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) VALUES(?,?)",build,22+n+2);
|
||||
if(path==2)rollups.prepareBuild(version,build);
|
||||
for(var scope:scopes) for(var order:ProductTableOrder.values()) {
|
||||
var expected=metrics.calculateTable(scope,actor).rows().stream().sorted(order.comparator()).toList();
|
||||
var actual=new ArrayList<MetricsService.MetricRow>();
|
||||
int pages=(expected.size()+2)/3;
|
||||
for(int page=1;page<=pages+1;page++) {
|
||||
var result=metrics.calculateProductPage(scope,actor,page,3,order.wire());
|
||||
assertEquals(expected.size(),result.total(),"total path "+path);
|
||||
assertTrue(result.rows().size()<=3);
|
||||
actual.addAll(result.rows());
|
||||
}
|
||||
assertEquals(normalized(expected),normalized(actual),"path="+path+" order="+order+" scope="+scope);
|
||||
}
|
||||
}
|
||||
when(actor.getDepartments()).thenReturn(List.of("A"));
|
||||
var authorized=metrics.calculateTable(base,actor).rows().stream().sorted(ProductTableOrder.PRODUCT.comparator()).toList();
|
||||
assertEquals(normalized(authorized),normalized(metrics.calculateProductPage(base,actor,1,100,"productAsc").rows()));
|
||||
}
|
||||
|
||||
@Test void pageSelectionDoesNotMultiplyUnicodeEquivalentBucketsAndValidatesLimits() {
|
||||
var base=query("product",true,Map.of());
|
||||
var expected=metrics.calculateTable(base,actor).rows().getFirst();
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("black_rate"));
|
||||
assertEquals(normalized(metrics.calculateTable(base,actor).rows().stream().sorted(ProductTableOrder.PRODUCT.comparator()).toList()),
|
||||
normalized(metrics.calculateProductPage(base,actor,1,100,"brightQualifiedDesc").rows()),"Hidden weights must not leak through ordering");
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
|
||||
var limited=metrics.calculatePairedProductSnapshot(base,actor,List.of(expected.key(),expected.key()),()->"source");
|
||||
assertEquals(normalized(List.of(expected)),normalized(limited.metrics().rows()));
|
||||
assertTrue(metrics.calculatePairedProductSnapshot(base,actor,List.of(),()->"source").metrics().rows().isEmpty());
|
||||
for(int page:new int[]{0,-1,1_000_001})assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,page,15,"productAsc"));
|
||||
for(int size:new int[]{0,101})assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,1,size,"productAsc"));
|
||||
assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,1,15,"product; DROP TABLE"));
|
||||
assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(query("overall",true,Map.of()),actor,1,15,"productAsc"));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user