优化:产品查询数据库分页并修复报告测试契约

This commit is contained in:
2026-11-14 14:34:56 +08:00
parent d948e1e2a4
commit 8f1b790b1d
27 changed files with 576 additions and 76 deletions
@@ -158,6 +158,12 @@ public class AnalysisFineDefectsService {
/** One authorized scan (or rollup query) for every period/object and all visible children. */
@Transactional(readOnly = true, isolation = Isolation.REPEATABLE_READ)
public Map<String, Object> grouped(Map<String, Object> input, AppUserPrincipal actor, String dimension, String granularity) {
return grouped(input,actor,dimension,granularity,null);
}
public Map<String,Object> grouped(Map<String,Object> input,AppUserPrincipal actor,String dimension,String granularity,List<String> productKeys) {
productKeys=cn.yunsheng.quality.metrics.ProductBuckets.parse(productKeys);
if(productKeys!=null&&!dimension.equals("product"))throw bad("产品桶只能用于产品细分表");
Set<String> allowed = governance.allowedFields(actor);
Request request = parse(input, allowed, true);
String bucket = switch (dimension) {
@@ -184,6 +190,7 @@ public class AnalysisFineDefectsService {
if (request.productSearch() != null) {
predicate.append(" AND LOCATE(LOWER(?),LOWER(COALESCE(f.product,'')))>0"); args.add(request.productSearch());
}
cn.yunsheng.quality.metrics.ProductBuckets.append(predicate,args,productKeys);
List<DefectField> fields = applicableFields(request,allowed);
List<Category> categories = categories(request.stage());
Set<String> components = new LinkedHashSet<>();
@@ -32,13 +32,17 @@ public class DefectDrilldownController {
String granularity=String.valueOf(input.getOrDefault("trendGranularity","month"));
if(!Set.of("period","overall","department","group","product","projectGroup").contains(dimension))throw bad("细分维度无效");
if(!Set.of("month","quarter","year").contains(granularity))throw bad("趋势粒度无效");
Map<String,Object> scope=new LinkedHashMap<>(input);scope.remove("trendGranularity");scope.remove("view");scope.put("dimension","defect");scope.put("mode","top");scope.put("limit",100);
List<String> productKeys=cn.yunsheng.quality.metrics.ProductBuckets.parse(input.get("productKeys"));
if(productKeys!=null&&!dimension.equals("product"))throw bad("产品桶只能用于产品细分表");
Map<String,Object> scope=new LinkedHashMap<>(input);scope.remove("productKeys");scope.remove("trendGranularity");scope.remove("view");scope.put("dimension","defect");scope.put("mode","top");scope.put("limit",100);
// Reuse the source parser to reject unknown filters and validate the selected category/stage.
AnalysisFineDefectsService.Request request=AnalysisFineDefectsService.parse(scope,governance.allowedFields(actor),true);
if(request.category()==null)throw bad("必须指定不良分类");
var query=new MetricsService.MetricsQuery(number(scope,"year"),number(scope,"month"),number(scope,"quarter"),request.department(),request.group(),request.product(),request.projectGroup(),dimension.equals("period")?"overall":dimension,request.includeUnassigned(),request.releaseId(),granularity,AnalysisExplorationTools.exclusions(scope.get("exclusions")))
.withProductSearch(request.productSearch()).withExtraFilters(scope.get("extraFilters")).withColumnFilters(scope.get("columnFilters")).withMonthRange(number(scope,"monthStart"),number(scope,"monthEnd")).withStage(request.stage());
var paired=metrics.calculatePairedSnapshot(query,actor,dimension.equals("period"),() -> fine.grouped(scope,actor,dimension,granularity));
var paired=productKeys==null
? metrics.calculatePairedSnapshot(query,actor,dimension.equals("period"),() -> fine.grouped(scope,actor,dimension,granularity))
: metrics.calculatePairedProductSnapshot(query,actor,productKeys,() -> fine.grouped(scope,actor,dimension,granularity,productKeys));
// Refresh the visible fields after the snapshot; a concurrent policy revocation cannot release old fields.
governance.requireDetail(actor);governance.requireBoundAction("QUERY","query.detail");
Set<String> currentAllowed=governance.allowedFields(actor);
@@ -109,6 +109,40 @@ public class MetricsController {
return responseAccess.visibleTrend(metrics.calculateTrend(query, actor), actor);
}
/** 产品分页接口:范围、授权和正式规则与主查询一致。 */
@GetMapping("/products")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN', 'DATA_ADMIN', 'USER')")
public MetricsService.ProductPage productsHttp(@RequestParam(required = false) Integer year,
@RequestParam(required = false) Integer month,
@RequestParam(required = false) Integer monthStart,
@RequestParam(required = false) Integer monthEnd,
@RequestParam(required = false) String quarter,
@RequestParam(required = false) String department,
@RequestParam(required = false) String group,
@RequestParam(required = false) String product,
@RequestParam(required = false) String productSearch,
@RequestParam(required = false) String projectGroup,
@RequestParam(defaultValue = "product") String view,
@RequestParam(defaultValue = "true") boolean includeUnassigned,
@RequestParam(required = false) String releaseId,
@RequestParam(defaultValue = "month") String trendGranularity,
@RequestParam(required = false) String extraFilters,
@RequestParam(required = false) String exclusions,
@RequestParam(required = false) String columnFilters,
@RequestParam(required = false) String stage,
@RequestParam(defaultValue="1") int page,
@RequestParam(defaultValue="15") int pageSize,
@RequestParam(defaultValue="brightQualifiedDesc") String order,
@AuthenticationPrincipal AppUserPrincipal actor) {
Integer quarterNumber = parseQuarter(quarter);
MetricsService.MetricsQuery query = new MetricsService.MetricsQuery(year, month, quarterNumber, department,
group, product, projectGroup, view, includeUnassigned, releaseId, trendGranularity,
parseExclusions(exclusions));
query = query.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters)
.withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage);
return responseAccess.visibleProductPage(metrics.calculateProductPage(query,actor,page,pageSize,order),actor);
}
/** Existing direct callers retain the pre-range controller signature. */
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter,
String department, String group, String product, String productSearch,
@@ -142,15 +176,16 @@ public class MetricsController {
product, projectGroup, view, includeUnassigned, releaseId, trendGranularity, parseExclusions(exclusions))
.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters).withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage);
MetricsService.MetricSnapshot visible = responseAccess.visible(tableOnly ? metrics.calculateTable(query,actor) : metrics.calculate(query,actor), actor);
if (rowLimit == null) return visible;
if (rowLimit < 1 || rowLimit > 500 || rowOffset < 0) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
if (rowOrder != null && !rowOrder.equals("scrapDesc")) throw new IllegalArgumentException("rowOrder 无效");
if (rowLimit == null && rowOrder == null) return visible;
if (rowLimit != null && (rowLimit < 1 || rowLimit > 500 || rowOffset < 0)) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
ProductTableOrder productOrder=rowOrder==null||rowOrder.equals("scrapDesc")?null:ProductTableOrder.parse(rowOrder);
List<MetricsService.MetricRow> ordered = new ArrayList<>(visible.rows());
if ("scrapDesc".equals(rowOrder)) ordered.sort(Comparator
.comparing(this::totalScrap).reversed()
.thenComparing(row -> row.key() == null ? "" : row.key()));
int start = Math.min(rowOffset, ordered.size());
int end = (int) Math.min((long) start + rowLimit, ordered.size());
if(productOrder!=null)ordered.sort(productOrder.comparator());
int start = rowLimit==null?0:Math.min(rowOffset, ordered.size());
int end = rowLimit==null?ordered.size():(int) Math.min((long) start + rowLimit, ordered.size());
Map<String, Object> context = new LinkedHashMap<>(visible.context());
context.put("rowCount", ordered.size());
return new MetricsService.MetricSnapshot(visible.release(), context, visible.summary(), ordered.subList(start, end),
@@ -31,6 +31,17 @@ public class MetricsResponseAccess {
return new MetricSnapshot(filtered.release(), filtered.context(), hideDefects(filtered.summary()), List.of(), List.of(), defects, filtered.coverage());
}
/** 分页结果返回前重新核验当前用户、字段和明细服务绑定。 */
public MetricsService.ProductPage visibleProductPage(MetricsService.ProductPage page,AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);
if(!Boolean.TRUE.equals(governance.policy(actor.getId().toString()).get("detailAllowed")))
return new MetricsService.ProductPage(page.release(),page.context(),List.of(),0,page.page(),page.pageSize(),page.order());
governance.requireBoundAction("QUERY","query.detail");
Set<String> allowed=governance.allowedFields(actor);
var rows=allowed.contains("*")?page.rows():page.rows().stream().map(row->restrictDefects(row,allowed)).toList();
return new MetricsService.ProductPage(page.release(),page.context(),rows,page.total(),page.page(),page.pageSize(),page.order());
}
/** 趋势查询沿用明细授权和指标字段可见性边界。 */
public MetricTrendSnapshot visibleTrend(MetricTrendSnapshot snapshot, AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);
@@ -87,6 +87,54 @@ public class MetricsService {
return withConsistentSnapshot(() -> calculateForReleaseSnapshot(query, actor, release, tableOnly));
}
public record ProductPage(Map<String,Object> release,Map<String,Object> context,List<MetricRow> rows,long total,int page,int pageSize,String order) {}
/** 数据库选出当前页的产品桶,再按正式规则计算这些桶的完整分量。 */
public ProductPage calculateProductPage(MetricsQuery query,AppUserPrincipal actor,int page,int pageSize,String orderName) {
if(actor==null)throw new ResponseStatusException(HttpStatus.FORBIDDEN,"指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor);governance.requireBoundAction("QUERY","query.aggregate");validate(query);
if(!"product".equals(query.view()))throw new IllegalArgumentException("产品分页只支持product视图");
if(page<1||page>1_000_000||pageSize<1||pageSize>100)throw new IllegalArgumentException("页码或每页数量无效");
ProductTableOrder requestedOrder=ProductTableOrder.parse(orderName);
ProductTableOrder order=requestedOrder.visibleTo(governance.allowedFields(actor));
String releaseId=query.releaseId()==null||query.releaseId().isBlank()?currentReleaseId():query.releaseId();
if(releaseId==null){var empty=emptySnapshot(query);return new ProductPage(empty.release(),empty.context(),List.of(),0,page,pageSize,requestedOrder.wire());}
ReleaseInfo release=release(releaseId,actor);
return withConsistentSnapshot(()->{
boolean unrolled=hasUnrolledBuilds(releaseId),sales=salesOrderRollupReady(releaseId,query,actor);
boolean source=!query.extraFilters().isEmpty()||!query.columnFilters().isEmpty();
var branches=new ArrayList<String>();var args=new ArrayList<Object>();
for(String stage:List.of("black","bright")) {
if(query.stage()!=null&&!query.stage().equals(stage))continue;
if(!source||sales)addProductRankBranch(branches,args,stageScope(stage,releaseId,query,actor,false,sales));
if(!sales&&(source||unrolled))addProductRankBranch(branches,args,stageScope(stage,releaseId,query,actor,true,false));
}
String candidates="WITH candidates AS ("+String.join(" UNION ALL ",branches)+") ";
String ranked="SELECT ANY_VALUE(bucket_key) AS bucket_key,"+order.rankExpression()+" AS sort_weight FROM candidates GROUP BY CAST(bucket_key AS BINARY)";
String sql=candidates+", ranked AS ("+ranked+") SELECT bucket_key,COUNT(*) OVER() AS row_count FROM ranked ORDER BY "+order.sqlOrder()+" LIMIT ? OFFSET ?";
var pageArgs=new ArrayList<Object>(args);pageArgs.add(pageSize);pageArgs.add((long)(page-1)*pageSize);
var keys=new ArrayList<String>();long[] count={0};
jdbc.query(sql,(RowCallbackHandler)rs->{keys.add(rs.getString("bucket_key"));count[0]=rs.getLong("row_count");},pageArgs.toArray());
if(keys.isEmpty()) {
Long total=jdbc.queryForObject(candidates+"SELECT COUNT(*) FROM (SELECT CAST(bucket_key AS BINARY) FROM candidates GROUP BY CAST(bucket_key AS BINARY)) all_products",Long.class,args.toArray());
count[0]=total==null?0:total;
}
var snapshot=calculateForReleaseSnapshot(query,actor,release,true,keys);
var byKey=new LinkedHashMap<String,MetricRow>();snapshot.rows().forEach(row->byKey.put(row.key(),row));
var rows=keys.stream().map(byKey::get).filter(Objects::nonNull).toList();
return new ProductPage(snapshot.release(),snapshot.context(),rows,count[0],page,pageSize,requestedOrder.wire());
});
}
private void addProductRankBranch(List<String> branches,List<Object> args,StageScope scope) {
boolean black=scope.stage().equals("black");
String nil="CAST(NULL AS DECIMAL(38,8))";
branches.add("SELECT "+ProductBuckets.EXPRESSION+" AS bucket_key,"+(black?"SUM(f.total_weight)":nil)+" AS black_total,"+
(black?nil:"SUM(f.qualified_weight)")+" AS bright_good,"+(black?nil:"SUM(f.scrap_weight)")+" AS bright_scrap "+
"FROM release_member rm JOIN "+scope.table()+" f"+scope.hint()+" ON f.build_id=rm.transform_build_id"+scope.where()+" GROUP BY bucket_key");
args.addAll(scope.args());
}
/** 只按时间桶生成趋势,不构造总体摘要或产品/分组维度行。 */
public MetricTrendSnapshot calculateTrend(MetricsQuery query, AppUserPrincipal actor) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "指标查询必须绑定当前有效用户");
@@ -105,6 +153,15 @@ public class MetricsService {
/** Authorize before opening the read snapshot, so service-binding locks do not span source scans. */
public <T> PairedSnapshot<T> calculatePairedSnapshot(MetricsQuery query, AppUserPrincipal actor,
boolean trendOnly, Supplier<T> source) {
return calculatePairedSnapshot(query,actor,trendOnly,source,null);
}
public <T> PairedSnapshot<T> calculatePairedProductSnapshot(MetricsQuery query,AppUserPrincipal actor,List<String> productKeys,Supplier<T> source) {
if(!"product".equals(query.view()))throw new IllegalArgumentException("产品桶范围无效");
return calculatePairedSnapshot(query,actor,false,source,ProductBuckets.parse(productKeys));
}
private <T> PairedSnapshot<T> calculatePairedSnapshot(MetricsQuery query,AppUserPrincipal actor,boolean trendOnly,Supplier<T> source,List<String> productKeys) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN,"指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor);
governance.requireBoundAction("QUERY","query.aggregate");
@@ -114,7 +171,7 @@ public class MetricsService {
ReleaseInfo release=release(releaseId,actor);
return withConsistentSnapshot(() -> trendOnly
? new PairedSnapshot<T>(null,calculateTrendForReleaseSnapshot(query,actor,release),source.get())
: new PairedSnapshot<T>(calculateForReleaseSnapshot(query,actor,release),null,source.get()));
: new PairedSnapshot<T>(calculateForReleaseSnapshot(query,actor,release,productKeys!=null,productKeys),null,source.get()));
}
private MetricTrendSnapshot calculateTrendForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor,
@@ -257,9 +314,13 @@ public class MetricsService {
}
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release, boolean tableOnly) {
return calculateForReleaseSnapshot(query,actor,release,tableOnly,null);
}
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release, boolean tableOnly,List<String> productKeys) {
String releaseId = release.id();
boolean hasUnrolledBuilds = hasUnrolledBuilds(releaseId);
AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds, null, !tableOnly);
AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds, null, !tableOnly,productKeys);
String ruleVersion=jdbc.queryForObject("SELECT rule_version FROM data_release WHERE id=?",String.class,releaseId);
ConfigurableRuleEngine rules=rulesForVersion(ruleVersion);
MetricRow summary = rules.applyMetrics(aggregates.summary().toMetricRow("overall", "总体", null, null, null));
@@ -268,7 +329,7 @@ public class MetricsService {
MetricRow adjusted=rules.applyMetrics(new MetricRow(row.key(),row.label(),row.department(),row.group(),row.product(),row.projectGroup(),row.compositeRate(),row.black(),row.bright(),row.coverage(),row.compositeRateWithoutHeadquartersScrap()));
return new TrendMetricRow(row.month(),adjusted.key(),adjusted.label(),adjusted.department(),adjusted.group(),adjusted.product(),adjusted.projectGroup(),adjusted.compositeRate(),adjusted.black(),adjusted.bright(),adjusted.coverage(),adjusted.compositeRateWithoutHeadquartersScrap(),row.period(),row.periodLabel(),row.granularity());
}).toList();
Map<String, Object> coverage = queryCoverage(releaseId, query, actor, hasUnrolledBuilds);
Map<String, Object> coverage = productKeys==null?queryCoverage(releaseId, query, actor, hasUnrolledBuilds):Map.of();
Map<String, Object> context = context(query, query.includeUnassigned());
context.put("ruleVersionId",ruleVersion==null?BusinessRowMapper.RULE_VERSION:ruleVersion);
context.put("calculationRules",rules.calculationRules());context.put("metricFormulaDescriptions",rules.formulaDescriptions());
@@ -376,24 +437,29 @@ public class MetricsService {
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments, boolean includePeriods) {
return queryAggregates(releaseId,query,actor,hasUnrolledBuilds,canonicalGroupDepartments,includePeriods,null);
}
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments, boolean includePeriods,List<String> productKeys) {
AggregateSet result = new AggregateSet();
boolean salesRollup = salesOrderRollupReady(releaseId, query, actor);
boolean hasSourceFilters = !query.extraFilters().isEmpty() || !query.columnFilters().isEmpty();
if (!hasSourceFilters || salesRollup) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods);
aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods,productKeys);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods,productKeys);
}
if (!salesRollup && (hasUnrolledBuilds || hasSourceFilters)) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods);
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods,productKeys);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods,productKeys);
}
return result;
}
private void aggregateStage(AggregateSet result, String stage, String view, String releaseId,
MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup,
Map<String, String> canonicalGroupDepartments, boolean includePeriods) {
if(query.stage()!=null&&!query.stage().equals(stage))return;
Map<String, String> canonicalGroupDepartments, boolean includePeriods,List<String> productKeys) {
if(query.stage()!=null&&!query.stage().equals(stage)||productKeys!=null&&productKeys.isEmpty())return;
String table = factFallback ? (stage.equals("black") ? "fact_black" : "fact_bright") : (salesRollup ? "sales_order_metric_rollup" : "metric_rollup");
String dimensionColumn = view.equals("overall") ? null : switch (view) {
case "department" -> "f.department";
@@ -407,22 +473,11 @@ public class MetricsService {
boolean canonicalGroupRows = "group".equals(view) && canonicalGroupDepartments != null;
String ownerDepartment = canonicalGroupRows ? ", f.department COLLATE utf8mb4_0900_as_cs AS owner_department" : "";
String aggregateKeys = canonicalGroupRows ? "trend_period, bucket_key, owner_department" : "trend_period, bucket_key";
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
if (salesRollup) new SalesOrderMetricRollups(jdbc).append(where, args, query.extraFilters());
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
StageScope scope=stageScope(stage,releaseId,query,actor,factFallback,salesRollup);
List<Object> args=new ArrayList<>(scope.args());
StringBuilder limitedWhere=new StringBuilder(scope.where());
ProductBuckets.append(limitedWhere,args,productKeys);
String where=limitedWhere.toString();
String trendBucket = includePeriods ? trendBucketExpression(query.trendGranularity()) : "'all'";
String sql = """
SELECT %s AS trend_period, %s AS bucket_key%s,
@@ -463,6 +518,29 @@ public class MetricsService {
}, args.toArray());
}
private record StageScope(String stage,String table,String hint,String where,List<Object> args) {}
private StageScope stageScope(String stage,String releaseId,MetricsQuery query,AppUserPrincipal actor,boolean factFallback,boolean salesRollup) {
String table=factFallback?(stage.equals("black")?"fact_black":"fact_bright"):(salesRollup?"sales_order_metric_rollup":"metric_rollup");
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
if (salesRollup) new SalesOrderMetricRollups(jdbc).append(where, args, query.extraFilters());
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
return new StageScope(stage,table,factFilterHint(stage,factFallback,query),where.toString(),List.copyOf(args));
}
private String componentSum(String key) {
String path = "$." + key;
String extracted = "JSON_EXTRACT(f.components, '" + path + "')";
@@ -0,0 +1,27 @@
package cn.yunsheng.quality.metrics;
import java.util.*;
/** 已授权范围内的产品桶限制;空字符串与NULL桶保留各自语义。 */
public final class ProductBuckets {
public static final String EXPRESSION="COALESCE(f.product COLLATE utf8mb4_0900_as_cs, '__UNASSIGNED__') COLLATE utf8mb4_0900_as_cs";
private ProductBuckets() {}
public static List<String> parse(Object input) {
if(input==null)return null;
if(!(input instanceof List<?> values)||values.size()>100)throw new IllegalArgumentException("产品桶必须为不超过100项的列表");
var result=new LinkedHashSet<String>();
for(Object value:values) {
if(!(value instanceof String key)||key.length()>512)throw new IllegalArgumentException("产品桶名称无效");
result.add(key);
}
return List.copyOf(result);
}
public static void append(StringBuilder where,List<Object> args,List<String> keys) {
if(keys==null)return;
if(keys.isEmpty()){where.append(" AND 1=0");return;}
// STRCMP avoids hash-join comparisons dropping canonically equivalent Unicode text.
where.append(" AND (").append(String.join(" OR ",Collections.nCopies(keys.size(),
"STRCMP("+EXPRESSION+",CONVERT(? USING utf8mb4) COLLATE utf8mb4_0900_as_cs)=0"))).append(')');
args.addAll(keys);
}
}
@@ -0,0 +1,47 @@
package cn.yunsheng.quality.metrics;
import java.math.BigDecimal;
import java.util.Comparator;
/** SQL分页与全量导出共用的重量排序约定;比率仍由正式规则计算。 */
public enum ProductTableOrder {
BRIGHT_QUALIFIED("brightQualifiedDesc", "SUM(bright_good)"),
BLACK_TOTAL("blackTotalDesc", "SUM(black_total)"),
BRIGHT_TOTAL("brightTotalDesc", "CASE WHEN SUM(bright_good) IS NULL AND SUM(bright_scrap) IS NULL THEN NULL ELSE COALESCE(SUM(bright_good),0)+COALESCE(SUM(bright_scrap),0) END"),
PRODUCT("productAsc", null);
private final String wire;
private final String expression;
ProductTableOrder(String wire, String expression) { this.wire=wire; this.expression=expression; }
public String wire() { return wire; }
public static ProductTableOrder parse(String value) {
for (var order:values()) if (order.wire.equals(value)) return order;
throw new IllegalArgumentException("产品排序无效");
}
public String rankExpression() { return expression==null?"NULL":expression; }
public String sqlOrder() {
// UTF-16字节序与既有Java对象键次序一致,同重量时固定顺序。
return (expression==null?"":"sort_weight DESC, ")+"CAST(CONVERT(bucket_key USING utf16) AS BINARY) ASC";
}
public ProductTableOrder visibleTo(java.util.Set<String> fields) {
String field=switch(this) {
case BRIGHT_QUALIFIED -> "bright_qualified_weight";
case BLACK_TOTAL -> "black_total_weight";
case BRIGHT_TOTAL -> "bright_total_weight";
case PRODUCT -> "*";
};
return fields.contains("*")||fields.contains(field)?this:PRODUCT;
}
private BigDecimal weight(MetricsService.MetricRow row) {
return switch(this) {
case BRIGHT_QUALIFIED -> row.bright().qualifiedWeight();
case BLACK_TOTAL -> row.black().totalWeight();
case BRIGHT_TOTAL -> row.bright().totalWeight();
case PRODUCT -> null;
};
}
public Comparator<MetricsService.MetricRow> comparator() {
Comparator<MetricsService.MetricRow> keys=Comparator.comparing(row->row.key()==null?"":row.key());
return this==PRODUCT?keys:Comparator.comparing(this::weight,Comparator.nullsLast(Comparator.reverseOrder())).thenComparing(keys);
}
}
@@ -84,4 +84,19 @@ class MetricsResponseAccessTest {
assertTrue(aggregateOnly.trend().isEmpty());
verify(governance, never()).requireBoundAction("QUERY", "query.detail");
}
@Test void productPageRechecksCurrentDetailAndFieldPolicyAndHidesCountsWithoutDetailAccess() {
var page=new ProductPage(Map.of("id","r"),Map.of(),List.of(row),120,2,15,"productAsc");
var access=new MetricsResponseAccess(governance);
when(governance.policy(actor.getId().toString())).thenReturn(Map.of("detailAllowed",true));
when(governance.allowedFields(actor)).thenReturn(Set.of("black_total_weight"));
var visible=access.visibleProductPage(page,actor);
assertEquals(120,visible.total());assertEquals(2,visible.page());
assertEquals(stage.totalWeight(),visible.rows().getFirst().black().totalWeight());
assertNull(visible.rows().getFirst().bright().totalWeight());
verify(governance).requireUserActivePolicy(actor);
when(governance.policy(actor.getId().toString())).thenReturn(Map.of("detailAllowed",false));
var revoked=access.visibleProductPage(page,actor);
assertTrue(revoked.rows().isEmpty());assertEquals(0,revoked.total());
}
}
@@ -93,4 +93,59 @@ class SalesOrderMetricRollupsMySqlTest {
rollups.refreshCurrentBuild(build);assertFalse(rollups.ready(release,query.extraFilters(),true,Set.of("*")));
assertEquals(0,jdbc.queryForObject("SELECT COUNT(*) FROM sales_order_metric_rollup_build WHERE version_id=?",Integer.class,version));
}
@Test void databaseProductPagesMatchFormalRowsAcrossRawRollupsFiltersAndStableNullTies() {
// Include blanks, a missing stage, equal weights and UTF-16 ordering edges.
int n=0;
for(String product:List.of("", " ", "edge-😀", "edge-\uE000", "edge-é", "edge-e\u0301", "edge-A", "edge-a")) {
jdbc.update("INSERT INTO fact_bright(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',?,100,1,999,JSON_OBJECT('销售订单号','order-0'),JSON_OBJECT('bright_processing_scrap',1))",build,product);
n++;
}
jdbc.update("INSERT INTO fact_black(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',NULL,NULL,0,0,JSON_OBJECT(),JSON_OBJECT())",build);
// Stage-specific representatives remain distinct exactly as in the existing formal table.
jdbc.update("INSERT INTO fact_black(build_id,month_key,department,group_name,product,qualified_weight,scrap_weight,total_weight,raw_values,components) VALUES(?,'2026-01','A','g',?,50,1,51,JSON_OBJECT(),JSON_OBJECT())",build,"edge-e\u0301");
var base=query("product",true,Map.of());
var scopes=List.of(base,base.withStage("bright"),base.withStage("black"),base.withMonthRange(1,1),base.withProductSearch("edge"),
query("product",false,Map.of(SalesOrderTypes.KEY,List.of("标准销售订单"))),
query("product",true,Map.of(SalesOrderTypes.EXCLUDE_KEY,List.of("样品销售订单"))));
// No rollups -> facts; regular rollups; then sales-order rollups.
for(int path=0;path<3;path++) {
if(path==1) for(String stage:List.of("black","bright")) {
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT build_id,?,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,1,0,0,0 FROM fact_"+stage,stage);
}
if(path==1)jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) VALUES(?,?)",build,22+n+2);
if(path==2)rollups.prepareBuild(version,build);
for(var scope:scopes) for(var order:ProductTableOrder.values()) {
var expected=metrics.calculateTable(scope,actor).rows().stream().sorted(order.comparator()).toList();
var actual=new ArrayList<MetricsService.MetricRow>();
int pages=(expected.size()+2)/3;
for(int page=1;page<=pages+1;page++) {
var result=metrics.calculateProductPage(scope,actor,page,3,order.wire());
assertEquals(expected.size(),result.total(),"total path "+path);
assertTrue(result.rows().size()<=3);
actual.addAll(result.rows());
}
assertEquals(normalized(expected),normalized(actual),"path="+path+" order="+order+" scope="+scope);
}
}
when(actor.getDepartments()).thenReturn(List.of("A"));
var authorized=metrics.calculateTable(base,actor).rows().stream().sorted(ProductTableOrder.PRODUCT.comparator()).toList();
assertEquals(normalized(authorized),normalized(metrics.calculateProductPage(base,actor,1,100,"productAsc").rows()));
}
@Test void pageSelectionDoesNotMultiplyUnicodeEquivalentBucketsAndValidatesLimits() {
var base=query("product",true,Map.of());
var expected=metrics.calculateTable(base,actor).rows().getFirst();
when(governance.allowedFields(actor)).thenReturn(Set.of("black_rate"));
assertEquals(normalized(metrics.calculateTable(base,actor).rows().stream().sorted(ProductTableOrder.PRODUCT.comparator()).toList()),
normalized(metrics.calculateProductPage(base,actor,1,100,"brightQualifiedDesc").rows()),"Hidden weights must not leak through ordering");
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
var limited=metrics.calculatePairedProductSnapshot(base,actor,List.of(expected.key(),expected.key()),()->"source");
assertEquals(normalized(List.of(expected)),normalized(limited.metrics().rows()));
assertTrue(metrics.calculatePairedProductSnapshot(base,actor,List.of(),()->"source").metrics().rows().isEmpty());
for(int page:new int[]{0,-1,1_000_001})assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,page,15,"productAsc"));
for(int size:new int[]{0,101})assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,1,size,"productAsc"));
assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(base,actor,1,15,"product; DROP TABLE"));
assertThrows(IllegalArgumentException.class,()->metrics.calculateProductPage(query("overall",true,Map.of()),actor,1,15,"productAsc"));
}
}