优化:查询按范围分步加载并精简表格聚合

This commit is contained in:
2026-11-14 12:34:50 +08:00
parent 381901184c
commit 374f08c526
11 changed files with 301 additions and 42 deletions
@@ -70,10 +70,12 @@ public class MetricsController {
@RequestParam(required=false) String exclusions,
@RequestParam(required=false) String columnFilters,
@RequestParam(required=false) String stage,
@RequestParam(defaultValue="full") String part,
@AuthenticationPrincipal AppUserPrincipal actor) {
if (!List.of("full", "table").contains(part)) throw new IllegalArgumentException("part 必须是 full 或 table");
return calculateRangeStage(year, month, monthStart, monthEnd, quarter, department, group, product, productSearch,
projectGroup, view, includeUnassigned, releaseId, trendGranularity, rowLimit, rowOffset, rowOrder,
extraFilters, exclusions, columnFilters, stage, actor);
extraFilters, exclusions, columnFilters, stage, actor, "table".equals(part));
}
/** 轻量趋势接口:查询参数与正式指标接口一致,但只返回版本、上下文、趋势和覆盖信息。 */
@@ -131,10 +133,15 @@ public class MetricsController {
}
private MetricsService.MetricSnapshot calculateRangeStage(Integer year,Integer month,Integer monthStart,Integer monthEnd,String quarter,String department,String group,String product,String productSearch,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Integer rowLimit,int rowOffset,String rowOrder,String extraFilters,String exclusions,String columnFilters,String stage,AppUserPrincipal actor) {
return calculateRangeStage(year,month,monthStart,monthEnd,quarter,department,group,product,productSearch,projectGroup,view,includeUnassigned,releaseId,trendGranularity,rowLimit,rowOffset,rowOrder,extraFilters,exclusions,columnFilters,stage,actor,false);
}
private MetricsService.MetricSnapshot calculateRangeStage(Integer year,Integer month,Integer monthStart,Integer monthEnd,String quarter,String department,String group,String product,String productSearch,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Integer rowLimit,int rowOffset,String rowOrder,String extraFilters,String exclusions,String columnFilters,String stage,AppUserPrincipal actor,boolean tableOnly) {
Integer quarterNumber = parseQuarter(quarter);
MetricsService.MetricSnapshot visible = responseAccess.visible(metrics.calculate(new MetricsService.MetricsQuery(year, month, quarterNumber, department, group,
MetricsService.MetricsQuery query = new MetricsService.MetricsQuery(year, month, quarterNumber, department, group,
product, projectGroup, view, includeUnassigned, releaseId, trendGranularity, parseExclusions(exclusions))
.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters).withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage), actor), actor);
.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters).withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage);
MetricsService.MetricSnapshot visible = responseAccess.visible(tableOnly ? metrics.calculateTable(query,actor) : metrics.calculate(query,actor), actor);
if (rowLimit == null) return visible;
if (rowLimit < 1 || rowLimit > 500 || rowOffset < 0) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
if (rowOrder != null && !rowOrder.equals("scrapDesc")) throw new IllegalArgumentException("rowOrder 无效");
@@ -67,6 +67,15 @@ public class MetricsService {
}
public MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor) {
return calculate(query, actor, false);
}
/** 仅按对象汇总表格,省去时间与对象交叉分组的中间行。 */
public MetricSnapshot calculateTable(MetricsQuery query, AppUserPrincipal actor) {
return calculate(query, actor, true);
}
private MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor, boolean tableOnly) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor);
governance.requireBoundAction("QUERY", "query.aggregate");
@@ -75,7 +84,7 @@ public class MetricsService {
? currentReleaseId() : query.releaseId();
if (releaseId == null) return emptySnapshot(query);
ReleaseInfo release = release(releaseId, actor);
return calculateForRelease(query, actor, release);
return withConsistentSnapshot(() -> calculateForReleaseSnapshot(query, actor, release, tableOnly));
}
/** 只按时间桶生成趋势,不构造总体摘要或产品/分组维度行。 */
@@ -244,9 +253,13 @@ public class MetricsService {
}
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release) {
return calculateForReleaseSnapshot(query, actor, release, false);
}
private MetricSnapshot calculateForReleaseSnapshot(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release, boolean tableOnly) {
String releaseId = release.id();
boolean hasUnrolledBuilds = hasUnrolledBuilds(releaseId);
AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds);
AggregateSet aggregates = queryAggregates(releaseId, query, actor, hasUnrolledBuilds, null, !tableOnly);
String ruleVersion=jdbc.queryForObject("SELECT rule_version FROM data_release WHERE id=?",String.class,releaseId);
ConfigurableRuleEngine rules=rulesForVersion(ruleVersion);
MetricRow summary = rules.applyMetrics(aggregates.summary().toMetricRow("overall", "总体", null, null, null));
@@ -358,23 +371,28 @@ public class MetricsService {
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments) {
return queryAggregates(releaseId, query, actor, hasUnrolledBuilds, canonicalGroupDepartments, true);
}
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments, boolean includePeriods) {
AggregateSet result = new AggregateSet();
boolean salesRollup = salesOrderRollupReady(releaseId, query, actor);
boolean hasSourceFilters = !query.extraFilters().isEmpty() || !query.columnFilters().isEmpty();
if (!hasSourceFilters || salesRollup) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments);
aggregateStage(result, "black", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, salesRollup, canonicalGroupDepartments, includePeriods);
}
if (!salesRollup && (hasUnrolledBuilds || hasSourceFilters)) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments);
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, false, canonicalGroupDepartments, includePeriods);
}
return result;
}
private void aggregateStage(AggregateSet result, String stage, String view, String releaseId,
MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup,
Map<String, String> canonicalGroupDepartments) {
Map<String, String> canonicalGroupDepartments, boolean includePeriods) {
if(query.stage()!=null&&!query.stage().equals(stage))return;
String table = factFallback ? (stage.equals("black") ? "fact_black" : "fact_bright") : (salesRollup ? "sales_order_metric_rollup" : "metric_rollup");
String dimensionColumn = view.equals("overall") ? null : switch (view) {
@@ -405,7 +423,7 @@ public class MetricsService {
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
String trendBucket = trendBucketExpression(query.trendGranularity());
String trendBucket = includePeriods ? trendBucketExpression(query.trendGranularity()) : "'all'";
String sql = """
SELECT %s AS trend_period, %s AS bucket_key%s,
SUM(f.qualified_weight) AS qualified_weight,
@@ -437,7 +455,7 @@ public class MetricsService {
String key = rs.getString("bucket_key");
AggregateStats stats = new AggregateStats().read(rs, stage);
result.summary().merge(stage, stats);
result.periods().computeIfAbsent(period, ignored -> new AggregateStats()).merge(stage, stats);
if (includePeriods) result.periods().computeIfAbsent(period, ignored -> new AggregateStats()).merge(stage, stats);
if (dimensionColumn != null) {
if (!canonicalGroupRows || canonicalGroupOwnerMatches(canonicalGroupDepartments, key, rs.getString("owner_department")))
result.dimensions().computeIfAbsent(key, ignored -> new AggregateStats()).merge(stage, stats);
@@ -51,4 +51,31 @@ class MetricsTrendControllerTest {
assertEquals(Map.of("customerCode", List.of("000123")), value.extraFilters());
assertEquals("src_any", value.columnFilters().getFirst().get("field"));
}
@Test void tableProjectionKeepsSourceScopeAndUsesTheSameResponseAuthorization() {
MetricsService metrics = mock(MetricsService.class);
MetricsResponseAccess access = mock(MetricsResponseAccess.class);
AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "table", "unused", "表格", true, false,
List.of("USER"), List.of("D"), List.of("G"), List.of());
var response = new MetricsService.MetricSnapshot(Map.of(), Map.of(), null, List.of(), List.of(), Map.of(), Map.of());
when(metrics.calculateTable(any(), eq(actor))).thenReturn(response);
when(access.visible(response, actor)).thenReturn(response);
var actual = new MetricsController(metrics, access).calculateHttp(
2026, null, 3, 9, null, "D", "G", "P", "18X15", "PG", "product", false,
"release-a", "quarter", null, 0, null, "{\"customerCode\":[\"000123\"]}",
"{\"product\":[\"P2\"]}",
"[{\"field\":\"src_any\",\"stages\":[\"black\"],\"operator\":\"contains\",\"values\":[\"000123\"]}]",
"black", "table", actor);
assertEquals(response, actual);
var query = ArgumentCaptor.forClass(MetricsService.MetricsQuery.class);
verify(metrics).calculateTable(query.capture(), eq(actor));
verify(access).visible(response, actor);
var value = query.getValue();
assertEquals(3, value.monthStart()); assertEquals(9, value.monthEnd());
assertEquals("black", value.stage()); assertEquals("P", value.product());
assertEquals("D", value.department()); assertEquals("G", value.group());
assertEquals("18X15", value.productSearch()); assertEquals("PG", value.projectGroup());
assertEquals(Map.of("product", List.of("P2")), value.exclusions());
assertEquals(Map.of("customerCode", List.of("000123")), value.extraFilters());
assertEquals("contains", value.columnFilters().getFirst().get("operator"));
}
}
@@ -49,6 +49,29 @@ class MetricsTrendServiceTest {
assertTrue(trendSql.stream().allMatch(sql -> !sql.contains("bucket_key") && !sql.contains("f.product")));
}
@Test void tableProjectionKeepsFormalRowsSummaryAndCoverageWithoutTimeByProductGrouping() {
JdbcTemplate jdbc = jdbc(true, true);
var governance = mock(GovernanceAccessService.class);
MetricsService service = new MetricsService(jdbc, governance);
var query = new MetricsService.MetricsQuery(2026, null, null, null, null, null, null,
"product", true, RELEASE, "month", Map.of()).withMonthRange(2, 5).withStage("bright");
var full = service.calculate(query, actor);
clearInvocations(jdbc, governance);
var table = service.calculateTable(query, actor);
assertEquals(full.summary(), table.summary());
assertEquals(full.rows(), table.rows());
assertEquals(full.defects(), table.defects());
assertEquals(full.coverage(), table.coverage());
assertEquals(full.context(), table.context());
assertTrue(table.trend().isEmpty());
var sql = capturedSql(jdbc).stream().filter(value -> value.contains("AS bucket_key")).toList();
assertEquals(1, sql.size());
assertTrue(sql.getFirst().contains("SELECT 'all' AS trend_period"));
assertTrue(sql.getFirst().contains("f.month_key BETWEEN ? AND ?"));
verify(governance).requireUserActivePolicy(actor);
verify(governance).requireBoundAction("QUERY", "query.aggregate");
}
@Test void rangeStageAndFiltersReachOnlyTheRequestedTrendAndCoverageQueries() {
JdbcTemplate jdbc = jdbc(true, true);
MetricsService service = new MetricsService(jdbc, mock(GovernanceAccessService.class));