perf: preaggregate compact defect category statistics for product drilldown

This commit is contained in:
2026-11-13 14:46:55 +08:00
parent 92b1c88b31
commit a4eabeb7f2
8 changed files with 212 additions and 5 deletions
@@ -0,0 +1,107 @@
package cn.yunsheng.quality.analysis;
import static cn.yunsheng.quality.analysis.AnalysisFineDefectsService.*;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowCallbackHandler;
import tools.jackson.databind.json.JsonMapper;
/** Category payloads retain the exact counters and original authorization dimensions. */
final class FineDefectCompactRollups {
private static final int FORMAT = 1;
private final JdbcTemplate jdbc;
private final JsonMapper json;
FineDefectCompactRollups(JdbcTemplate jdbc, JsonMapper json) { this.jdbc=jdbc; this.json=json; }
void invalidate(String build) {
jdbc.update("DELETE FROM fine_defect_compact_build WHERE build_id=?",build);
}
void refreshBuild(String build) {
invalidate(build);
jdbc.update("DELETE FROM fine_defect_compact WHERE build_id=?",build);
Integer ready=jdbc.queryForObject("SELECT COUNT(*) FROM fine_defect_rollup_build WHERE build_id=?",Integer.class,build);
if(ready==null||ready!=1)return;
insert(build,"",List.of());
jdbc.update("INSERT INTO fine_defect_compact_build(build_id,format_version) VALUES(?,?)",build,FORMAT);
}
void refreshGroups(String build,String stage,String predicate,List<Object> scopeArgs) {
List<Object> args=new ArrayList<>(List.of(build,stage));args.addAll(scopeArgs);
jdbc.update("DELETE f FROM fine_defect_compact f WHERE f.build_id=? AND f.stage=? AND ("+predicate+")",args.toArray());
List<Object> filters=new ArrayList<>(List.of(stage));filters.addAll(scopeArgs);
insert(build," AND f.stage=? AND ("+predicate+")",filters);
// A partial update cannot make a previously incomplete build ready.
}
private void insert(String build,String predicate,List<Object> scopeArgs) {
List<Object> args=new ArrayList<>(List.of(build));args.addAll(scopeArgs);
String category=categoryExpression();
jdbc.update("""
INSERT INTO fine_defect_compact(build_id,stage,month_key,department,group_name,project_group,
product,category,source_row_count,stats,payload_bytes)
SELECT build_id,stage,month_key,department,group_name,project_group,product,category,
source_row_count,stats,OCTET_LENGTH(CAST(stats AS CHAR)) FROM (
SELECT f.build_id,f.stage,ANY_VALUE(f.month_key) AS month_key,
ANY_VALUE(f.department) AS department,ANY_VALUE(f.group_name) AS group_name,
ANY_VALUE(f.project_group) AS project_group,ANY_VALUE(f.product) AS product,
"""+category+" AS category,"+"""
MAX(f.source_row_count) AS source_row_count,
JSON_ARRAYAGG(JSON_ARRAY(f.stat_kind,f.field_key,CAST(f.weight_sum AS CHAR),
f.present_rows,f.missing_rows,f.invalid_rows,f.negative_rows)) AS stats
FROM fine_defect_rollup f WHERE f.build_id=? AND f.grain='detail'
"""+predicate+"""
GROUP BY f.build_id,f.stage,CAST(f.month_key AS BINARY),CAST(f.department AS BINARY),
CAST(f.group_name AS BINARY),CAST(f.project_group AS BINARY),CAST(f.product AS BINARY),category
) packed WHERE category IS NOT NULL
""",args.toArray());
}
private static String categoryExpression() {
StringBuilder source=new StringBuilder("CASE f.field_key"),formal=new StringBuilder("CASE f.field_key");
for(String stage:List.of("black","bright")) {
for(var field:fields(stage))source.append(" WHEN ").append(literal(field.key())).append(" THEN ").append(literal(field.category()));
for(var category:categories(stage))if(category.available())formal.append(" WHEN ").append(literal(category.componentCode())).append(" THEN ").append(literal(category.name()));
}
return "CASE f.stat_kind WHEN 'source' THEN "+source+" END WHEN 'formal' THEN "+formal+" END WHEN 'expected' THEN f.field_key END";
}
private static String literal(String value) { return "'"+value.replace("'","''")+"'"; }
boolean read(Map<String,Aggregator> groups,Supplier<Aggregator> create,Request request,Set<String> allowed,
String predicate,List<Object> scopeArgs,String bucket) {
Integer missing=jdbc.queryForObject("""
SELECT COUNT(*) FROM release_member rm
LEFT JOIN fine_defect_compact_build cb ON cb.build_id=rm.transform_build_id AND cb.format_version=?
WHERE rm.release_id=? AND cb.build_id IS NULL
""",Integer.class,FORMAT,scopeArgs.getFirst());
if(missing==null||missing>0)return false;
List<Object> args=new ArrayList<>(scopeArgs);args.add(request.stage());args.add(request.category());
String from=" FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id AND b.source_revision_id=rm.source_revision_id"
+" JOIN source_revision sr ON sr.id=rm.source_revision_id JOIN fine_defect_compact f ON f.build_id=rm.transform_build_id "
+predicate+" AND f.stage=? AND f.category=?";
// Bound each JSON result before aggregation. Oversized buckets use the existing tall-stat query.
Integer oversized=jdbc.queryForObject("SELECT COUNT(*) FROM (SELECT "+bucket+" AS bucket_key"+from
+" GROUP BY bucket_key HAVING SUM(f.payload_bytes)+4*COUNT(*)>LEAST(4194304,@@max_allowed_packet DIV 4)) oversized",Integer.class,args.toArray());
if(oversized!=null&&oversized>0)return false;
jdbc.query("SELECT "+bucket+" AS bucket_key,JSON_ARRAYAGG(f.stats) AS parts,SUM(f.source_row_count) AS source_row_count"
+from+" GROUP BY bucket_key",(RowCallbackHandler) rs -> {
Aggregator aggregate=groups.computeIfAbsent(rs.getString("bucket_key"),key -> create.get());
long rows=rs.getLong("source_row_count");
// SQL groups with the same collation as formal metrics. Decimal strings avoid float conversion.
for(var part:json.readTree(rs.getString("parts")))for(var stat:part) {
aggregate.mergeRollup(new Aggregator.RollupStat(stat.get(0).asString(),stat.get(1).asString(),
new BigDecimal(stat.get(2).asString()),stat.get(3).longValue(),stat.get(4).longValue(),
stat.get(5).longValue(),stat.get(6).longValue(),rows),allowed);
}
},args.toArray());
return true;
}
}
@@ -25,6 +25,15 @@ public class FineDefectRollupBackfill implements ApplicationRunner {
""", (rs, row) -> rs.getString(1)); """, (rs, row) -> rs.getString(1));
long start = System.nanoTime(); long start = System.nanoTime();
for (String build : missing) rollups.refreshBuild(build); for (String build : missing) rollups.refreshBuild(build);
var compactMissing=jdbc.query("""
SELECT DISTINCT b.id FROM current_release c JOIN release_member rm ON rm.release_id=c.release_id
JOIN transform_build b ON b.id=rm.transform_build_id
JOIN fine_defect_rollup_build rb ON rb.build_id=b.id
LEFT JOIN fine_defect_compact_build cb ON cb.build_id=b.id AND cb.format_version=1
WHERE c.dataset_id='quality' AND b.status='COMPLETED' AND cb.build_id IS NULL ORDER BY b.id
""",(rs,row) -> rs.getString(1));
for(String build:compactMissing)rollups.refreshCompactBuild(build);
if(!compactMissing.isEmpty())LOG.info("Compact fine defect backfill: builds={}, elapsedMs={}",compactMissing.size(),(System.nanoTime()-start)/1_000_000L);
if (!missing.isEmpty()) LOG.info("Fine defect rollup backfill: builds={}, elapsedMs={}", missing.size(), (System.nanoTime() - start) / 1_000_000L); if (!missing.isEmpty()) LOG.info("Fine defect rollup backfill: builds={}, elapsedMs={}", missing.size(), (System.nanoTime() - start) / 1_000_000L);
} }
} }
@@ -23,11 +23,13 @@ import static cn.yunsheng.quality.analysis.AnalysisFineDefectsService.*;
public class FineDefectRollups { public class FineDefectRollups {
private final JdbcTemplate jdbc; private final JdbcTemplate jdbc;
private final JsonMapper json; private final JsonMapper json;
private final FineDefectCompactRollups compact;
public FineDefectRollups(JdbcTemplate jdbc, JsonMapper json) { this.jdbc = jdbc; this.json = json; } public FineDefectRollups(JdbcTemplate jdbc, JsonMapper json) { this.jdbc = jdbc; this.json = json; this.compact = new FineDefectCompactRollups(jdbc,json); }
@Transactional @Transactional
public void refreshBuild(String build) { public void refreshBuild(String build) {
compact.invalidate(build);
jdbc.update("DELETE FROM fine_defect_rollup_build WHERE build_id=?", build); jdbc.update("DELETE FROM fine_defect_rollup_build WHERE build_id=?", build);
jdbc.update("DELETE FROM fine_defect_rollup WHERE build_id=?", build); jdbc.update("DELETE FROM fine_defect_rollup WHERE build_id=?", build);
long rows = 0; long rows = 0;
@@ -39,8 +41,10 @@ public class FineDefectRollups {
} }
refreshOrganizations(build, "", List.of()); refreshOrganizations(build, "", List.of());
jdbc.update("INSERT INTO fine_defect_rollup_build(build_id,source_row_count) VALUES(?,?)", build, rows); jdbc.update("INSERT INTO fine_defect_rollup_build(build_id,source_row_count) VALUES(?,?)", build, rows);
compact.refreshBuild(build);
} }
@Transactional
public void copyBuild(String parent, String child) { public void copyBuild(String parent, String child) {
Integer ready = jdbc.queryForObject("SELECT COUNT(*) FROM fine_defect_rollup_build WHERE build_id=?", Integer.class, parent); Integer ready = jdbc.queryForObject("SELECT COUNT(*) FROM fine_defect_rollup_build WHERE build_id=?", Integer.class, parent);
if (ready == null || ready != 1) { refreshBuild(child); return; } if (ready == null || ready != 1) { refreshBuild(child); return; }
@@ -52,8 +56,10 @@ public class FineDefectRollups {
FROM fine_defect_rollup WHERE build_id=? FROM fine_defect_rollup WHERE build_id=?
""", child, parent); """, child, parent);
jdbc.update("INSERT INTO fine_defect_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM fine_defect_rollup_build WHERE build_id=?", child, parent); jdbc.update("INSERT INTO fine_defect_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM fine_defect_rollup_build WHERE build_id=?", child, parent);
compact.refreshBuild(child);
} }
@Transactional
public void refreshGroups(String build, Set<GroupKey> keys) { public void refreshGroups(String build, Set<GroupKey> keys) {
for (String stage : List.of("black", "bright")) { for (String stage : List.of("black", "bright")) {
var selected = keys.stream().filter(k -> stage.equals(k.stage())).toList(); var selected = keys.stream().filter(k -> stage.equals(k.stage())).toList();
@@ -65,8 +71,9 @@ public class FineDefectRollups {
List<Object> deletedArgs = new ArrayList<>(List.of(build, stage)); deletedArgs.addAll(args); List<Object> deletedArgs = new ArrayList<>(List.of(build, stage)); deletedArgs.addAll(args);
jdbc.update("DELETE f FROM fine_defect_rollup f WHERE f.build_id=? AND f.stage=? AND f.grain='detail' AND (" + predicate + ")", deletedArgs.toArray()); jdbc.update("DELETE f FROM fine_defect_rollup f WHERE f.build_id=? AND f.stage=? AND f.grain='detail' AND (" + predicate + ")", deletedArgs.toArray());
var groups = aggregate(build, stage, " AND (" + predicate + ")", args); var groups = aggregate(build, stage, " AND (" + predicate + ")", args);
if (!representable(groups)) { jdbc.update("DELETE FROM fine_defect_rollup_build WHERE build_id=?", build); return; } if (!representable(groups)) { jdbc.update("DELETE FROM fine_defect_rollup_build WHERE build_id=?", build); compact.invalidate(build); return; }
insert(build, groups); insert(build, groups);
compact.refreshGroups(build,stage,predicate,args);
} }
var organizations = keys.stream().map(k -> new GroupKey(k.stage(), k.month(), k.department(), k.group(), null, null)).distinct().toList(); var organizations = keys.stream().map(k -> new GroupKey(k.stage(), k.month(), k.department(), k.group(), null, null)).distinct().toList();
if (!organizations.isEmpty()) { if (!organizations.isEmpty()) {
@@ -78,6 +85,9 @@ public class FineDefectRollups {
} }
} }
@Transactional
public void refreshCompactBuild(String build) { compact.refreshBuild(build); }
private void refreshOrganizations(String build, String predicate, List<Object> scopeArgs) { private void refreshOrganizations(String build, String predicate, List<Object> scopeArgs) {
List<Object> args = new ArrayList<>(List.of(build)); args.addAll(scopeArgs); List<Object> args = new ArrayList<>(List.of(build)); args.addAll(scopeArgs);
jdbc.update("DELETE f FROM fine_defect_rollup f WHERE f.build_id=? AND f.grain='organization'" + predicate, args.toArray()); jdbc.update("DELETE f FROM fine_defect_rollup f WHERE f.build_id=? AND f.grain='organization'" + predicate, args.toArray());
@@ -130,6 +140,7 @@ public class FineDefectRollups {
String bucket, String dimension) { String bucket, String dimension) {
Integer missing=jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm LEFT JOIN fine_defect_rollup_build rb ON rb.build_id=rm.transform_build_id WHERE rm.release_id=? AND rb.build_id IS NULL",Integer.class,scopeArgs.getFirst()); Integer missing=jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm LEFT JOIN fine_defect_rollup_build rb ON rb.build_id=rm.transform_build_id WHERE rm.release_id=? AND rb.build_id IS NULL",Integer.class,scopeArgs.getFirst());
if(missing==null||missing>0)return false; if(missing==null||missing>0)return false;
if(dimension.equals("product")&&compact.read(groups,create,request,allowed,predicate,scopeArgs,bucket))return true;
boolean organization=!predicate.contains("f.product")&&!predicate.contains("f.project_group")&&!List.of("product","projectGroup").contains(dimension); boolean organization=!predicate.contains("f.product")&&!predicate.contains("f.project_group")&&!List.of("product","projectGroup").contains(dimension);
List<Object> args=new ArrayList<>(scopeArgs);args.add(request.stage());args.add(organization?"organization":"detail"); List<Object> args=new ArrayList<>(scopeArgs);args.add(request.stage());args.add(organization?"organization":"detail");
List<String> selected=new ArrayList<>(); List<String> selected=new ArrayList<>();
@@ -31,12 +31,14 @@ public class MetricsRollupService {
public record GroupKey(String stage,String month,String department,String group,String project,String product) {} public record GroupKey(String stage,String month,String department,String group,String project,String product) {}
/** 归类维护沿用旧汇总;仅重建归属发生变化的分组,重量计算结果不变。 */ /** 归类维护沿用旧汇总;仅重建归属发生变化的分组,重量计算结果不变。 */
@Transactional
public void copyBuild(String parent,String child) { public void copyBuild(String parent,String child) {
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent); jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent);
if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总"); if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总");
fineDefects.copyBuild(parent, child); fineDefects.copyBuild(parent, child);
} }
@Transactional
public void refreshGroups(String build,java.util.Set<GroupKey> keys) { public void refreshGroups(String build,java.util.Set<GroupKey> keys) {
for(String stage:List.of("black","bright")) { for(String stage:List.of("black","bright")) {
var selected=keys.stream().filter(k->stage.equals(k.stage())).toList();if(selected.isEmpty())continue; var selected=keys.stream().filter(k->stage.equals(k.stage())).toList();if(selected.isEmpty())continue;
@@ -0,0 +1,23 @@
-- Query-oriented category payloads over the existing exact source statistics.
CREATE TABLE fine_defect_compact_build (
build_id CHAR(36) COLLATE utf8mb4_0900_ai_ci NOT NULL PRIMARY KEY,
format_version INT NOT NULL,
CONSTRAINT fk_fine_compact_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs;
CREATE TABLE fine_defect_compact (
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
build_id CHAR(36) COLLATE utf8mb4_0900_ai_ci NOT NULL,
stage VARCHAR(16) NOT NULL,
month_key CHAR(7) NOT NULL,
department VARCHAR(190) NULL,
group_name VARCHAR(190) NULL,
project_group VARCHAR(190) NULL,
product VARCHAR(190) NULL,
category VARCHAR(32) NOT NULL,
source_row_count BIGINT NOT NULL,
stats JSON NOT NULL,
payload_bytes INT NOT NULL,
KEY ix_fine_compact_scope (build_id,stage,category,month_key),
CONSTRAINT fk_fine_compact_fact_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_as_cs;
@@ -43,7 +43,7 @@ class FineDefectRollupsMySqlTest {
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))"); jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))");
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))"); jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))"); for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))");
for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql")) { for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql", "V42__fine_defect_compact_rollups.sql")) {
try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) { try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) {
for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql); for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql);
} }
@@ -111,6 +111,49 @@ class FineDefectRollupsMySqlTest {
assertEquals(canonical(source.grouped(input,actor,"period","year")),canonical(fast.grouped(input,actor,"period","year"))); assertEquals(canonical(source.grouped(input,actor,"period","year")),canonical(fast.grouped(input,actor,"period","year")));
} }
@Test void productDrilldownUsesCompactStatisticsWithoutChangingScopedOrRestrictedResults() {
var input=request("black",Map.of("category","责废"));
var expected=source.grouped(input,actor,"product","month");clearInvocations(jdbc);
assertEquals(canonical(expected),canonical(fast.grouped(input,actor,"product","month")));
var queries=org.mockito.ArgumentCaptor.forClass(String.class);
verify(jdbc,atLeastOnce()).query(queries.capture(),any(RowCallbackHandler.class),any(Object[].class));
assertTrue(queries.getAllValues().stream().anyMatch(sql -> sql.contains("JOIN fine_defect_compact f")));
for(var scope:List.<Map<String,Object>>of(Map.of("department","A"),Map.of("group","G"),Map.of("includeUnassigned",false),Map.of("productSearch","p0"))) {
var scoped=request("black",scope);scoped.put("category","责废");
assertEquals(canonical(source.grouped(scoped,actor,"product","month")),canonical(fast.grouped(scoped,actor,"product","month")));
}
var viewer=new AppUserPrincipal(UUID.randomUUID(),"scoped-viewer","","Viewer",true,false,List.of("USER"),List.of("A"),List.of("G"),List.of());
when(governance.allowedFields(viewer)).thenReturn(Set.of("责废尺寸重量"));
assertEquals(canonical(source.grouped(input,viewer,"product","month")),canonical(fast.grouped(input,viewer,"product","month")));
}
@Test void compactBackfillPreservesUnicodeEquivalentDimensionGroupsAndExactRowCounts() {
seed("black","2026-01","A","G","Café",Map.of("责废尺寸重量","0.100000000000000001"),Map.of("black_responsibility_scrap",1));
seed("black","2026-01","A","G","Cafe\u0301",Map.of("责废尺寸重量","0.200000000000000002"),Map.of("black_responsibility_scrap",1));
metrics.refreshBuild(build);
jdbc.update("DELETE FROM fine_defect_compact_build WHERE build_id=?",build);
jdbc.update("DELETE FROM fine_defect_compact WHERE build_id=?",build);
var input=request("black",Map.of("category","责废","productSearch","caf"));
var expected=fast.grouped(input,actor,"product","month"); // Existing SQL-collation tall path.
rollups.refreshCompactBuild(build);
var actual=fast.grouped(input,actor,"product","month");
var groups=(Map<?,?>)actual.get("groups");assertEquals(1,groups.size());
// SQL may choose either equivalent spelling as the group label.
assertEquals(canonical(new ArrayList<>(((Map<?,?>)expected.get("groups")).values())),canonical(new ArrayList<>(groups.values())));
assertEquals(2L,((Map<?,?>)groups.values().iterator().next()).get("sourceRows"));
}
@Test void missingVersionedCompactOrOversizedBucketFallsBackToExactTallStatistics() {
var input=request("black",Map.of("category","责废"));
var expected=source.grouped(input,actor,"product","month");
jdbc.update("UPDATE fine_defect_compact SET payload_bytes=5000000 WHERE build_id=?",build);
assertEquals(canonical(expected),canonical(fast.grouped(input,actor,"product","month")));
jdbc.update("UPDATE fine_defect_compact_build SET format_version=2 WHERE build_id=?",build);
assertEquals(canonical(expected),canonical(fast.grouped(input,actor,"product","month")));
jdbc.update("DELETE FROM fine_defect_compact_build WHERE build_id=?",build);
assertEquals(canonical(expected),canonical(fast.grouped(input,actor,"product","month")));
}
@Test void exactWeightsCountsReconciliationAndAllStandardScopesMatchTheSourceWithoutScanningFacts() { @Test void exactWeightsCountsReconciliationAndAllStandardScopesMatchTheSourceWithoutScanningFacts() {
for (String stage : List.of("black", "bright")) { for (String stage : List.of("black", "bright")) {
for (Map<String,Object> scope : List.<Map<String,Object>>of(Map.of(), Map.of("month", 1), Map.of("quarter", 1), for (Map<String,Object> scope : List.<Map<String,Object>>of(Map.of(), Map.of("month", 1), Map.of("quarter", 1),
@@ -151,6 +194,9 @@ class FineDefectRollupsMySqlTest {
equivalent(request("black", Map.of("group", "NEW")), actor); equivalent(request("black", Map.of("group", "NEW")), actor);
equivalent(request("black", Map.of("group", "G")), actor); equivalent(request("black", Map.of("group", "G")), actor);
equivalent(request("bright", Map.of()), actor); equivalent(request("bright", Map.of()), actor);
var detail=request("black",Map.of("category","责废"));
assertEquals(canonical(source.grouped(detail,actor,"product","month")),canonical(fast.grouped(detail,actor,"product","month")));
assertEquals(1,jdbc.queryForObject("SELECT COUNT(*) FROM fine_defect_compact_build WHERE build_id=? AND format_version=1",Integer.class,child));
} }
@Test void excludingASmallTypeSubtractsExactCountsAndWeightsAndLargeExclusionsKeepSourceFallback() { @Test void excludingASmallTypeSubtractsExactCountsAndWeightsAndLargeExclusionsKeepSourceFallback() {
@@ -184,7 +230,7 @@ class FineDefectRollupsMySqlTest {
String origin = System.getenv("QUALITY_FINE_ROLLUP_SOURCE_DATABASE"); String origin = System.getenv("QUALITY_FINE_ROLLUP_SOURCE_DATABASE");
assertTrue(origin.matches("[a-zA-Z0-9_]+") && !origin.equals(database)); assertTrue(origin.matches("[a-zA-Z0-9_]+") && !origin.equals(database));
String current = jdbc.queryForObject("SELECT release_id FROM " + origin + ".current_release WHERE dataset_id='quality'", String.class); String current = jdbc.queryForObject("SELECT release_id FROM " + origin + ".current_release WHERE dataset_id='quality'", String.class);
for (String table : List.of("fine_defect_rollup", "fine_defect_rollup_build", "metric_rollup", "metric_rollup_build", "fact_black", "fact_bright", "release_member", "transform_build", "source_revision")) jdbc.update("DELETE FROM " + table); for (String table : List.of("fine_defect_compact", "fine_defect_compact_build", "fine_defect_rollup", "fine_defect_rollup_build", "metric_rollup", "metric_rollup_build", "fact_black", "fact_bright", "release_member", "transform_build", "source_revision")) jdbc.update("DELETE FROM " + table);
jdbc.update("INSERT INTO transform_build SELECT b.id,b.source_revision_id FROM " + origin + ".transform_build b JOIN " + origin + ".release_member rm ON rm.transform_build_id=b.id WHERE rm.release_id=?", current); jdbc.update("INSERT INTO transform_build SELECT b.id,b.source_revision_id FROM " + origin + ".transform_build b JOIN " + origin + ".release_member rm ON rm.transform_build_id=b.id WHERE rm.release_id=?", current);
jdbc.update("INSERT INTO source_revision SELECT DISTINCT source_revision_id FROM transform_build"); jdbc.update("INSERT INTO source_revision SELECT DISTINCT source_revision_id FROM transform_build");
jdbc.update("INSERT INTO release_member SELECT ?,source_revision_id,transform_build_id FROM " + origin + ".release_member WHERE release_id=?", release, current); jdbc.update("INSERT INTO release_member SELECT ?,source_revision_id,transform_build_id FROM " + origin + ".release_member WHERE release_id=?", release, current);
+1 -1
View File
@@ -24,7 +24,7 @@ from pymysql.cursors import DictCursor
LABEL = 'com.lyunlab.quality-center-retention' LABEL = 'com.lyunlab.quality-center-retention'
SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService' SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED') TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED')
BUILD_CHILDREN = ('defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup', BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup',
'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build', 'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build',
'fact_black', 'fact_bright') 'fact_black', 'fact_bright')
OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion', OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion',
+9
View File
@@ -3,6 +3,15 @@
核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。 核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。
## 2026-11-13:细分表格预汇总优化
定位:年度全产品加工废细项需对约31万条细项统计重新分组,源查询约10秒,线上完整接口约11.76秒。继承柏拉图预汇总方式,新增按构建、阶段、月份、完整组织维度、产品和分类保存的紧凑统计;重量用十进制字符串,保留缺失/非法/负值及行数。查询继续使用原授权条件、正式版本绑定和正式分母,SQL保留原排序规则的产品桶语义。
生命周期:V42只新增两张派生汇总表;构建、复制与局部更新同事务维护。启动时仅补当前正式版本,可从既有统计生成,不重导原表。未完成、版本不符、扩展筛选或大桶超过预算时沿用原查询。保留清理识别新依赖;本轮不解除旧自动维护兼容保护。
验证:974项后端构建测试(812执行通过、162按条件跳过),专项隔离库/控制器/快照26项(25通过、1跳过),维护23项通过。隔离副本同范围核心SQL从9978ms降至1471ms,全部金额与计数一致,最大单桶13.9KB,自有库已清理。只读核对当前版本汇总,不等于私有原表全量回归。待发布后记录实际接口耗时。
## 2026-11-13:钻取路径移至趋势标题旁 ## 2026-11-13:钻取路径移至趋势标题旁
“全部分类 › 当前分类”已移到月/季/年度变化趋势标题旁,筛选项和图例间不再单独占行;产品汇总与黑片/亮片页共用。12项相关前端测试与生产构建通过,已发布2026.11.13.3,功能提交 `79edb3f`。线上确认路径位于标题行,返回功能保持。只更新前端;本地提交,未推送Gitea。截图:`output/20261113-defect-drilldown/live-heading.png`。 “全部分类 › 当前分类”已移到月/季/年度变化趋势标题旁,筛选项和图例间不再单独占行;产品汇总与黑片/亮片页共用。12项相关前端测试与生产构建通过,已发布2026.11.13.3,功能提交 `79edb3f`。线上确认路径位于标题行,返回功能保持。只更新前端;本地提交,未推送Gitea。截图:`output/20261113-defect-drilldown/live-heading.png`。