From ea84e26b2f3d7983e6675fa70596b56038e6044a Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 16:52:27 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=EF=BC=9A=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E7=AD=9B=E9=80=89=E8=B6=85=E6=97=B6=E4=B8=8E?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E6=89=AB=E6=8F=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../quality/filters/SourceColumnQueries.java | 40 +++++++++---------- .../quality/metrics/MetricsService.java | 33 ++++++++++----- .../analysis/FineDefectRollupsMySqlTest.java | 2 +- .../filters/SourceColumnQueriesMySqlTest.java | 37 ++++++++++++++--- .../filters/SourceColumnQueriesTest.java | 9 ++--- docs/当前进度.md | 6 +++ web/src/api.ts | 5 ++- 7 files changed, 89 insertions(+), 43 deletions(-) diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java index 908634f..2f07d49 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java @@ -60,6 +60,16 @@ public class SourceColumnQueries { } return List.copyOf(result); } + // 只提取一次,并区分 JSON null 与字符串“null”,供筛选、候选值和成员读取共用。 + private static final String SOURCE_TEXT = "JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON)))"; + private static final String SOURCE_VALUE = "NULLIF(TRIM("+SOURCE_TEXT+"), '') COLLATE utf8mb4_0900_as_cs"; + + /** 原始字段筛选按发布构建读取,避免优化器从整年月份范围扫描大 JSON 原表。 */ + public static String factIndexHint(String stage) { + if(!List.of("black","bright").contains(stage))throw bad("数据阶段无效"); + return " FORCE INDEX (uq_fact_"+stage+"_origin)"; + } + private static final List STAGES = List.of("black", "bright"); private static final List OPERATORS = List.of("in", "not_in", "is_empty", "not_empty"); private static final Map CANONICAL_COLUMNS = Map.of( @@ -100,7 +110,7 @@ public class SourceColumnQueries { "f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs"); String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) " + "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id " - + "FROM release_member rm JOIN fact_" + stage + " f2 " + + "FROM release_member rm JOIN fact_" + stage + " f2" + factIndexHint(stage) + " " + "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id" + representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative " + "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id"; @@ -240,10 +250,7 @@ public class SourceColumnQueries { String expression; Object column = ((Map) field.get("columns")).get(stage); if ("source_value".equals(field.get("type"))) { - expression = "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL " - + "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL " - + "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END " - + "COLLATE utf8mb4_0900_as_cs"; + expression = SOURCE_VALUE; } else { String factColumn = CANONICAL_COLUMNS.get(key); if (factColumn == null) throw bad("规范维度无效"); @@ -255,19 +262,19 @@ public class SourceColumnQueries { if (operator.equals("in")) { where.append(expression).append(" IN (") .append(String.join(",", Collections.nCopies(values.size(), "?"))).append(')'); - if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column)); + if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column)); } else { where.append('(').append(expression).append(" IS NULL OR ").append(expression).append(" NOT IN (") .append(String.join(",", Collections.nCopies(values.size(), "?"))).append("))"); if ("source_value".equals(field.get("type"))) { - bindSourcePaths(args, String.valueOf(column)); - bindSourcePaths(args, String.valueOf(column)); + bindSourcePath(args, String.valueOf(column)); + bindSourcePath(args, String.valueOf(column)); } } args.addAll(values); } else { where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL"); - if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column)); + if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column)); } where.append(')'); } @@ -578,12 +585,7 @@ public class SourceColumnQueries { } String path = jsonPath(column); args.add(path); - args.add(path); - args.add(path); - return "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL " - + "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL " - + "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END " - + "COLLATE utf8mb4_0900_as_cs"; + return SOURCE_VALUE; } private static List combine(List first, List second) { @@ -601,9 +603,7 @@ public class SourceColumnQueries { } String path = jsonPath(column); args.add(path); - args.add(path); - return "CASE WHEN JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL " - + "ELSE JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?)) END COLLATE utf8mb4_0900_as_cs"; + return SOURCE_TEXT+" COLLATE utf8mb4_0900_as_cs"; } private static String tupleColumns(int count) { @@ -814,11 +814,9 @@ public class SourceColumnQueries { catch (Exception e) { throw new IllegalArgumentException("来源列名无法编码", e); } } - private void bindSourcePaths(List args, String header) { + private void bindSourcePath(List args, String header) { String path = jsonPath(header); args.add(path); - args.add(path); - args.add(path); } private static ResponseStatusException bad(String message) { diff --git a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java index d474a43..f5102f7 100644 --- a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java +++ b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java @@ -39,6 +39,7 @@ public class MetricsService { private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128; private final JdbcTemplate jdbc; private final GovernanceAccessService governance; + private final SourceColumnQueries sourceColumns; private final TransactionTemplate readSnapshot; private final TransactionTemplate governanceSnapshot; @@ -46,6 +47,7 @@ public class MetricsService { public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance, PlatformTransactionManager transactionManager) { this.jdbc = jdbc; this.governance = governance; + this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); this.readSnapshot = new TransactionTemplate(transactionManager); this.readSnapshot.setIsolationLevel(TransactionDefinition.ISOLATION_REPEATABLE_READ); this.readSnapshot.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); @@ -56,7 +58,9 @@ public class MetricsService { /** Keeps package-level in-memory aggregation tests independent of a database transaction manager. */ public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance) { - this.jdbc = jdbc; this.governance = governance; this.readSnapshot = null; this.governanceSnapshot = null; + this.jdbc = jdbc; this.governance = governance; + this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); + this.readSnapshot = null; this.governanceSnapshot = null; } public MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor) { @@ -158,7 +162,7 @@ public class MetricsService { if (factFallback) { new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor); if (!query.columnFilters().isEmpty()) - new SourceColumnQueries(jdbc, governance, JsonMapper.builder().build()) + sourceColumns .append(where, args, stage, query.columnFilters(), releaseId, actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", @@ -180,7 +184,7 @@ public class MetricsService { %s AS bright_processing_scrap, %s AS bright_diffusion_scrap, %s AS row_count - FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id + FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id %s GROUP BY trend_period ORDER BY trend_period @@ -189,7 +193,7 @@ public class MetricsService { componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"), componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"), componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"), - countExpression, table, where); + countExpression, table, factFilterHint(stage,factFallback,query), where); jdbc.query(sql, (RowCallbackHandler) rs -> { String period = rs.getString("trend_period"); AggregateStats stats = new AggregateStats().read(rs, stage); @@ -393,7 +397,7 @@ public class MetricsService { appendExclusions(where, args, query); if(factFallback) { new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor); - if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor); + if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); if (!effectiveIncludeUnassigned(query)) @@ -413,7 +417,7 @@ public class MetricsService { %s AS bright_processing_scrap, %s AS bright_diffusion_scrap, %s AS row_count - FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id + FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id %s GROUP BY %s ORDER BY %s @@ -423,7 +427,7 @@ public class MetricsService { componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"), componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"), componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"), - factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, where, + factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, factFilterHint(stage,factFallback,query), where, aggregateKeys, aggregateKeys); jdbc.query(sql, (RowCallbackHandler) rs -> { String period = rs.getString("trend_period"); @@ -595,6 +599,15 @@ public class MetricsService { "sourceMonths", sourceMonths); } + private static String factFilterHint(String stage,boolean factFallback,MetricsQuery query) { + if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return ""; + // 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。 + if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null + ||query.department()!=null||query.group()!=null||query.product()!=null||query.projectGroup()!=null + ||query.productSearch()!=null)return ""; + return SourceColumnQueries.factIndexHint(stage); + } + private void coverageStage(Map months, String stage, String releaseId, MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup) { if(query.stage()!=null&&!query.stage().equals(stage))return; @@ -610,7 +623,7 @@ public class MetricsService { appendExclusions(where, args, query); if(factFallback) { new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor); - if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor); + if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); String countExpression = factFallback ? "COUNT(*)" : "SUM(f.source_row_count)"; @@ -625,11 +638,11 @@ public class MetricsService { String sql = """ SELECT f.month_key, %s AS row_count, %s AS excluded_blank_rows, %s AS unmapped_rows, %s AS project_unconfigured_rows - FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id + FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id %s GROUP BY f.month_key ORDER BY f.month_key - """.formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, where); + """.formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, factFilterHint(stage,factFallback,query), where); jdbc.query(sql, (RowCallbackHandler) rs -> { CoverageMonth coverage = months.computeIfAbsent(rs.getString("month_key"), ignored -> new CoverageMonth()); long rowCount = rs.getLong("row_count"); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java index 5725f18..c8df7f4 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java @@ -42,7 +42,7 @@ class FineDefectRollupsMySqlTest { jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)"); jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))"); jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))"); - for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))"); + for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,sheet_name VARCHAR(80),physical_row_no INT,KEY ix_fact_" + stage + "_month_dims(build_id,month_key),UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))"); for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql", "V42__fine_defect_compact_rollups.sql")) { try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) { for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 6e3216d..3457798 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -115,6 +115,7 @@ class SourceColumnQueriesMySqlTest { for(Read read:representatives) { String stage=read.sql().contains("fact_black f2")?"black":"bright"; assertTrue(read.sql().contains("GROUP BY f2.build_id,f2.source_revision_id")); + assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表"); StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"); List args=new java.util.ArrayList<>(); DataScope.append(scope,args,actor,"f2.department COLLATE utf8mb4_0900_as_cs","f2.group_name COLLATE utf8mb4_0900_as_cs");args.add(RELEASE); @@ -157,6 +158,23 @@ class SourceColumnQueriesMySqlTest { assertThrows(ResponseStatusException.class, () -> queries.validate(RELEASE, unauthorized, actor)); } + @Test void sourceNullNormalizationMatchesThePreviousExpressionForEveryOperator() { + String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key"); + for(String raw:List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[]}","{\"委托单\":{\"n\":1}}")) + insert("bright","build-bright",BRIGHT_REVISION,"部门A","组A",raw); + String old="CASE WHEN JSON_EXTRACT(f.raw_values,'$.\"委托单\"') IS NULL OR JSON_TYPE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))='NULL' THEN NULL ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))),'') END COLLATE utf8mb4_0900_as_cs"; + for(String operator:List.of("in","not_in","is_empty","not_empty")) { + Map rule=new java.util.LinkedHashMap<>(Map.of("field",key,"stages",List.of("bright"),"operator",operator)); + List args=new java.util.ArrayList<>();String condition; + if(operator.equals("in")||operator.equals("not_in")) { + rule.put("values",List.of("null","0007"));args.add("null");args.add("0007"); + condition=operator.equals("in")?old+" IN (?,?)":"("+old+" IS NULL OR "+old+" NOT IN (?,?))"; + } else condition=old+(operator.equals("is_empty")?" IS NULL":" IS NOT NULL"); + Long expected=jdbc.queryForObject("SELECT COUNT(*) FROM fact_bright f WHERE f.build_id='build-bright' AND f.department='部门A' AND f.group_name='组A' AND ("+condition+")",Long.class,args.toArray()); + assertEquals(expected,countFiltered("bright",SourceColumnQueries.parse(List.of(rule))),operator); + } + } + @Test void explicitMappingWorksAcrossStagesAndMemberPageCountsMergedTuplesBeforePaging() { List> catalog = queries.catalog(RELEASE, actor); String mappedKey = (String) catalog.stream().filter(item -> item.get("label").equals("跨阶段委托号")) @@ -252,12 +270,19 @@ class SourceColumnQueriesMySqlTest { jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE); jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))"); for(String stage:List.of("black","bright")) { - jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT"); + jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON"); jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=?,scrap_weight=?,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id",stage.equals("black")?90:80,stage.equals("black")?10:20); } jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black"); jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT"); - var service=new cn.yunsheng.quality.metrics.MetricsService(jdbc,governance); + var headerReads=new java.util.concurrent.atomic.AtomicInteger(); + var capturing=new JdbcTemplate(jdbc.getDataSource()) { + @Override public List query(String sql,org.springframework.jdbc.core.RowMapper mapper,Object... args) { + if(sql.contains("JSON_KEYS(f.raw_values)"))headerReads.incrementAndGet(); + return super.query(sql,mapper,args); + } + }; + var service=new cn.yunsheng.quality.metrics.MetricsService(capturing,governance); var releases=mock(cn.yunsheng.quality.release.ReleaseService.class); when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE); var mapper=JsonMapper.builder().build(); @@ -272,6 +297,7 @@ class SourceColumnQueriesMySqlTest { assertEquals(stage.equals("black")?legacy.summary().black():legacy.summary().bright(),stage.equals("black")?selected.summary().black():selected.summary().bright()); assertEquals(0L,selected.summary().coverage().get(stage.equals("black")?"brightRows":"blackRows")); assertEquals(1,selected.trend().size()); + assertEquals(1,service.calculateTrend(base.withStage(stage),actor).trend().size()); var detail=records.recordsFiltered(stage,2026,1,null,"部门A","组A",null,null,null,false,RELEASE,1,50,null,null,null,"asc",null,null,mapper.writeValueAsString(List.of(filter)),actor); assertEquals(1L,((Number)detail.get("total")).longValue()); java.nio.file.Path file=exporter.exportFiltered(2026,1,null,"部门A","组A",null,null,false,RELEASE,actor,Map.of(),null,stage,List.of(filter)); @@ -282,6 +308,7 @@ class SourceColumnQueriesMySqlTest { var other=base.withColumnFilters(List.of(filter)).withStage(stage); assertThrows(ResponseStatusException.class,()->other.withStage(stage.equals("black")?"bright":"black")); } + assertEquals(2,headerReads.get(),"汇总、趋势与覆盖检查应复用同一份授权字段目录"); } @Test void continuousRangeDetailsAndBothExcelSheetsExcludeOutsideMonthsAndUnauthorizedRows() throws Exception { @@ -289,7 +316,7 @@ class SourceColumnQueriesMySqlTest { when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE); var mapper=JsonMapper.builder().build(); for(String stage:List.of("black","bright")) { - jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT"); + jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON"); jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=90,scrap_weight=10,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id"); String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION; for(String month:List.of("2026-03","2026-09","2026-10","2025-03")) { @@ -336,7 +363,7 @@ class SourceColumnQueriesMySqlTest { jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))"); for(String stage:List.of("black","bright")) { jdbc.execute("DELETE FROM fact_"+stage); - jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT"); + jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON"); } jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black"); jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT"); @@ -395,7 +422,7 @@ class SourceColumnQueriesMySqlTest { jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))"); for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7)," - + "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL)"); + + "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL,sheet_name VARCHAR(80),physical_row_no INT,UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))"); jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64) PRIMARY KEY,label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)"); } diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java index 899cb9b..5acd01f 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java @@ -143,9 +143,9 @@ class SourceColumnQueriesTest { queries.append(where, args, "black", filters, "release", actor); - assertTrue(where.toString().contains("JSON_EXTRACT(f.raw_values, ?) IS NULL")); + assertTrue(where.toString().contains("NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON))")); assertTrue(where.toString().contains("COLLATE utf8mb4_0900_as_cs IN (?,?)")); - assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "00007", "x' OR 1=1 --"), args); + assertEquals(List.of("$.\"委托单\"", "00007", "x' OR 1=1 --"), args); assertFalse(where.toString().contains("委托单")); assertFalse(where.toString().contains("x' OR 1=1")); } @@ -160,7 +160,7 @@ class SourceColumnQueriesTest { StringBuilder where = new StringBuilder(); queries.append(where, args, "black", filters, "release", actor); assertTrue(where.toString().contains(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL")); - assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\""), args); + assertEquals(List.of("$.\"委托单\""), args); } List> filters = SourceColumnQueries.parse(List.of(Map.of( "field", SOURCE, "stages", List.of("bright"), "operator", "is_empty"))); @@ -179,8 +179,7 @@ class SourceColumnQueriesTest { assertTrue(where.toString().contains("IS NULL OR")); assertTrue(where.toString().contains(" NOT IN (?)")); - assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", - "$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "000123"), args); + assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "000123"), args); } @Test void rejectsInvalidStageAndMissingFieldsInTheActiveCatalog() { diff --git a/docs/当前进度.md b/docs/当前进度.md index 7185e66..6f2c007 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -3,6 +3,12 @@ 核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。 +## 2026-11-13:来源字段筛选超时修复 + +定位:客户等原始字段筛选时,汇总、趋势和覆盖检查重复读取字段目录,数据库对年度查询选择了原表扫描,约48秒超过页面30秒期限。复用服务级字段目录缓存,目录及无窄范围限制的年度来源筛选使用既有构建索引;JSON值每次表达式只提取一次,保留空值、字符串和大小写语义。单月、季度、产品及扩展条件保留原查询计划。查询超时提示改为重试或调整范围,不再误报连接断开。权限和正式计算规则不变,无新增表或索引。 + +验证:后端完整构建975项(812执行通过、163按条件跳过);隔离MySQL及快照专项34项(33通过、1跳过),前端相关17项及生产构建通过。独立复核无实质问题;已验证主目录本轮6文件与交付目录一致,新旧运行包数据库迁移文件一致。准备发布2026.11.13.5,线上同条件性能及一致性验收待完成;本轮尚未同步Gitea。 + ## 2026-11-13:Gitea 同步交付 本次将已发布的来源明细与期间范围修复、换年保持产品层级、查询默认年度、不良率曲线细分、表格及 Excel 联动、趋势与表格独立加载、标题旁钻取路径,以及紧凑预汇总提速整合同步至 `lyun/quality_analyse` 的 `main`。保留各次发布的提交记录,线上版本为2026.11.13.4,本次仓库同步不重复部署。 diff --git a/web/src/api.ts b/web/src/api.ts index 3d3f5d1..e0a6c56 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -122,7 +122,10 @@ export async function request(path: string, init: RequestInit = {}, timeoutMs } catch (error) { handleAccessFailure(error); - if (error instanceof Error && error.name === 'RequestTimeoutError') throw new ApiError(408, '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。'); + if (error instanceof Error && error.name === 'RequestTimeoutError') { + const query = /^\/api\/(?:metrics(?:\/|$)|records(?:\/|$)|analysis\/(?:source\/|defects(?:\/|$)))/.test(requestPath(path)); + throw new ApiError(408, query ? '查询处理超时,请重试或调整筛选范围。' : '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。'); + } throw error; } }