From ea84e26b2f3d7983e6675fa70596b56038e6044a Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 16:52:27 +0800 Subject: [PATCH 01/11] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=EF=BC=9A=E6=9D=A5?= =?UTF-8?q?=E6=BA=90=E5=AD=97=E6=AE=B5=E7=AD=9B=E9=80=89=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E4=B8=8E=E9=87=8D=E5=A4=8D=E6=89=AB=E6=8F=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../quality/filters/SourceColumnQueries.java | 40 +++++++++---------- .../quality/metrics/MetricsService.java | 33 ++++++++++----- .../analysis/FineDefectRollupsMySqlTest.java | 2 +- .../filters/SourceColumnQueriesMySqlTest.java | 37 ++++++++++++++--- .../filters/SourceColumnQueriesTest.java | 9 ++--- docs/当前进度.md | 6 +++ web/src/api.ts | 5 ++- 7 files changed, 89 insertions(+), 43 deletions(-) diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java index 908634f..2f07d49 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java @@ -60,6 +60,16 @@ public class SourceColumnQueries { } return List.copyOf(result); } + // 只提取一次,并区分 JSON null 与字符串“null”,供筛选、候选值和成员读取共用。 + private static final String SOURCE_TEXT = "JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON)))"; + private static final String SOURCE_VALUE = "NULLIF(TRIM("+SOURCE_TEXT+"), '') COLLATE utf8mb4_0900_as_cs"; + + /** 原始字段筛选按发布构建读取,避免优化器从整年月份范围扫描大 JSON 原表。 */ + public static String factIndexHint(String stage) { + if(!List.of("black","bright").contains(stage))throw bad("数据阶段无效"); + return " FORCE INDEX (uq_fact_"+stage+"_origin)"; + } + private static final List STAGES = List.of("black", "bright"); private static final List OPERATORS = List.of("in", "not_in", "is_empty", "not_empty"); private static final Map CANONICAL_COLUMNS = Map.of( @@ -100,7 +110,7 @@ public class SourceColumnQueries { "f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs"); String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) " + "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id " - + "FROM release_member rm JOIN fact_" + stage + " f2 " + + "FROM release_member rm JOIN fact_" + stage + " f2" + factIndexHint(stage) + " " + "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id" + representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative " + "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id"; @@ -240,10 +250,7 @@ public class SourceColumnQueries { String expression; Object column = ((Map) field.get("columns")).get(stage); if ("source_value".equals(field.get("type"))) { - expression = "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL " - + "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL " - + "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END " - + "COLLATE utf8mb4_0900_as_cs"; + expression = SOURCE_VALUE; } else { String factColumn = CANONICAL_COLUMNS.get(key); if (factColumn == null) throw bad("规范维度无效"); @@ -255,19 +262,19 @@ public class SourceColumnQueries { if (operator.equals("in")) { where.append(expression).append(" IN (") .append(String.join(",", Collections.nCopies(values.size(), "?"))).append(')'); - if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column)); + if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column)); } else { where.append('(').append(expression).append(" IS NULL OR ").append(expression).append(" NOT IN (") .append(String.join(",", Collections.nCopies(values.size(), "?"))).append("))"); if ("source_value".equals(field.get("type"))) { - bindSourcePaths(args, String.valueOf(column)); - bindSourcePaths(args, String.valueOf(column)); + bindSourcePath(args, String.valueOf(column)); + bindSourcePath(args, String.valueOf(column)); } } args.addAll(values); } else { where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL"); - if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column)); + if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column)); } where.append(')'); } @@ -578,12 +585,7 @@ public class SourceColumnQueries { } String path = jsonPath(column); args.add(path); - args.add(path); - args.add(path); - return "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL " - + "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL " - + "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END " - + "COLLATE utf8mb4_0900_as_cs"; + return SOURCE_VALUE; } private static List combine(List first, List second) { @@ -601,9 +603,7 @@ public class SourceColumnQueries { } String path = jsonPath(column); args.add(path); - args.add(path); - return "CASE WHEN JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL " - + "ELSE JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?)) END COLLATE utf8mb4_0900_as_cs"; + return SOURCE_TEXT+" COLLATE utf8mb4_0900_as_cs"; } private static String tupleColumns(int count) { @@ -814,11 +814,9 @@ public class SourceColumnQueries { catch (Exception e) { throw new IllegalArgumentException("来源列名无法编码", e); } } - private void bindSourcePaths(List args, String header) { + private void bindSourcePath(List args, String header) { String path = jsonPath(header); args.add(path); - args.add(path); - args.add(path); } private static ResponseStatusException bad(String message) { diff --git a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java index d474a43..f5102f7 100644 --- a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java +++ b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java @@ -39,6 +39,7 @@ public class MetricsService { private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128; private final JdbcTemplate jdbc; private final GovernanceAccessService governance; + private final SourceColumnQueries sourceColumns; private final TransactionTemplate readSnapshot; private final TransactionTemplate governanceSnapshot; @@ -46,6 +47,7 @@ public class MetricsService { public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance, PlatformTransactionManager transactionManager) { this.jdbc = jdbc; this.governance = governance; + this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); this.readSnapshot = new TransactionTemplate(transactionManager); this.readSnapshot.setIsolationLevel(TransactionDefinition.ISOLATION_REPEATABLE_READ); this.readSnapshot.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); @@ -56,7 +58,9 @@ public class MetricsService { /** Keeps package-level in-memory aggregation tests independent of a database transaction manager. */ public MetricsService(JdbcTemplate jdbc, GovernanceAccessService governance) { - this.jdbc = jdbc; this.governance = governance; this.readSnapshot = null; this.governanceSnapshot = null; + this.jdbc = jdbc; this.governance = governance; + this.sourceColumns = new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); + this.readSnapshot = null; this.governanceSnapshot = null; } public MetricSnapshot calculate(MetricsQuery query, AppUserPrincipal actor) { @@ -158,7 +162,7 @@ public class MetricsService { if (factFallback) { new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor); if (!query.columnFilters().isEmpty()) - new SourceColumnQueries(jdbc, governance, JsonMapper.builder().build()) + sourceColumns .append(where, args, stage, query.columnFilters(), releaseId, actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", @@ -180,7 +184,7 @@ public class MetricsService { %s AS bright_processing_scrap, %s AS bright_diffusion_scrap, %s AS row_count - FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id + FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id %s GROUP BY trend_period ORDER BY trend_period @@ -189,7 +193,7 @@ public class MetricsService { componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"), componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"), componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"), - countExpression, table, where); + countExpression, table, factFilterHint(stage,factFallback,query), where); jdbc.query(sql, (RowCallbackHandler) rs -> { String period = rs.getString("trend_period"); AggregateStats stats = new AggregateStats().read(rs, stage); @@ -393,7 +397,7 @@ public class MetricsService { appendExclusions(where, args, query); if(factFallback) { new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor); - if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor); + if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); if (!effectiveIncludeUnassigned(query)) @@ -413,7 +417,7 @@ public class MetricsService { %s AS bright_processing_scrap, %s AS bright_diffusion_scrap, %s AS row_count - FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id + FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id %s GROUP BY %s ORDER BY %s @@ -423,7 +427,7 @@ public class MetricsService { componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"), componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"), componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"), - factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, where, + factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, factFilterHint(stage,factFallback,query), where, aggregateKeys, aggregateKeys); jdbc.query(sql, (RowCallbackHandler) rs -> { String period = rs.getString("trend_period"); @@ -595,6 +599,15 @@ public class MetricsService { "sourceMonths", sourceMonths); } + private static String factFilterHint(String stage,boolean factFallback,MetricsQuery query) { + if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return ""; + // 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。 + if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null + ||query.department()!=null||query.group()!=null||query.product()!=null||query.projectGroup()!=null + ||query.productSearch()!=null)return ""; + return SourceColumnQueries.factIndexHint(stage); + } + private void coverageStage(Map months, String stage, String releaseId, MetricsQuery query, AppUserPrincipal actor, boolean factFallback, boolean salesRollup) { if(query.stage()!=null&&!query.stage().equals(stage))return; @@ -610,7 +623,7 @@ public class MetricsService { appendExclusions(where, args, query); if(factFallback) { new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor); - if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor); + if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); String countExpression = factFallback ? "COUNT(*)" : "SUM(f.source_row_count)"; @@ -625,11 +638,11 @@ public class MetricsService { String sql = """ SELECT f.month_key, %s AS row_count, %s AS excluded_blank_rows, %s AS unmapped_rows, %s AS project_unconfigured_rows - FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id + FROM release_member rm JOIN %s f%s ON f.build_id=rm.transform_build_id %s GROUP BY f.month_key ORDER BY f.month_key - """.formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, where); + """.formatted(countExpression, excludedExpression, unmappedExpression, projectExpression, table, factFilterHint(stage,factFallback,query), where); jdbc.query(sql, (RowCallbackHandler) rs -> { CoverageMonth coverage = months.computeIfAbsent(rs.getString("month_key"), ignored -> new CoverageMonth()); long rowCount = rs.getLong("row_count"); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java index 5725f18..c8df7f4 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java @@ -42,7 +42,7 @@ class FineDefectRollupsMySqlTest { jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)"); jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))"); jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))"); - for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))"); + for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,sheet_name VARCHAR(80),physical_row_no INT,KEY ix_fact_" + stage + "_month_dims(build_id,month_key),UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))"); for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql", "V42__fine_defect_compact_rollups.sql")) { try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) { for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 6e3216d..3457798 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -115,6 +115,7 @@ class SourceColumnQueriesMySqlTest { for(Read read:representatives) { String stage=read.sql().contains("fact_black f2")?"black":"bright"; assertTrue(read.sql().contains("GROUP BY f2.build_id,f2.source_revision_id")); + assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表"); StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"); List args=new java.util.ArrayList<>(); DataScope.append(scope,args,actor,"f2.department COLLATE utf8mb4_0900_as_cs","f2.group_name COLLATE utf8mb4_0900_as_cs");args.add(RELEASE); @@ -157,6 +158,23 @@ class SourceColumnQueriesMySqlTest { assertThrows(ResponseStatusException.class, () -> queries.validate(RELEASE, unauthorized, actor)); } + @Test void sourceNullNormalizationMatchesThePreviousExpressionForEveryOperator() { + String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key"); + for(String raw:List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[]}","{\"委托单\":{\"n\":1}}")) + insert("bright","build-bright",BRIGHT_REVISION,"部门A","组A",raw); + String old="CASE WHEN JSON_EXTRACT(f.raw_values,'$.\"委托单\"') IS NULL OR JSON_TYPE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))='NULL' THEN NULL ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values,'$.\"委托单\"'))),'') END COLLATE utf8mb4_0900_as_cs"; + for(String operator:List.of("in","not_in","is_empty","not_empty")) { + Map rule=new java.util.LinkedHashMap<>(Map.of("field",key,"stages",List.of("bright"),"operator",operator)); + List args=new java.util.ArrayList<>();String condition; + if(operator.equals("in")||operator.equals("not_in")) { + rule.put("values",List.of("null","0007"));args.add("null");args.add("0007"); + condition=operator.equals("in")?old+" IN (?,?)":"("+old+" IS NULL OR "+old+" NOT IN (?,?))"; + } else condition=old+(operator.equals("is_empty")?" IS NULL":" IS NOT NULL"); + Long expected=jdbc.queryForObject("SELECT COUNT(*) FROM fact_bright f WHERE f.build_id='build-bright' AND f.department='部门A' AND f.group_name='组A' AND ("+condition+")",Long.class,args.toArray()); + assertEquals(expected,countFiltered("bright",SourceColumnQueries.parse(List.of(rule))),operator); + } + } + @Test void explicitMappingWorksAcrossStagesAndMemberPageCountsMergedTuplesBeforePaging() { List> catalog = queries.catalog(RELEASE, actor); String mappedKey = (String) catalog.stream().filter(item -> item.get("label").equals("跨阶段委托号")) @@ -252,12 +270,19 @@ class SourceColumnQueriesMySqlTest { jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE); jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))"); for(String stage:List.of("black","bright")) { - jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT"); + jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON"); jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=?,scrap_weight=?,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id",stage.equals("black")?90:80,stage.equals("black")?10:20); } jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black"); jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT"); - var service=new cn.yunsheng.quality.metrics.MetricsService(jdbc,governance); + var headerReads=new java.util.concurrent.atomic.AtomicInteger(); + var capturing=new JdbcTemplate(jdbc.getDataSource()) { + @Override public List query(String sql,org.springframework.jdbc.core.RowMapper mapper,Object... args) { + if(sql.contains("JSON_KEYS(f.raw_values)"))headerReads.incrementAndGet(); + return super.query(sql,mapper,args); + } + }; + var service=new cn.yunsheng.quality.metrics.MetricsService(capturing,governance); var releases=mock(cn.yunsheng.quality.release.ReleaseService.class); when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE); var mapper=JsonMapper.builder().build(); @@ -272,6 +297,7 @@ class SourceColumnQueriesMySqlTest { assertEquals(stage.equals("black")?legacy.summary().black():legacy.summary().bright(),stage.equals("black")?selected.summary().black():selected.summary().bright()); assertEquals(0L,selected.summary().coverage().get(stage.equals("black")?"brightRows":"blackRows")); assertEquals(1,selected.trend().size()); + assertEquals(1,service.calculateTrend(base.withStage(stage),actor).trend().size()); var detail=records.recordsFiltered(stage,2026,1,null,"部门A","组A",null,null,null,false,RELEASE,1,50,null,null,null,"asc",null,null,mapper.writeValueAsString(List.of(filter)),actor); assertEquals(1L,((Number)detail.get("total")).longValue()); java.nio.file.Path file=exporter.exportFiltered(2026,1,null,"部门A","组A",null,null,false,RELEASE,actor,Map.of(),null,stage,List.of(filter)); @@ -282,6 +308,7 @@ class SourceColumnQueriesMySqlTest { var other=base.withColumnFilters(List.of(filter)).withStage(stage); assertThrows(ResponseStatusException.class,()->other.withStage(stage.equals("black")?"bright":"black")); } + assertEquals(2,headerReads.get(),"汇总、趋势与覆盖检查应复用同一份授权字段目录"); } @Test void continuousRangeDetailsAndBothExcelSheetsExcludeOutsideMonthsAndUnauthorizedRows() throws Exception { @@ -289,7 +316,7 @@ class SourceColumnQueriesMySqlTest { when(releases.resolvePublishedRelease(RELEASE)).thenReturn(RELEASE); var mapper=JsonMapper.builder().build(); for(String stage:List.of("black","bright")) { - jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT"); + jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON"); jdbc.update("UPDATE fact_"+stage+" SET qualified_weight=90,scrap_weight=10,total_weight=100,components=JSON_OBJECT(),sheet_name='fixture',physical_row_no=id"); String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION; for(String month:List.of("2026-03","2026-09","2026-10","2025-03")) { @@ -336,7 +363,7 @@ class SourceColumnQueriesMySqlTest { jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))"); for(String stage:List.of("black","bright")) { jdbc.execute("DELETE FROM fact_"+stage); - jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON,ADD sheet_name VARCHAR(80),ADD physical_row_no INT"); + jdbc.execute("ALTER TABLE fact_"+stage+" ADD qualified_weight DECIMAL(28,10),ADD scrap_weight DECIMAL(28,10),ADD total_weight DECIMAL(28,10),ADD components JSON"); } jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black"); jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT"); @@ -395,7 +422,7 @@ class SourceColumnQueriesMySqlTest { jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))"); for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7)," - + "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL)"); + + "department VARCHAR(190),group_name VARCHAR(190),product VARCHAR(190),project_group VARCHAR(190),raw_values JSON NOT NULL,sheet_name VARCHAR(80),physical_row_no INT,UNIQUE KEY uq_fact_"+stage+"_origin(build_id,sheet_name,physical_row_no))"); jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64) PRIMARY KEY,label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)"); } diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java index 899cb9b..5acd01f 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java @@ -143,9 +143,9 @@ class SourceColumnQueriesTest { queries.append(where, args, "black", filters, "release", actor); - assertTrue(where.toString().contains("JSON_EXTRACT(f.raw_values, ?) IS NULL")); + assertTrue(where.toString().contains("NULLIF(JSON_EXTRACT(f.raw_values, ?), CAST('null' AS JSON))")); assertTrue(where.toString().contains("COLLATE utf8mb4_0900_as_cs IN (?,?)")); - assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "00007", "x' OR 1=1 --"), args); + assertEquals(List.of("$.\"委托单\"", "00007", "x' OR 1=1 --"), args); assertFalse(where.toString().contains("委托单")); assertFalse(where.toString().contains("x' OR 1=1")); } @@ -160,7 +160,7 @@ class SourceColumnQueriesTest { StringBuilder where = new StringBuilder(); queries.append(where, args, "black", filters, "release", actor); assertTrue(where.toString().contains(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL")); - assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\""), args); + assertEquals(List.of("$.\"委托单\""), args); } List> filters = SourceColumnQueries.parse(List.of(Map.of( "field", SOURCE, "stages", List.of("bright"), "operator", "is_empty"))); @@ -179,8 +179,7 @@ class SourceColumnQueriesTest { assertTrue(where.toString().contains("IS NULL OR")); assertTrue(where.toString().contains(" NOT IN (?)")); - assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", - "$.\"委托单\"", "$.\"委托单\"", "$.\"委托单\"", "000123"), args); + assertEquals(List.of("$.\"委托单\"", "$.\"委托单\"", "000123"), args); } @Test void rejectsInvalidStageAndMissingFieldsInTheActiveCatalog() { diff --git a/docs/当前进度.md b/docs/当前进度.md index 7185e66..6f2c007 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -3,6 +3,12 @@ 核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。 +## 2026-11-13:来源字段筛选超时修复 + +定位:客户等原始字段筛选时,汇总、趋势和覆盖检查重复读取字段目录,数据库对年度查询选择了原表扫描,约48秒超过页面30秒期限。复用服务级字段目录缓存,目录及无窄范围限制的年度来源筛选使用既有构建索引;JSON值每次表达式只提取一次,保留空值、字符串和大小写语义。单月、季度、产品及扩展条件保留原查询计划。查询超时提示改为重试或调整范围,不再误报连接断开。权限和正式计算规则不变,无新增表或索引。 + +验证:后端完整构建975项(812执行通过、163按条件跳过);隔离MySQL及快照专项34项(33通过、1跳过),前端相关17项及生产构建通过。独立复核无实质问题;已验证主目录本轮6文件与交付目录一致,新旧运行包数据库迁移文件一致。准备发布2026.11.13.5,线上同条件性能及一致性验收待完成;本轮尚未同步Gitea。 + ## 2026-11-13:Gitea 同步交付 本次将已发布的来源明细与期间范围修复、换年保持产品层级、查询默认年度、不良率曲线细分、表格及 Excel 联动、趋势与表格独立加载、标题旁钻取路径,以及紧凑预汇总提速整合同步至 `lyun/quality_analyse` 的 `main`。保留各次发布的提交记录,线上版本为2026.11.13.4,本次仓库同步不重复部署。 diff --git a/web/src/api.ts b/web/src/api.ts index 3d3f5d1..e0a6c56 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -122,7 +122,10 @@ export async function request(path: string, init: RequestInit = {}, timeoutMs } catch (error) { handleAccessFailure(error); - if (error instanceof Error && error.name === 'RequestTimeoutError') throw new ApiError(408, '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。'); + if (error instanceof Error && error.name === 'RequestTimeoutError') { + const query = /^\/api\/(?:metrics(?:\/|$)|records(?:\/|$)|analysis\/(?:source\/|defects(?:\/|$)))/.test(requestPath(path)); + throw new ApiError(408, query ? '查询处理超时,请重试或调整筛选范围。' : '连接质量数据服务超时,请点击重新连接;服务恢复后可继续使用。'); + } throw error; } } From 5068b8fbc9716c3588e892619c555cab23afad6f Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 16:56:38 +0800 Subject: [PATCH 02/11] =?UTF-8?q?=E6=96=87=E6=A1=A3=EF=BC=9A=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E6=9D=A5=E6=BA=90=E7=AD=9B=E9=80=89=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=9A=84=E7=BA=BF=E4=B8=8A=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/当前进度.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/当前进度.md b/docs/当前进度.md index 6f2c007..89fad32 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -7,7 +7,7 @@ 定位:客户等原始字段筛选时,汇总、趋势和覆盖检查重复读取字段目录,数据库对年度查询选择了原表扫描,约48秒超过页面30秒期限。复用服务级字段目录缓存,目录及无窄范围限制的年度来源筛选使用既有构建索引;JSON值每次表达式只提取一次,保留空值、字符串和大小写语义。单月、季度、产品及扩展条件保留原查询计划。查询超时提示改为重试或调整范围,不再误报连接断开。权限和正式计算规则不变,无新增表或索引。 -验证:后端完整构建975项(812执行通过、163按条件跳过);隔离MySQL及快照专项34项(33通过、1跳过),前端相关17项及生产构建通过。独立复核无实质问题;已验证主目录本轮6文件与交付目录一致,新旧运行包数据库迁移文件一致。准备发布2026.11.13.5,线上同条件性能及一致性验收待完成;本轮尚未同步Gitea。 +验证:后端完整构建975项(812执行通过、163按条件跳过);隔离MySQL及快照专项34项(33通过、1跳过),前端相关17项及生产构建通过。独立复核无实质问题;已验证主目录本轮6文件与交付目录一致,新旧运行包数据库迁移文件一致。已发布2026.11.13.5(功能提交 `ea84e26`)。线上同条件汇总/趋势从48282/48281ms降至17818/17816ms,完整结果规范化校验值均一致;页面正常加载时两接口约5528/5529ms,无失败提示。实际来源明细返回29条、121个字段,Excel下载有效。数据库迁移文件不变,MySQL原进程和DSH配置保留。只读同条件复测不等于私有原表全量回归。本轮独立分支已本地提交,尚未同步Gitea;私有验收材料位于 `output/20261113-source-filter-timeout/`。 ## 2026-11-13:Gitea 同步交付 From 7782e33becb69d8ef0b104f748a0dde5cc64504b Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 17:56:18 +0800 Subject: [PATCH 03/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=EF=BC=9A=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E6=9D=A5=E6=BA=90=E7=AD=9B=E9=80=89=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=E4=B8=8E=E5=90=8E=E5=8F=B0=E6=9E=84=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 + .../analysis/AnalysisExplorationTools.java | 2 +- .../analysis/AnalysisFineDefectsService.java | 2 +- .../AnalysisInvestigationService.java | 2 +- .../AnalysisSourceQueryController.java | 10 +- .../analytics/AnalyticsController.java | 2 +- .../filters/FilterFieldsController.java | 4 +- .../quality/filters/QueryFilterFields.java | 11 +- .../quality/filters/SourceColumnQueries.java | 31 ++- .../quality/filters/SourceFilterIndex.java | 122 +++++++++++ .../filters/SourceFilterIndexBackfill.java | 194 ++++++++++++++++++ .../quality/metrics/MetricsRollupService.java | 41 ++++ .../quality/metrics/MetricsService.java | 14 +- .../quality/records/RecordController.java | 9 +- .../records/RecordsWorkbookExportService.java | 16 +- .../quality/workflow/DataWorkflowService.java | 10 +- .../migration/V43__source_filter_indexes.sql | 39 ++++ .../filters/SourceColumnQueriesMySqlTest.java | 131 +++++++++++- .../SourceFilterIndexBackfillTest.java | 117 +++++++++++ .../metrics/MetricsRollupServiceTest.java | 74 +++++++ deploy/macos/retention.py | 20 +- deploy/macos/test_retention.py | 34 ++- deploy/macos/test_revision_retention.py | 38 +++- docs/当前进度.md | 8 + scripts/revision_retention.py | 10 +- web/src/StageProductPage.tsx | 44 ++-- web/src/StageSourceFilters.tsx | 8 +- web/src/api.ts | 2 +- .../stage_product_filter_speed.test.mjs | 53 +++++ 29 files changed, 997 insertions(+), 59 deletions(-) create mode 100644 backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java create mode 100644 backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfill.java create mode 100644 backend/src/main/resources/db/migration/V43__source_filter_indexes.sql create mode 100644 backend/src/test/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfillTest.java create mode 100644 web/verification/stage_product_filter_speed.test.mjs diff --git a/README.md b/README.md index 145eebc..27f5e66 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,14 @@ DSH CLI 固定为 `0.2.2-rc.1`,依赖与官方 Python SDK 校验记录在 `run 底部表格同步展开当前分类的细项,重量/废率切换与 Excel 导出跟随当前显示;产品和期间行维度保持不变。细项使用正式阶段同期总重量作分母,月、季、年均先汇总分量再计算;没有来源或分母时显示缺失,真实零显示零。细项与正式分类总量不一致时提示覆盖不足,不补造分类。 +## 筛选索引与查询加载 + +来源字段的等于、排除和空值筛选共用按构建冻结的值字典及记录索引;汇总、趋势、来源明细和导出沿用当前授权与正式指标计算。源字段值保留空值、大小写及前导零语义,旧配置筛选的字符串 `null` 约定不变。下拉框使用只读候选模式,只取一页及是否还有;需要精确总数的原接口继续保留。 + +启动时在单线程后台补建当前完整正式版本的索引,导入及归类在事务提交后进入同一去重队列。完整构建标记提交后才启用;未完成、失败或未补建的历史构建沿用原始查询,不阻止启动和正式数据发布。索引增加持久存储及后台构建成本,规模取决于非空字段数和不同值数量;高基数宽表应先测容量。保留清理随无引用构建回收索引;升级后原维护兼容保护继续生效。 + +黑片/亮片页的普通查询在数据版本已解析后独立加载,来源字段条件仍等待对应版本的字段目录;产品搜索不因无关元数据失败而禁用。 + ## 旧版本时间迁移 连接与 MySQL 会话现在统一使用 UTC,界面仍按本地时区显示。V25 只校正旧代码按 UTC 墙上时间写入的截止/有效期、报告调度及报告租约列(前者由 Java 写入,租约使用 SQL UTC_TIMESTAMP),不改 SQL CURRENT_TIMESTAMP 自动生成的创建时间,也不修改业务日期或指标。若旧库存在此类值,迁移要求先备份并核实旧会话相对 UTC 的偏移,然后设置 `MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS`(旧会话 UTC 为 `0`,旧会话中国标准时间为 `28800`)。本机启动可在私有 local-config 中记录 `legacyTimestampOffsetSeconds`;不能仅凭电脑当前时区猜测。全新空库不需要该参数;迁移标记和历史值修改在同一事务提交,防止提交确认丢失后重复平移。还原旧备份时应重新核实其原运行时区。 diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisExplorationTools.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisExplorationTools.java index 378c182..9c7c77f 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisExplorationTools.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisExplorationTools.java @@ -369,7 +369,7 @@ public class AnalysisExplorationTools { SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value); SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value); appendDetailExclusions(sql, args, query); - new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor); + new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor,releaseId); DataScope.append(sql, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); boolean includeUnassigned = query.includeUnassigned() diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisFineDefectsService.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisFineDefectsService.java index 40d1c8e..a1087b2 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisFineDefectsService.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisFineDefectsService.java @@ -276,7 +276,7 @@ public class AnalysisFineDefectsService { SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value); if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__")) predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')"); - new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor); + new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor,releaseId); DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); } diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisInvestigationService.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisInvestigationService.java index 3774df4..62c0bff 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisInvestigationService.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisInvestigationService.java @@ -90,7 +90,7 @@ public class AnalysisInvestigationService { SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x); SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x); if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')"); - new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor); + new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor,release); DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); } if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());} diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryController.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryController.java index 3b973c7..7d49ff3 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryController.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryController.java @@ -43,12 +43,16 @@ public class AnalysisSourceQueryController { @PostMapping("/values") public Map values(@RequestBody Map input,@AuthenticationPrincipal AppUserPrincipal actor) { String scope=captureScope(actor); - if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset").containsAll(input.keySet()))throw bad("候选值请求包含未知字段"); + if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset","candidateOnly").containsAll(input.keySet()))throw bad("候选值请求包含未知字段"); String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null)); Map context=context(input); if(!(input.get("field") instanceof String field)||field.isBlank()||input.get("search")!=null&&!(input.get("search") instanceof String))throw bad("候选字段或搜索值无效"); - Map result=new LinkedHashMap<>(columns.values(release,context,field,strings(input.get("stages"),2),Objects.toString(input.get("search"),null), - integer(input.getOrDefault("limit",50),1,200),integer(input.getOrDefault("offset",0),0,100000),actor)); + if(input.containsKey("candidateOnly")&&!(input.get("candidateOnly") instanceof Boolean))throw bad("候选分页参数无效"); + List stages=strings(input.get("stages"),2);String search=Objects.toString(input.get("search"),null); + int limit=integer(input.getOrDefault("limit",50),1,200),offset=integer(input.getOrDefault("offset",0),0,100000); + Map result=new LinkedHashMap<>(Boolean.TRUE.equals(input.get("candidateOnly")) + ? columns.values(release,context,field,stages,search,limit,offset,true,actor) + : columns.values(release,context,field,stages,search,limit,offset,actor)); result.put("period",input.get("period"));result.put("filters",input.get("filters")); result.put("values",AnalysisProtocolService.list(result.get("rows")).stream().map(row->AnalysisProtocolService.map(row).get("value")).toList()); result.put("candidateOnly",true);reauthorize(actor,scope,false);return result; diff --git a/backend/src/main/java/cn/yunsheng/quality/analytics/AnalyticsController.java b/backend/src/main/java/cn/yunsheng/quality/analytics/AnalyticsController.java index 56130b1..35e314b 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analytics/AnalyticsController.java +++ b/backend/src/main/java/cn/yunsheng/quality/analytics/AnalyticsController.java @@ -111,7 +111,7 @@ public class AnalyticsController { throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前字段授权不包含所选报废类别"); List args = new ArrayList<>(extra.isEmpty()?List.of(selected, selected):List.of(selected)); StringBuilder where = baseWhere(args, year, month, quarter, department, group, product, projectGroup, includeUnassigned, actor); - new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor); + new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor,selected); List aggregated; if (dimension.equals("category")) { List selectedComponents = selectedCategory == null ? visibleComponents : List.of(selectedCategory); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/FilterFieldsController.java b/backend/src/main/java/cn/yunsheng/quality/filters/FilterFieldsController.java index 073a6cc..95f24b9 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/FilterFieldsController.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/FilterFieldsController.java @@ -36,9 +36,11 @@ public class FilterFieldsController { governance.requireUserActivePolicy(actor); Map> result=new LinkedHashMap<>(); String release=releases.resolvePublishedRelease(null); + SourceFilterIndex index=new SourceFilterIndex(jdbc); for(String stage:List.of("black","bright")) { // Raw source headers are stable within each build. Read a representative row of every current build. - List raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2 WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release); + String hint=index.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":""; + List raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2"+hint+" WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release); Set columns=new TreeSet<>(); var json=tools.jackson.databind.json.JsonMapper.builder().build(); Set allowed=governance.allowedFields(actor); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/QueryFilterFields.java b/backend/src/main/java/cn/yunsheng/quality/filters/QueryFilterFields.java index c297342..42bf310 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/QueryFilterFields.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/QueryFilterFields.java @@ -48,6 +48,9 @@ public class QueryFilterFields { return Collections.unmodifiableMap(result); } public void append(StringBuilder where,List args,String stage,Map> filters,AppUserPrincipal actor) { + append(where,args,stage,filters,actor,null); + } + public void append(StringBuilder where,List args,String stage,Map> filters,AppUserPrincipal actor,String releaseId) { if(filters==null || filters.isEmpty())return; Map fields=new HashMap<>(); visible(actor).forEach(f->fields.put(f.key(),f)); for(var entry:filters.entrySet()) { @@ -61,6 +64,7 @@ public class QueryFilterFields { boolean empty=entry.getValue().contains(EMPTY); if(empty && !f.allowEmpty())throw bad(f.label()+"不允许筛选空值"); String column=stage.equals("black")?f.blackColumn():f.brightColumn(); + if(releaseId!=null&&new SourceFilterIndex(jdbc).appendLegacy(where,args,stage,releaseId,column,entry.getValue()))continue; // Paths and selected values are bound parameters; never interpolate configured identifiers. String expression="NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs"; List values=entry.getValue().stream().filter(v->!v.equals(EMPTY)).toList(); @@ -90,10 +94,15 @@ public class QueryFilterFields { StringBuilder scope=new StringBuilder(" WHERE rm.release_id=?"); List scoped=new ArrayList<>();scoped.add(releaseId); DataScope.append(scope,scoped,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); + var index=new SourceFilterIndex(jdbc); + if(index.ready(releaseId)) { + selects.add(index.candidateSelect(stage,column,scope.toString(),scoped,args).replace("AS candidate_value","AS value")); + continue; + } selects.add("SELECT NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs AS value FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+scope); args.add(path(column));args.addAll(scoped); } - String predicate=" WHERE value IS NOT NULL"; + String predicate=" WHERE value IS NOT NULL AND CAST(value AS BINARY)<>CAST('null' AS BINARY)"; if(search!=null && !search.isBlank()) { predicate+=" AND LOCATE(LOWER(?),LOWER(value))>0";args.add(search); } List found=jdbc.query("SELECT DISTINCT value FROM ("+String.join(" UNION ALL ",selects)+") choices"+predicate+" ORDER BY value LIMIT 201",(rs,n)->rs.getString(1),args.toArray()); return Map.of("values",found.stream().limit(200).toList(),"hasMore",found.size()>200); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java index 2f07d49..cde99bc 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java @@ -80,11 +80,17 @@ public class SourceColumnQueries { private final JdbcTemplate jdbc; private final GovernanceAccessService governance; private final JsonMapper json; + private SourceFilterIndex filterIndex; + @org.springframework.beans.factory.annotation.Autowired + public void setFilterIndex(SourceFilterIndex index) { this.filterIndex=index; } + public boolean indexed(String release) { return filterIndex.ready(release); } + public SourceColumnQueries(JdbcTemplate jdbc, GovernanceAccessService governance, JsonMapper json) { this.jdbc = jdbc; this.governance = governance; this.json = json; + this.filterIndex = new SourceFilterIndex(jdbc); } /** @@ -110,7 +116,7 @@ public class SourceColumnQueries { "f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs"); String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) " + "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id " - + "FROM release_member rm JOIN fact_" + stage + " f2" + factIndexHint(stage) + " " + + "FROM release_member rm JOIN fact_" + stage + " f2" + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage)) + " " + "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id" + representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative " + "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id"; @@ -249,6 +255,8 @@ public class SourceColumnQueries { String operator = (String) rule.get("operator"); String expression; Object column = ((Map) field.get("columns")).get(stage); + @SuppressWarnings("unchecked") List selected = (List) rule.getOrDefault("values",List.of()); + if("source_value".equals(field.get("type")) && filterIndex.append(where,args,stage,releaseId,String.valueOf(column),operator,selected))continue; if ("source_value".equals(field.get("type"))) { expression = SOURCE_VALUE; } else { @@ -383,12 +391,16 @@ public class SourceColumnQueries { public Map values(String releaseId, Map context, String fieldKey, List stages, String search, int limit, int offset, AppUserPrincipal actor) { + return values(releaseId,context,fieldKey,stages,search,limit,offset,false,actor); + } + public Map values(String releaseId,Map context,String fieldKey,List stages, + String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) { requireQueryAccess(actor); - return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,actor)); + return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,candidateOnly,actor)); } private Map valuesSnapshot(String releaseId,Map context,String fieldKey, - List stages,String search,int limit,int offset,AppUserPrincipal actor) { + List stages,String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) { if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效"); if (fieldKey == null || fieldKey.isBlank()) throw bad("来源列标识无效"); if (stages == null || stages.isEmpty() || stages.size() > STAGES.size() @@ -419,6 +431,10 @@ public class SourceColumnQueries { StringBuilder where = new StringBuilder(" WHERE rm.release_id=?"); List whereArgs = new ArrayList<>(List.of(releaseId)); appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor); + if("source_value".equals(field.get("type")) && filterIndex.ready(releaseId)) { + selects.add(filterIndex.candidateSelect(stage,String.valueOf(columns.get(stage)),where.toString(),whereArgs,unionArgs)); + continue; + } String fact = stage.equals("black") ? "fact_black" : "fact_bright"; selects.add("SELECT " + expression + " AS candidate_value " + "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id " @@ -437,6 +453,13 @@ public class SourceColumnQueries { } String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition; String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value"; + if(candidateOnly) { + String sql="SELECT candidate_value FROM ("+candidates+") matches GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?"; + List parameters=combine(unionArgs,searchArgs);parameters.add(limit+1);parameters.add(offset); + List> found=pages.get(new QueryKey(sql,parameters,"candidates:"+fieldKey),()->List.copyOf(jdbc.query(sql,(rs,row)->Map.of("value",rs.getString(1)),parameters.toArray()))); + boolean more=found.size()>limit; + return Map.of("rows",List.copyOf(found.subList(0,Math.min(limit,found.size()))),"hasMore",more,"limit",limit,"offset",offset,"pageComplete",true,"complete",offset==0&&!more,"releaseId",releaseId,"field",fieldKey,"selectedStages",List.copyOf(stages)); + } long total = total("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", combine(unionArgs,searchArgs)); List> rows = page("SELECT candidate_value FROM (" + candidates + ") matches " + "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?", @@ -563,7 +586,7 @@ public class SourceColumnQueries { List groups = parseSelections(context.dimensions().get("group"), "group"); if (!context.includeUnassigned() && !groups.contains("__UNASSIGNED__")) where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')"); - new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor); + new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor,releaseId); if (!context.columnFilters().isEmpty()) append(where, args, stage, context.columnFilters(), releaseId, actor); DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java new file mode 100644 index 0000000..72628f2 --- /dev/null +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java @@ -0,0 +1,122 @@ +package cn.yunsheng.quality.filters; + +import java.util.List; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; + +/** Sparse, build-owned source values. Every consumer still authorizes and scopes the actual facts. */ +@Service +public class SourceFilterIndex { + public record BuildAvailable(String buildId) {} + private static final int FORMAT = 1; + private final JdbcTemplate jdbc; + private volatile Boolean available; + public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; } + + public boolean available() { + if (available == null) { + Integer count=jdbc.queryForObject("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('source_filter_build','source_filter_field','source_filter_value','source_filter_member')",Integer.class); + available=count!=null&&count==4; + } + return available; + } + + public boolean ready(String release) { + if(release==null||!available())return false; + // Only successful immutable builds are retained; a pending backfill is checked again later. + Integer missing=jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm LEFT JOIN source_filter_build sb ON sb.build_id=rm.transform_build_id AND sb.format_version=? WHERE rm.release_id=? AND sb.build_id IS NULL",Integer.class,FORMAT,release); + if(missing==null||missing!=0)return false; + return true; + } + + /** Idempotent and atomic with fact construction, including callers which instantiate this helper. */ + public void ensureBuild(String build) { + if(!available())return; + var tx=new TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())); + tx.executeWithoutResult(status->{ + jdbc.queryForObject("SELECT id FROM transform_build WHERE id=? FOR UPDATE",String.class,build); + Integer done=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build WHERE build_id=? AND format_version=?",Integer.class,build,FORMAT); + if(done!=null&&done>0)return; + jdbc.update("DELETE m FROM source_filter_member m JOIN source_filter_field sf ON sf.id=m.field_id WHERE sf.build_id=?",build); + jdbc.update("DELETE v FROM source_filter_value v JOIN source_filter_field sf ON sf.id=v.field_id WHERE sf.build_id=?",build); + jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build); + jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build); + // Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source. + jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash)) ENGINE=InnoDB"); + try { + for(String stage:List.of("black","bright"))buildStage(build,stage); + jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT); + } finally { jdbc.execute("DROP TEMPORARY TABLE source_filter_staging"); } + }); + } + + private void buildStage(String build,String stage) { + var bounds=jdbc.queryForMap("SELECT MIN(id) AS first_id,MAX(id) AS last_id FROM fact_"+stage+" FORCE INDEX(uq_fact_"+stage+"_origin) WHERE build_id=?",build); + if(bounds.get("first_id")==null)return; + long after=((Number)bounds.get("first_id")).longValue()-1; + long last=((Number)bounds.get("last_id")).longValue(); + while(after ids=jdbc.queryForList("SELECT id FROM fact_"+stage+" FORCE INDEX(PRIMARY) WHERE build_id=? AND id>? AND id<=? ORDER BY id LIMIT 500",Long.class,build,after,last); + if(ids.isEmpty())return; + long end=ids.getLast(); + String rows=" FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE WHERE f.build_id=? AND f.id>? AND f.id<=?"; + jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end); + String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')"; + String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?"; + jdbc.update("DELETE FROM source_filter_staging"); + jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end); + jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash"); + Integer collisions=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)",Integer.class); + if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布"); + jdbc.update("INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash"); + after=end; + } + } + + /** Same-row membership is shared by metrics, trends, details and exports; empty values are sparse. */ + public boolean append(StringBuilder where,List args,String stage,String release,String column, + String operator,List values) { + if(!ready(release))return false; + where.append(" AND "); + membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"), + operator.equals("in")||operator.equals("not_in")?values:null,false); + return true; + } + + /** Legacy configured filters classify only the byte-exact literal string "null" as empty. */ + public boolean appendLegacy(StringBuilder where,List args,String stage,String release,String column,List selected) { + if(!ready(release))return false; + var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList(); + where.append(" AND ("); + if(values.isEmpty())where.append("FALSE"); + else membership(where,args,stage,release,column,false,values,true); + if(selected.contains(QueryFilterFields.EMPTY)) { + where.append(" OR ");membership(where,args,stage,release,column,true,null,true); + } + where.append(')');return true; + } + + private void membership(StringBuilder where,List args,String stage,String release,String column, + boolean negative,List values,boolean legacy) { + where.append("f.id ").append(negative?"NOT IN (":"IN (") + .append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin"); + args.add(release);args.add(stage);args.add(column); + if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)"); + if(values!=null) { + where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')'); + args.addAll(values); + } + where.append(')'); + } + + /** Dictionary-first candidates short-circuit after one authorized matching member per value. */ + public String candidateSelect(String stage,String column,String scope,List scopeArgs,List args) { + args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs); + return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id " + +"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id " + +scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)"; + } +} diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfill.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfill.java new file mode 100644 index 0000000..a7af05c --- /dev/null +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfill.java @@ -0,0 +1,194 @@ +package cn.yunsheng.quality.filters; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.context.event.EventListener; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +/** Asynchronously builds current and newly completed snapshots; unindexed queries retain their raw-value fallback. */ +@Component +public class SourceFilterIndexBackfill implements ApplicationRunner { + private static final Logger LOG = LoggerFactory.getLogger(SourceFilterIndexBackfill.class); + private static final int MAX_PENDING_BUILDS = 1024; + + private final JdbcTemplate jdbc; + private final SourceFilterIndex index; + private final ExecutorService worker; + private final Object pendingLock = new Object(); + private final LinkedHashSet pending = new LinkedHashSet<>(); + private boolean drainScheduled; + private String inFlight; + + @Autowired + public SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index) { + this(jdbc, index, createWorker()); + } + + SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index, ExecutorService worker) { + this.jdbc = jdbc; + this.index = index; + this.worker = worker; + } + + @Override + public void run(ApplicationArguments args) { + List currentBuilds; + try { + if (!index.available()) return; + currentBuilds = jdbc.query(""" + SELECT DISTINCT b.id + FROM current_release c + JOIN data_release r ON r.id=c.release_id + JOIN release_member rm ON rm.release_id=r.id + JOIN transform_build b ON b.id=rm.transform_build_id + WHERE c.dataset_id='quality' + AND r.status='PUBLISHED' + AND r.complete=TRUE + AND b.status='COMPLETED' + ORDER BY b.id + """, (rs, row) -> rs.getString(1)); + } catch (RuntimeException failure) { + LOG.error("Source filter index backfill discovery failed; raw-value fallback remains available", failure); + return; + } + if (currentBuilds.isEmpty()) return; + int accepted = enqueue(currentBuilds); + LOG.info("Source filter index backfill queued: discovered={}, accepted={}", currentBuilds.size(), accepted); + } + + @EventListener + public void onBuildAvailable(SourceFilterIndex.BuildAvailable event) { + if (event == null || event.buildId() == null || event.buildId().isBlank()) return; + int accepted = enqueue(List.of(event.buildId())); + if (accepted > 0) LOG.info("Source filter index build queued after commit: build={}", event.buildId()); + } + + private int enqueue(List builds) { + boolean schedule = false; + int accepted = 0; + int deferred = 0; + synchronized (pendingLock) { + for (String build : builds) { + if (build == null || build.isBlank() || build.equals(inFlight) || pending.contains(build)) continue; + if (pending.size() >= MAX_PENDING_BUILDS) { + deferred++; + continue; + } + pending.add(build); + accepted++; + } + if (!pending.isEmpty() && !drainScheduled) { + drainScheduled = true; + schedule = true; + } + } + if (deferred > 0) { + LOG.warn("Source filter index queue is full; {} builds keep raw-value fallback", deferred); + } + if (schedule) scheduleDrain(); + return accepted; + } + + private void scheduleDrain() { + try { + worker.execute(this::drainPending); + } catch (RejectedExecutionException failure) { + int queued; + synchronized (pendingLock) { + drainScheduled = false; + queued = pending.size(); + } + LOG.error("Source filter index drain could not be scheduled; {} builds keep raw-value fallback", queued, failure); + } + } + + private void drainPending() { + long started = System.nanoTime(); + int succeeded = 0; + int failed = 0; + while (true) { + String buildId; + synchronized (pendingLock) { + if (pending.isEmpty()) { + drainScheduled = false; + break; + } + var iterator = pending.iterator(); + buildId = iterator.next(); + iterator.remove(); + inFlight = buildId; + } + if (Thread.currentThread().isInterrupted()) { + synchronized (pendingLock) { + pending.add(buildId); + inFlight = null; + drainScheduled = false; + } + LOG.info("Source filter index drain interrupted; {} builds remain queued", pendingCount()); + return; + } + + long buildStarted = System.nanoTime(); + try { + index.ensureBuild(buildId); + succeeded++; + LOG.info("Source filter index build completed: build={}, queuedRemaining={}, elapsedMs={}", buildId, + pendingCount(), (System.nanoTime() - buildStarted) / 1_000_000L); + } catch (RuntimeException failure) { + if (Thread.currentThread().isInterrupted()) { + synchronized (pendingLock) { + pending.add(buildId); + inFlight = null; + drainScheduled = false; + } + LOG.info("Source filter index drain interrupted while building {}; {} builds remain queued", buildId, + pendingCount()); + return; + } + failed++; + LOG.error("Source filter index build failed: build={}; raw-value fallback remains available, continuing", + buildId, failure); + } finally { + synchronized (pendingLock) { inFlight = null; } + } + } + LOG.info("Source filter index drain finished: succeeded={}, failed={}, elapsedMs={}", succeeded, failed, + (System.nanoTime() - started) / 1_000_000L); + } + + private int pendingCount() { + synchronized (pendingLock) { return pending.size(); } + } + + @PreDestroy + public void stopWorker() { + worker.shutdownNow(); + try { + if (!worker.awaitTermination(2, TimeUnit.SECONDS)) { + LOG.warn("Source filter index backfill worker did not stop within two seconds"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + + private static ExecutorService createWorker() { + return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1), runnable -> { + Thread thread = new Thread(runnable, "source-filter-index-backfill"); + thread.setDaemon(true); + return thread; + }, new ThreadPoolExecutor.AbortPolicy()); + } +} diff --git a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java index f29284b..6c6b7c7 100644 --- a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java +++ b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsRollupService.java @@ -1,13 +1,21 @@ package cn.yunsheng.quality.metrics; +import cn.yunsheng.quality.filters.SourceFilterIndex; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; /** Builds immutable, per-transform-build source-component rollups for read-only metric queries. */ @Service public class MetricsRollupService { + private static final Logger LOG = LoggerFactory.getLogger(MetricsRollupService.class); static final String DIMENSION_COLLATION = "utf8mb4_0900_as_cs"; private static final List FACT_TABLES = List.of("fact_black", "fact_bright"); private static final String COMPONENTS = "JSON_OBJECT(" + @@ -22,10 +30,18 @@ public class MetricsRollupService { private final JdbcTemplate jdbc; private final cn.yunsheng.quality.analysis.FineDefectRollups fineDefects; + private final SourceFilterIndex sourceFilterIndex; + private ApplicationEventPublisher eventPublisher; public MetricsRollupService(JdbcTemplate jdbc) { this.jdbc = jdbc; this.fineDefects = new cn.yunsheng.quality.analysis.FineDefectRollups(jdbc, tools.jackson.databind.json.JsonMapper.builder().build()); + this.sourceFilterIndex = new SourceFilterIndex(jdbc); + } + + @Autowired + public void setApplicationEventPublisher(ApplicationEventPublisher eventPublisher) { + this.eventPublisher = eventPublisher; } public record GroupKey(String stage,String month,String department,String group,String project,String product) {} @@ -36,6 +52,7 @@ public class MetricsRollupService { jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent); if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总"); fineDefects.copyBuild(parent, child); + announceSourceFilterBuild(child); } @Transactional @@ -74,6 +91,30 @@ public class MetricsRollupService { """, buildId, buildId, buildId); fineDefects.refreshBuild(buildId); new SalesOrderMetricRollups(jdbc).refreshCurrentBuild(buildId); + announceSourceFilterBuild(buildId); + } + + void announceSourceFilterBuild(String buildId) { + if (eventPublisher == null) { + // Directly constructed fixtures have no application event bus; retain + // their explicit synchronous behavior (legacy schemas no-op safely). + sourceFilterIndex.ensureBuild(buildId); + return; + } + Runnable publish = () -> { + try { + eventPublisher.publishEvent(new SourceFilterIndex.BuildAvailable(buildId)); + } catch (RuntimeException failure) { + LOG.error("Source filter index notification failed for {}; raw-value fallback remains available", buildId, failure); + } + }; + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override public void afterCommit() { publish.run(); } + }); + } else { + publish.run(); + } } static String rollupInsertSql(String table) { diff --git a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java index f5102f7..647e524 100644 --- a/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java +++ b/backend/src/main/java/cn/yunsheng/quality/metrics/MetricsService.java @@ -39,7 +39,10 @@ public class MetricsService { private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128; private final JdbcTemplate jdbc; private final GovernanceAccessService governance; - private final SourceColumnQueries sourceColumns; + private SourceColumnQueries sourceColumns; + @Autowired + public void setSourceColumns(SourceColumnQueries sourceColumns) { this.sourceColumns = sourceColumns; } + private final TransactionTemplate readSnapshot; private final TransactionTemplate governanceSnapshot; @@ -160,7 +163,7 @@ public class MetricsService { appendDimensions(where, args, query); appendExclusions(where, args, query); if (factFallback) { - new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor); + new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor,releaseId); if (!query.columnFilters().isEmpty()) sourceColumns .append(where, args, stage, query.columnFilters(), releaseId, actor); @@ -396,7 +399,7 @@ public class MetricsService { appendDimensions(where, args, query); appendExclusions(where, args, query); if(factFallback) { - new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor); + new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId); if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); @@ -599,7 +602,8 @@ public class MetricsService { "sourceMonths", sourceMonths); } - private static String factFilterHint(String stage,boolean factFallback,MetricsQuery query) { + private String factFilterHint(String stage,boolean factFallback,MetricsQuery query) { + if(sourceColumns.indexed(query.releaseId()))return ""; if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return ""; // 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。 if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null @@ -622,7 +626,7 @@ public class MetricsService { appendDimensions(where, args, query); appendExclusions(where, args, query); if(factFallback) { - new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor); + new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId); if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor); } DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); diff --git a/backend/src/main/java/cn/yunsheng/quality/records/RecordController.java b/backend/src/main/java/cn/yunsheng/quality/records/RecordController.java index e589804..c526ad6 100644 --- a/backend/src/main/java/cn/yunsheng/quality/records/RecordController.java +++ b/backend/src/main/java/cn/yunsheng/quality/records/RecordController.java @@ -53,6 +53,10 @@ public class RecordController { private final JsonMapper mapper; private final GovernanceAccessService governance; private final RecordsWorkbookExportService workbookExport; + private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns; + @org.springframework.beans.factory.annotation.Autowired + public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; } + public RecordController(JdbcTemplate jdbc, ReleaseService releases, JsonMapper mapper, GovernanceAccessService governance, RecordsWorkbookExportService workbookExport) { @@ -61,6 +65,7 @@ public class RecordController { this.mapper = mapper; this.governance = governance; this.workbookExport = workbookExport; + this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper); } @@ -156,10 +161,10 @@ public class RecordController { args.add(search); args.add(jsonPath(selectedField)); } - new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor); + new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor,selectedRelease); var sourceFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters); for(var rule:sourceFilters)if(!List.of(stage).containsAll((List)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段"); - if(!sourceFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,stage,sourceFilters,selectedRelease,actor); + if(!sourceFilters.isEmpty())sourceColumns.append(predicate,args,stage,sourceFilters,selectedRelease,actor); DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); String countSql = """ diff --git a/backend/src/main/java/cn/yunsheng/quality/records/RecordsWorkbookExportService.java b/backend/src/main/java/cn/yunsheng/quality/records/RecordsWorkbookExportService.java index 2a742c3..6df156f 100644 --- a/backend/src/main/java/cn/yunsheng/quality/records/RecordsWorkbookExportService.java +++ b/backend/src/main/java/cn/yunsheng/quality/records/RecordsWorkbookExportService.java @@ -52,6 +52,10 @@ public class RecordsWorkbookExportService { private final ReleaseService releases; private final GovernanceAccessService governance; private final JsonMapper mapper; + private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns; + @org.springframework.beans.factory.annotation.Autowired + public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; } + public RecordsWorkbookExportService(JdbcTemplate jdbc, ReleaseService releases, GovernanceAccessService governance, JsonMapper mapper) { @@ -59,6 +63,7 @@ public class RecordsWorkbookExportService { this.releases = releases; this.governance = governance; this.mapper = mapper; + this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper); } public Path export(Integer year, Integer month, Integer quarter, String department, String group, @@ -120,8 +125,11 @@ public class RecordsWorkbookExportService { LinkedHashSet rawFields = new LinkedHashSet<>(); if (releaseId != null) { QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product, - projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd); - scan(rawScope, rs -> rawFields.addAll(visibleRaw(readJson(rs.getString(1)), allowedFields).keySet())); + projectGroup, includeUnassigned, "CAST(JSON_KEYS(f.raw_values) AS CHAR)", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd); + scan(rawScope, rs -> { + Object decoded=mapper.readValue(rs.getString(1),Object.class); + if(decoded instanceof List keys)for(Object key:keys)if(key instanceof String name&&(allowedFields.contains("*")||allowedFields.contains(name)))rawFields.add(name); + }); } boolean showOrderType=SalesOrderTypes.allowed(allowedFields); if (rawFields.size() + HEADER_COUNT + (showOrderType?1:0) > MAX_COLUMNS) @@ -167,8 +175,8 @@ public class RecordsWorkbookExportService { SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value); if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__")) predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')"); - new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor); - if(!columnFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor); + new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor,releaseId); + if(!columnFilters.isEmpty())sourceColumns.append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor); DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs"); String columns = select == null ? "f.raw_values" : select; diff --git a/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java b/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java index 6121127..99f230a 100644 --- a/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java +++ b/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java @@ -15,6 +15,7 @@ import cn.yunsheng.quality.ingestion.ImportWorker; import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild; import cn.yunsheng.quality.ingestion.MappingValueCodec; import cn.yunsheng.quality.metrics.MetricsService; +import cn.yunsheng.quality.metrics.MetricsRollupService; import cn.yunsheng.quality.release.ReleaseService; import cn.yunsheng.quality.source.SourceCatalogService; import cn.yunsheng.quality.source.SourceDeletionService; @@ -66,6 +67,7 @@ public class DataWorkflowService { private final TaskExecutor executor; private final TransactionTemplate transaction; private final MetricsService metrics; + private MetricsRollupService rollups; private final AtomicReference runningOperation = new AtomicReference<>(); private final AtomicReference runningLease = new AtomicReference<>(); @@ -108,6 +110,12 @@ public class DataWorkflowService { this.executor = executor; this.transaction = new TransactionTemplate(manager); this.metrics=metrics; + this.rollups=new MetricsRollupService(jdbc); + } + + @Autowired + public void setMetricsRollupService(MetricsRollupService rollups) { + this.rollups = rollups; } public Map status(AppUserPrincipal actor) { @@ -376,7 +384,7 @@ public class DataWorkflowService { sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor); updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED"); var selectedRules=unclassifiedRules; - var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor)); + var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,rollups).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor)); jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token); publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false); jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token); diff --git a/backend/src/main/resources/db/migration/V43__source_filter_indexes.sql b/backend/src/main/resources/db/migration/V43__source_filter_indexes.sql new file mode 100644 index 0000000..5eeea39 --- /dev/null +++ b/backend/src/main/resources/db/migration/V43__source_filter_indexes.sql @@ -0,0 +1,39 @@ +CREATE TABLE source_filter_build ( + build_id CHAR(36) NOT NULL PRIMARY KEY, + format_version INT NOT NULL, + CONSTRAINT fk_source_filter_build FOREIGN KEY (build_id) REFERENCES transform_build(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE source_filter_field ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + build_id CHAR(36) NOT NULL, + stage VARCHAR(16) NOT NULL, + column_name LONGTEXT COLLATE utf8mb4_0900_bin NOT NULL, + name_hash BINARY(32) NOT NULL, + UNIQUE KEY uq_source_filter_field (build_id,stage,name_hash), + CONSTRAINT fk_source_filter_field_build FOREIGN KEY (build_id) REFERENCES transform_build(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE source_filter_value ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + field_id BIGINT NOT NULL, + value_hash BINARY(32) NOT NULL, + value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL, + UNIQUE KEY uq_source_filter_value_hash (field_id,value_hash), + KEY ix_source_filter_value_text (field_id,value(128)), + CONSTRAINT fk_source_filter_value_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE source_filter_member ( + field_id BIGINT NOT NULL, + fact_id BIGINT NOT NULL, + value_id BIGINT NOT NULL, + PRIMARY KEY (field_id,fact_id), + KEY ix_source_filter_member_value (value_id,fact_id), + CONSTRAINT fk_source_filter_member_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id), + CONSTRAINT fk_source_filter_member_value FOREIGN KEY (value_id) REFERENCES source_filter_value(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- Cover representative-row selection without reading every row's large source JSON. +ALTER TABLE fact_black ADD KEY ix_black_source_catalog (build_id,source_revision_id,department,group_name,id); +ALTER TABLE fact_bright ADD KEY ix_bright_source_catalog (build_id,source_revision_id,department,group_name,id); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 3457798..6ced49e 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -265,7 +265,9 @@ class SourceColumnQueriesMySqlTest { return fields.stream().filter(item -> item.get("label").equals(label)).findFirst().orElse(null); } - @Test void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters() throws Exception { + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(booleans={false,true}) + void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed) throws Exception { jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))"); jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE); jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))"); @@ -275,6 +277,7 @@ class SourceColumnQueriesMySqlTest { } jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black"); jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT"); + if(indexed){installFilterIndex();var index=new SourceFilterIndex(jdbc);for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);} var headerReads=new java.util.concurrent.atomic.AtomicInteger(); var capturing=new JdbcTemplate(jdbc.getDataSource()) { @Override public List query(String sql,org.springframework.jdbc.core.RowMapper mapper,Object... args) { @@ -417,6 +420,132 @@ class SourceColumnQueriesMySqlTest { assertThrows(ResponseStatusException.class,()->controller.query(input,actor)); } + @Test void indexedFiltersCandidatesAndTuplesMatchRawSemanticsAndStayAuthorized() throws Exception { + when(governance.allowedFields(actor)).thenReturn(Set.of("*")); + List samples=List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[1,2]}","{\"委托单\":{\"a\":1}}","{\"委托单\":\"é\"}","{\"委托单\":\"é\"}","{\"委托单\":\"A\"}","{\"委托单\":\"a\"}","{\"委托单\":\" padded \"}","{\"委托单\":\"null\"}"); + for(String raw:samples)insert("black","build-black",BLACK_REVISION,"部门A","组A",raw); + String longKey="来源".repeat(300); + insert("black","build-black",BLACK_REVISION,"部门A","组A",JsonMapper.builder().build().writeValueAsString(Map.of(longKey,"long-key-value"))); + insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}"); + String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key"); + List> rules=new java.util.ArrayList<>(); + for(String op:List.of("in","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}")) + rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value))); + var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList(); + var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor); + var rawMembers=queries.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor); + jdbc.update("INSERT INTO query_filter_field VALUES ('plain','原始委托单','委托单','委托单',TRUE,TRUE,1)"); + List> legacySelections=List.of(List.of("null"),List.of("0007"),List.of(QueryFilterFields.EMPTY),List.of("0007",QueryFilterFields.EMPTY)); + var rawLegacy=legacySelections.stream().map(this::legacyCount).toList(); + installFilterIndex(); + var index=new SourceFilterIndex(jdbc); + for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build); + var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); + assertTrue(indexed.indexed(RELEASE)); + assertEquals(rawLegacy,legacySelections.stream().map(this::legacyCount).toList()); + assertEquals("long-key-value",jdbc.queryForObject("SELECT sv.value FROM source_filter_field sf JOIN source_filter_value sv ON sv.field_id=sf.id WHERE sf.column_name=? AND sf.build_id='build-black'",String.class,longKey)); + var combined=List.of(Map.of("field",key,"stages",List.of("black"),"operator","in","values",List.of("0007")),Map.of("field",field(indexed.catalog(RELEASE,actor),"加工单位").get("key"),"stages",List.of("black"),"operator","in","values",List.of("机二"))); + assertEquals(0L,indexed.memberPage(RELEASE,Map.of("year",2026,"columnFilters",combined),List.of(key),List.of("black"),50,0,actor).get("totalCount")); + assertEquals(List.of(),indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),"hidden-only-value",50,0,true,actor).get("rows")); + jdbc.update("INSERT INTO transform_build VALUES ('build-child')"); + jdbc.update("INSERT INTO fact_black(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT 'build-child',source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no FROM fact_black WHERE build_id='build-black'"); + index.ensureBuild("build-child"); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member sm JOIN source_filter_field sf ON sf.id=sm.field_id JOIN fact_black f ON f.id=sm.fact_id WHERE sf.build_id='build-child' AND f.build_id<>'build-child'",Long.class)); + assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList()); + assertEquals(rawValues,indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor)); + assertEquals(rawMembers,indexed.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor)); + var page=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,2,0,true,actor); + assertEquals(2,((List)page.get("rows")).size());assertEquals(true,page.get("hasMore"));assertFalse(page.containsKey("totalCount")); + long members=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class); + index.ensureBuild("build-black");assertEquals(members,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class)); + when(governance.allowedFields(actor)).thenReturn(Set.of("加工单位")); + assertThrows(ResponseStatusException.class,()->indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),null,50,0,true,actor)); + when(governance.allowedFields(actor)).thenReturn(Set.of("*")); + jdbc.update("DELETE FROM source_filter_build WHERE build_id='build-black'"); + assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList()); + } + + @Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception { + installFilterIndex(); + var fail=new java.util.concurrent.atomic.AtomicBoolean(true); + var failing=new JdbcTemplate(jdbc.getDataSource()) { + @Override public int update(String sql) { + if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure"); + return super.update(sql); + } + @Override public int update(String sql,Object... args) { + if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure"); + return super.update(sql,args); + } + }; + var index=new SourceFilterIndex(failing); + assertThrows(IllegalStateException.class,()->index.ensureBuild("build-black")); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build",Long.class)); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_field",Long.class)); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_value",Long.class)); + assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class)); + assertFalse(index.ready(RELEASE)); + for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build); + assertTrue(index.ready(RELEASE)); + } + + @Test + @EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_FILTER_BENCHMARK",matches="true") + void syntheticWideSourceIndexBenchmark() throws Exception { + when(governance.allowedFields(actor)).thenReturn(Set.of("*")); + int size=Integer.getInteger("quality.benchmarkRows",10000); + jdbc.execute("CREATE TABLE benchmark_digit(n INT PRIMARY KEY)");for(int n=0;n<10;n++)jdbc.update("INSERT INTO benchmark_digit VALUES (?)",n); + for(String stage:List.of("black","bright")) { + jdbc.execute("DELETE FROM fact_"+stage); + StringBuilder raw=new StringBuilder("JSON_OBJECT('客户名称',CONCAT('客户',MOD(n,300))"); + for(int f=0;f<120;f++)raw.append(",'字段").append(f).append("',CONCAT('value-").append(f).append("-',").append(stage.equals("black")?"MOD(n,50)":"n").append(")"); + raw.append(')'); + String numbers="SELECT a.n+10*b.n+100*c.n+1000*d.n+10000*e.n AS n FROM benchmark_digit a CROSS JOIN benchmark_digit b CROSS JOIN benchmark_digit c CROSS JOIN benchmark_digit d CROSS JOIN benchmark_digit e LIMIT "+size; + jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT ?,?,'2026-01','部门A','组A','P-1','项目A',"+raw+",'data',n+1 FROM ("+numbers+") numbers","build-"+stage,stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION); + } + var rawQuery=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); + String field=(String)field(rawQuery.catalog(RELEASE,actor),"客户名称").get("key"); + var old=new java.util.LinkedHashMap(); + for(String stage:List.of("black","bright")) { + long start=System.nanoTime();var result=rawQuery.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor); + old.put(stage,Map.of("ms",(System.nanoTime()-start)/1_000_000L,"result",result)); + } + installFilterIndex();var index=new SourceFilterIndex(jdbc); + for(String build:List.of("build-black","build-bright","build-hidden")) { + long start=System.nanoTime();index.ensureBuild(build);System.out.println("FILTER_BENCH_BUILD "+build+" rows="+size+" ms="+(System.nanoTime()-start)/1_000_000L); + } + var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()); + for(String stage:List.of("black","bright")) { + long start=System.nanoTime();var result=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor); + long elapsed=(System.nanoTime()-start)/1_000_000L; + assertEquals(((Map)old.get(stage)).get("result"),result); + System.out.println("FILTER_BENCH_QUERY "+stage+" rawMs="+((Map)old.get(stage)).get("ms")+" indexMs="+elapsed); + } + for(String table:List.of("source_filter_field","source_filter_value","source_filter_member"))jdbc.execute("ANALYZE TABLE "+table); + System.out.println("FILTER_BENCH_SPACE "+jdbc.queryForList("SELECT table_name,table_rows,data_length,index_length FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name LIKE 'source_filter_%'")); + } + + private long legacyCount(List selected) { + var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList(List.of(RELEASE)); + new QueryFilterFields(jdbc,governance).append(where,args,"black",Map.of("plain",selected),actor,RELEASE); + DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); + return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray()); + } + + private long filteredCount(SourceColumnQueries source,Map rule) { + var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList(List.of(RELEASE)); + source.append(where,args,"black",List.of(rule),RELEASE,actor); + DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs"); + return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray()); + } + + private void installFilterIndex() throws Exception { + jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)"); + for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id); + String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/V43__source_filter_indexes.sql")); + for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement); + } + private void schema() { jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))"); jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))"); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfillTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfillTest.java new file mode 100644 index 0000000..d11499d --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceFilterIndexBackfillTest.java @@ -0,0 +1,117 @@ +package cn.yunsheng.quality.filters; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.util.ArrayDeque; +import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; + +class SourceFilterIndexBackfillTest { + @Test + void skipsWhenIndexTablesAreUnavailable() { + JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class); + SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class); + ManualExecutor worker = new ManualExecutor(); + doReturn(false).when(index).available(); + var runner = new SourceFilterIndexBackfill(jdbc, index, worker); + + runner.run(null); + + verify(jdbc, never()).query(anyString(), org.mockito.ArgumentMatchers.>any()); + verify(index, never()).ensureBuild(anyString()); + assertEquals(0, worker.queued()); + runner.stopWorker(); + } + + @Test + void coalescesEventsDuringDrainAndContinuesAfterAnIndividualFailure() { + JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class); + SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class); + ManualExecutor worker = new ManualExecutor(); + doReturn(true).when(index).available(); + doReturn(List.of("failed-build", "ready-build")).when(jdbc) + .query(anyString(), org.mockito.ArgumentMatchers.>any()); + var runner = new SourceFilterIndexBackfill(jdbc, index, worker); + doAnswer(invocation -> { + runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain")); + runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain")); + runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("failed-build")); + throw new IllegalStateException("synthetic index failure"); + }).when(index).ensureBuild("failed-build"); + + runner.run(null); + + assertEquals(1, worker.queued()); + verify(index, never()).ensureBuild(anyString()); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(jdbc).query(sql.capture(), org.mockito.ArgumentMatchers.>any()); + assertTrue(sql.getValue().contains("c.dataset_id='quality'")); + assertTrue(sql.getValue().contains("r.status='PUBLISHED'")); + assertTrue(sql.getValue().contains("r.complete=TRUE")); + assertTrue(sql.getValue().contains("b.status='COMPLETED'")); + + worker.runNext(); + + verify(index).ensureBuild("failed-build"); + verify(index).ensureBuild("ready-build"); + verify(index).ensureBuild("event-during-drain"); + assertEquals(0, worker.queued()); + runner.stopWorker(); + assertTrue(worker.isShutdown()); + } + + @Test + void componentHasAnAutowiredProductionConstructorAndCleanShutdown() { + try (var context = new AnnotationConfigApplicationContext()) { + context.registerBean(JdbcTemplate.class, () -> org.mockito.Mockito.mock(JdbcTemplate.class)); + context.registerBean(SourceFilterIndex.class, () -> org.mockito.Mockito.mock(SourceFilterIndex.class)); + context.register(SourceFilterIndexBackfill.class); + context.refresh(); + + assertNotNull(context.getBean(SourceFilterIndexBackfill.class)); + } + } + + private static final class ManualExecutor extends AbstractExecutorService { + private final ArrayDeque tasks = new ArrayDeque<>(); + private boolean shutdown; + + @Override public void shutdown() { shutdown = true; } + + @Override public List shutdownNow() { + shutdown = true; + List pending = List.copyOf(tasks); + tasks.clear(); + return pending; + } + + @Override public boolean isShutdown() { return shutdown; } + + @Override public boolean isTerminated() { return shutdown && tasks.isEmpty(); } + + @Override public boolean awaitTermination(long timeout, TimeUnit unit) { return isTerminated(); } + + @Override public void execute(Runnable command) { + if (shutdown) throw new RejectedExecutionException("executor is stopped"); + tasks.add(command); + } + + int queued() { return tasks.size(); } + + void runNext() { tasks.removeFirst().run(); } + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsRollupServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsRollupServiceTest.java index afb4255..934ea69 100644 --- a/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsRollupServiceTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsRollupServiceTest.java @@ -1,8 +1,21 @@ package cn.yunsheng.quality.metrics; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import cn.yunsheng.quality.filters.SourceFilterIndex; import org.junit.jupiter.api.Test; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; class MetricsRollupServiceTest { @Test @@ -17,4 +30,65 @@ class MetricsRollupServiceTest { assertTrue(sql.contains("GROUP BY f.build_id, f.month_key " + collation)); assertTrue(sql.contains("f.group_name " + collation + ", f.project_group " + collation)); } + + @Test + void publishesFilterBuildAvailableOnlyAfterTheOuterTransactionCommits() { + var service = new MetricsRollupService(mock(JdbcTemplate.class)); + var publisher = mock(ApplicationEventPublisher.class); + service.setApplicationEventPublisher(publisher); + String build = "11111111-1111-4111-8111-111111111111"; + + TransactionSynchronizationManager.initSynchronization(); + try { + service.announceSourceFilterBuild(build); + verifyNoInteractions(publisher); + var synchronizations = TransactionSynchronizationManager.getSynchronizations(); + assertEquals(1, synchronizations.size()); + for (TransactionSynchronization synchronization : synchronizations) synchronization.afterCommit(); + verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build)); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + } + + @Test + void publishesImmediatelyWhenThereIsNoTransactionSynchronization() { + var service = new MetricsRollupService(mock(JdbcTemplate.class)); + var publisher = mock(ApplicationEventPublisher.class); + service.setApplicationEventPublisher(publisher); + String build = "22222222-2222-4222-8222-222222222222"; + + service.announceSourceFilterBuild(build); + + verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build)); + } + + @Test + void notificationFailureAfterCommitDoesNotEscapeTheTransactionSynchronization() { + var service = new MetricsRollupService(mock(JdbcTemplate.class)); + var publisher = mock(ApplicationEventPublisher.class); + service.setApplicationEventPublisher(publisher); + String build = "33333333-3333-4333-8333-333333333333"; + var event = new SourceFilterIndex.BuildAvailable(build); + doThrow(new IllegalStateException("synthetic event delivery failure")).when(publisher).publishEvent(event); + + TransactionSynchronizationManager.initSynchronization(); + try { + service.announceSourceFilterBuild(build); + var synchronizations = TransactionSynchronizationManager.getSynchronizations(); + assertDoesNotThrow(() -> synchronizations.get(0).afterCommit()); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + } + + @Test + void directFixtureConstructionKeepsTheSynchronousLegacySchemaFallback() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + var service = new MetricsRollupService(jdbc); + + service.announceSourceFilterBuild("44444444-4444-4444-8444-444444444444"); + + verify(jdbc).queryForObject(contains("information_schema.tables"), eq(Integer.class)); + } } diff --git a/deploy/macos/retention.py b/deploy/macos/retention.py index 6af57b5..0a4708f 100644 --- a/deploy/macos/retention.py +++ b/deploy/macos/retention.py @@ -26,6 +26,7 @@ SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService' TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED') BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup', 'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build', + 'source_filter_member', 'source_filter_value', 'source_filter_field', 'source_filter_build', 'fact_black', 'fact_bright') OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion', 'release_action', 'candidate_reconciliation') @@ -253,7 +254,24 @@ class Database: self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid)) for t in BUILD_CHILDREN: if t in self.columns: - count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,)) + if t == 'source_filter_member': + # Members have no build_id; drain each owned field in + # order without aliases because the exclusion uses LOCK TABLES. + count = 0 + for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)): + count = self.execute('DELETE FROM source_filter_member WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],)) + if count: + break + elif t == 'source_filter_value': + # Values are likewise owned through fields and are deleted + # only after all field memberships have been drained. + count = 0 + for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)): + count = self.execute('DELETE FROM source_filter_value WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],)) + if count: + break + else: + count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,)) if count: return count, 0 self.execute('DELETE FROM transform_build WHERE id=%s', (bid,)) diff --git a/deploy/macos/test_retention.py b/deploy/macos/test_retention.py index 3f33398..8b4f411 100644 --- a/deploy/macos/test_retention.py +++ b/deploy/macos/test_retention.py @@ -16,6 +16,7 @@ from unittest.mock import patch ROOT=Path(__file__).resolve().parents[2] sys.path.insert(0,str(ROOT/'scripts')) import backup_retention as br +import managed_test_resources spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py')) ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret) @@ -91,14 +92,18 @@ class DatabaseRetentionTests(unittest.TestCase): cfg=json.loads((ROOT/'.runtime/local-config.json').read_text()) cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True) cls.name='quality_retention_test_'+uuid.uuid4().hex[:12] + managed_test_resources.create(ROOT,cls.name,cls.execute_admin) with cls.admin.cursor() as c: - c.execute('CREATE DATABASE '+ret.ident(cls.name)) c.execute('USE '+ret.ident(cls.name)) ddl=[ 'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))', 'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))', 'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)', 'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))', + 'CREATE TABLE source_filter_build(build_id varchar(36) PRIMARY KEY,format_version int,FOREIGN KEY(build_id) REFERENCES transform_build(id))', + 'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))', + 'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))', + 'CREATE TABLE source_filter_member(field_id bigint NOT NULL,fact_id bigint NOT NULL,value_id bigint NOT NULL,PRIMARY KEY(field_id,fact_id),KEY(value_id,fact_id),FOREIGN KEY(field_id) REFERENCES source_filter_field(id),FOREIGN KEY(value_id) REFERENCES source_filter_value(id))', 'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))', 'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))', 'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)', @@ -115,14 +120,25 @@ class DatabaseRetentionTests(unittest.TestCase): @classmethod def tearDownClass(cls): - with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name)) + managed_test_resources.drop(ROOT,cls.name,cls.execute_admin) cls.admin.close() + @classmethod + def execute_admin(cls,sql,args=None): + result='' + statements=[part.strip() for part in sql.split(';') if part.strip()] + with cls.admin.cursor() as cursor: + for statement in statements: + cursor.execute(statement,args if len(statements)==1 else None) + result='\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount) + return result + def setUp(self): self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name) self.db=ret.Database(ROOT,self.name);self.db.root=self.root self.db.execute('SET FOREIGN_KEY_CHECKS=0') - for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table)) + for table in self.db.tables: + if table!='__quality_test_owner':self.db.execute('TRUNCATE TABLE '+ret.ident(table)) self.db.execute('SET FOREIGN_KEY_CHECKS=1') self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')") self.db.execute("INSERT INTO current_release VALUES('quality','current')") @@ -187,14 +203,24 @@ class DatabaseRetentionTests(unittest.TestCase): def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self): self.retire_old();state={'builds':{'obsolete':1}} + self.db.execute("INSERT INTO source_filter_build VALUES('obsolete',1)") + self.db.execute("INSERT INTO source_filter_field VALUES(9,'obsolete','black','空字段',%s),(10,'obsolete','black','客户',%s),(11,'obsolete','black','工单',%s)", (b'0'*32,b'1'*32,b'2'*32)) + self.db.execute("INSERT INTO source_filter_value VALUES(100,10,%s,'客户A'),(101,10,%s,'客户B'),(102,11,%s,'工单A')", (b'1'*32,b'2'*32,b'3'*32)) + self.db.execute("INSERT INTO source_filter_member VALUES(10,1,100),(10,2,101),(11,3,102)") with self.db.frozen('current'): self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0)) + self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_member')[0]['n'],2) + self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_value')[0]['n'],3) + self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_field')[0]['n'],3) + self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_build')[0]['n'],1) old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0] self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64) self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,)) - for _ in range(4): + for _ in range(20): with self.db.frozen('current'):self.db.build_step(state,1800,4000,1) self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")) + for table in ('source_filter_member','source_filter_value','source_filter_field','source_filter_build'): + self.assertFalse(self.db.rows('SELECT * FROM '+ret.ident(table))) self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'")) self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'")) diff --git a/deploy/macos/test_revision_retention.py b/deploy/macos/test_revision_retention.py index ee95b87..a11c258 100644 --- a/deploy/macos/test_revision_retention.py +++ b/deploy/macos/test_revision_retention.py @@ -13,6 +13,7 @@ from unittest.mock import patch ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'scripts')) import backup_retention as br +import managed_test_resources import revision_retention as revisions spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py')) ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret) @@ -26,12 +27,15 @@ class SourceRevisionTests(unittest.TestCase): cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root', password=cfg['rootPassword'], autocommit=True) cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12] + managed_test_resources.create(ROOT, cls.name, cls.execute_admin) ddl = [ 'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))', 'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))', 'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))', "CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)", 'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', + 'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))', + 'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))', 'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', 'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', 'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))', @@ -42,23 +46,34 @@ class SourceRevisionTests(unittest.TestCase): 'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))'] try: with cls.admin.cursor() as c: - c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name)) + c.execute('USE ' + ret.ident(cls.name)) for statement in ddl: c.execute(statement + ' ENGINE=InnoDB') except BaseException: - with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name)) + managed_test_resources.drop(ROOT, cls.name, cls.execute_admin) cls.admin.close(); raise @classmethod def tearDownClass(cls): - with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name)) + managed_test_resources.drop(ROOT, cls.name, cls.execute_admin) cls.admin.close() + @classmethod + def execute_admin(cls, sql, args=None): + result = '' + statements = [part.strip() for part in sql.split(';') if part.strip()] + with cls.admin.cursor() as cursor: + for statement in statements: + cursor.execute(statement, args if len(statements) == 1 else None) + result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount) + return result + def setUp(self): self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve() self.db = ret.Database(ROOT, self.name); self.db.root = self.root self.db.execute('SET FOREIGN_KEY_CHECKS=0') - for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table)) + for table in self.db.tables: + if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table)) self.db.execute('SET FOREIGN_KEY_CHECKS=1') self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')") self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')") @@ -121,6 +136,21 @@ class SourceRevisionTests(unittest.TestCase): self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0) self.assertTrue(self.present()); self.assertTrue(path.exists()) + def test_indexed_business_values_do_not_count_as_revision_metadata(self): + path = self.add_revision('indexed-only', b'indexed-only workbook') + self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,)) + self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,)) + self.collect(100); result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 1) + self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists()) + + def test_indexed_field_names_do_not_count_as_revision_metadata(self): + path = self.add_revision('indexed-field-only', b'field-only workbook') + self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,)) + self.collect(100); result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 1) + self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists()) + def test_new_reference_resets_continuous_grace(self): self.add_revision(); self.collect(100) self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500) diff --git a/docs/当前进度.md b/docs/当前进度.md index 89fad32..aee40fb 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -3,6 +3,14 @@ 核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。 +## 2026-11-13:统一筛选提速(发布验收中) + +实现:V43新增按构建保存的来源字段字典和稀疏记录索引,及两阶段字段目录覆盖索引;来源筛选、配置筛选、候选、指标、趋势、明细与明细导出共用。来源列查询实例统一复用,候选模式省去精确总数,产品页常规查询不再等待慢目录,本字段选中值变化不重复读取自己的候选;明细导出先只读字段名再读业务行。 + +生命周期:启动仅在单线程后台补当前完整发布版本;导入和归类通过提交后事件进入同一有界去重队列。完整标记提交后才启用,故障保留原始查询,业务事务不等待索引。旧配置 `null`、来源字符串 `null`、空值、Unicode、同一事实行和当前权限分别保持;构建保留清理识别4张新表,未解除维护兼容保护。 + +验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 + ## 2026-11-13:来源字段筛选超时修复 定位:客户等原始字段筛选时,汇总、趋势和覆盖检查重复读取字段目录,数据库对年度查询选择了原表扫描,约48秒超过页面30秒期限。复用服务级字段目录缓存,目录及无窄范围限制的年度来源筛选使用既有构建索引;JSON值每次表达式只提取一次,保留空值、字符串和大小写语义。单月、季度、产品及扩展条件保留原查询计划。查询超时提示改为重试或调整范围,不再误报连接断开。权限和正式计算规则不变,无新增表或索引。 diff --git a/scripts/revision_retention.py b/scripts/revision_retention.py index 6714c50..2596802 100644 --- a/scripts/revision_retention.py +++ b/scripts/revision_retention.py @@ -9,9 +9,11 @@ import uuid # These are measured business values, not documents that refer to source IDs. -VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'), - ('fact_bright', 'raw_values'), ('fact_bright', 'components'), - ('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')} +BUSINESS_VALUE_COLUMNS = {('fact_black', 'raw_values'), ('fact_black', 'components'), + ('fact_bright', 'raw_values'), ('fact_bright', 'components'), + ('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components'), + ('source_filter_field', 'column_name'), + ('source_filter_value', 'value')} TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'} @@ -54,7 +56,7 @@ class Cleaner: def metadata_mentions(self, needles, excluded=None): for table, columns in self.db.column_types.items(): for column, kind in columns.items(): - if kind not in TEXT_TYPES or (table, column) in VALUE_JSON: + if kind not in TEXT_TYPES or (table, column) in BUSINESS_VALUE_COLUMNS: continue conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles] args = list(needles) diff --git a/web/src/StageProductPage.tsx b/web/src/StageProductPage.tsx index 578011a..f645fa4 100644 --- a/web/src/StageProductPage.tsx +++ b/web/src/StageProductPage.tsx @@ -37,7 +37,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU const [moreFiltersOpen,setMoreFiltersOpen]=useState(false); const [extraFilters,setExtraFilters]=useState>(initial.extraFilters||{}); const [extraFields,setExtraFields]=useState([]); - const [options,setOptions]=useState(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');const [supportLoading,setSupportLoading]=useState(true); + const [options,setOptions]=useState(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState(''); const [applied,setApplied]=useState(null);const [revision,setRevision]=useState(0);const [response,setResponse]=useState<{key:string;value:MetricsResponse}|null>(null);const [loading,setLoading]=useState(false);const [failure,setFailure]=useState<{key:string;message:string}|null>(null); const [defectDisplay,setDefectDisplay]=useState<'weight'|'rate'>('weight'); const [fullFields,setFullFields]=useState(true);const [rateBasis,setRateBasis]=useState<'original'|'withoutHeadquartersScrap'>('original');const [basisOpen,setBasisOpen]=useState(false);const [trendMode,setTrendMode]=useState<'production'|'defects'|'data'>('production'); @@ -50,29 +50,39 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU const supportGeneration=useRef(0);const supportController=useRef(null); const loadSupport=useCallback(async()=>{ const generation=++supportGeneration.current;supportController.current?.abort();const controller=new AbortController();supportController.current=controller; - setSupportLoading(true);setSupportError(''); - try{const [choices,source,filterFields]=await Promise.all([api.options(controller.signal),api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal),api.filterFields(false,controller.signal)]);if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);setCatalog(source);setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));} - catch(error){if(generation===supportGeneration.current&&!onUnauthorized(error))setSupportError(error instanceof Error?error.message:'查询条件读取失败');} - finally{if(generation===supportGeneration.current)setSupportLoading(false);} + setSupportError(''); + const fail=(error:unknown)=>{if(generation===supportGeneration.current&&!controller.signal.aborted&&!onUnauthorized(error))setSupportError(previous=>previous||(error instanceof Error?error.message:'查询条件读取失败'));}; + const requests=[ + api.options(controller.signal).then(choices=>{if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);}).catch(fail), + api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal).then(source=>{if(generation===supportGeneration.current)setCatalog(source);}).catch(fail), + api.filterFields(false,controller.signal).then(filterFields=>{if(generation===supportGeneration.current)setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));}).catch(fail), + ]; + await Promise.all(requests); },[onUnauthorized,initial.releaseId,currentReleaseId]); useEffect(()=>{if(active)void loadSupport();return()=>{supportGeneration.current++;supportController.current?.abort();};},[active,refreshKey,loadSupport]); const fields=useMemo(()=>catalog?.sourceFields.filter(field=>field.type==='source_value'&&field.stages.includes(stage))||[],[catalog,stage]); const exactProductSearch=useMemo(()=>(options?.products||[]).find(name=>name.trim().toLocaleLowerCase()===appliedSearch.trim().toLocaleLowerCase())||'', [options,appliedSearch]); const queryProducts=useMemo(()=>drilledProduct?[drilledProduct]:exactProductSearch?product.length&&!product.includes(exactProductSearch)?['__codex_no_product_search_match__']:[exactProductSearch]:product,[drilledProduct,exactProductSearch,product]); useEffect(()=>{setDrilledProduct('');},[department,group,product,projectGroup,sourceFilters,extraFilters]); - const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:initial.releaseId||currentReleaseId||catalog?.releaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,catalog,stage,sourceFilters,extraFilters,initial.releaseId,currentReleaseId]); + const queryReleaseId=initial.releaseId||(currentReleaseId!==undefined?currentReleaseId||undefined:catalog?.releaseId); + const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,queryReleaseId,stage,sourceFilters,extraFilters]); + const catalogMatchesDraft=!!catalog&&!!draft.releaseId&&catalog.releaseId===draft.releaseId; + const queryReady=(query:QueryParams)=>!!queryReleaseId&&query.releaseId===queryReleaseId&&(!query.productSearch||!!options)&&(!(query.columnFilters||[]).length||!!catalog&&catalog.releaseId===query.releaseId); + const draftReady=queryReady(draft); + const appliedReady=!!applied&&queryReady(applied); // Apply organizational/source filters directly, like the other query pages. Product text search keeps its explicit submit. - useEffect(()=>{if(active&&catalog&&!supportLoading&&!supportError)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,catalog,draft,supportLoading,supportError]); + useEffect(()=>{if(active&&draftReady)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,draft,draftReady]); const queryKey=JSON.stringify({query:applied,revision});const current=response?.key===queryKey?response.value:null;const error=failure?.key===queryKey?failure.message:''; useEffect(()=>{ - if(!active||!applied||supportError)return;const controller=new AbortController();setLoading(true);setFailure(null); + if(!active||!applied||!appliedReady)return;const controller=new AbortController();setLoading(true);setFailure(null); void api.metrics(applied,controller.signal).then(value=>{if(!controller.signal.aborted)setResponse({key:queryKey,value});}).catch(e=>{if(!controller.signal.aborted&&!onUnauthorized(e))setFailure({key:queryKey,message:e instanceof Error?e.message:'查询失败'});}).finally(()=>{if(!controller.signal.aborted)setLoading(false);}); return()=>controller.abort(); - },[active,queryKey,onUnauthorized,supportError]); + },[active,queryKey,onUnauthorized,appliedReady]); useEffect(()=>{if(!active||!applied)return;const bytes=new TextEncoder().encode(JSON.stringify({query:queryForLocation(applied,initial.releaseId)}));const encoded=btoa(Array.from(bytes,b=>String.fromCharCode(b)).join('')).replace(/=/g,'').replace(/\+/g,'-').replace(/\//g,'_');const url=`?page=${stage==='black'?'blackProduct':'brightProduct'}&context=${encoded}`;window.history.replaceState({},'',`${location.pathname}${url}`);onLocationChange(url);},[active,applied,stage]); const trendQuery=useMemo(()=>applied?{...applied,month:undefined,quarter:undefined,...(trendGranularity==='year'?{year:undefined}:{}),trendGranularity}:null,[applied,trendGranularity]); const trendKey=JSON.stringify({query:trendQuery,revision}); - useEffect(()=>{if(!active||!trendQuery||supportError)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,supportError]); + const trendReady=!!trendQuery&&queryReady(trendQuery); + useEffect(()=>{if(!active||!trendQuery||!trendReady)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,trendReady]); const visibleTrend=useMemo(()=>fullYearTrend(trendResponse?.key===trendKey?trendResponse.value.trend:[],applied?.year||year,trendGranularity),[trendResponse,trendKey,applied?.year,year,trendGranularity]); const displayedTrend=useRetainedResult(trendResponse?.key===trendKey?visibleTrend:null)||visibleTrend; const selectedTrendPeriod=trendGranularity==='month'&&applied?.month @@ -80,12 +90,14 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU : trendGranularity==='quarter'&&applied?.quarter ? `${applied.year}-Q${applied.quarter}` : undefined; const selectTrendPeriod=(value:number|string)=>{automaticPeriod.current=false;const next=nextTrendPeriod(value,{period,year,month,quarter});setYear(next.year);setMonth(next.month);setQuarter(next.quarter);setPeriod(next.period);}; const dirty=!!applied&&JSON.stringify(draft)!==JSON.stringify(applied); - const queryBusy=supportLoading||loading||(!error&&!supportError&&(dirty||!current)); - const queryFailed=!!(error||supportError); + const metadataPending=!draftReady; + const metadataFailed=!draftReady&&!!supportError; + const queryBusy=!metadataFailed&&(metadataPending||loading||(!error&&(dirty||!current))); + const queryFailed=!!error||metadataFailed; const submit=()=>{automaticPeriod.current=false;setAppliedSearch(search.trim());setDrilledProduct('');setRevision(v=>v+1);}; const setTrendObject=(value?:string)=>{const selected=value||'';setDrilledProduct('');setSearch(selected);setAppliedSearch(selected);}; const returnToProducts=()=>{setDrilledProduct('');setSearch(drillParentSearch.current);setAppliedSearch(drillParentSearch.current);if(trendMode!=='defects')setTrendMode('production');}; - const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:catalog?.releaseId},stage,[]));setRevision(v=>v+1);}; + const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,[]));setRevision(v=>v+1);}; const groups=(options?.groups||[]).filter(name=>!department.length||department.includes(options?.groupDepartments?.[name]||'')); const paths=options?.organizationPaths||[];const products=department.length||group.length?[...new Set(paths.filter(path=>(!department.length||department.includes(path.department||''))&&(!group.length||group.includes(path.group||''))).map(path=>path.product).filter((v):v is string=>!!v))]:options?.products||[]; const trendOptions=useMemo(()=>[...products.map(value=>({label:value,value})),...(drilledProduct==='__UNASSIGNED__'?[{label:'未分类',value:drilledProduct}]:[])],[products,drilledProduct]); @@ -124,9 +136,9 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
{queryBusy?'正在查询':queryFailed?'查询失败':'查询完成'}
{moreFiltersOpen&&
- {extraFields.map(field=>)} + {draft.releaseId&&extraFields.map(field=>)} {options?.projectGroupStatus==='READY'&&select('项目组',projectGroup,options.projectGroups||[],setProjectGroup,true)} - {catalog&&} + {catalogMatchesDraft&&}
} {error&&setRevision(v=>v+1)}>重试}/>} @@ -150,7 +162,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU } - busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!catalog||!!supportError} + busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!options} fullFields={fullFields} onFullFields={setFullFields} defectDisplay={defectDisplay} onDefectDisplay={setDefectDisplay} returnControls={drilledProduct&&} exporting={exporting} exportDisabled={!!defectDrill.category&&(!defectDrill.table||defectDrill.pending||!!defectDrill.error)||!current||loading||!tableReady||!table?.rows.length} onExport={()=>void runExport()}> diff --git a/web/src/StageSourceFilters.tsx b/web/src/StageSourceFilters.tsx index 8653fec..303e7fd 100644 --- a/web/src/StageSourceFilters.tsx +++ b/web/src/StageSourceFilters.tsx @@ -11,13 +11,15 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S const [search,setSearch]=useState('');const [choices,setChoices]=useState([]);const [busy,setBusy]=useState(false);const [error,setError]=useState('');const [more,setMore]=useState(false);const [revision,setRevision]=useState(0); const {active}=usePageCache(); const showBusy=useDelayedIndicator(busy); - const contextKey=JSON.stringify(query);const current=useRef(0); + const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query; + const period={year,...(month?{month}:{}),...(quarter?{quarter}:{})}; + const filters={...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)}; + const contextKey=JSON.stringify({releaseId,period,filters,field:field.key,stage});const current=useRef(0); useEffect(()=>{ if(!active)return; const generation=++current.current,controller=new AbortController();setBusy(true);setError('');setChoices([]); const timer=window.setTimeout(()=>{ - const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query; - void api.sourceColumnValues({releaseId,period:{year,...(month?{month}:{}),...(quarter?{quarter}:{})},filters:{...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)},field:field.key,stages:[stage],search,limit:50,offset:0},controller.signal) + void api.sourceColumnValues({releaseId,period,filters,field:field.key,stages:[stage],search,limit:50,offset:0,candidateOnly:true},controller.signal) .then(result=>{if(current.current!==generation)return;setChoices(result.values.filter((v):v is string=>typeof v==='string'));setMore(result.hasMore===true||(result.total??0)>50);}) .catch(failure=>{if(controller.signal.aborted||current.current!==generation)return;if(!onUnauthorized(failure))setError(failure instanceof Error?failure.message:'选项读取失败');}) .finally(()=>{if(current.current===generation)setBusy(false);}); diff --git a/web/src/api.ts b/web/src/api.ts index e0a6c56..63db6e0 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -239,7 +239,7 @@ export const api = { }, activateSalesOrderTypes(versionId:string,expectedVersion:string) { return mutate<{versionId:string}>('/api/sales-order-types/activate',{versionId,expectedVersion}); }, sourceColumnCatalog(releaseId?: string, signal?: AbortSignal) { return request<{releaseId:string;sourceFields:SourceColumnField[]}>(`/api/analysis/source/catalog?${toQuery({releaseId})}`, { signal }); }, - sourceColumnValues(body: Record, signal?: AbortSignal) { return queryPost<{values:Array;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); }, + sourceColumnValues(body: Record & {candidateOnly?:true}, signal?: AbortSignal) { return queryPost<{values:Array;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); }, workspaceState() { return request<{ authRevision: string; dataRevision: string; currentReleaseId: string | null }>('/api/workspace/state', {}, 8000); }, filterFields(manage = false, signal?: AbortSignal) { return request(`/api/filter-fields?manage=${manage}`, { signal }); }, filterFieldOptions(key: string, search = '', signal?: AbortSignal,releaseId?:string,stage?:string) { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search,releaseId,stage})}`, { signal }); }, diff --git a/web/verification/stage_product_filter_speed.test.mjs b/web/verification/stage_product_filter_speed.test.mjs new file mode 100644 index 0000000..182d16e --- /dev/null +++ b/web/verification/stage_product_filter_speed.test.mjs @@ -0,0 +1,53 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import ts from 'typescript'; + +const read=name=>fs.readFileSync(new URL(`../src/${name}`,import.meta.url),'utf8'); + +test('stage product common queries use a resolved release and do not wait for source catalog when unfiltered',()=>{ + const source=read('StageProductPage.tsx'); + const releaseExpression=source.match(/const queryReleaseId=([^;]+);/)[1]; + const resolvedRelease=new Function('initial','currentReleaseId','catalog',`return ${releaseExpression}`); + assert.equal(resolvedRelease({releaseId:'historical'},'current',{releaseId:'catalog'}),'historical'); + assert.equal(resolvedRelease({},'current',{releaseId:'catalog'}),'current'); + assert.equal(resolvedRelease({},null,{releaseId:'stale'}),undefined); + assert.equal(resolvedRelease({},undefined,{releaseId:'resolved'}),'resolved'); + const expression=source.match(/const queryReady=\(query:QueryParams\)=>((?:[^;]|\n)+);/)[1]; + const ready=new Function('query','queryReleaseId','catalog','options',`return ${expression}`); + assert.equal(ready({releaseId:'r1',columnFilters:[]},'r1',null,null),true); + assert.equal(ready({releaseId:'r1',columnFilters:[]},undefined,null,null),false); + assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',null,null),false); + assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',{releaseId:'r1'},null),true); + assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r2',{releaseId:'r1'},null),false); + assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,null),false); + assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,{}),true); + assert.match(source,/if\(active&&draftReady\)setApplied/); + assert.doesNotMatch(source,/if\(!active\|\|!applied\|\|supportError\)/); +}); + +test('source option request context excludes its own selected values and tracks other filters',()=>{ + const source=read('StageSourceFilters.tsx'); + const declarations=source.match(/ const \{year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters\}=query;\n const period=.*;\n const filters=.*;\n const contextKey=[^;]+;/)[0]; + const run=new Function('query','field','stage',`${ts.transpileModule(declarations,{compilerOptions:{target:ts.ScriptTarget.ES2022}}).outputText};return {filters,contextKey};`); + const base={year:2026,month:4,releaseId:'r1',department:'D1',columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['North']}]}; + const first=run(base,{key:'customer'},'black'); + const ownValueChanged=run({...base,columnFilters:[{field:'customer',values:['OTHER']},{field:'region',values:['North']}]},{key:'customer'},'black'); + const otherValueChanged=run({...base,columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['South']}]},{key:'customer'},'black'); + assert.deepEqual(first.filters.columnFilters,[{field:'region',values:['North']}]); + assert.equal(first.contextKey,ownValueChanged.contextKey); + assert.notEqual(first.contextKey,otherValueChanged.contextKey); +}); + +test('source value dropdowns request candidate-only results while preserving search and debounce behavior',()=>{ + const source=read('StageSourceFilters.tsx'); + assert.match(source,/sourceColumnValues\(\{releaseId,period,filters,field:field\.key,stages:\[stage\],search,limit:50,offset:0,candidateOnly:true\}/); + assert.match(source,/search\?300:0/); + assert.match(source,/result\.values\.filter\(\(v\):v is string=>typeof v==='string'\)/); + assert.match(read('api.ts'),/sourceColumnValues\(body: Record & \{candidateOnly\?:true\}/); +}); + + +test('ordinary product search only depends on product options', () => { + assert.match(read('StageProductPage.tsx'), /searchUnavailable=\{!options\}/); +}); From 5e02c6ad6667dff9b3a6a28d899843e8b76830a7 Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 18:20:32 +0800 Subject: [PATCH 04/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=EF=BC=9A=E7=AD=9B?= =?UTF-8?q?=E9=80=89=E7=B4=A2=E5=BC=95=E6=8C=89=E6=9A=82=E5=AD=98=E7=AA=97?= =?UTF-8?q?=E5=8F=A3=E8=AF=BB=E5=8F=96=E5=AD=97=E5=85=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../quality/filters/SourceFilterIndex.java | 8 +++++--- .../filters/SourceColumnQueriesMySqlTest.java | 20 +++++++++++++++++++ docs/当前进度.md | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java index 72628f2..df529c0 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java @@ -11,6 +11,8 @@ import org.springframework.jdbc.datasource.DataSourceTransactionManager; public class SourceFilterIndex { public record BuildAvailable(String buildId) {} private static final int FORMAT = 1; + static final String MEMBER_INSERT_SQL="INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash"; + static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)"; private final JdbcTemplate jdbc; private volatile Boolean available; public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; } @@ -44,7 +46,7 @@ public class SourceFilterIndex { jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build); jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build); // Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source. - jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash)) ENGINE=InnoDB"); + jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB"); try { for(String stage:List.of("black","bright"))buildStage(build,stage); jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT); @@ -69,9 +71,9 @@ public class SourceFilterIndex { jdbc.update("DELETE FROM source_filter_staging"); jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end); jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash"); - Integer collisions=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)",Integer.class); + Integer collisions=jdbc.queryForObject(COLLISION_SQL,Integer.class); if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布"); - jdbc.update("INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash"); + jdbc.update(MEMBER_INSERT_SQL); after=end; } } diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 6ced49e..06d8b6e 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -465,6 +465,26 @@ class SourceColumnQueriesMySqlTest { assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList()); } + @Test void memberBuildPlansStartFromTheBoundedStageWindowRatherThanTheGrowingDictionary() throws Exception { + installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black"); + jdbc.execute((org.springframework.jdbc.core.ConnectionCallback)connection->{ + try(var statement=connection.createStatement()) { + statement.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB"); + try { + statement.execute("INSERT INTO source_filter_staging SELECT v.field_id,m.fact_id,v.value,v.value_hash FROM source_filter_value v JOIN source_filter_member m ON m.value_id=v.id LIMIT 4"); + for(String sql:List.of(SourceFilterIndex.MEMBER_INSERT_SQL,SourceFilterIndex.COLLISION_SQL)) { + try(var rows=statement.executeQuery("EXPLAIN "+sql)) { + String first=null; + while(rows.next())if(List.of("s","v").contains(rows.getString("table"))){first=rows.getString("table");break;} + assertEquals("s",first,"每批构建只能从有界临时窗口开始,不能反复扫描历史字典"); + } + } + } finally {statement.execute("DROP TEMPORARY TABLE source_filter_staging");} + } + return null; + }); + } + @Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception { installFilterIndex(); var fail=new java.util.concurrent.atomic.AtomicBoolean(true); diff --git a/docs/当前进度.md b/docs/当前进度.md index aee40fb..096b0d2 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -9,7 +9,7 @@ 生命周期:启动仅在单线程后台补当前完整发布版本;导入和归类通过提交后事件进入同一有界去重队列。完整标记提交后才启用,故障保留原始查询,业务事务不等待索引。旧配置 `null`、来源字符串 `null`、空值、Unicode、同一事实行和当前权限分别保持;构建保留清理识别4张新表,未解除维护兼容保护。 -验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 +验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上2026.11.13.6已生效,后台补建期间发现每批全扫历史字典,实时计划约85万词条先行;改为500条事实窗口驱动及临时覆盖索引,专项47项(46通过、1跳过)及独立审查通过,准备2026.11.13.7补丁。已完成索引格式不变可复用。补建期间临时将数据库缓存从256MB调到1GB,验收前必须恢复原值;记录保存在本机私有发布目录。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 ## 2026-11-13:来源字段筛选超时修复 From 7bd9e51004c0e7db47c532ab2d2183c903214fb6 Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 18:37:11 +0800 Subject: [PATCH 05/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=EF=BC=9A=E6=8C=89?= =?UTF-8?q?=E6=89=B9=E6=AC=A1=E9=87=8D=E7=BD=AE=E7=AD=9B=E9=80=89=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=E4=B8=B4=E6=97=B6=E7=AA=97=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cn/yunsheng/quality/filters/SourceFilterIndex.java | 9 ++++++--- .../quality/filters/SourceColumnQueriesMySqlTest.java | 9 +++++---- docs/当前进度.md | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java index df529c0..02dbb19 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java @@ -11,6 +11,7 @@ import org.springframework.jdbc.datasource.DataSourceTransactionManager; public class SourceFilterIndex { public record BuildAvailable(String buildId) {} private static final int FORMAT = 1; + static final String CREATE_STAGE_SQL="CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB"; static final String MEMBER_INSERT_SQL="INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash"; static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)"; private final JdbcTemplate jdbc; @@ -46,11 +47,10 @@ public class SourceFilterIndex { jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build); jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build); // Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source. - jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB"); try { for(String stage:List.of("black","bright"))buildStage(build,stage); jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT); - } finally { jdbc.execute("DROP TEMPORARY TABLE source_filter_staging"); } + } finally { jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging"); } }); } @@ -68,7 +68,10 @@ public class SourceFilterIndex { jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end); String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')"; String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?"; - jdbc.update("DELETE FROM source_filter_staging"); + // DROP/CREATE TEMPORARY do not commit the owning build transaction. Reset the + // physical table too: DELETE retains undo/deleted records until that transaction ends. + jdbc.execute("DROP TEMPORARY TABLE IF EXISTS source_filter_staging"); + jdbc.execute(CREATE_STAGE_SQL); jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end); jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash"); Integer collisions=jdbc.queryForObject(COLLISION_SQL,Integer.class); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 06d8b6e..4d1a681 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -469,7 +469,7 @@ class SourceColumnQueriesMySqlTest { installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black"); jdbc.execute((org.springframework.jdbc.core.ConnectionCallback)connection->{ try(var statement=connection.createStatement()) { - statement.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash,fact_id)) ENGINE=InnoDB"); + statement.execute(SourceFilterIndex.CREATE_STAGE_SQL); try { statement.execute("INSERT INTO source_filter_staging SELECT v.field_id,m.fact_id,v.value,v.value_hash FROM source_filter_value v JOIN source_filter_member m ON m.value_id=v.id LIMIT 4"); for(String sql:List.of(SourceFilterIndex.MEMBER_INSERT_SQL,SourceFilterIndex.COLLISION_SQL)) { @@ -487,14 +487,15 @@ class SourceColumnQueriesMySqlTest { @Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception { installFilterIndex(); - var fail=new java.util.concurrent.atomic.AtomicBoolean(true); + for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}"); + var memberWrites=new java.util.concurrent.atomic.AtomicInteger(); var failing=new JdbcTemplate(jdbc.getDataSource()) { @Override public int update(String sql) { - if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure"); + if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure"); return super.update(sql); } @Override public int update(String sql,Object... args) { - if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure"); + if(sql.startsWith("INSERT INTO source_filter_member")&&memberWrites.incrementAndGet()==2)throw new IllegalStateException("synthetic member write failure"); return super.update(sql,args); } }; diff --git a/docs/当前进度.md b/docs/当前进度.md index 096b0d2..23cc969 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -9,7 +9,7 @@ 生命周期:启动仅在单线程后台补当前完整发布版本;导入和归类通过提交后事件进入同一有界去重队列。完整标记提交后才启用,故障保留原始查询,业务事务不等待索引。旧配置 `null`、来源字符串 `null`、空值、Unicode、同一事实行和当前权限分别保持;构建保留清理识别4张新表,未解除维护兼容保护。 -验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上2026.11.13.6已生效,后台补建期间发现每批全扫历史字典,实时计划约85万词条先行;改为500条事实窗口驱动及临时覆盖索引,专项47项(46通过、1跳过)及独立审查通过,准备2026.11.13.7补丁。已完成索引格式不变可复用。补建期间临时将数据库缓存从256MB调到1GB,验收前必须恢复原值;记录保存在本机私有发布目录。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 +验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上2026.11.13.6已生效,后台补建期间发现每批全扫历史字典,实时计划约85万词条先行;改为500条事实窗口驱动及临时覆盖索引,专项47项(46通过、1跳过)及独立审查通过,已发布2026.11.13.7(`5e02c6a`),完整构建986项(819通过、167跳过)。进一步将每批临时表物理重建,避免长事务中DELETE遗留窗口历史;第二窗口故障回滚及独立审查通过,准备2026.11.13.8。已完成索引格式不变可复用。补建期间临时将数据库缓存从256MB调到1GB,验收前必须恢复原值;记录保存在本机私有发布目录。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 ## 2026-11-13:来源字段筛选超时修复 From fa70dc65049989d3d8a74327b42987ac370aa1bf Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 19:19:18 +0800 Subject: [PATCH 06/11] Cover source candidate scope checks without wide fact row reads --- .../quality/filters/SourceFilterIndex.java | 11 ++++++++- .../migration/V44__source_candidate_scope.sql | 6 +++++ .../filters/SourceColumnQueriesMySqlTest.java | 23 +++++++++++++++++-- 3 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 backend/src/main/resources/db/migration/V44__source_candidate_scope.sql diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java index 02dbb19..3c56947 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java @@ -16,6 +16,7 @@ public class SourceFilterIndex { static final String COLLISION_SQL="SELECT COUNT(*) FROM source_filter_staging s STRAIGHT_JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)"; private final JdbcTemplate jdbc; private volatile Boolean available; + private volatile Boolean candidateScopeAvailable; public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; } public boolean available() { @@ -26,6 +27,14 @@ public class SourceFilterIndex { return available; } + private String candidateFactHint(String stage) { + if (candidateScopeAvailable == null) { + Integer count=jdbc.queryForObject("SELECT COUNT(DISTINCT table_name) FROM information_schema.statistics WHERE table_schema=DATABASE() AND ((table_name='fact_black' AND index_name='ix_black_candidate_scope') OR (table_name='fact_bright' AND index_name='ix_bright_candidate_scope'))",Integer.class); + candidateScopeAvailable=count!=null&&count==2; + } + return candidateScopeAvailable ? " FORCE INDEX (ix_"+stage+"_candidate_scope)" : ""; + } + public boolean ready(String release) { if(release==null||!available())return false; // Only successful immutable builds are retained; a pending backfill is checked again later. @@ -121,7 +130,7 @@ public class SourceFilterIndex { public String candidateSelect(String stage,String column,String scope,List scopeArgs,List args) { args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs); return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id " - +"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id " + +"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f"+candidateFactHint(stage)+" ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id " +scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)"; } } diff --git a/backend/src/main/resources/db/migration/V44__source_candidate_scope.sql b/backend/src/main/resources/db/migration/V44__source_candidate_scope.sql new file mode 100644 index 0000000..a928672 --- /dev/null +++ b/backend/src/main/resources/db/migration/V44__source_candidate_scope.sql @@ -0,0 +1,6 @@ +-- Keep common candidate authorization/date/product checks off the wide JSON clustered rows. +-- The fact table remains authoritative. Changes to its dimensions update this index automatically. +ALTER TABLE fact_black ADD INDEX ix_black_candidate_scope + (id,build_id,source_revision_id,month_key,department,group_name,product); +ALTER TABLE fact_bright ADD INDEX ix_bright_candidate_scope + (id,build_id,source_revision_id,month_key,department,group_name,product); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 4d1a681..82b0239 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -485,6 +485,23 @@ class SourceColumnQueriesMySqlTest { }); } + @Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception { + installFilterIndex();var index=new SourceFilterIndex(jdbc); + for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build); + var scopeArgs=new java.util.ArrayList(List.of(RELEASE,"2026-01","2026-12","部门A","组A","P-1")); + var args=new java.util.ArrayList(); + String sql=index.candidateSelect("black","委托单"," WHERE rm.release_id=? AND f.month_key BETWEEN ? AND ? AND f.department=? AND f.group_name=? AND f.product=?",scopeArgs,args); + assertEquals(Set.of("0007","0008"),new java.util.HashSet<>(jdbc.queryForList(sql,String.class,args.toArray()))); + boolean covering=false; + for(var row:jdbc.queryForList("EXPLAIN "+sql,args.toArray())) { + if("f".equals(row.get("table"))) { + assertEquals("ix_black_candidate_scope",row.get("key")); + covering=String.valueOf(row.get("Extra")).contains("Using index"); + } + } + assertTrue(covering,"候选值的日期、权限和产品校验不应读取宽 JSON 原始行"); + } + @Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception { installFilterIndex(); for(int n=0;n<501;n++)insert("black","build-black",BLACK_REVISION,"部门A","组A","{\"委托单\":\"two-window-row\"}"); @@ -563,8 +580,10 @@ class SourceColumnQueriesMySqlTest { private void installFilterIndex() throws Exception { jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)"); for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id); - String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/V43__source_filter_indexes.sql")); - for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement); + for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql")) { + String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration)); + for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement); + } } private void schema() { From 20f1c7d0e2dc5f0d366411c2807b4e549180492e Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 20:14:51 +0800 Subject: [PATCH 07/11] Seek the first visible source row without scanning all fact ids --- .../quality/filters/SourceColumnQueries.java | 12 ++++---- .../V45__source_catalog_first_visible.sql | 6 ++++ .../filters/SourceColumnQueriesMySqlTest.java | 28 +++++++++++++++++-- 3 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 backend/src/main/resources/db/migration/V45__source_catalog_first_visible.sql diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java index cde99bc..18fe501 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java @@ -110,16 +110,16 @@ public class SourceColumnQueries { Map stagePresent = new LinkedHashMap<>(); for (String stage : STAGES) { Set headers = new LinkedHashSet<>(); - StringBuilder representativeScope = new StringBuilder(" WHERE rm.release_id=?"); + StringBuilder representativeScope = new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"); List args = new ArrayList<>(List.of(releaseId)); DataScope.append(representativeScope, args, actor, "f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs"); + // Stop at the first authorized fact of each build instead of aggregating every fact id. String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) " - + "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id " - + "FROM release_member rm JOIN fact_" + stage + " f2" + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage)) + " " - + "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id" - + representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative " - + "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id"; + + "FROM (SELECT DISTINCT transform_build_id,source_revision_id FROM release_member WHERE release_id=?) rm " + + "JOIN fact_" + stage + " f ON f.id=(SELECT f2.id FROM fact_" + stage + " f2" + + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage)) + + representativeScope + " ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1) ORDER BY rm.transform_build_id"; List keyLists = rawHeaders.get(new QueryKey(sql,args,"raw-headers"), () -> Collections.unmodifiableList(new ArrayList<>(jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray())))); stagePresent.put(stage, !keyLists.isEmpty()); diff --git a/backend/src/main/resources/db/migration/V45__source_catalog_first_visible.sql b/backend/src/main/resources/db/migration/V45__source_catalog_first_visible.sql new file mode 100644 index 0000000..5e78c2f --- /dev/null +++ b/backend/src/main/resources/db/migration/V45__source_catalog_first_visible.sql @@ -0,0 +1,6 @@ +-- Equality on the owning build/revision followed by fact id supports the first visible row. +-- Keep authorization dimensions covered, without reading wide JSON rows while seeking it. +ALTER TABLE fact_black DROP INDEX ix_black_source_catalog, + ADD INDEX ix_black_source_catalog (build_id,source_revision_id,id,department,group_name); +ALTER TABLE fact_bright DROP INDEX ix_bright_source_catalog, + ADD INDEX ix_bright_source_catalog (build_id,source_revision_id,id,department,group_name); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 82b0239..3926326 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -101,7 +101,7 @@ class SourceColumnQueriesMySqlTest { assertTrue((Boolean)page.get("pageComplete")); } - @Test void groupedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() { + @Test void orderedCatalogKeepsTheSameAuthorizedRepresentativeRowsAsTheLegacyQuery() { record Read(String sql,Object[] args,List result) {} List reads=new java.util.ArrayList<>(); var capturing=new JdbcTemplate(jdbc.getDataSource()) { @@ -114,7 +114,7 @@ class SourceColumnQueriesMySqlTest { assertEquals(2,representatives.size()); for(Read read:representatives) { String stage=read.sql().contains("fact_black f2")?"black":"bright"; - assertTrue(read.sql().contains("GROUP BY f2.build_id,f2.source_revision_id")); + assertTrue(read.sql().contains("ORDER BY f2.build_id,f2.source_revision_id,f2.id LIMIT 1")); assertTrue(read.sql().contains("FORCE INDEX (uq_fact_"+stage+"_origin)"),"来源目录必须按已发布构建读取索引,避免扫描整张原表"); StringBuilder scope=new StringBuilder(" WHERE f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"); List args=new java.util.ArrayList<>(); @@ -485,6 +485,28 @@ class SourceColumnQueriesMySqlTest { }); } + @Test void catalogStopsAtTheFirstAuthorizedFactWithoutSortingEveryFactId() throws Exception { + installFilterIndex(); + var captured=new java.util.ArrayList();var parameters=new java.util.ArrayList(); + var tracking=new JdbcTemplate(jdbc.getDataSource()) { + @Override public List query(String sql,org.springframework.jdbc.core.RowMapper mapper,Object... args) { + if(sql.startsWith("SELECT CAST(JSON_KEYS")){captured.add(sql);parameters.add(args);} + return super.query(sql,mapper,args); + } + }; + var catalog=new SourceColumnQueries(tracking,governance,JsonMapper.builder().build()).catalog(RELEASE,actor); + assertNotNull(field(catalog,"委托单")); + assertNull(field(catalog,"组织外行列"),"首条全局记录不可见时,必须继续寻找当前用户的第一条可见记录"); + assertNull(field(catalog,"其他组织列")); + assertEquals(2,captured.size()); + for(int i=0;iline.contains("Sort:")&&line.contains("f2.")),plan); + assertTrue(plan.contains("Covering index lookup on f2 using ix_"),plan); + } + } + @Test void candidateScopeLookupUsesACoveringIndexAndPreservesAuthorizedValues() throws Exception { installFilterIndex();var index=new SourceFilterIndex(jdbc); for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build); @@ -580,7 +602,7 @@ class SourceColumnQueriesMySqlTest { private void installFilterIndex() throws Exception { jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)"); for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id); - for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql")) { + for(String migration:List.of("V43__source_filter_indexes.sql","V44__source_candidate_scope.sql","V45__source_catalog_first_visible.sql")) { String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/"+migration)); for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement); } From 7cae4cfbdfc19790a1ce251f446a8c328b9ee064 Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 20:32:16 +0800 Subject: [PATCH 08/11] Record accepted filter-speed release and measured index storage --- docs/当前进度.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/当前进度.md b/docs/当前进度.md index 23cc969..eac97da 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -1,15 +1,23 @@ # 当前进度 -核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。 +核实日期:2026年11月13日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。 -## 2026-11-13:统一筛选提速(发布验收中) +## 2026-11-13:统一筛选提速(已发布验收) -实现:V43新增按构建保存的来源字段字典和稀疏记录索引,及两阶段字段目录覆盖索引;来源筛选、配置筛选、候选、指标、趋势、明细与明细导出共用。来源列查询实例统一复用,候选模式省去精确总数,产品页常规查询不再等待慢目录,本字段选中值变化不重复读取自己的候选;明细导出先只读字段名再读业务行。 +已发布2026.11.13.10,功能提交 `20f1c7d`;当前正式版52/52个构建的筛选索引全部就绪。新增按构建保存的来源字段字典与稀疏记录索引,来源/配置筛选、候选、指标、趋势、明细及明细导出共用;正式算法、当前字段/组织权限和发布版绑定保持。候选模式只取上限加一,不再为下拉精确计数;常规页面查询不等待无关目录,本字段改选不重复读取自己的候选,明细导出先只读字段名。 -生命周期:启动仅在单线程后台补当前完整发布版本;导入和归类通过提交后事件进入同一有界去重队列。完整标记提交后才启用,故障保留原始查询,业务事务不等待索引。旧配置 `null`、来源字符串 `null`、空值、Unicode、同一事实行和当前权限分别保持;构建保留清理识别4张新表,未解除维护兼容保护。 +生命周期:启动和构建提交后通过同一单线程有界去重队列补索引,完整提交后才启用;失败或尚未就绪保留原查询,导入事务不等待索引。构建过程按500条事实窗口驱动字典关联,并物理重建临时窗口,避免回扫增长字典和长事务临时表累积。保留清理已识别4张新增表,未解除维护兼容保护。 -验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上2026.11.13.6已生效,后台补建期间发现每批全扫历史字典,实时计划约85万词条先行;改为500条事实窗口驱动及临时覆盖索引,专项47项(46通过、1跳过)及独立审查通过,已发布2026.11.13.7(`5e02c6a`),完整构建986项(819通过、167跳过)。进一步将每批临时表物理重建,避免长事务中DELETE遗留窗口历史;第二窗口故障回滚及独立审查通过,准备2026.11.13.8。已完成索引格式不变可复用。补建期间临时将数据库缓存从256MB调到1GB,验收前必须恢复原值;记录保存在本机私有发布目录。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。 +首次加载还存在两条慢路径:候选为校验年份/可见范围反复读取宽原始行,V44新增薄覆盖索引;字段目录扫描约110万事实只为找每构建首条可见记录,V45改索引列序及带完整权限的有序LIMIT1查询。实际计划回归要求覆盖读取、不排序全部事实、找到首条即停;与旧MIN查询直接对照原始字段目录,保留空/隐藏构建和重复成员语义。 + +验证:前端相关46项及生产构建通过;完整后端988项(819执行通过、169按条件跳过),专项隔离MySQL/服务49项(48通过、1基准跳过),保留清理34项通过,多次独立审查无阻断。隔离场景覆盖黑片/亮片、全部4类来源运算、旧配置null和字面字符串null、Unicode、同一事实行、多字段、当前权限、导出、跨窗口故障回滚与重试。未执行私有原表全量回归。 + +线上实测在恢复原256MiB数据库缓存、后端重新启动且无并行测试时完成:同一客户筛选首次汇总1366ms、趋势347ms、来源明细513ms、原候选条件381ms、去掉自身约束的完整候选首开3546ms;月度458ms、季度369ms。原始超时场景汇总/趋势约48.3秒;前一版索引已全就绪时首次汇总仍18843ms,此次首条目录优化后降到1366ms。汇总、趋势、29条来源明细的完整规范化结果与原基准一致,月/季与原路径一致。Excel约3479ms,1张表、29条明细、A1:EA30,ZIP/XML有效;页面查询完成,来源抽屉载入121字段、无失败提示,点击产品与来源范围一致。以上是代表场景的只读重放,不是数据库完整冷启动或所有字段组合的固定耗时承诺。 + +空间实测:新增字典/成员等4表文件共13,727,023,104字节,事实表4个覆盖索引页约372,064,256字节;本口径合计约14.10GB(13.13GiB),不含既有原表、redo/undo和临时文件,也不等同于数据库目录净增差值。此次覆盖全部原始字段,空间显著高于先前紧凑柏拉图汇总。补建期间临时1GiB缓存已恢复256MiB,核验16384页且调整完成。数据库原进程和DSH配置保留,无业务版本指针变更。 + +实现与验收记录已本地提交;本轮新优化尚未同步Gitea。私有请求、结果校验及页面截图保存在 `output/20261113-filter-speed/`,不进入仓库。主工作目录已有历史改动保持,当前交付在独立分支 `codex/unified-filter-speed-20261113`。 ## 2026-11-13:来源字段筛选超时修复 From 3c022e86147139c6bd3ea89dac312393342c16e7 Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 21:23:12 +0800 Subject: [PATCH 09/11] Add literal contains filters across source queries and analysis context --- .../AnalysisProtocolHistoryFacts.java | 2 +- .../quality/filters/SourceColumnQueries.java | 16 ++++++--- .../quality/filters/SourceFilterIndex.java | 17 ++++++--- .../filters/SourceColumnQueriesMySqlTest.java | 35 ++++++++++++++++--- .../filters/SourceColumnQueriesTest.java | 20 +++++++++-- scripts/nao_agent_task.py | 8 +++-- scripts/nao_evidence_report.py | 3 +- scripts/nao_history_report.py | 3 +- scripts/nao_quality_tools.py | 4 +-- scripts/nao_query_contract.py | 10 +++--- verification/test_nao_evidence_report.py | 9 +++++ verification/test_nao_history_report.py | 12 ++++--- verification/test_nao_quality_tools.py | 8 +++-- verification/test_source_column_protocol.py | 31 ++++++++++++++-- web/src/StageSourceFilters.tsx | 2 +- web/src/analysisVisualsModel.ts | 8 ++--- web/src/types.ts | 2 +- web/verification/analysis_visuals.test.mjs | 9 +++++ web/verification/source_contains.test.mjs | 30 ++++++++++++++++ web/verification/stage_product_query.test.mjs | 6 ++++ 20 files changed, 194 insertions(+), 41 deletions(-) create mode 100644 web/verification/source_contains.test.mjs diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisProtocolHistoryFacts.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisProtocolHistoryFacts.java index 986a072..c13b7eb 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisProtocolHistoryFacts.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisProtocolHistoryFacts.java @@ -891,7 +891,7 @@ final class AnalysisProtocolHistoryFacts { String label=text(columnLabels.get(field),historyFieldLabel(section,sources,field)); if(label.isBlank())label="授权列"; List prefix=strings(rule.get("stages")).stream().map(AnalysisProtocolHistoryFacts::stageLabel).toList(); - String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";}; + String operator=switch(text(rule.get("operator"),"")){case "in"->"包含";case "contains"->"包含文字";case "not_in"->"不含";case "is_empty"->"为空";case "not_empty"->"非空";default->"筛选";}; List values=scopeLiterals(rule.get("values")); StringBuilder part=new StringBuilder();if(!prefix.isEmpty())part.append(String.join("、",prefix)).append(' '); part.append(label).append(' ').append(operator);if(!values.isEmpty())part.append(' ').append(String.join("、",values));parts.add(part.toString()); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java index 18fe501..e4d607b 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceColumnQueries.java @@ -71,7 +71,7 @@ public class SourceColumnQueries { } private static final List STAGES = List.of("black", "bright"); - private static final List OPERATORS = List.of("in", "not_in", "is_empty", "not_empty"); + private static final List OPERATORS = List.of("in", "contains", "not_in", "is_empty", "not_empty"); private static final Map CANONICAL_COLUMNS = Map.of( "product", "product", "department", "department", "group", "group_name", "projectGroup", "project_group"); private static final Map CANONICAL_LABELS = Map.of( @@ -208,12 +208,13 @@ public class SourceColumnQueries { normalizedStages.add(value); } List values = new ArrayList<>(); - if (operator.equals("in") || operator.equals("not_in")) { + if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) { if (!(rule.get("values") instanceof List rawValues) || rawValues.isEmpty() || rawValues.size() > 50) - throw bad("in/not_in 筛选必须提供1到50个字符串值"); + throw bad("文本筛选必须提供1到50个字符串值"); for (Object value : rawValues) { if (!(value instanceof String text) || text.isEmpty() || text.length() > 256) throw bad("来源列筛选值必须是1到256字符的完整字符串;空值请使用空值操作符"); + if(operator.equals("contains")&&text.isBlank())throw bad("包含筛选的关键词不能全为空白"); if (!values.contains(text)) values.add(text); } } else if (rule.containsKey("values")) { @@ -223,7 +224,7 @@ public class SourceColumnQueries { normalized.put("field", field); normalized.put("stages", List.copyOf(normalizedStages)); normalized.put("operator", operator); - if (operator.equals("in") || operator.equals("not_in")) normalized.put("values", List.copyOf(values)); + if (operator.equals("in") || operator.equals("not_in") || operator.equals("contains")) normalized.put("values", List.copyOf(values)); parsed.add(Collections.unmodifiableMap(normalized)); } return List.copyOf(parsed); @@ -280,6 +281,13 @@ public class SourceColumnQueries { } } args.addAll(values); + } else if (operator.equals("contains")) { + for(int i=0;i0)where.append(" OR "); + where.append("LOCATE(?,").append(expression).append(")>0"); + args.add(selected.get(i)); + if("source_value".equals(field.get("type")))bindSourcePath(args,String.valueOf(column)); + } } else { where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL"); if ("source_value".equals(field.get("type"))) bindSourcePath(args, String.valueOf(column)); diff --git a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java index 3c56947..3902b0b 100644 --- a/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java +++ b/backend/src/main/java/cn/yunsheng/quality/filters/SourceFilterIndex.java @@ -96,7 +96,7 @@ public class SourceFilterIndex { if(!ready(release))return false; where.append(" AND "); membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"), - operator.equals("in")||operator.equals("not_in")?values:null,false); + operator.equals("in")||operator.equals("not_in")||operator.equals("contains")?values:null,false,operator.equals("contains")); return true; } @@ -106,20 +106,27 @@ public class SourceFilterIndex { var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList(); where.append(" AND ("); if(values.isEmpty())where.append("FALSE"); - else membership(where,args,stage,release,column,false,values,true); + else membership(where,args,stage,release,column,false,values,true,false); if(selected.contains(QueryFilterFields.EMPTY)) { - where.append(" OR ");membership(where,args,stage,release,column,true,null,true); + where.append(" OR ");membership(where,args,stage,release,column,true,null,true,false); } where.append(')');return true; } private void membership(StringBuilder where,List args,String stage,String release,String column, - boolean negative,List values,boolean legacy) { + boolean negative,List values,boolean legacy,boolean contains) { where.append("f.id ").append(negative?"NOT IN (":"IN (") .append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin"); args.add(release);args.add(stage);args.add(column); if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)"); - if(values!=null) { + if(contains) { + where.append(" AND ("); + for(int i=0;i0)where.append(" OR "); + where.append("LOCATE(?,sv.value COLLATE utf8mb4_0900_as_cs)>0");args.add(values.get(i)); + } + where.append(')'); + } else if(values!=null) { where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')'); args.addAll(values); } diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 3926326..89fb661 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -266,8 +266,8 @@ class SourceColumnQueriesMySqlTest { } @org.junit.jupiter.params.ParameterizedTest - @org.junit.jupiter.params.provider.ValueSource(booleans={false,true}) - void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed) throws Exception { + @org.junit.jupiter.params.provider.CsvSource({"false,in","true,in","false,contains","true,contains"}) + void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed,String operator) throws Exception { jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))"); jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE); jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))"); @@ -293,7 +293,7 @@ class SourceColumnQueriesMySqlTest { var records=new cn.yunsheng.quality.records.RecordController(jdbc,releases,mapper,governance,exporter); String fieldKey=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key"); for(String stage:List.of("black","bright")) { - var filter=Map.of("field",fieldKey,"stages",List.of(stage),"operator","in","values",List.of("0007")); + var filter=Map.of("field",fieldKey,"stages",List.of(stage),"operator",operator,"values",List.of(operator.equals("contains")?"007":"0007")); var base=new cn.yunsheng.quality.metrics.MetricsService.MetricsQuery(Integer.valueOf(2026),Integer.valueOf(1),null,"部门A","组A",null,null,"product",false,RELEASE,"month",Map.of()).withColumnFilters(List.of(filter)); var legacy=service.calculate(base,actor);var selected=service.calculate(base.withStage(stage),actor); assertEquals(1,selected.rows().size());assertEquals(1L,((Number)selected.coverage().get("includedRows")).longValue()); @@ -429,7 +429,7 @@ class SourceColumnQueriesMySqlTest { insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}"); String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key"); List> rules=new java.util.ArrayList<>(); - for(String op:List.of("in","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}")) + for(String op:List.of("in","contains","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}")) rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value))); var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList(); var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor); @@ -465,6 +465,33 @@ class SourceColumnQueriesMySqlTest { assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList()); } + @Test void containsMatchesLiteralSubstringsWithOrSemanticsInBothStagesAndBothQueryPaths() throws Exception { + when(governance.allowedFields(actor)).thenReturn(Set.of("*")); + var mapper=JsonMapper.builder().build(); + for(String stage:List.of("black","bright")) { + String revision=stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION; + for(String value:List.of("xxAlpha%_\\tail","中文日立公司","prefix x' OR 1=1 -- suffix","000123","null")) + insert(stage,"build-"+stage,revision,"部门A","组A",mapper.writeValueAsString(Map.of("委托单",value))); + insert(stage,"build-"+stage,revision,"部门B","组B",mapper.writeValueAsString(Map.of("委托单","日立-Alpha-hidden"))); + } + String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key"); + List> needles=List.of(List.of("Alpha"),List.of("alpha"),List.of("%"),List.of("_"),List.of("\\"),List.of("日立"),List.of("x' OR 1=1 --"),List.of("001"),List.of("null"),List.of("日立","Alpha")); + List expected=List.of(1L,0L,1L,1L,1L,1L,1L,1L,1L,2L); + for(boolean indexed:List.of(false,true)) { + if(indexed) { + installFilterIndex();var index=new SourceFilterIndex(jdbc); + for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build); + queries=new SourceColumnQueries(jdbc,governance,mapper); + } + for(String stage:List.of("black","bright"))for(int n=0;nof("field",key,"stages",List.of(stage),"operator","contains","values",needles.get(n)); + assertEquals(expected.get(n),countFiltered(stage,List.of(rule)),stage+" indexed="+indexed+" keywords="+needles.get(n)); + } + var canonical=Map.of("field","product","stages",List.of("black"),"operator","contains","values",List.of("P-")); + assertTrue(countFiltered("black",List.of(canonical))>0); + } + } + @Test void memberBuildPlansStartFromTheBoundedStageWindowRatherThanTheGrowingDictionary() throws Exception { installFilterIndex();var index=new SourceFilterIndex(jdbc);index.ensureBuild("build-black"); jdbc.execute((org.springframework.jdbc.core.ConnectionCallback)connection->{ diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java index 5acd01f..507bd4b 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java @@ -32,7 +32,7 @@ class SourceColumnQueriesTest { @Test void rejectsUnknownShapeOperatorsEmptyValuesAndMoreThanFiftyValues() { assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(Map.of())); assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of( - "field", SOURCE, "stages", List.of("black"), "operator", "contains", "values", List.of("x"))))); + "field", SOURCE, "stages", List.of("black"), "operator", "starts_with", "values", List.of("x"))))); assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of( "field", SOURCE, "stages", List.of("black"), "operator", "in", "values", List.of(""))))); assertThrows(ResponseStatusException.class, () -> SourceColumnQueries.parse(List.of(Map.of( @@ -42,6 +42,22 @@ class SourceColumnQueriesTest { "field", SOURCE, "stages", List.of("bright"), "operator", "not_in", "values", tooMany)))); } + @Test void containsKeepsLiteralKeywordsAndBindsEachJsonPathAfterItsNeedle() { + var rule=Map.of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",List.of("000","%_\\","x' OR 1=1 --","000")); + var parsed=SourceColumnQueries.parse(List.of(rule)); + assertEquals(List.of("000","%_\\","x' OR 1=1 --"),parsed.getFirst().get("values")); + var source=spy(new SourceColumnQueries(mock(JdbcTemplate.class),governance,JsonMapper.builder().build())); + doReturn(List.of(sourceField())).when(source).scopedCatalog("release",actor); + var sql=new StringBuilder();var args=new ArrayList(); + source.append(sql,args,"black",parsed,"release",actor); + assertEquals(List.of("000","$.\"委托单\"","%_\\","$.\"委托单\"","x' OR 1=1 --","$.\"委托单\""),args); + assertEquals(3,sql.toString().split("LOCATE",-1).length-1); + assertFalse(sql.toString().contains("x' OR")); + for(Object values:List.of(List.of(),List.of(""),List.of(" "),List.of("x".repeat(257)),List.of(7))) { + assertThrows(ResponseStatusException.class,()->SourceColumnQueries.parse(List.of(Map.of("field",SOURCE,"stages",List.of("black"),"operator","contains","values",values)))); + } + } + @Test void monthRangeUsesInclusiveYearBoundariesAndKeepsLegacyMonthPredicate() { Map range=SourceColumnQueries.period(Map.of("period",Map.of("year",2026,"monthStart",3,"monthEnd",9))); StringBuilder rangeSql=new StringBuilder();List rangeArgs=new ArrayList<>(); @@ -222,6 +238,6 @@ class SourceColumnQueriesTest { columns.put("black", stage.equals("black") ? header : null); columns.put("bright", stage.equals("bright") ? header : null); return Map.of("key", key, "label", header, "columns", columns, "stages", List.of(stage), - "type", "source_value", "operators", List.of("in", "not_in", "is_empty", "not_empty")); + "type", "source_value", "operators", List.of("in", "contains", "not_in", "is_empty", "not_empty")); } } diff --git a/scripts/nao_agent_task.py b/scripts/nao_agent_task.py index f24fd13..1f5cef6 100644 --- a/scripts/nao_agent_task.py +++ b/scripts/nao_agent_task.py @@ -144,6 +144,8 @@ def _resolve_source_column_values(task, *, release_id, resolver): if not isinstance(rules, list): return None for rule_index, rule in enumerate(rules): + # Candidate lookup confirms complete values only. A contains keyword is + # intentionally not required to equal a full candidate source value. if rule.get('operator') not in ('in', 'not_in'): continue field_key = rule['field'] @@ -1212,7 +1214,7 @@ def plan_task(*, base_url, api_key, model, question, context, previous=None, can if aliases and field.get('type')=='source_value':item['aliases']=aliases compact_fields.append(item) compact_catalog['sourceFields']=compact_fields - compact_catalog['sourceColumnOperators']=['in','not_in','is_empty','not_empty'] + compact_catalog['sourceColumnOperators']=['in','not_in','contains','is_empty','not_empty'] payload['businessCatalog']=compact_catalog system += ('\n综合以上生成报告应整合previousVerifiedTurns中多个已完成轮次,historyMode=all、freshQuery=false;不得把最新局部追问当作全部历史。' '只有用户要求更新数据时才freshQuery=true。报告与复述不自动开展新调查。' @@ -1292,9 +1294,9 @@ JSON字段:goal(完整当前目标),kind(analysis/metrics/members/explain/re filterUpdates允许department/group/product/projectGroup/extraFilters/exclusions/includeUnassigned。null明确清除该条件,{}表示没有变化;单一维度用目录原名字符串,extraFilters和exclusions用字段到值数组。额外字段仅限businessCatalog.extraFields;客户代码使用extraFilters.customerCode,保留完整字符串。未知字段不能丢弃或扩大范围,应澄清能力限制。汽车交通对应department=新能源汽车磁钢,国内/海外才是其业务分组。 productMatch为ambiguous且未选择时,只需列出授权候选并needsClarification=true,不选择或合并后缀、不换成全部产品。unique使用完整原名;18×15×4不能匹配18×15×4.1。用户选择前轮候选时保留原年份、指标和趋势目标。newTask=true仅用于用户明确另开无关任务,并声明新期间和条件。 今年、上月等使用serverCalendar;最新数据月才用目录,不猜测缺失期间。真正缺少期间或对象有歧义才澄清。普通读取业务指标freshQuery=true。 -原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/is_empty/not_empty","values":[完整原文]}。in/not_in要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。大小写、空格、前导零都是字面值,不做归一化。规则AND、values OR;每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。 +原表列规则必须只使用businessCatalog.sourceFields中的稳定key;字段label供显示,columns按black/bright给出不同原列名。每条columnFilters规则为{"field":"目录key","stages":["black"或"bright"或两者],"operator":"in/not_in/contains/is_empty/not_empty","values":[完整原文或包含关键字]}。in/not_in/contains要求1到50个不超过256字符的文本值;is_empty/not_empty不得带values。规则AND、values OR;文本大小写、空格、前导零均保留原文,不对contains关键字归一化;contains是原文的字面子串筛选,不是通配符或正则。每次变更给出整份规则列表,不能只增量拼接。用户明确只查黑片/亮片或分别筛选时,规则stages分别填写;没有提出阶段差异时覆盖本次所选指标阶段,综合默认black和bright。任何字段/阶段列未授权、缺失或目录不可用都不得忽略条件,必须澄清或说明限制。 只问报废率时fields=[scrapRate];同时要求分子分母、计算基数、报废总量或总重时,分别保留对应重量字段;要求分子分母时fields=[scrapRate,scrapWeight,totalWeight]。不能只在goal中提及而漏掉实际fields。分子为现有正式报废总量,分母为该阶段正式总重,黑片使用现有修正总重。利用料、本部报废、过账外废沿用现有分量逻辑,不使用1-合格率,也不平均原表百分比。未指定阶段的报废率返回black和bright两个阶段;没有统一综合报废率定义,用户明确要综合报废率时请求其定义,不转成综合合格率。 -按用户原文提出in/not_in文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。 +按用户原文提出in/not_in完整文本值;不要仅因字段目录没有列出取值就请求澄清。平台会在语义复核及产品对象确认之后,自动用同期间、同范围授权候选精确核验in/not_in取值。只有大小写、空格、前导零完全相同才执行;近似或无法核验时由平台安全转为澄清并清空本轮规则,模型不猜选近似值、不声称已查过目录取值。contains关键字是子串而非完整候选值,不做等值候选核验,按用户原文交由正式查询执行字面子串筛选。真正字段、阶段、对象或操作含义有歧义时才由规划器提出澄清。候选搜索不作为正式结果证据。 kind=members仅用于列出有哪些委托单/批号/规格等唯一成员,不表示原始行;memberFields使用当前目录key且最多5个,必须与用户明确要列出的对象语义相符,不能把筛选字段当输出成员。裸“规格”表示产品规格,要选择目录中真实产品规格列;不能用图号或客户代码代替。明确问图号时选择图号列,明确问客户规格时选择客户规格列。若用户明确给出“成品规格”等目录label,优先精确匹配该label;裸“规格”有多个候选或没有可用字段时,needsClarification=true、memberFields=[],保留原目标并澄清。并列、否定或复合字段意图由规划器按用户表达处理,不能补入已取消字段。stages必须明确为一个或两个阶段,期间与普通filters照常保留。需要澄清时stages可为空,本轮不查询并保留目标。members的fields=[]、verifySource=false、fullDetails=false、freshQuery=true,不计算KPI、不改成metrics;清单由平台直接渲染,无需撰写业务数值。 单期投入重量排名或明确单阶段报废排名:kind=analysis、dimensions为实际product/department/group、fields仅totalWeight/scrapWeight、dimensionStages明确各维度black/bright、trendRequired=false;未限定阶段的报废排名使用queryPlan ranking、fields=[rate,scrapWeight]、stages=[]、dimensionStages对应维度=[];全量正式分母不随Top展示数量改变。普通年度重量排名默认periodCoverage=available,注明已发布月份;只有用户明确要求完整全年齐全时用complete。排名与趋势同时要求时仍保留真实要求,不删除trendRequired。 能力边界必须忠实表达:排名/缺陷保留analysis及真实dimensions和dimensionStages;两个独立期间比较保留comparisonRequired=true和comparisonPeriod;源表核验为source_verify及verifySource=true;原始逐行记录为details及fullDetails=true;报告为report。不得为了适配已接入能力而改写成普通指标。不得把“有哪些”理解成所有原始字段。后端负责说明仍未接入的能力。 diff --git a/scripts/nao_evidence_report.py b/scripts/nao_evidence_report.py index b4b8eb1..ddd9978 100644 --- a/scripts/nao_evidence_report.py +++ b/scripts/nao_evidence_report.py @@ -154,7 +154,8 @@ def _scope_label(filters): if key=='columnFilters': for rule in value: stage='、'.join({'black':'黑片','bright':'亮片'}.get(s,s) for s in rule.get('stages',[])) - operation={'in':'为','not_in':'排除','is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'') + operation={'in':'为','not_in':'排除','contains':'包含片段', + 'is_empty':'为空','not_empty':'不为空'}.get(rule.get('operator'),'') parts.append(stage+'来源字段'+operation+'、'.join(rule.get('values',[]))) else: parts.append(names[key]+'='+str(value)) return ';'.join(parts) if parts else '当前授权范围内总体(默认不含未分类)' diff --git a/scripts/nao_history_report.py b/scripts/nao_history_report.py index e59ad65..c2faacb 100644 --- a/scripts/nao_history_report.py +++ b/scripts/nao_history_report.py @@ -850,7 +850,8 @@ def _history_scope_caption(snapshot, section): parts.append('包含未分配项') column_filters = filters.get('columnFilters') if isinstance(column_filters, list): - operator_labels = {'in': '包含', 'not_in': '不含', 'is_empty': '为空', 'not_empty': '非空'} + operator_labels = {'in': '为', 'not_in': '排除', 'contains': '包含片段', + 'is_empty': '为空', 'not_empty': '非空'} for rule in column_filters: if not isinstance(rule, dict): continue diff --git a/scripts/nao_quality_tools.py b/scripts/nao_quality_tools.py index 48e1d43..42cacde 100644 --- a/scripts/nao_quality_tools.py +++ b/scripts/nao_quality_tools.py @@ -115,11 +115,11 @@ QUALITY_DATA_SCHEMA: dict[str, Any] = { "field": {"type": "string", "minLength": 1}, "stages": {"type": "array", "minItems": 1, "maxItems": 2, "items": {"type": "string", "enum": ["black", "bright"]}}, - "operator": {"type": "string", "enum": ["in", "not_in", "is_empty", "not_empty"]}, + "operator": {"type": "string", "enum": ["in", "not_in", "contains", "is_empty", "not_empty"]}, "values": {"type": "array", "minItems": 1, "maxItems": 50, "items": {"type": "string", "maxLength": 256}}, }, "required": ["field", "stages", "operator"]}, - "description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR,文本大小写、空格及前导零按原文匹配。is_empty/not_empty不得带values。"}, + "description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR;in/not_in匹配完整文本值,contains按原文作字面子串匹配(MySQL LOCATE),不做关键字归一化,大小写、空格及前导零保留原文。每条文本规则提供1到50个不超过256字符的值;is_empty/not_empty不得带values。"}, "view": {"type": "string", "enum": ["overall", "department", "group", "product", "projectGroup"], "description": "query 的正式分组视图;changes 仅支持非 overall 维度。"}, "stage": {"type": ["string", "null"], "enum": ["black", "bright", None], "description": "top/details/defects/investigate必须明确black/bright。rank单阶段传black/bright;rank的metric=scrapWeight且stage省略或null时,合计黑片+亮片报废总重全量排序,并保留正式综合合格率。投入重量rank仍必须指定阶段。changes可选,未传为综合范围。"}, "dimension": {"type": "string", "enum": ["category", "defect", "product", "department", "group", "projectGroup"], "description": "top模式仅category/product/department/group/projectGroup,不能用defect;具体缺陷TOP必须使用mode=defects、dimension=defect。product是规格。changes 使用 product/department/group/projectGroup。defects 使用 defect 或 product;未指定具体 defect 字段时,省略 dimension 默认按 defect 查询阶段具体缺陷。"}, diff --git a/scripts/nao_query_contract.py b/scripts/nao_query_contract.py index e149d6a..732b9dc 100644 --- a/scripts/nao_query_contract.py +++ b/scripts/nao_query_contract.py @@ -3,7 +3,7 @@ import copy DEPARTMENT_ALIASES = {'汽车交通':'新能源汽车磁钢'} SOURCE_STAGES = {'black', 'bright'} -COLUMN_OPERATORS = {'in', 'not_in', 'is_empty', 'not_empty'} +COLUMN_OPERATORS = {'in', 'not_in', 'contains', 'is_empty', 'not_empty'} def source_field_labels(source_fields): @@ -35,11 +35,13 @@ def parse_column_filters(value): raise ValueError(f'{name}.stages必须是非空black/bright数组。') if operator not in COLUMN_OPERATORS:raise ValueError(f'{name}.operator不在当前字段支持目录中。') clean={'field':key,'stages':list(stages),'operator':operator} - if operator in ('in','not_in'): + if operator in ('in','not_in','contains'): values=rule.get('values') if (not isinstance(values,list) or not 1<=len(values)<=50 - or any(not isinstance(item,str) or item=='' or len(item)>256 for item in values)): - raise ValueError(f'{name}.values必须是1到50个不超过256字符的完整文本值。') + or any(not isinstance(item,str) or item=='' + or operator=='contains' and not item.strip() + or len(item)>256 for item in values)): + raise ValueError(f'{name}.values必须是1到50个不超过256字符的文本值或包含关键字。') clean['values']=list(dict.fromkeys(values)) elif 'values' in rule:raise ValueError(f'{name}的空值操作不接受values。') result.append(clean) diff --git a/verification/test_nao_evidence_report.py b/verification/test_nao_evidence_report.py index 70adbd2..026225c 100644 --- a/verification/test_nao_evidence_report.py +++ b/verification/test_nao_evidence_report.py @@ -34,6 +34,15 @@ def coverage_comparison(): class BoundReportTest(unittest.TestCase): + def test_scope_label_distinguishes_full_value_from_substring_filters(self): + from nao_evidence_report import _scope_label + label = _scope_label({'columnFilters': [ + {'field': 'orderNo', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7']}, + {'field': 'orderNo', 'stages': ['bright'], 'operator': 'contains', 'values': ['07']}, + ]}) + self.assertIn('黑片来源字段为WO-7', label) + self.assertIn('亮片来源字段包含片段07', label) + def test_priority_summary_keeps_three_signals_and_rate_denominators_separate(self): signals={'scrapIncrease':[{'label':'放量规格','scrapDelta':120}], 'currentScrap':[{'label':'存量规格','current':{'scrapWeight':500}}], diff --git a/verification/test_nao_history_report.py b/verification/test_nao_history_report.py index 89753a6..18f28b5 100644 --- a/verification/test_nao_history_report.py +++ b/verification/test_nao_history_report.py @@ -456,9 +456,11 @@ class HistoryReportExecutionTest(unittest.TestCase): first_section = candidate['sections'][0] first_section['filters']['columnFilters'] = [{ 'field': 'src_work_order', 'stages': ['black'], 'operator': 'in', 'values': ['WO-7'], + }, { + 'field': 'src_contains_code', 'stages': ['bright'], 'operator': 'contains', 'values': ['07'], }] set_source_task_context(candidate, SOURCE_A, - {'sourceFieldLabels': {'src_work_order': '黑片委托单号'}}) + {'sourceFieldLabels': {'src_work_order': '黑片委托单号', 'src_contains_code': '客户代码'}}) set_source_task_context(candidate, SOURCE_B, {'sourceFieldLabels': {'src_work_order': '亮片委托单号'}}) fact = candidate['facts'][0] @@ -476,12 +478,14 @@ class HistoryReportExecutionTest(unittest.TestCase): projected = _history_model_fact_projection(candidate, [fact])[0] self.assertIn('产业部 消费电子产业部', projected['modelScope']['displayScope']) self.assertIn('业务分组 消费电子-3C', projected['modelScope']['displayScope']) - self.assertIn('黑片委托单号 包含 WO-7', projected['modelScope']['displayScope']) + self.assertIn('黑片委托单号 为 WO-7', projected['modelScope']['displayScope']) + self.assertIn('亮片 客户代码 包含片段 07', projected['modelScope']['displayScope']) self.assertIn('消费电子产业部', projected['modelText']) self.assertIn('消费电子-3C', projected['modelText']) self.assertIn('黑片委托单号', projected['modelText']) self.assertIn('WO-7', projected['modelText']) - for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:', 'src_work_order'): + for machine_label in ('筛选 {', 'group:', 'department:', 'columnFilters:', + 'src_work_order', 'src_contains_code'): self.assertNotIn(machine_label, projected['modelText']) self.assertNotIn(machine_label, json.dumps(projected['modelScope'], ensure_ascii=False)) self.assertEqual(projected['text'], source_text) @@ -581,7 +585,7 @@ class HistoryReportExecutionTest(unittest.TestCase): goals = payload['historySourceGoals'] self.assertTrue(any('产业部 消费电子产业部' in goal['displayScopeLabel'] and '业务分组 消费电子-3C' in goal['displayScopeLabel'] - and '黑片委托单号 包含 WO-7' in goal['displayScopeLabel'] + and '黑片委托单号 为 WO-7' in goal['displayScopeLabel'] for goal in goals)) self.assertTrue(all('filters' not in goal and '_sourceTurnId' not in goal for goal in goals)) self.assertTrue(all('filters' not in scope and 'sourceTurnId' not in scope diff --git a/verification/test_nao_quality_tools.py b/verification/test_nao_quality_tools.py index c8cb9d7..ba6f170 100644 --- a/verification/test_nao_quality_tools.py +++ b/verification/test_nao_quality_tools.py @@ -284,6 +284,10 @@ class QualityToolsTests(unittest.TestCase): self.assertEqual(definition["inputSchema"]["properties"]["mode"]["enum"], ["catalog", "query", "rank", "compare", "top", "changes", "defects", "details", "investigate"]) self.assertIn("defect", definition["inputSchema"]["properties"]) + column_filter = definition["inputSchema"]["properties"]["columnFilters"] + self.assertEqual(column_filter["items"]["properties"]["operator"]["enum"], + ["in", "not_in", "contains", "is_empty", "not_empty"]) + self.assertIn("MySQL LOCATE", column_filter["description"]) self.assertFalse(definition["inputSchema"]["additionalProperties"]) self.assertEqual(definition["inputSchema"]["allOf"][0]["then"]["required"], ["releaseId"]) @@ -529,11 +533,11 @@ class QualityToolsTests(unittest.TestCase): if path == "/api/analysis/tools/catalog": response["sourceFields"] = [{"key": "customerCode", "label": "客户代码", "type": "source_value", "stages": ["black"], - "columns": {"black": "customer_code"}, "operators": ["in"]}] + "columns": {"black": "customer_code"}, "operators": ["in", "contains"]}] return response api = AuthorizedColumnApi() tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name) - rule = {"field": "customerCode", "stages": ["black"], "operator": "in", "values": ["C-17"]} + rule = {"field": "customerCode", "stages": ["black"], "operator": "contains", "values": ["C-17"]} result = tools.quality_data({"mode": "defects", "releaseId": RELEASE, "period": {"year": 2026, "monthStart": 3, "monthEnd": 9}, "filters": {"exclusions": {"product": ["规格甲"]}}, "columnFilters": [rule], diff --git a/verification/test_source_column_protocol.py b/verification/test_source_column_protocol.py index 801fa3f..1bac357 100644 --- a/verification/test_source_column_protocol.py +++ b/verification/test_source_column_protocol.py @@ -25,7 +25,7 @@ from test_nao_quality_tools import FakeApi, RELEASE def source_field(key, label, black, bright, stages=('black', 'bright')): return {'key': key, 'label': label, 'columns': {'black': black, 'bright': bright}, 'stages': list(stages), 'type': 'source_value', - 'operators': ['in', 'not_in', 'is_empty', 'not_empty']} + 'operators': ['in', 'not_in', 'contains', 'is_empty', 'not_empty']} SOURCE_FIELDS = [ @@ -349,6 +349,15 @@ class SourceColumnProtocolTests(unittest.TestCase): with self.assertRaisesRegex(TaskUnavailable, '不超过256字符'): protocol_task(columnFilters=[{**rule, 'values': ['X' * 257]}]) + def test_contains_keywords_are_preserved_as_literal_or_values(self): + rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains', + 'values': [' 001', 'A|B', 'abC']} + task = protocol_task(columnFilters=[rule]) + self.assertEqual(task['filters']['columnFilters'], [rule]) + for invalid in ([], [''], [' '], ['x' * 257], [1], ['one'] * 51): + with self.subTest(invalid=invalid), self.assertRaises(TaskUnavailable): + protocol_task(columnFilters=[{**rule, 'values': invalid}]) + def test_protocol_presentation_enums_default_consistently_when_omitted_or_null(self): base = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'], period={'year': 2026, 'month': 3}, comparisonRequired=False, @@ -396,6 +405,10 @@ class SourceColumnProtocolTests(unittest.TestCase): self.assertIn('periodCoverage(available/complete,可省略或null', prompt) self.assertIn('freshQuery(bool)', prompt) self.assertIn('needsClarification(bool)', prompt) + self.assertIn('in/not_in/contains/is_empty/not_empty', prompt) + self.assertIn('contains关键字是子串而非完整候选值', prompt) + self.assertEqual(model_call.call_args_list[0].kwargs['payload']['businessCatalog']['sourceColumnOperators'], + ['in', 'not_in', 'contains', 'is_empty', 'not_empty']) def test_three_arbitrary_source_keys_keep_literal_values_and_stage_scopes(self): task = protocol_task() @@ -429,7 +442,7 @@ class SourceColumnProtocolTests(unittest.TestCase): def test_unauthorized_unknown_and_missing_stage_columns_fail_closed(self): for bad in ([{**RULES[0], 'field': 'notInDirectory'}], [{**RULES[0], 'field': 'blackOnlyCode', 'stages': ['bright']}], - [{**RULES[0], 'operator': 'contains'}]): + [{**RULES[0], 'operator': 'starts_with'}]): with self.subTest(bad=bad), self.assertRaises(TaskUnavailable): normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'], period={'year': 2026}, columnFilters=bad), '筛选原表字段', @@ -529,6 +542,20 @@ class SourceColumnProtocolTests(unittest.TestCase): self.assertEqual(customer_lookup['filters']['columnFilters'], [ {'field': 'dispatchOrder', 'stages': ['bright'], 'operator': 'not_empty'}]) + def test_contains_keyword_skips_exact_candidate_value_validation(self): + from nao_agent_task import _resolve_source_column_values + rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'contains', + 'values': ['001']} + task = protocol_task(columnFilters=[rule]) + calls = [] + def resolver(**contract): + calls.append(contract) + raise AssertionError('contains is a substring, not a complete candidate value') + result = _resolve_source_column_values(task, release_id=RELEASE, resolver=resolver) + self.assertEqual(result, {'status': 'exact'}) + self.assertEqual(calls, []) + self.assertEqual(task['filters']['columnFilters'], [rule]) + def test_missing_literal_offers_same_scope_alternatives_without_substitution(self): from nao_agent_task import _resolve_source_column_values task = protocol_task(columnFilters=[RULES[0]]) diff --git a/web/src/StageSourceFilters.tsx b/web/src/StageSourceFilters.tsx index 303e7fd..039b8af 100644 --- a/web/src/StageSourceFilters.tsx +++ b/web/src/StageSourceFilters.tsx @@ -32,6 +32,6 @@ export function StageSourceFilters({stage,fields,query,value,onChange,onUnauthor const selected=value.map(rule=>rule.field);const patch=(key:string,change:Partial)=>onChange(value.map(rule=>rule.field===key?{...rule,...change}:rule)); return
{part!=='rows'&&patch(rule.field,{operator,values:[]})}/>{['in','not_in'].includes(rule.operator)?patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:}
:onChange(value.filter(item=>item.field!==rule.field))}>移除}/>;})} + {part!=='picker'&&value.map(rule=>{const field=fields.find(item=>item.key===rule.field);return field?
{field.label}patch(rule.field,{values:values.filter(value=>value.trim().length>0)})} options={[]}/>:['in','not_in'].includes(rule.operator)?patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:}
:onChange(value.filter(item=>item.field!==rule.field))}>移除}/>;})} ; } diff --git a/web/src/analysisVisualsModel.ts b/web/src/analysisVisualsModel.ts index c5b8f5b..3cd106e 100644 --- a/web/src/analysisVisualsModel.ts +++ b/web/src/analysisVisualsModel.ts @@ -7,7 +7,7 @@ export type AnalysisVisualSectionKey = 'overview' | 'stage' | 'department' | 'gr export type AnalysisVisualDimension = 'department' | 'group' | 'product' | 'category' | 'defect'; export type AnalysisExclusionDimension = 'department' | 'group' | 'product' | 'projectGroup'; export type AnalysisExclusions = Partial>; -export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] } +export interface SourceColumnFilter { field: string; stages: Array<'black' | 'bright'>; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] } export type AnalysisVisualContextValue = string | number | boolean | Array | Record | Record | SourceColumnFilter[] | AnalysisExclusions; export type AnalysisVisualContext = Record; @@ -253,8 +253,8 @@ function safeContext(value: unknown): AnalysisVisualContext | null { || typeof rule.field !== 'string' || !/^[A-Za-z0-9_]{1,64}$/.test(rule.field) || !Array.isArray(rule.stages) || !rule.stages.length || rule.stages.length > 2 || new Set(rule.stages).size !== rule.stages.length || rule.stages.some(stage => stage !== 'black' && stage !== 'bright') - || !['in', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null; - const needsValues = rule.operator === 'in' || rule.operator === 'not_in'; + || !['in', 'contains', 'not_in', 'is_empty', 'not_empty'].includes(String(rule.operator))) return null; + const needsValues = rule.operator === 'in' || rule.operator === 'not_in' || rule.operator === 'contains'; if (needsValues && (!Array.isArray(rule.values) || !rule.values.length || rule.values.length > 50 || rule.values.some(value => typeof value !== 'string' || !value.trim() || value.length > 256))) return null; if (!needsValues && rule.values !== undefined) return null; filters.push({ field: rule.field, stages: [...rule.stages] as SourceColumnFilter['stages'], operator: rule.operator as SourceColumnFilter['operator'], ...(needsValues ? { values: [...rule.values as string[]] } : {}) }); @@ -1397,7 +1397,7 @@ export function columnFilterSummary(value: unknown, labelValue: unknown): string if (!isRecord(rule) || typeof rule.field !== 'string' || !Array.isArray(rule.stages)) return []; const stage = rule.stages.map(item => item === 'black' ? '黑片' : item === 'bright' ? '亮片' : '').filter(Boolean).join('、'); const label = typeof labels[rule.field] === 'string' ? labels[rule.field] : '来源字段'; - const operation = ({ in: '为', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record)[String(rule.operator)]; + const operation = ({ in: '为', contains: '包含', not_in: '排除', is_empty: '为空', not_empty: '不为空' } as Record)[String(rule.operator)]; if (!operation) return []; const values = Array.isArray(rule.values) ? rule.values.filter(item => typeof item === 'string').join('、') : ''; return [`${stage} ${label}${operation}${values}`.trim()]; diff --git a/web/src/types.ts b/web/src/types.ts index 5026d0a..7ba4d9a 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -3,7 +3,7 @@ export type ViewKind = 'overall' | 'department' | 'group' | 'product' | 'project export type PeriodKind = 'year' | 'quarter' | 'month'; export interface SourceColumnField { key: string; label: string; type: string; stages: Stage[]; columns: Partial> } -export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] } +export interface SourceColumnFilter { field: string; stages: Stage[]; operator: 'in' | 'contains' | 'not_in' | 'is_empty' | 'not_empty'; values?: string[] } export interface QueryParams { year: number; diff --git a/web/verification/analysis_visuals.test.mjs b/web/verification/analysis_visuals.test.mjs index 837a554..ffb4601 100644 --- a/web/verification/analysis_visuals.test.mjs +++ b/web/verification/analysis_visuals.test.mjs @@ -1042,3 +1042,12 @@ test('adaptive trends cover every series, preserve missing values, and keep non- assert.equal(formatVisualValue(.9639601707533932, 'ratio'), '96.40%'); assert.equal(formatVisualValue(3, 'kg'), '3.00 kg'); }); + + test('contains keywords survive analysis scope and display as substring filters',()=>{ + const rule={field:'src_any',stages:['bright'],operator:'contains',values:['日立','%_']}; + const parsed=parseAnalysisVisuals([visual({scope:{columnFilters:[rule],columnLabels:{src_any:'客户'}}})]); + assert.equal(parsed.visuals.length,1); + assert.deepEqual(parsed.visuals[0].scope.columnFilters,[rule]); + assert.match(columnFilterSummary([rule],{src_any:'客户'})[0],/包含.*日立/); + assert.equal(parseAnalysisVisuals([visual({scope:{columnFilters:[{...rule,values:[]}]}})]).visuals.length,0); + }); diff --git a/web/verification/source_contains.test.mjs b/web/verification/source_contains.test.mjs new file mode 100644 index 0000000..7100661 --- /dev/null +++ b/web/verification/source_contains.test.mjs @@ -0,0 +1,30 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import ts from 'typescript'; + +const source=fs.readFileSync(new URL('../src/StageSourceFilters.tsx',import.meta.url),'utf8'); +const output=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX,target:ts.ScriptTarget.ES2022}}).outputText; +const jsx=(type,props)=>({type,props});const exports={}; +new Function('require','exports',output)(name=>{ + if(name==='react/jsx-runtime')return {jsx,jsxs:jsx}; + if(name==='antd')return {Select:'Select',Button:'Button',Alert:'Alert',Spin:'Spin'}; + if(name==='@ant-design/icons')return {CloseOutlined:'CloseOutlined'}; + if(name==='./readableSelectPopup')return {readableSelectPopup:{}}; + return {}; +},exports); +const nodes=value=>!value||typeof value!=='object'?[]:[value,...[value.props?.children].flat(Infinity).flatMap(nodes)]; +const field={key:'customer',label:'客户名称',type:'source_value',stages:['bright'],columns:{bright:'客户名称'}}; + +test('contains is selectable and accepts bounded free keywords without a value-candidate lookup',()=>{ + let changed; + const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立']}; + const root=exports.StageSourceFilters({stage:'bright',fields:[field],query:{year:2026},value:[rule],onChange:value=>{changed=value;},onUnauthorized:()=>false}); + const all=nodes(root);const operation=all.find(x=>x.type==='Select'&&x.props['aria-label']==='客户名称筛选方式'); + assert.ok(operation.props.options.some(x=>x.value==='contains'&&x.label==='包含')); + const keywords=all.find(x=>x.type==='Select'&&x.props['aria-label']==='客户名称包含文字'); + assert.equal(keywords.props.mode,'tags');assert.equal(keywords.props.maxCount,50);assert.equal(keywords.props.maxLength,256);assert.deepEqual(keywords.props.options,[]); + keywords.props.onChange(['日立','%_',' ']); + assert.deepEqual(changed,[{...rule,values:['日立','%_']}]); + operation.props.onChange('in');assert.deepEqual(changed,[{...rule,operator:'in',values:[]}]); +}); diff --git a/web/verification/stage_product_query.test.mjs b/web/verification/stage_product_query.test.mjs index 5242320..b8f76c8 100644 --- a/web/verification/stage_product_query.test.mjs +++ b/web/verification/stage_product_query.test.mjs @@ -30,3 +30,9 @@ test('stage query includes unclassified source rows even when an old link exclud assert.equal(base.includeUnassigned,false); } }); + +test('contains keywords are retained while an unfilled contains rule is inactive',()=>{ + const rule={field:'customer',stages:['bright'],operator:'contains',values:['日立','%_']}; + assert.deepEqual(activeStageFilters([rule],'bright'),[rule]); + assert.deepEqual(activeStageFilters([{...rule,values:[]}],'bright'),[]); +}); From f8432b4e02fb9eb32c4bd2444bce170f1d104928 Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 21:51:31 +0800 Subject: [PATCH 10/11] Bound contains keyword input and clear duplicate search drafts --- web/src/StageSourceFilters.tsx | 8 +++++++- web/verification/source_contains.test.mjs | 10 ++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/web/src/StageSourceFilters.tsx b/web/src/StageSourceFilters.tsx index 039b8af..ea18ba4 100644 --- a/web/src/StageSourceFilters.tsx +++ b/web/src/StageSourceFilters.tsx @@ -28,10 +28,16 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S },[active,field.key,stage,contextKey,search,revision,onUnauthorized]); return setSearch(text.slice(0,256).replace(/[\uD800-\uDBFF]$/u,''))} onBlur={()=>setSearch('')} + onChange={values=>{onChange(values.filter(value=>value.trim().length>0&&value.length<=256));setSearch('');}} options={[]}/>; +} export function StageSourceFilters({stage,fields,query,value,onChange,onUnauthorized,part='both'}:{part?:'both'|'picker'|'rows';stage:Stage;fields:SourceColumnField[];query:QueryParams;value:SourceColumnFilter[];onChange:(value:SourceColumnFilter[])=>void;onUnauthorized:(error:unknown)=>boolean}) { const selected=value.map(rule=>rule.field);const patch=(key:string,change:Partial)=>onChange(value.map(rule=>rule.field===key?{...rule,...change}:rule)); return
{part!=='rows'&&patch(rule.field,{operator,values:[]})}/>{rule.operator==='contains'?patch(rule.field,{operator,values:[]})}/>{rule.operator==='contains'?patch(rule.field,{values})}/>:['in','not_in'].includes(rule.operator)?patch(rule.field,{values})} onUnauthorized={onUnauthorized}/>:}
:onChange(value.filter(item=>item.field!==rule.field))}>移除}/>;})} ; } diff --git a/web/verification/source_contains.test.mjs b/web/verification/source_contains.test.mjs index 7100661..5ac90bc 100644 --- a/web/verification/source_contains.test.mjs +++ b/web/verification/source_contains.test.mjs @@ -5,9 +5,11 @@ import ts from 'typescript'; const source=fs.readFileSync(new URL('../src/StageSourceFilters.tsx',import.meta.url),'utf8'); const output=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,jsx:ts.JsxEmit.ReactJSX,target:ts.ScriptTarget.ES2022}}).outputText; +let lastSearch; const jsx=(type,props)=>({type,props});const exports={}; new Function('require','exports',output)(name=>{ if(name==='react/jsx-runtime')return {jsx,jsxs:jsx}; + if(name==='react')return {useState:initial=>[initial,value=>{lastSearch=value;} ]}; if(name==='antd')return {Select:'Select',Button:'Button',Alert:'Alert',Spin:'Spin'}; if(name==='@ant-design/icons')return {CloseOutlined:'CloseOutlined'}; if(name==='./readableSelectPopup')return {readableSelectPopup:{}}; @@ -22,8 +24,12 @@ test('contains is selectable and accepts bounded free keywords without a value-c const root=exports.StageSourceFilters({stage:'bright',fields:[field],query:{year:2026},value:[rule],onChange:value=>{changed=value;},onUnauthorized:()=>false}); const all=nodes(root);const operation=all.find(x=>x.type==='Select'&&x.props['aria-label']==='客户名称筛选方式'); assert.ok(operation.props.options.some(x=>x.value==='contains'&&x.label==='包含')); - const keywords=all.find(x=>x.type==='Select'&&x.props['aria-label']==='客户名称包含文字'); - assert.equal(keywords.props.mode,'tags');assert.equal(keywords.props.maxCount,50);assert.equal(keywords.props.maxLength,256);assert.deepEqual(keywords.props.options,[]); + const free=all.find(x=>typeof x.type==='function'&&x.type.name==='ContainsValues'); + const keywords=free.type(free.props); + assert.equal(keywords.props.mode,'tags');assert.equal(keywords.props.maxCount,50);assert.deepEqual(keywords.props.options,[]); + keywords.props.onSearch('x'.repeat(257));assert.equal(lastSearch.length,256); + keywords.props.onSearch('x'.repeat(255)+'😀');assert.equal(lastSearch,'x'.repeat(255)); + keywords.props.onSearch('日立');keywords.props.onBlur();assert.equal(lastSearch,''); keywords.props.onChange(['日立','%_',' ']); assert.deepEqual(changed,[{...rule,values:['日立','%_']}]); operation.props.onChange('in');assert.deepEqual(changed,[{...rule,operator:'in',values:[]}]); From f47341f3f0177610af4708457ed7f365b1b44dbc Mon Sep 17 00:00:00 2001 From: lyun Date: Fri, 13 Nov 2026 22:10:50 +0800 Subject: [PATCH 11/11] Document contains filter usage and live acceptance --- README.md | 2 +- docs/当前进度.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 27f5e66..278d896 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ DSH CLI 固定为 `0.2.2-rc.1`,依赖与官方 Python SDK 校验记录在 `run ## 筛选索引与查询加载 -来源字段的等于、排除和空值筛选共用按构建冻结的值字典及记录索引;汇总、趋势、来源明细和导出沿用当前授权与正式指标计算。源字段值保留空值、大小写及前导零语义,旧配置筛选的字符串 `null` 约定不变。下拉框使用只读候选模式,只取一页及是否还有;需要精确总数的原接口继续保留。 +来源字段支持等于、包含、排除和空值筛选;“包含”输入关键词后按回车生效,多个关键词任一命中,区分大小写,`%`、`_` 按普通文字匹配,每个关键词最多256字符、最多50个。各类筛选共用按构建冻结的值字典及记录索引;汇总、趋势、来源明细和导出沿用当前授权与正式指标计算。源字段值保留空值、大小写及前导零语义,旧配置筛选的字符串 `null` 约定不变。下拉框使用只读候选模式,只取一页及是否还有;需要精确总数的原接口继续保留。 启动时在单线程后台补建当前完整正式版本的索引,导入及归类在事务提交后进入同一去重队列。完整构建标记提交后才启用;未完成、失败或未补建的历史构建沿用原始查询,不阻止启动和正式数据发布。索引增加持久存储及后台构建成本,规模取决于非空字段数和不同值数量;高基数宽表应先测容量。保留清理随无引用构建回收索引;升级后原维护兼容保护继续生效。 diff --git a/docs/当前进度.md b/docs/当前进度.md index eac97da..c1717d2 100644 --- a/docs/当前进度.md +++ b/docs/当前进度.md @@ -3,6 +3,18 @@ 核实日期:2026年11月13日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。 +## 2026-11-13:来源字段“包含”筛选(已发布) + +来源字段操作菜单新增“包含”:自由输入关键词按回车,多个关键词按OR匹配;不同筛选条件之间仍为AND。后台用绑定参数的LOCATE执行字面子串匹配,大小写及正式来源值语义保持,`%`、`_`、反斜杠和引号不是通配或SQL内容。支持1–50个非空白关键词,每个最多256字符;前端控制搜索长度并避免截断代理对,重复关键词失焦后清空草稿。 + +汇总、趋势、来源明细与Excel统一走同一解析/授权/查询逻辑,兼容原始JSON回退与现有索引。DSH活跃共享契约、图表上下文和历史/证据显示同步接受contains;关键词不做完整候选值的等值核验。打包只替换5个相关DSH业务模块,保留当前已验收的流式助手和原固定发行版;模型配置及状态不变,未调用在线模型。不新增迁移、表或索引,不重建既有52个构建。 + +验证:前端61项及类型/生产构建通过;完整后端990项(820执行通过、170按条件跳过);专项隔离MySQL及服务45项(44通过、1基准跳过),覆盖两阶段、原表/索引、规范维度、大小写、特殊字符、零/字符串null/空值、组织权限,以及汇总/趋势/明细/Excel。DSH契约/显示Python135项(133通过、2跳过),应用构建16项通过,新打包应用的contains契约及固定运行时检查通过,独立审查无阻断。未执行私有原表全量回归或在线模型验收。 + +线上关键词命中2个完整客户值,contains与对应in集合的汇总业务数据、趋势及来源结果一致;代表查询约1965/721/465ms。来源29条、121个原始字段,导出29条且ZIP/XML有效;首次导出12907ms、复用后844ms。实际菜单、键盘提交、多关键词字面值、257字符粘贴限制及重复失焦清理已检查,页面无失败提示。前端2026.11.13.12(`f8432b4`),后端功能提交`3c022e8`,后续仅前端输入边界修正,数据库进程保持;DSH固定引擎、模型配置及状态保持。 + +代码已在独立分支 `codex/source-filter-contains-20261113` 本地提交,本轮未同步Gitea。私有请求、结果及效果截图位于 `output/20261113-source-contains/`,不进入Git。 + ## 2026-11-13:统一筛选提速(已发布验收) 已发布2026.11.13.10,功能提交 `20f1c7d`;当前正式版52/52个构建的筛选索引全部就绪。新增按构建保存的来源字段字典与稀疏记录索引,来源/配置筛选、候选、指标、趋势、明细及明细导出共用;正式算法、当前字段/组织权限和发布版绑定保持。候选模式只取上限加一,不再为下拉精确计数;常规页面查询不等待无关目录,本字段改选不重复读取自己的候选,明细导出先只读字段名。