优化:统一来源筛选索引与后台构建
This commit is contained in:
@@ -58,6 +58,14 @@ DSH CLI 固定为 `0.2.2-rc.1`,依赖与官方 Python SDK 校验记录在 `run
|
||||
|
||||
底部表格同步展开当前分类的细项,重量/废率切换与 Excel 导出跟随当前显示;产品和期间行维度保持不变。细项使用正式阶段同期总重量作分母,月、季、年均先汇总分量再计算;没有来源或分母时显示缺失,真实零显示零。细项与正式分类总量不一致时提示覆盖不足,不补造分类。
|
||||
|
||||
## 筛选索引与查询加载
|
||||
|
||||
来源字段的等于、排除和空值筛选共用按构建冻结的值字典及记录索引;汇总、趋势、来源明细和导出沿用当前授权与正式指标计算。源字段值保留空值、大小写及前导零语义,旧配置筛选的字符串 `null` 约定不变。下拉框使用只读候选模式,只取一页及是否还有;需要精确总数的原接口继续保留。
|
||||
|
||||
启动时在单线程后台补建当前完整正式版本的索引,导入及归类在事务提交后进入同一去重队列。完整构建标记提交后才启用;未完成、失败或未补建的历史构建沿用原始查询,不阻止启动和正式数据发布。索引增加持久存储及后台构建成本,规模取决于非空字段数和不同值数量;高基数宽表应先测容量。保留清理随无引用构建回收索引;升级后原维护兼容保护继续生效。
|
||||
|
||||
黑片/亮片页的普通查询在数据版本已解析后独立加载,来源字段条件仍等待对应版本的字段目录;产品搜索不因无关元数据失败而禁用。
|
||||
|
||||
## 旧版本时间迁移
|
||||
|
||||
连接与 MySQL 会话现在统一使用 UTC,界面仍按本地时区显示。V25 只校正旧代码按 UTC 墙上时间写入的截止/有效期、报告调度及报告租约列(前者由 Java 写入,租约使用 SQL UTC_TIMESTAMP),不改 SQL CURRENT_TIMESTAMP 自动生成的创建时间,也不修改业务日期或指标。若旧库存在此类值,迁移要求先备份并核实旧会话相对 UTC 的偏移,然后设置 `MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS`(旧会话 UTC 为 `0`,旧会话中国标准时间为 `28800`)。本机启动可在私有 local-config 中记录 `legacyTimestampOffsetSeconds`;不能仅凭电脑当前时区猜测。全新空库不需要该参数;迁移标记和历史值修改在同一事务提交,防止提交确认丢失后重复平移。还原旧备份时应重新核实其原运行时区。
|
||||
|
||||
@@ -369,7 +369,7 @@ public class AnalysisExplorationTools {
|
||||
SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value);
|
||||
SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value);
|
||||
appendDetailExclusions(sql, args, query);
|
||||
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor);
|
||||
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor,releaseId);
|
||||
DataScope.append(sql, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
|
||||
"f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
boolean includeUnassigned = query.includeUnassigned()
|
||||
|
||||
@@ -276,7 +276,7 @@ public class AnalysisFineDefectsService {
|
||||
SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value);
|
||||
if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))
|
||||
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
|
||||
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor);
|
||||
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor,releaseId);
|
||||
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
|
||||
"f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
}
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ public class AnalysisInvestigationService {
|
||||
SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x);
|
||||
SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x);
|
||||
if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
|
||||
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor);
|
||||
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor,release);
|
||||
DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
}
|
||||
if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());}
|
||||
|
||||
+7
-3
@@ -43,12 +43,16 @@ public class AnalysisSourceQueryController {
|
||||
@PostMapping("/values")
|
||||
public Map<String,Object> values(@RequestBody Map<String,Object> input,@AuthenticationPrincipal AppUserPrincipal actor) {
|
||||
String scope=captureScope(actor);
|
||||
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
|
||||
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset","candidateOnly").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
|
||||
String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null));
|
||||
Map<String,Object> context=context(input);
|
||||
if(!(input.get("field") instanceof String field)||field.isBlank()||input.get("search")!=null&&!(input.get("search") instanceof String))throw bad("候选字段或搜索值无效");
|
||||
Map<String,Object> result=new LinkedHashMap<>(columns.values(release,context,field,strings(input.get("stages"),2),Objects.toString(input.get("search"),null),
|
||||
integer(input.getOrDefault("limit",50),1,200),integer(input.getOrDefault("offset",0),0,100000),actor));
|
||||
if(input.containsKey("candidateOnly")&&!(input.get("candidateOnly") instanceof Boolean))throw bad("候选分页参数无效");
|
||||
List<String> stages=strings(input.get("stages"),2);String search=Objects.toString(input.get("search"),null);
|
||||
int limit=integer(input.getOrDefault("limit",50),1,200),offset=integer(input.getOrDefault("offset",0),0,100000);
|
||||
Map<String,Object> result=new LinkedHashMap<>(Boolean.TRUE.equals(input.get("candidateOnly"))
|
||||
? columns.values(release,context,field,stages,search,limit,offset,true,actor)
|
||||
: columns.values(release,context,field,stages,search,limit,offset,actor));
|
||||
result.put("period",input.get("period"));result.put("filters",input.get("filters"));
|
||||
result.put("values",AnalysisProtocolService.list(result.get("rows")).stream().map(row->AnalysisProtocolService.map(row).get("value")).toList());
|
||||
result.put("candidateOnly",true);reauthorize(actor,scope,false);return result;
|
||||
|
||||
@@ -111,7 +111,7 @@ public class AnalyticsController {
|
||||
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前字段授权不包含所选报废类别");
|
||||
List<Object> args = new ArrayList<>(extra.isEmpty()?List.of(selected, selected):List.of(selected));
|
||||
StringBuilder where = baseWhere(args, year, month, quarter, department, group, product, projectGroup, includeUnassigned, actor);
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor);
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor,selected);
|
||||
List<AggregateTop> aggregated;
|
||||
if (dimension.equals("category")) {
|
||||
List<Component> selectedComponents = selectedCategory == null ? visibleComponents : List.of(selectedCategory);
|
||||
|
||||
@@ -36,9 +36,11 @@ public class FilterFieldsController {
|
||||
governance.requireUserActivePolicy(actor);
|
||||
Map<String,List<String>> result=new LinkedHashMap<>();
|
||||
String release=releases.resolvePublishedRelease(null);
|
||||
SourceFilterIndex index=new SourceFilterIndex(jdbc);
|
||||
for(String stage:List.of("black","bright")) {
|
||||
// Raw source headers are stable within each build. Read a representative row of every current build.
|
||||
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2 WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
|
||||
String hint=index.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":"";
|
||||
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2"+hint+" WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
|
||||
Set<String> columns=new TreeSet<>();
|
||||
var json=tools.jackson.databind.json.JsonMapper.builder().build();
|
||||
Set<String> allowed=governance.allowedFields(actor);
|
||||
|
||||
@@ -48,6 +48,9 @@ public class QueryFilterFields {
|
||||
return Collections.unmodifiableMap(result);
|
||||
}
|
||||
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor) {
|
||||
append(where,args,stage,filters,actor,null);
|
||||
}
|
||||
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor,String releaseId) {
|
||||
if(filters==null || filters.isEmpty())return;
|
||||
Map<String,Field> fields=new HashMap<>(); visible(actor).forEach(f->fields.put(f.key(),f));
|
||||
for(var entry:filters.entrySet()) {
|
||||
@@ -61,6 +64,7 @@ public class QueryFilterFields {
|
||||
boolean empty=entry.getValue().contains(EMPTY);
|
||||
if(empty && !f.allowEmpty())throw bad(f.label()+"不允许筛选空值");
|
||||
String column=stage.equals("black")?f.blackColumn():f.brightColumn();
|
||||
if(releaseId!=null&&new SourceFilterIndex(jdbc).appendLegacy(where,args,stage,releaseId,column,entry.getValue()))continue;
|
||||
// Paths and selected values are bound parameters; never interpolate configured identifiers.
|
||||
String expression="NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs";
|
||||
List<String> values=entry.getValue().stream().filter(v->!v.equals(EMPTY)).toList();
|
||||
@@ -90,10 +94,15 @@ public class QueryFilterFields {
|
||||
StringBuilder scope=new StringBuilder(" WHERE rm.release_id=?");
|
||||
List<Object> scoped=new ArrayList<>();scoped.add(releaseId);
|
||||
DataScope.append(scope,scoped,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
var index=new SourceFilterIndex(jdbc);
|
||||
if(index.ready(releaseId)) {
|
||||
selects.add(index.candidateSelect(stage,column,scope.toString(),scoped,args).replace("AS candidate_value","AS value"));
|
||||
continue;
|
||||
}
|
||||
selects.add("SELECT NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs AS value FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+scope);
|
||||
args.add(path(column));args.addAll(scoped);
|
||||
}
|
||||
String predicate=" WHERE value IS NOT NULL";
|
||||
String predicate=" WHERE value IS NOT NULL AND CAST(value AS BINARY)<>CAST('null' AS BINARY)";
|
||||
if(search!=null && !search.isBlank()) { predicate+=" AND LOCATE(LOWER(?),LOWER(value))>0";args.add(search); }
|
||||
List<String> found=jdbc.query("SELECT DISTINCT value FROM ("+String.join(" UNION ALL ",selects)+") choices"+predicate+" ORDER BY value LIMIT 201",(rs,n)->rs.getString(1),args.toArray());
|
||||
return Map.of("values",found.stream().limit(200).toList(),"hasMore",found.size()>200);
|
||||
|
||||
@@ -80,11 +80,17 @@ public class SourceColumnQueries {
|
||||
private final JdbcTemplate jdbc;
|
||||
private final GovernanceAccessService governance;
|
||||
private final JsonMapper json;
|
||||
private SourceFilterIndex filterIndex;
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
public void setFilterIndex(SourceFilterIndex index) { this.filterIndex=index; }
|
||||
public boolean indexed(String release) { return filterIndex.ready(release); }
|
||||
|
||||
|
||||
public SourceColumnQueries(JdbcTemplate jdbc, GovernanceAccessService governance, JsonMapper json) {
|
||||
this.jdbc = jdbc;
|
||||
this.governance = governance;
|
||||
this.json = json;
|
||||
this.filterIndex = new SourceFilterIndex(jdbc);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -110,7 +116,7 @@ public class SourceColumnQueries {
|
||||
"f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) "
|
||||
+ "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id "
|
||||
+ "FROM release_member rm JOIN fact_" + stage + " f2" + factIndexHint(stage) + " "
|
||||
+ "FROM release_member rm JOIN fact_" + stage + " f2" + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage)) + " "
|
||||
+ "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"
|
||||
+ representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative "
|
||||
+ "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id";
|
||||
@@ -249,6 +255,8 @@ public class SourceColumnQueries {
|
||||
String operator = (String) rule.get("operator");
|
||||
String expression;
|
||||
Object column = ((Map<?, ?>) field.get("columns")).get(stage);
|
||||
@SuppressWarnings("unchecked") List<String> selected = (List<String>) rule.getOrDefault("values",List.of());
|
||||
if("source_value".equals(field.get("type")) && filterIndex.append(where,args,stage,releaseId,String.valueOf(column),operator,selected))continue;
|
||||
if ("source_value".equals(field.get("type"))) {
|
||||
expression = SOURCE_VALUE;
|
||||
} else {
|
||||
@@ -383,12 +391,16 @@ public class SourceColumnQueries {
|
||||
public Map<String, Object> values(String releaseId, Map<String, Object> context, String fieldKey,
|
||||
List<String> stages, String search, int limit, int offset,
|
||||
AppUserPrincipal actor) {
|
||||
return values(releaseId,context,fieldKey,stages,search,limit,offset,false,actor);
|
||||
}
|
||||
public Map<String,Object> values(String releaseId,Map<String,Object> context,String fieldKey,List<String> stages,
|
||||
String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
|
||||
requireQueryAccess(actor);
|
||||
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,actor));
|
||||
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,candidateOnly,actor));
|
||||
}
|
||||
|
||||
private Map<String,Object> valuesSnapshot(String releaseId,Map<String,Object> context,String fieldKey,
|
||||
List<String> stages,String search,int limit,int offset,AppUserPrincipal actor) {
|
||||
List<String> stages,String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
|
||||
if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效");
|
||||
if (fieldKey == null || fieldKey.isBlank()) throw bad("来源列标识无效");
|
||||
if (stages == null || stages.isEmpty() || stages.size() > STAGES.size()
|
||||
@@ -419,6 +431,10 @@ public class SourceColumnQueries {
|
||||
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
|
||||
List<Object> whereArgs = new ArrayList<>(List.of(releaseId));
|
||||
appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor);
|
||||
if("source_value".equals(field.get("type")) && filterIndex.ready(releaseId)) {
|
||||
selects.add(filterIndex.candidateSelect(stage,String.valueOf(columns.get(stage)),where.toString(),whereArgs,unionArgs));
|
||||
continue;
|
||||
}
|
||||
String fact = stage.equals("black") ? "fact_black" : "fact_bright";
|
||||
selects.add("SELECT " + expression + " AS candidate_value "
|
||||
+ "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id "
|
||||
@@ -437,6 +453,13 @@ public class SourceColumnQueries {
|
||||
}
|
||||
String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition;
|
||||
String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value";
|
||||
if(candidateOnly) {
|
||||
String sql="SELECT candidate_value FROM ("+candidates+") matches GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?";
|
||||
List<Object> parameters=combine(unionArgs,searchArgs);parameters.add(limit+1);parameters.add(offset);
|
||||
List<Map<String,Object>> found=pages.get(new QueryKey(sql,parameters,"candidates:"+fieldKey),()->List.copyOf(jdbc.query(sql,(rs,row)->Map.of("value",rs.getString(1)),parameters.toArray())));
|
||||
boolean more=found.size()>limit;
|
||||
return Map.of("rows",List.copyOf(found.subList(0,Math.min(limit,found.size()))),"hasMore",more,"limit",limit,"offset",offset,"pageComplete",true,"complete",offset==0&&!more,"releaseId",releaseId,"field",fieldKey,"selectedStages",List.copyOf(stages));
|
||||
}
|
||||
long total = total("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", combine(unionArgs,searchArgs));
|
||||
List<Map<String, Object>> rows = page("SELECT candidate_value FROM (" + candidates + ") matches "
|
||||
+ "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?",
|
||||
@@ -563,7 +586,7 @@ public class SourceColumnQueries {
|
||||
List<String> groups = parseSelections(context.dimensions().get("group"), "group");
|
||||
if (!context.includeUnassigned() && !groups.contains("__UNASSIGNED__"))
|
||||
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
|
||||
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor);
|
||||
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor,releaseId);
|
||||
if (!context.columnFilters().isEmpty()) append(where, args, stage, context.columnFilters(), releaseId, actor);
|
||||
DataScope.append(where, args, actor,
|
||||
"f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package cn.yunsheng.quality.filters;
|
||||
|
||||
import java.util.List;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
|
||||
|
||||
/** Sparse, build-owned source values. Every consumer still authorizes and scopes the actual facts. */
|
||||
@Service
|
||||
public class SourceFilterIndex {
|
||||
public record BuildAvailable(String buildId) {}
|
||||
private static final int FORMAT = 1;
|
||||
private final JdbcTemplate jdbc;
|
||||
private volatile Boolean available;
|
||||
public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; }
|
||||
|
||||
public boolean available() {
|
||||
if (available == null) {
|
||||
Integer count=jdbc.queryForObject("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('source_filter_build','source_filter_field','source_filter_value','source_filter_member')",Integer.class);
|
||||
available=count!=null&&count==4;
|
||||
}
|
||||
return available;
|
||||
}
|
||||
|
||||
public boolean ready(String release) {
|
||||
if(release==null||!available())return false;
|
||||
// Only successful immutable builds are retained; a pending backfill is checked again later.
|
||||
Integer missing=jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm LEFT JOIN source_filter_build sb ON sb.build_id=rm.transform_build_id AND sb.format_version=? WHERE rm.release_id=? AND sb.build_id IS NULL",Integer.class,FORMAT,release);
|
||||
if(missing==null||missing!=0)return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Idempotent and atomic with fact construction, including callers which instantiate this helper. */
|
||||
public void ensureBuild(String build) {
|
||||
if(!available())return;
|
||||
var tx=new TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource()));
|
||||
tx.executeWithoutResult(status->{
|
||||
jdbc.queryForObject("SELECT id FROM transform_build WHERE id=? FOR UPDATE",String.class,build);
|
||||
Integer done=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build WHERE build_id=? AND format_version=?",Integer.class,build,FORMAT);
|
||||
if(done!=null&&done>0)return;
|
||||
jdbc.update("DELETE m FROM source_filter_member m JOIN source_filter_field sf ON sf.id=m.field_id WHERE sf.build_id=?",build);
|
||||
jdbc.update("DELETE v FROM source_filter_value v JOIN source_filter_field sf ON sf.id=v.field_id WHERE sf.build_id=?",build);
|
||||
jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build);
|
||||
jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build);
|
||||
// Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source.
|
||||
jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash)) ENGINE=InnoDB");
|
||||
try {
|
||||
for(String stage:List.of("black","bright"))buildStage(build,stage);
|
||||
jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT);
|
||||
} finally { jdbc.execute("DROP TEMPORARY TABLE source_filter_staging"); }
|
||||
});
|
||||
}
|
||||
|
||||
private void buildStage(String build,String stage) {
|
||||
var bounds=jdbc.queryForMap("SELECT MIN(id) AS first_id,MAX(id) AS last_id FROM fact_"+stage+" FORCE INDEX(uq_fact_"+stage+"_origin) WHERE build_id=?",build);
|
||||
if(bounds.get("first_id")==null)return;
|
||||
long after=((Number)bounds.get("first_id")).longValue()-1;
|
||||
long last=((Number)bounds.get("last_id")).longValue();
|
||||
while(after<last) {
|
||||
if(Thread.currentThread().isInterrupted())throw new IllegalStateException("来源筛选索引构建已中断");
|
||||
List<Long> ids=jdbc.queryForList("SELECT id FROM fact_"+stage+" FORCE INDEX(PRIMARY) WHERE build_id=? AND id>? AND id<=? ORDER BY id LIMIT 500",Long.class,build,after,last);
|
||||
if(ids.isEmpty())return;
|
||||
long end=ids.getLast();
|
||||
String rows=" FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE WHERE f.build_id=? AND f.id>? AND f.id<=?";
|
||||
jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end);
|
||||
String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')";
|
||||
String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?";
|
||||
jdbc.update("DELETE FROM source_filter_staging");
|
||||
jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end);
|
||||
jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash");
|
||||
Integer collisions=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)",Integer.class);
|
||||
if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布");
|
||||
jdbc.update("INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash");
|
||||
after=end;
|
||||
}
|
||||
}
|
||||
|
||||
/** Same-row membership is shared by metrics, trends, details and exports; empty values are sparse. */
|
||||
public boolean append(StringBuilder where,List<Object> args,String stage,String release,String column,
|
||||
String operator,List<String> values) {
|
||||
if(!ready(release))return false;
|
||||
where.append(" AND ");
|
||||
membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"),
|
||||
operator.equals("in")||operator.equals("not_in")?values:null,false);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Legacy configured filters classify only the byte-exact literal string "null" as empty. */
|
||||
public boolean appendLegacy(StringBuilder where,List<Object> args,String stage,String release,String column,List<String> selected) {
|
||||
if(!ready(release))return false;
|
||||
var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList();
|
||||
where.append(" AND (");
|
||||
if(values.isEmpty())where.append("FALSE");
|
||||
else membership(where,args,stage,release,column,false,values,true);
|
||||
if(selected.contains(QueryFilterFields.EMPTY)) {
|
||||
where.append(" OR ");membership(where,args,stage,release,column,true,null,true);
|
||||
}
|
||||
where.append(')');return true;
|
||||
}
|
||||
|
||||
private void membership(StringBuilder where,List<Object> args,String stage,String release,String column,
|
||||
boolean negative,List<String> values,boolean legacy) {
|
||||
where.append("f.id ").append(negative?"NOT IN (":"IN (")
|
||||
.append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin");
|
||||
args.add(release);args.add(stage);args.add(column);
|
||||
if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)");
|
||||
if(values!=null) {
|
||||
where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')');
|
||||
args.addAll(values);
|
||||
}
|
||||
where.append(')');
|
||||
}
|
||||
|
||||
/** Dictionary-first candidates short-circuit after one authorized matching member per value. */
|
||||
public String candidateSelect(String stage,String column,String scope,List<Object> scopeArgs,List<Object> args) {
|
||||
args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs);
|
||||
return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id "
|
||||
+"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id "
|
||||
+scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package cn.yunsheng.quality.filters;
|
||||
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Asynchronously builds current and newly completed snapshots; unindexed queries retain their raw-value fallback. */
|
||||
@Component
|
||||
public class SourceFilterIndexBackfill implements ApplicationRunner {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SourceFilterIndexBackfill.class);
|
||||
private static final int MAX_PENDING_BUILDS = 1024;
|
||||
|
||||
private final JdbcTemplate jdbc;
|
||||
private final SourceFilterIndex index;
|
||||
private final ExecutorService worker;
|
||||
private final Object pendingLock = new Object();
|
||||
private final LinkedHashSet<String> pending = new LinkedHashSet<>();
|
||||
private boolean drainScheduled;
|
||||
private String inFlight;
|
||||
|
||||
@Autowired
|
||||
public SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index) {
|
||||
this(jdbc, index, createWorker());
|
||||
}
|
||||
|
||||
SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index, ExecutorService worker) {
|
||||
this.jdbc = jdbc;
|
||||
this.index = index;
|
||||
this.worker = worker;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
List<String> currentBuilds;
|
||||
try {
|
||||
if (!index.available()) return;
|
||||
currentBuilds = jdbc.query("""
|
||||
SELECT DISTINCT b.id
|
||||
FROM current_release c
|
||||
JOIN data_release r ON r.id=c.release_id
|
||||
JOIN release_member rm ON rm.release_id=r.id
|
||||
JOIN transform_build b ON b.id=rm.transform_build_id
|
||||
WHERE c.dataset_id='quality'
|
||||
AND r.status='PUBLISHED'
|
||||
AND r.complete=TRUE
|
||||
AND b.status='COMPLETED'
|
||||
ORDER BY b.id
|
||||
""", (rs, row) -> rs.getString(1));
|
||||
} catch (RuntimeException failure) {
|
||||
LOG.error("Source filter index backfill discovery failed; raw-value fallback remains available", failure);
|
||||
return;
|
||||
}
|
||||
if (currentBuilds.isEmpty()) return;
|
||||
int accepted = enqueue(currentBuilds);
|
||||
LOG.info("Source filter index backfill queued: discovered={}, accepted={}", currentBuilds.size(), accepted);
|
||||
}
|
||||
|
||||
@EventListener
|
||||
public void onBuildAvailable(SourceFilterIndex.BuildAvailable event) {
|
||||
if (event == null || event.buildId() == null || event.buildId().isBlank()) return;
|
||||
int accepted = enqueue(List.of(event.buildId()));
|
||||
if (accepted > 0) LOG.info("Source filter index build queued after commit: build={}", event.buildId());
|
||||
}
|
||||
|
||||
private int enqueue(List<String> builds) {
|
||||
boolean schedule = false;
|
||||
int accepted = 0;
|
||||
int deferred = 0;
|
||||
synchronized (pendingLock) {
|
||||
for (String build : builds) {
|
||||
if (build == null || build.isBlank() || build.equals(inFlight) || pending.contains(build)) continue;
|
||||
if (pending.size() >= MAX_PENDING_BUILDS) {
|
||||
deferred++;
|
||||
continue;
|
||||
}
|
||||
pending.add(build);
|
||||
accepted++;
|
||||
}
|
||||
if (!pending.isEmpty() && !drainScheduled) {
|
||||
drainScheduled = true;
|
||||
schedule = true;
|
||||
}
|
||||
}
|
||||
if (deferred > 0) {
|
||||
LOG.warn("Source filter index queue is full; {} builds keep raw-value fallback", deferred);
|
||||
}
|
||||
if (schedule) scheduleDrain();
|
||||
return accepted;
|
||||
}
|
||||
|
||||
private void scheduleDrain() {
|
||||
try {
|
||||
worker.execute(this::drainPending);
|
||||
} catch (RejectedExecutionException failure) {
|
||||
int queued;
|
||||
synchronized (pendingLock) {
|
||||
drainScheduled = false;
|
||||
queued = pending.size();
|
||||
}
|
||||
LOG.error("Source filter index drain could not be scheduled; {} builds keep raw-value fallback", queued, failure);
|
||||
}
|
||||
}
|
||||
|
||||
private void drainPending() {
|
||||
long started = System.nanoTime();
|
||||
int succeeded = 0;
|
||||
int failed = 0;
|
||||
while (true) {
|
||||
String buildId;
|
||||
synchronized (pendingLock) {
|
||||
if (pending.isEmpty()) {
|
||||
drainScheduled = false;
|
||||
break;
|
||||
}
|
||||
var iterator = pending.iterator();
|
||||
buildId = iterator.next();
|
||||
iterator.remove();
|
||||
inFlight = buildId;
|
||||
}
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
synchronized (pendingLock) {
|
||||
pending.add(buildId);
|
||||
inFlight = null;
|
||||
drainScheduled = false;
|
||||
}
|
||||
LOG.info("Source filter index drain interrupted; {} builds remain queued", pendingCount());
|
||||
return;
|
||||
}
|
||||
|
||||
long buildStarted = System.nanoTime();
|
||||
try {
|
||||
index.ensureBuild(buildId);
|
||||
succeeded++;
|
||||
LOG.info("Source filter index build completed: build={}, queuedRemaining={}, elapsedMs={}", buildId,
|
||||
pendingCount(), (System.nanoTime() - buildStarted) / 1_000_000L);
|
||||
} catch (RuntimeException failure) {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
synchronized (pendingLock) {
|
||||
pending.add(buildId);
|
||||
inFlight = null;
|
||||
drainScheduled = false;
|
||||
}
|
||||
LOG.info("Source filter index drain interrupted while building {}; {} builds remain queued", buildId,
|
||||
pendingCount());
|
||||
return;
|
||||
}
|
||||
failed++;
|
||||
LOG.error("Source filter index build failed: build={}; raw-value fallback remains available, continuing",
|
||||
buildId, failure);
|
||||
} finally {
|
||||
synchronized (pendingLock) { inFlight = null; }
|
||||
}
|
||||
}
|
||||
LOG.info("Source filter index drain finished: succeeded={}, failed={}, elapsedMs={}", succeeded, failed,
|
||||
(System.nanoTime() - started) / 1_000_000L);
|
||||
}
|
||||
|
||||
private int pendingCount() {
|
||||
synchronized (pendingLock) { return pending.size(); }
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void stopWorker() {
|
||||
worker.shutdownNow();
|
||||
try {
|
||||
if (!worker.awaitTermination(2, TimeUnit.SECONDS)) {
|
||||
LOG.warn("Source filter index backfill worker did not stop within two seconds");
|
||||
}
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
private static ExecutorService createWorker() {
|
||||
return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1), runnable -> {
|
||||
Thread thread = new Thread(runnable, "source-filter-index-backfill");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
}, new ThreadPoolExecutor.AbortPolicy());
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,21 @@
|
||||
package cn.yunsheng.quality.metrics;
|
||||
|
||||
import cn.yunsheng.quality.filters.SourceFilterIndex;
|
||||
import java.util.List;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
/** Builds immutable, per-transform-build source-component rollups for read-only metric queries. */
|
||||
@Service
|
||||
public class MetricsRollupService {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(MetricsRollupService.class);
|
||||
static final String DIMENSION_COLLATION = "utf8mb4_0900_as_cs";
|
||||
private static final List<String> FACT_TABLES = List.of("fact_black", "fact_bright");
|
||||
private static final String COMPONENTS = "JSON_OBJECT(" +
|
||||
@@ -22,10 +30,18 @@ public class MetricsRollupService {
|
||||
|
||||
private final JdbcTemplate jdbc;
|
||||
private final cn.yunsheng.quality.analysis.FineDefectRollups fineDefects;
|
||||
private final SourceFilterIndex sourceFilterIndex;
|
||||
private ApplicationEventPublisher eventPublisher;
|
||||
|
||||
public MetricsRollupService(JdbcTemplate jdbc) {
|
||||
this.jdbc = jdbc;
|
||||
this.fineDefects = new cn.yunsheng.quality.analysis.FineDefectRollups(jdbc, tools.jackson.databind.json.JsonMapper.builder().build());
|
||||
this.sourceFilterIndex = new SourceFilterIndex(jdbc);
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public void setApplicationEventPublisher(ApplicationEventPublisher eventPublisher) {
|
||||
this.eventPublisher = eventPublisher;
|
||||
}
|
||||
|
||||
public record GroupKey(String stage,String month,String department,String group,String project,String product) {}
|
||||
@@ -36,6 +52,7 @@ public class MetricsRollupService {
|
||||
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent);
|
||||
if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总");
|
||||
fineDefects.copyBuild(parent, child);
|
||||
announceSourceFilterBuild(child);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
@@ -74,6 +91,30 @@ public class MetricsRollupService {
|
||||
""", buildId, buildId, buildId);
|
||||
fineDefects.refreshBuild(buildId);
|
||||
new SalesOrderMetricRollups(jdbc).refreshCurrentBuild(buildId);
|
||||
announceSourceFilterBuild(buildId);
|
||||
}
|
||||
|
||||
void announceSourceFilterBuild(String buildId) {
|
||||
if (eventPublisher == null) {
|
||||
// Directly constructed fixtures have no application event bus; retain
|
||||
// their explicit synchronous behavior (legacy schemas no-op safely).
|
||||
sourceFilterIndex.ensureBuild(buildId);
|
||||
return;
|
||||
}
|
||||
Runnable publish = () -> {
|
||||
try {
|
||||
eventPublisher.publishEvent(new SourceFilterIndex.BuildAvailable(buildId));
|
||||
} catch (RuntimeException failure) {
|
||||
LOG.error("Source filter index notification failed for {}; raw-value fallback remains available", buildId, failure);
|
||||
}
|
||||
};
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@Override public void afterCommit() { publish.run(); }
|
||||
});
|
||||
} else {
|
||||
publish.run();
|
||||
}
|
||||
}
|
||||
|
||||
static String rollupInsertSql(String table) {
|
||||
|
||||
@@ -39,7 +39,10 @@ public class MetricsService {
|
||||
private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128;
|
||||
private final JdbcTemplate jdbc;
|
||||
private final GovernanceAccessService governance;
|
||||
private final SourceColumnQueries sourceColumns;
|
||||
private SourceColumnQueries sourceColumns;
|
||||
@Autowired
|
||||
public void setSourceColumns(SourceColumnQueries sourceColumns) { this.sourceColumns = sourceColumns; }
|
||||
|
||||
private final TransactionTemplate readSnapshot;
|
||||
private final TransactionTemplate governanceSnapshot;
|
||||
|
||||
@@ -160,7 +163,7 @@ public class MetricsService {
|
||||
appendDimensions(where, args, query);
|
||||
appendExclusions(where, args, query);
|
||||
if (factFallback) {
|
||||
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor);
|
||||
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor,releaseId);
|
||||
if (!query.columnFilters().isEmpty())
|
||||
sourceColumns
|
||||
.append(where, args, stage, query.columnFilters(), releaseId, actor);
|
||||
@@ -396,7 +399,7 @@ public class MetricsService {
|
||||
appendDimensions(where, args, query);
|
||||
appendExclusions(where, args, query);
|
||||
if(factFallback) {
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
|
||||
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
|
||||
}
|
||||
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
@@ -599,7 +602,8 @@ public class MetricsService {
|
||||
"sourceMonths", sourceMonths);
|
||||
}
|
||||
|
||||
private static String factFilterHint(String stage,boolean factFallback,MetricsQuery query) {
|
||||
private String factFilterHint(String stage,boolean factFallback,MetricsQuery query) {
|
||||
if(sourceColumns.indexed(query.releaseId()))return "";
|
||||
if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return "";
|
||||
// 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。
|
||||
if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null
|
||||
@@ -622,7 +626,7 @@ public class MetricsService {
|
||||
appendDimensions(where, args, query);
|
||||
appendExclusions(where, args, query);
|
||||
if(factFallback) {
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
|
||||
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
|
||||
}
|
||||
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
|
||||
@@ -53,6 +53,10 @@ public class RecordController {
|
||||
private final JsonMapper mapper;
|
||||
private final GovernanceAccessService governance;
|
||||
private final RecordsWorkbookExportService workbookExport;
|
||||
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
|
||||
|
||||
|
||||
public RecordController(JdbcTemplate jdbc, ReleaseService releases, JsonMapper mapper,
|
||||
GovernanceAccessService governance, RecordsWorkbookExportService workbookExport) {
|
||||
@@ -61,6 +65,7 @@ public class RecordController {
|
||||
this.mapper = mapper;
|
||||
this.governance = governance;
|
||||
this.workbookExport = workbookExport;
|
||||
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
|
||||
}
|
||||
|
||||
|
||||
@@ -156,10 +161,10 @@ public class RecordController {
|
||||
args.add(search);
|
||||
args.add(jsonPath(selectedField));
|
||||
}
|
||||
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor);
|
||||
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor,selectedRelease);
|
||||
var sourceFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters);
|
||||
for(var rule:sourceFilters)if(!List.of(stage).containsAll((List<?>)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段");
|
||||
if(!sourceFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,stage,sourceFilters,selectedRelease,actor);
|
||||
if(!sourceFilters.isEmpty())sourceColumns.append(predicate,args,stage,sourceFilters,selectedRelease,actor);
|
||||
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
|
||||
String countSql = """
|
||||
|
||||
+12
-4
@@ -52,6 +52,10 @@ public class RecordsWorkbookExportService {
|
||||
private final ReleaseService releases;
|
||||
private final GovernanceAccessService governance;
|
||||
private final JsonMapper mapper;
|
||||
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
|
||||
|
||||
|
||||
public RecordsWorkbookExportService(JdbcTemplate jdbc, ReleaseService releases,
|
||||
GovernanceAccessService governance, JsonMapper mapper) {
|
||||
@@ -59,6 +63,7 @@ public class RecordsWorkbookExportService {
|
||||
this.releases = releases;
|
||||
this.governance = governance;
|
||||
this.mapper = mapper;
|
||||
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
|
||||
}
|
||||
|
||||
public Path export(Integer year, Integer month, Integer quarter, String department, String group,
|
||||
@@ -120,8 +125,11 @@ public class RecordsWorkbookExportService {
|
||||
LinkedHashSet<String> rawFields = new LinkedHashSet<>();
|
||||
if (releaseId != null) {
|
||||
QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product,
|
||||
projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd);
|
||||
scan(rawScope, rs -> rawFields.addAll(visibleRaw(readJson(rs.getString(1)), allowedFields).keySet()));
|
||||
projectGroup, includeUnassigned, "CAST(JSON_KEYS(f.raw_values) AS CHAR)", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd);
|
||||
scan(rawScope, rs -> {
|
||||
Object decoded=mapper.readValue(rs.getString(1),Object.class);
|
||||
if(decoded instanceof List<?> keys)for(Object key:keys)if(key instanceof String name&&(allowedFields.contains("*")||allowedFields.contains(name)))rawFields.add(name);
|
||||
});
|
||||
}
|
||||
boolean showOrderType=SalesOrderTypes.allowed(allowedFields);
|
||||
if (rawFields.size() + HEADER_COUNT + (showOrderType?1:0) > MAX_COLUMNS)
|
||||
@@ -167,8 +175,8 @@ public class RecordsWorkbookExportService {
|
||||
SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value);
|
||||
if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__"))
|
||||
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
|
||||
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor);
|
||||
if(!columnFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
|
||||
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor,releaseId);
|
||||
if(!columnFilters.isEmpty())sourceColumns.append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
|
||||
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
|
||||
"f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
String columns = select == null ? "f.raw_values" : select;
|
||||
|
||||
@@ -15,6 +15,7 @@ import cn.yunsheng.quality.ingestion.ImportWorker;
|
||||
import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild;
|
||||
import cn.yunsheng.quality.ingestion.MappingValueCodec;
|
||||
import cn.yunsheng.quality.metrics.MetricsService;
|
||||
import cn.yunsheng.quality.metrics.MetricsRollupService;
|
||||
import cn.yunsheng.quality.release.ReleaseService;
|
||||
import cn.yunsheng.quality.source.SourceCatalogService;
|
||||
import cn.yunsheng.quality.source.SourceDeletionService;
|
||||
@@ -66,6 +67,7 @@ public class DataWorkflowService {
|
||||
private final TaskExecutor executor;
|
||||
private final TransactionTemplate transaction;
|
||||
private final MetricsService metrics;
|
||||
private MetricsRollupService rollups;
|
||||
private final AtomicReference<String> runningOperation = new AtomicReference<>();
|
||||
private final AtomicReference<String> runningLease = new AtomicReference<>();
|
||||
|
||||
@@ -108,6 +110,12 @@ public class DataWorkflowService {
|
||||
this.executor = executor;
|
||||
this.transaction = new TransactionTemplate(manager);
|
||||
this.metrics=metrics;
|
||||
this.rollups=new MetricsRollupService(jdbc);
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public void setMetricsRollupService(MetricsRollupService rollups) {
|
||||
this.rollups = rollups;
|
||||
}
|
||||
|
||||
public Map<String, Object> status(AppUserPrincipal actor) {
|
||||
@@ -376,7 +384,7 @@ public class DataWorkflowService {
|
||||
sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor);
|
||||
updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED");
|
||||
var selectedRules=unclassifiedRules;
|
||||
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
|
||||
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,rollups).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
|
||||
jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token);
|
||||
publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false);
|
||||
jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token);
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
CREATE TABLE source_filter_build (
|
||||
build_id CHAR(36) NOT NULL PRIMARY KEY,
|
||||
format_version INT NOT NULL,
|
||||
CONSTRAINT fk_source_filter_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
CREATE TABLE source_filter_field (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
build_id CHAR(36) NOT NULL,
|
||||
stage VARCHAR(16) NOT NULL,
|
||||
column_name LONGTEXT COLLATE utf8mb4_0900_bin NOT NULL,
|
||||
name_hash BINARY(32) NOT NULL,
|
||||
UNIQUE KEY uq_source_filter_field (build_id,stage,name_hash),
|
||||
CONSTRAINT fk_source_filter_field_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
CREATE TABLE source_filter_value (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
field_id BIGINT NOT NULL,
|
||||
value_hash BINARY(32) NOT NULL,
|
||||
value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,
|
||||
UNIQUE KEY uq_source_filter_value_hash (field_id,value_hash),
|
||||
KEY ix_source_filter_value_text (field_id,value(128)),
|
||||
CONSTRAINT fk_source_filter_value_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
CREATE TABLE source_filter_member (
|
||||
field_id BIGINT NOT NULL,
|
||||
fact_id BIGINT NOT NULL,
|
||||
value_id BIGINT NOT NULL,
|
||||
PRIMARY KEY (field_id,fact_id),
|
||||
KEY ix_source_filter_member_value (value_id,fact_id),
|
||||
CONSTRAINT fk_source_filter_member_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id),
|
||||
CONSTRAINT fk_source_filter_member_value FOREIGN KEY (value_id) REFERENCES source_filter_value(id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
-- Cover representative-row selection without reading every row's large source JSON.
|
||||
ALTER TABLE fact_black ADD KEY ix_black_source_catalog (build_id,source_revision_id,department,group_name,id);
|
||||
ALTER TABLE fact_bright ADD KEY ix_bright_source_catalog (build_id,source_revision_id,department,group_name,id);
|
||||
+130
-1
@@ -265,7 +265,9 @@ class SourceColumnQueriesMySqlTest {
|
||||
return fields.stream().filter(item -> item.get("label").equals(label)).findFirst().orElse(null);
|
||||
}
|
||||
|
||||
@Test void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters() throws Exception {
|
||||
@org.junit.jupiter.params.ParameterizedTest
|
||||
@org.junit.jupiter.params.provider.ValueSource(booleans={false,true})
|
||||
void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed) throws Exception {
|
||||
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))");
|
||||
jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE);
|
||||
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
|
||||
@@ -275,6 +277,7 @@ class SourceColumnQueriesMySqlTest {
|
||||
}
|
||||
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
|
||||
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
|
||||
if(indexed){installFilterIndex();var index=new SourceFilterIndex(jdbc);for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);}
|
||||
var headerReads=new java.util.concurrent.atomic.AtomicInteger();
|
||||
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
|
||||
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
|
||||
@@ -417,6 +420,132 @@ class SourceColumnQueriesMySqlTest {
|
||||
assertThrows(ResponseStatusException.class,()->controller.query(input,actor));
|
||||
}
|
||||
|
||||
@Test void indexedFiltersCandidatesAndTuplesMatchRawSemanticsAndStayAuthorized() throws Exception {
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
|
||||
List<String> samples=List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[1,2]}","{\"委托单\":{\"a\":1}}","{\"委托单\":\"é\"}","{\"委托单\":\"é\"}","{\"委托单\":\"A\"}","{\"委托单\":\"a\"}","{\"委托单\":\" padded \"}","{\"委托单\":\"null\"}");
|
||||
for(String raw:samples)insert("black","build-black",BLACK_REVISION,"部门A","组A",raw);
|
||||
String longKey="来源".repeat(300);
|
||||
insert("black","build-black",BLACK_REVISION,"部门A","组A",JsonMapper.builder().build().writeValueAsString(Map.of(longKey,"long-key-value")));
|
||||
insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}");
|
||||
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
|
||||
List<Map<String,Object>> rules=new java.util.ArrayList<>();
|
||||
for(String op:List.of("in","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}"))
|
||||
rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value)));
|
||||
var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList();
|
||||
var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor);
|
||||
var rawMembers=queries.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor);
|
||||
jdbc.update("INSERT INTO query_filter_field VALUES ('plain','原始委托单','委托单','委托单',TRUE,TRUE,1)");
|
||||
List<List<String>> legacySelections=List.of(List.of("null"),List.of("0007"),List.of(QueryFilterFields.EMPTY),List.of("0007",QueryFilterFields.EMPTY));
|
||||
var rawLegacy=legacySelections.stream().map(this::legacyCount).toList();
|
||||
installFilterIndex();
|
||||
var index=new SourceFilterIndex(jdbc);
|
||||
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
|
||||
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
|
||||
assertTrue(indexed.indexed(RELEASE));
|
||||
assertEquals(rawLegacy,legacySelections.stream().map(this::legacyCount).toList());
|
||||
assertEquals("long-key-value",jdbc.queryForObject("SELECT sv.value FROM source_filter_field sf JOIN source_filter_value sv ON sv.field_id=sf.id WHERE sf.column_name=? AND sf.build_id='build-black'",String.class,longKey));
|
||||
var combined=List.of(Map.<String,Object>of("field",key,"stages",List.of("black"),"operator","in","values",List.of("0007")),Map.<String,Object>of("field",field(indexed.catalog(RELEASE,actor),"加工单位").get("key"),"stages",List.of("black"),"operator","in","values",List.of("机二")));
|
||||
assertEquals(0L,indexed.memberPage(RELEASE,Map.of("year",2026,"columnFilters",combined),List.of(key),List.of("black"),50,0,actor).get("totalCount"));
|
||||
assertEquals(List.of(),indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),"hidden-only-value",50,0,true,actor).get("rows"));
|
||||
jdbc.update("INSERT INTO transform_build VALUES ('build-child')");
|
||||
jdbc.update("INSERT INTO fact_black(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT 'build-child',source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no FROM fact_black WHERE build_id='build-black'");
|
||||
index.ensureBuild("build-child");
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member sm JOIN source_filter_field sf ON sf.id=sm.field_id JOIN fact_black f ON f.id=sm.fact_id WHERE sf.build_id='build-child' AND f.build_id<>'build-child'",Long.class));
|
||||
assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
|
||||
assertEquals(rawValues,indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor));
|
||||
assertEquals(rawMembers,indexed.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor));
|
||||
var page=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,2,0,true,actor);
|
||||
assertEquals(2,((List<?>)page.get("rows")).size());assertEquals(true,page.get("hasMore"));assertFalse(page.containsKey("totalCount"));
|
||||
long members=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class);
|
||||
index.ensureBuild("build-black");assertEquals(members,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("加工单位"));
|
||||
assertThrows(ResponseStatusException.class,()->indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),null,50,0,true,actor));
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
|
||||
jdbc.update("DELETE FROM source_filter_build WHERE build_id='build-black'");
|
||||
assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
|
||||
}
|
||||
|
||||
@Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception {
|
||||
installFilterIndex();
|
||||
var fail=new java.util.concurrent.atomic.AtomicBoolean(true);
|
||||
var failing=new JdbcTemplate(jdbc.getDataSource()) {
|
||||
@Override public int update(String sql) {
|
||||
if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure");
|
||||
return super.update(sql);
|
||||
}
|
||||
@Override public int update(String sql,Object... args) {
|
||||
if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure");
|
||||
return super.update(sql,args);
|
||||
}
|
||||
};
|
||||
var index=new SourceFilterIndex(failing);
|
||||
assertThrows(IllegalStateException.class,()->index.ensureBuild("build-black"));
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build",Long.class));
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_field",Long.class));
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_value",Long.class));
|
||||
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
|
||||
assertFalse(index.ready(RELEASE));
|
||||
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
|
||||
assertTrue(index.ready(RELEASE));
|
||||
}
|
||||
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_FILTER_BENCHMARK",matches="true")
|
||||
void syntheticWideSourceIndexBenchmark() throws Exception {
|
||||
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
|
||||
int size=Integer.getInteger("quality.benchmarkRows",10000);
|
||||
jdbc.execute("CREATE TABLE benchmark_digit(n INT PRIMARY KEY)");for(int n=0;n<10;n++)jdbc.update("INSERT INTO benchmark_digit VALUES (?)",n);
|
||||
for(String stage:List.of("black","bright")) {
|
||||
jdbc.execute("DELETE FROM fact_"+stage);
|
||||
StringBuilder raw=new StringBuilder("JSON_OBJECT('客户名称',CONCAT('客户',MOD(n,300))");
|
||||
for(int f=0;f<120;f++)raw.append(",'字段").append(f).append("',CONCAT('value-").append(f).append("-',").append(stage.equals("black")?"MOD(n,50)":"n").append(")");
|
||||
raw.append(')');
|
||||
String numbers="SELECT a.n+10*b.n+100*c.n+1000*d.n+10000*e.n AS n FROM benchmark_digit a CROSS JOIN benchmark_digit b CROSS JOIN benchmark_digit c CROSS JOIN benchmark_digit d CROSS JOIN benchmark_digit e LIMIT "+size;
|
||||
jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT ?,?,'2026-01','部门A','组A','P-1','项目A',"+raw+",'data',n+1 FROM ("+numbers+") numbers","build-"+stage,stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION);
|
||||
}
|
||||
var rawQuery=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
|
||||
String field=(String)field(rawQuery.catalog(RELEASE,actor),"客户名称").get("key");
|
||||
var old=new java.util.LinkedHashMap<String,Object>();
|
||||
for(String stage:List.of("black","bright")) {
|
||||
long start=System.nanoTime();var result=rawQuery.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
|
||||
old.put(stage,Map.of("ms",(System.nanoTime()-start)/1_000_000L,"result",result));
|
||||
}
|
||||
installFilterIndex();var index=new SourceFilterIndex(jdbc);
|
||||
for(String build:List.of("build-black","build-bright","build-hidden")) {
|
||||
long start=System.nanoTime();index.ensureBuild(build);System.out.println("FILTER_BENCH_BUILD "+build+" rows="+size+" ms="+(System.nanoTime()-start)/1_000_000L);
|
||||
}
|
||||
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
|
||||
for(String stage:List.of("black","bright")) {
|
||||
long start=System.nanoTime();var result=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
|
||||
long elapsed=(System.nanoTime()-start)/1_000_000L;
|
||||
assertEquals(((Map<?,?>)old.get(stage)).get("result"),result);
|
||||
System.out.println("FILTER_BENCH_QUERY "+stage+" rawMs="+((Map<?,?>)old.get(stage)).get("ms")+" indexMs="+elapsed);
|
||||
}
|
||||
for(String table:List.of("source_filter_field","source_filter_value","source_filter_member"))jdbc.execute("ANALYZE TABLE "+table);
|
||||
System.out.println("FILTER_BENCH_SPACE "+jdbc.queryForList("SELECT table_name,table_rows,data_length,index_length FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name LIKE 'source_filter_%'"));
|
||||
}
|
||||
|
||||
private long legacyCount(List<String> selected) {
|
||||
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
|
||||
new QueryFilterFields(jdbc,governance).append(where,args,"black",Map.of("plain",selected),actor,RELEASE);
|
||||
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
|
||||
}
|
||||
|
||||
private long filteredCount(SourceColumnQueries source,Map<String,Object> rule) {
|
||||
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
|
||||
source.append(where,args,"black",List.of(rule),RELEASE,actor);
|
||||
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
|
||||
}
|
||||
|
||||
private void installFilterIndex() throws Exception {
|
||||
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
|
||||
for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id);
|
||||
String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/V43__source_filter_indexes.sql"));
|
||||
for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement);
|
||||
}
|
||||
|
||||
private void schema() {
|
||||
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))");
|
||||
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))");
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package cn.yunsheng.quality.filters;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.AbstractExecutorService;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
|
||||
class SourceFilterIndexBackfillTest {
|
||||
@Test
|
||||
void skipsWhenIndexTablesAreUnavailable() {
|
||||
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
|
||||
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
|
||||
ManualExecutor worker = new ManualExecutor();
|
||||
doReturn(false).when(index).available();
|
||||
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
|
||||
|
||||
runner.run(null);
|
||||
|
||||
verify(jdbc, never()).query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
|
||||
verify(index, never()).ensureBuild(anyString());
|
||||
assertEquals(0, worker.queued());
|
||||
runner.stopWorker();
|
||||
}
|
||||
|
||||
@Test
|
||||
void coalescesEventsDuringDrainAndContinuesAfterAnIndividualFailure() {
|
||||
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
|
||||
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
|
||||
ManualExecutor worker = new ManualExecutor();
|
||||
doReturn(true).when(index).available();
|
||||
doReturn(List.of("failed-build", "ready-build")).when(jdbc)
|
||||
.query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
|
||||
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
|
||||
doAnswer(invocation -> {
|
||||
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
|
||||
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
|
||||
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("failed-build"));
|
||||
throw new IllegalStateException("synthetic index failure");
|
||||
}).when(index).ensureBuild("failed-build");
|
||||
|
||||
runner.run(null);
|
||||
|
||||
assertEquals(1, worker.queued());
|
||||
verify(index, never()).ensureBuild(anyString());
|
||||
ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
|
||||
verify(jdbc).query(sql.capture(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
|
||||
assertTrue(sql.getValue().contains("c.dataset_id='quality'"));
|
||||
assertTrue(sql.getValue().contains("r.status='PUBLISHED'"));
|
||||
assertTrue(sql.getValue().contains("r.complete=TRUE"));
|
||||
assertTrue(sql.getValue().contains("b.status='COMPLETED'"));
|
||||
|
||||
worker.runNext();
|
||||
|
||||
verify(index).ensureBuild("failed-build");
|
||||
verify(index).ensureBuild("ready-build");
|
||||
verify(index).ensureBuild("event-during-drain");
|
||||
assertEquals(0, worker.queued());
|
||||
runner.stopWorker();
|
||||
assertTrue(worker.isShutdown());
|
||||
}
|
||||
|
||||
@Test
|
||||
void componentHasAnAutowiredProductionConstructorAndCleanShutdown() {
|
||||
try (var context = new AnnotationConfigApplicationContext()) {
|
||||
context.registerBean(JdbcTemplate.class, () -> org.mockito.Mockito.mock(JdbcTemplate.class));
|
||||
context.registerBean(SourceFilterIndex.class, () -> org.mockito.Mockito.mock(SourceFilterIndex.class));
|
||||
context.register(SourceFilterIndexBackfill.class);
|
||||
context.refresh();
|
||||
|
||||
assertNotNull(context.getBean(SourceFilterIndexBackfill.class));
|
||||
}
|
||||
}
|
||||
|
||||
private static final class ManualExecutor extends AbstractExecutorService {
|
||||
private final ArrayDeque<Runnable> tasks = new ArrayDeque<>();
|
||||
private boolean shutdown;
|
||||
|
||||
@Override public void shutdown() { shutdown = true; }
|
||||
|
||||
@Override public List<Runnable> shutdownNow() {
|
||||
shutdown = true;
|
||||
List<Runnable> pending = List.copyOf(tasks);
|
||||
tasks.clear();
|
||||
return pending;
|
||||
}
|
||||
|
||||
@Override public boolean isShutdown() { return shutdown; }
|
||||
|
||||
@Override public boolean isTerminated() { return shutdown && tasks.isEmpty(); }
|
||||
|
||||
@Override public boolean awaitTermination(long timeout, TimeUnit unit) { return isTerminated(); }
|
||||
|
||||
@Override public void execute(Runnable command) {
|
||||
if (shutdown) throw new RejectedExecutionException("executor is stopped");
|
||||
tasks.add(command);
|
||||
}
|
||||
|
||||
int queued() { return tasks.size(); }
|
||||
|
||||
void runNext() { tasks.removeFirst().run(); }
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,21 @@
|
||||
package cn.yunsheng.quality.metrics;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.ArgumentMatchers.contains;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
|
||||
import cn.yunsheng.quality.filters.SourceFilterIndex;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
class MetricsRollupServiceTest {
|
||||
@Test
|
||||
@@ -17,4 +30,65 @@ class MetricsRollupServiceTest {
|
||||
assertTrue(sql.contains("GROUP BY f.build_id, f.month_key " + collation));
|
||||
assertTrue(sql.contains("f.group_name " + collation + ", f.project_group " + collation));
|
||||
}
|
||||
|
||||
@Test
|
||||
void publishesFilterBuildAvailableOnlyAfterTheOuterTransactionCommits() {
|
||||
var service = new MetricsRollupService(mock(JdbcTemplate.class));
|
||||
var publisher = mock(ApplicationEventPublisher.class);
|
||||
service.setApplicationEventPublisher(publisher);
|
||||
String build = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
try {
|
||||
service.announceSourceFilterBuild(build);
|
||||
verifyNoInteractions(publisher);
|
||||
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
|
||||
assertEquals(1, synchronizations.size());
|
||||
for (TransactionSynchronization synchronization : synchronizations) synchronization.afterCommit();
|
||||
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
|
||||
} finally {
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void publishesImmediatelyWhenThereIsNoTransactionSynchronization() {
|
||||
var service = new MetricsRollupService(mock(JdbcTemplate.class));
|
||||
var publisher = mock(ApplicationEventPublisher.class);
|
||||
service.setApplicationEventPublisher(publisher);
|
||||
String build = "22222222-2222-4222-8222-222222222222";
|
||||
|
||||
service.announceSourceFilterBuild(build);
|
||||
|
||||
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
|
||||
}
|
||||
|
||||
@Test
|
||||
void notificationFailureAfterCommitDoesNotEscapeTheTransactionSynchronization() {
|
||||
var service = new MetricsRollupService(mock(JdbcTemplate.class));
|
||||
var publisher = mock(ApplicationEventPublisher.class);
|
||||
service.setApplicationEventPublisher(publisher);
|
||||
String build = "33333333-3333-4333-8333-333333333333";
|
||||
var event = new SourceFilterIndex.BuildAvailable(build);
|
||||
doThrow(new IllegalStateException("synthetic event delivery failure")).when(publisher).publishEvent(event);
|
||||
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
try {
|
||||
service.announceSourceFilterBuild(build);
|
||||
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
|
||||
assertDoesNotThrow(() -> synchronizations.get(0).afterCommit());
|
||||
} finally {
|
||||
TransactionSynchronizationManager.clearSynchronization();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void directFixtureConstructionKeepsTheSynchronousLegacySchemaFallback() {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
var service = new MetricsRollupService(jdbc);
|
||||
|
||||
service.announceSourceFilterBuild("44444444-4444-4444-8444-444444444444");
|
||||
|
||||
verify(jdbc).queryForObject(contains("information_schema.tables"), eq(Integer.class));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
|
||||
TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED')
|
||||
BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup',
|
||||
'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build',
|
||||
'source_filter_member', 'source_filter_value', 'source_filter_field', 'source_filter_build',
|
||||
'fact_black', 'fact_bright')
|
||||
OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion',
|
||||
'release_action', 'candidate_reconciliation')
|
||||
@@ -253,7 +254,24 @@ class Database:
|
||||
self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid))
|
||||
for t in BUILD_CHILDREN:
|
||||
if t in self.columns:
|
||||
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
|
||||
if t == 'source_filter_member':
|
||||
# Members have no build_id; drain each owned field in
|
||||
# order without aliases because the exclusion uses LOCK TABLES.
|
||||
count = 0
|
||||
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
|
||||
count = self.execute('DELETE FROM source_filter_member WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
|
||||
if count:
|
||||
break
|
||||
elif t == 'source_filter_value':
|
||||
# Values are likewise owned through fields and are deleted
|
||||
# only after all field memberships have been drained.
|
||||
count = 0
|
||||
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
|
||||
count = self.execute('DELETE FROM source_filter_value WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
|
||||
if count:
|
||||
break
|
||||
else:
|
||||
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
|
||||
if count:
|
||||
return count, 0
|
||||
self.execute('DELETE FROM transform_build WHERE id=%s', (bid,))
|
||||
|
||||
@@ -16,6 +16,7 @@ from unittest.mock import patch
|
||||
ROOT=Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0,str(ROOT/'scripts'))
|
||||
import backup_retention as br
|
||||
import managed_test_resources
|
||||
spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py'))
|
||||
ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret)
|
||||
|
||||
@@ -91,14 +92,18 @@ class DatabaseRetentionTests(unittest.TestCase):
|
||||
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
|
||||
cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True)
|
||||
cls.name='quality_retention_test_'+uuid.uuid4().hex[:12]
|
||||
managed_test_resources.create(ROOT,cls.name,cls.execute_admin)
|
||||
with cls.admin.cursor() as c:
|
||||
c.execute('CREATE DATABASE '+ret.ident(cls.name))
|
||||
c.execute('USE '+ret.ident(cls.name))
|
||||
ddl=[
|
||||
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
|
||||
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
|
||||
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)',
|
||||
'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE source_filter_build(build_id varchar(36) PRIMARY KEY,format_version int,FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
|
||||
'CREATE TABLE source_filter_member(field_id bigint NOT NULL,fact_id bigint NOT NULL,value_id bigint NOT NULL,PRIMARY KEY(field_id,fact_id),KEY(value_id,fact_id),FOREIGN KEY(field_id) REFERENCES source_filter_field(id),FOREIGN KEY(value_id) REFERENCES source_filter_value(id))',
|
||||
'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)',
|
||||
@@ -115,14 +120,25 @@ class DatabaseRetentionTests(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name))
|
||||
managed_test_resources.drop(ROOT,cls.name,cls.execute_admin)
|
||||
cls.admin.close()
|
||||
|
||||
@classmethod
|
||||
def execute_admin(cls,sql,args=None):
|
||||
result=''
|
||||
statements=[part.strip() for part in sql.split(';') if part.strip()]
|
||||
with cls.admin.cursor() as cursor:
|
||||
for statement in statements:
|
||||
cursor.execute(statement,args if len(statements)==1 else None)
|
||||
result='\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
|
||||
return result
|
||||
|
||||
def setUp(self):
|
||||
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
|
||||
self.db=ret.Database(ROOT,self.name);self.db.root=self.root
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
|
||||
for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table))
|
||||
for table in self.db.tables:
|
||||
if table!='__quality_test_owner':self.db.execute('TRUNCATE TABLE '+ret.ident(table))
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
|
||||
self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')")
|
||||
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
|
||||
@@ -187,14 +203,24 @@ class DatabaseRetentionTests(unittest.TestCase):
|
||||
|
||||
def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self):
|
||||
self.retire_old();state={'builds':{'obsolete':1}}
|
||||
self.db.execute("INSERT INTO source_filter_build VALUES('obsolete',1)")
|
||||
self.db.execute("INSERT INTO source_filter_field VALUES(9,'obsolete','black','空字段',%s),(10,'obsolete','black','客户',%s),(11,'obsolete','black','工单',%s)", (b'0'*32,b'1'*32,b'2'*32))
|
||||
self.db.execute("INSERT INTO source_filter_value VALUES(100,10,%s,'客户A'),(101,10,%s,'客户B'),(102,11,%s,'工单A')", (b'1'*32,b'2'*32,b'3'*32))
|
||||
self.db.execute("INSERT INTO source_filter_member VALUES(10,1,100),(10,2,101),(11,3,102)")
|
||||
with self.db.frozen('current'):
|
||||
self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0))
|
||||
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_member')[0]['n'],2)
|
||||
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_value')[0]['n'],3)
|
||||
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_field')[0]['n'],3)
|
||||
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_build')[0]['n'],1)
|
||||
old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0]
|
||||
self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64)
|
||||
self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,))
|
||||
for _ in range(4):
|
||||
for _ in range(20):
|
||||
with self.db.frozen('current'):self.db.build_step(state,1800,4000,1)
|
||||
self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'"))
|
||||
for table in ('source_filter_member','source_filter_value','source_filter_field','source_filter_build'):
|
||||
self.assertFalse(self.db.rows('SELECT * FROM '+ret.ident(table)))
|
||||
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'"))
|
||||
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'"))
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from unittest.mock import patch
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
import backup_retention as br
|
||||
import managed_test_resources
|
||||
import revision_retention as revisions
|
||||
spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py'))
|
||||
ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret)
|
||||
@@ -26,12 +27,15 @@ class SourceRevisionTests(unittest.TestCase):
|
||||
cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root',
|
||||
password=cfg['rootPassword'], autocommit=True)
|
||||
cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
|
||||
managed_test_resources.create(ROOT, cls.name, cls.execute_admin)
|
||||
ddl = [
|
||||
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))',
|
||||
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))',
|
||||
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
|
||||
"CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)",
|
||||
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
|
||||
'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||
'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))',
|
||||
@@ -42,23 +46,34 @@ class SourceRevisionTests(unittest.TestCase):
|
||||
'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))']
|
||||
try:
|
||||
with cls.admin.cursor() as c:
|
||||
c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name))
|
||||
c.execute('USE ' + ret.ident(cls.name))
|
||||
for statement in ddl:
|
||||
c.execute(statement + ' ENGINE=InnoDB')
|
||||
except BaseException:
|
||||
with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name))
|
||||
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
|
||||
cls.admin.close(); raise
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name))
|
||||
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
|
||||
cls.admin.close()
|
||||
|
||||
@classmethod
|
||||
def execute_admin(cls, sql, args=None):
|
||||
result = ''
|
||||
statements = [part.strip() for part in sql.split(';') if part.strip()]
|
||||
with cls.admin.cursor() as cursor:
|
||||
for statement in statements:
|
||||
cursor.execute(statement, args if len(statements) == 1 else None)
|
||||
result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
|
||||
return result
|
||||
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve()
|
||||
self.db = ret.Database(ROOT, self.name); self.db.root = self.root
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
|
||||
for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
|
||||
for table in self.db.tables:
|
||||
if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
|
||||
self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')")
|
||||
self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')")
|
||||
@@ -121,6 +136,21 @@ class SourceRevisionTests(unittest.TestCase):
|
||||
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
||||
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
||||
|
||||
def test_indexed_business_values_do_not_count_as_revision_metadata(self):
|
||||
path = self.add_revision('indexed-only', b'indexed-only workbook')
|
||||
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,))
|
||||
self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,))
|
||||
self.collect(100); result = self.collect(2000)
|
||||
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||
self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists())
|
||||
|
||||
def test_indexed_field_names_do_not_count_as_revision_metadata(self):
|
||||
path = self.add_revision('indexed-field-only', b'field-only workbook')
|
||||
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,))
|
||||
self.collect(100); result = self.collect(2000)
|
||||
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||
self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists())
|
||||
|
||||
def test_new_reference_resets_continuous_grace(self):
|
||||
self.add_revision(); self.collect(100)
|
||||
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500)
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
核实日期:2026年11月12日(上海时区)。这是可更新的接续记录,不替代实际代码与运行状态。
|
||||
|
||||
|
||||
## 2026-11-13:统一筛选提速(发布验收中)
|
||||
|
||||
实现:V43新增按构建保存的来源字段字典和稀疏记录索引,及两阶段字段目录覆盖索引;来源筛选、配置筛选、候选、指标、趋势、明细与明细导出共用。来源列查询实例统一复用,候选模式省去精确总数,产品页常规查询不再等待慢目录,本字段选中值变化不重复读取自己的候选;明细导出先只读字段名再读业务行。
|
||||
|
||||
生命周期:启动仅在单线程后台补当前完整发布版本;导入和归类通过提交后事件进入同一有界去重队列。完整标记提交后才启用,故障保留原始查询,业务事务不等待索引。旧配置 `null`、来源字符串 `null`、空值、Unicode、同一事实行和当前权限分别保持;构建保留清理识别4张新表,未解除维护兼容保护。
|
||||
|
||||
验证:前端相关46项及生产构建通过;隔离MySQL覆盖索引与原始结果、黑片/亮片、明细和Excel,以及故障回滚和重试。保留清理34项通过。合成压力样本为两阶段各10000行×121列,索引约638MB,构建约73/183秒,候选约231/152ms;小规模热缓存原查询约58/47ms,因此不将该基准当作相对提速证明。完整后端构建985项(819执行通过、166按条件跳过),专项46项(45通过、1跳过)。线上同条件收益、实际空间仍在验收。本轮未同步Gitea。
|
||||
|
||||
## 2026-11-13:来源字段筛选超时修复
|
||||
|
||||
定位:客户等原始字段筛选时,汇总、趋势和覆盖检查重复读取字段目录,数据库对年度查询选择了原表扫描,约48秒超过页面30秒期限。复用服务级字段目录缓存,目录及无窄范围限制的年度来源筛选使用既有构建索引;JSON值每次表达式只提取一次,保留空值、字符串和大小写语义。单月、季度、产品及扩展条件保留原查询计划。查询超时提示改为重试或调整范围,不再误报连接断开。权限和正式计算规则不变,无新增表或索引。
|
||||
|
||||
@@ -9,9 +9,11 @@ import uuid
|
||||
|
||||
|
||||
# These are measured business values, not documents that refer to source IDs.
|
||||
VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'),
|
||||
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
|
||||
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')}
|
||||
BUSINESS_VALUE_COLUMNS = {('fact_black', 'raw_values'), ('fact_black', 'components'),
|
||||
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
|
||||
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components'),
|
||||
('source_filter_field', 'column_name'),
|
||||
('source_filter_value', 'value')}
|
||||
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
|
||||
|
||||
|
||||
@@ -54,7 +56,7 @@ class Cleaner:
|
||||
def metadata_mentions(self, needles, excluded=None):
|
||||
for table, columns in self.db.column_types.items():
|
||||
for column, kind in columns.items():
|
||||
if kind not in TEXT_TYPES or (table, column) in VALUE_JSON:
|
||||
if kind not in TEXT_TYPES or (table, column) in BUSINESS_VALUE_COLUMNS:
|
||||
continue
|
||||
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
|
||||
args = list(needles)
|
||||
|
||||
@@ -37,7 +37,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
const [moreFiltersOpen,setMoreFiltersOpen]=useState(false);
|
||||
const [extraFilters,setExtraFilters]=useState<Record<string,string[]>>(initial.extraFilters||{});
|
||||
const [extraFields,setExtraFields]=useState<FilterField[]>([]);
|
||||
const [options,setOptions]=useState<OptionsResponse|null>(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');const [supportLoading,setSupportLoading]=useState(true);
|
||||
const [options,setOptions]=useState<OptionsResponse|null>(null);const [catalog,setCatalog]=useState<{releaseId:string;sourceFields:SourceColumnField[]}|null>(null);const [supportError,setSupportError]=useState('');
|
||||
const [applied,setApplied]=useState<QueryParams|null>(null);const [revision,setRevision]=useState(0);const [response,setResponse]=useState<{key:string;value:MetricsResponse}|null>(null);const [loading,setLoading]=useState(false);const [failure,setFailure]=useState<{key:string;message:string}|null>(null);
|
||||
const [defectDisplay,setDefectDisplay]=useState<'weight'|'rate'>('weight');
|
||||
const [fullFields,setFullFields]=useState(true);const [rateBasis,setRateBasis]=useState<'original'|'withoutHeadquartersScrap'>('original');const [basisOpen,setBasisOpen]=useState(false);const [trendMode,setTrendMode]=useState<'production'|'defects'|'data'>('production');
|
||||
@@ -50,29 +50,39 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
const supportGeneration=useRef(0);const supportController=useRef<AbortController|null>(null);
|
||||
const loadSupport=useCallback(async()=>{
|
||||
const generation=++supportGeneration.current;supportController.current?.abort();const controller=new AbortController();supportController.current=controller;
|
||||
setSupportLoading(true);setSupportError('');
|
||||
try{const [choices,source,filterFields]=await Promise.all([api.options(controller.signal),api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal),api.filterFields(false,controller.signal)]);if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);setCatalog(source);setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));}
|
||||
catch(error){if(generation===supportGeneration.current&&!onUnauthorized(error))setSupportError(error instanceof Error?error.message:'查询条件读取失败');}
|
||||
finally{if(generation===supportGeneration.current)setSupportLoading(false);}
|
||||
setSupportError('');
|
||||
const fail=(error:unknown)=>{if(generation===supportGeneration.current&&!controller.signal.aborted&&!onUnauthorized(error))setSupportError(previous=>previous||(error instanceof Error?error.message:'查询条件读取失败'));};
|
||||
const requests=[
|
||||
api.options(controller.signal).then(choices=>{if(generation!==supportGeneration.current)return;if(automaticPeriod.current){const available=defaultQueryMonth(choices);setMonth(available.month);automaticPeriod.current=false;}setOptions(choices);}).catch(fail),
|
||||
api.sourceColumnCatalog(initial.releaseId||currentReleaseId||undefined,controller.signal).then(source=>{if(generation===supportGeneration.current)setCatalog(source);}).catch(fail),
|
||||
api.filterFields(false,controller.signal).then(filterFields=>{if(generation===supportGeneration.current)setExtraFields(filterFields.filter(field=>field.key==='salesOrderType'));}).catch(fail),
|
||||
];
|
||||
await Promise.all(requests);
|
||||
},[onUnauthorized,initial.releaseId,currentReleaseId]);
|
||||
useEffect(()=>{if(active)void loadSupport();return()=>{supportGeneration.current++;supportController.current?.abort();};},[active,refreshKey,loadSupport]);
|
||||
const fields=useMemo(()=>catalog?.sourceFields.filter(field=>field.type==='source_value'&&field.stages.includes(stage))||[],[catalog,stage]);
|
||||
const exactProductSearch=useMemo(()=>(options?.products||[]).find(name=>name.trim().toLocaleLowerCase()===appliedSearch.trim().toLocaleLowerCase())||'', [options,appliedSearch]);
|
||||
const queryProducts=useMemo(()=>drilledProduct?[drilledProduct]:exactProductSearch?product.length&&!product.includes(exactProductSearch)?['__codex_no_product_search_match__']:[exactProductSearch]:product,[drilledProduct,exactProductSearch,product]);
|
||||
useEffect(()=>{setDrilledProduct('');},[department,group,product,projectGroup,sourceFilters,extraFilters]);
|
||||
const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:initial.releaseId||currentReleaseId||catalog?.releaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,catalog,stage,sourceFilters,extraFilters,initial.releaseId,currentReleaseId]);
|
||||
const queryReleaseId=initial.releaseId||(currentReleaseId!==undefined?currentReleaseId||undefined:catalog?.releaseId);
|
||||
const draft=useMemo(()=>stageProductQuery({year,...(period==='month'?{month}:period==='quarter'?{quarter}:{}),department:encodeQueryValueList(department),group:encodeQueryValueList(group),product:encodeQueryValueList(queryProducts),projectGroup:encodeQueryValueList(projectGroup),productSearch:drilledProduct?undefined:appliedSearch.trim()||undefined,includeUnassigned,extraFilters,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,sourceFilters),[year,period,month,quarter,department,group,queryProducts,drilledProduct,projectGroup,appliedSearch,includeUnassigned,queryReleaseId,stage,sourceFilters,extraFilters]);
|
||||
const catalogMatchesDraft=!!catalog&&!!draft.releaseId&&catalog.releaseId===draft.releaseId;
|
||||
const queryReady=(query:QueryParams)=>!!queryReleaseId&&query.releaseId===queryReleaseId&&(!query.productSearch||!!options)&&(!(query.columnFilters||[]).length||!!catalog&&catalog.releaseId===query.releaseId);
|
||||
const draftReady=queryReady(draft);
|
||||
const appliedReady=!!applied&&queryReady(applied);
|
||||
// Apply organizational/source filters directly, like the other query pages. Product text search keeps its explicit submit.
|
||||
useEffect(()=>{if(active&&catalog&&!supportLoading&&!supportError)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,catalog,draft,supportLoading,supportError]);
|
||||
useEffect(()=>{if(active&&draftReady)setApplied(previous=>JSON.stringify(previous)===JSON.stringify(draft)?previous:draft);},[active,draft,draftReady]);
|
||||
const queryKey=JSON.stringify({query:applied,revision});const current=response?.key===queryKey?response.value:null;const error=failure?.key===queryKey?failure.message:'';
|
||||
useEffect(()=>{
|
||||
if(!active||!applied||supportError)return;const controller=new AbortController();setLoading(true);setFailure(null);
|
||||
if(!active||!applied||!appliedReady)return;const controller=new AbortController();setLoading(true);setFailure(null);
|
||||
void api.metrics(applied,controller.signal).then(value=>{if(!controller.signal.aborted)setResponse({key:queryKey,value});}).catch(e=>{if(!controller.signal.aborted&&!onUnauthorized(e))setFailure({key:queryKey,message:e instanceof Error?e.message:'查询失败'});}).finally(()=>{if(!controller.signal.aborted)setLoading(false);});
|
||||
return()=>controller.abort();
|
||||
},[active,queryKey,onUnauthorized,supportError]);
|
||||
},[active,queryKey,onUnauthorized,appliedReady]);
|
||||
useEffect(()=>{if(!active||!applied)return;const bytes=new TextEncoder().encode(JSON.stringify({query:queryForLocation(applied,initial.releaseId)}));const encoded=btoa(Array.from(bytes,b=>String.fromCharCode(b)).join('')).replace(/=/g,'').replace(/\+/g,'-').replace(/\//g,'_');const url=`?page=${stage==='black'?'blackProduct':'brightProduct'}&context=${encoded}`;window.history.replaceState({},'',`${location.pathname}${url}`);onLocationChange(url);},[active,applied,stage]);
|
||||
const trendQuery=useMemo(()=>applied?{...applied,month:undefined,quarter:undefined,...(trendGranularity==='year'?{year:undefined}:{}),trendGranularity}:null,[applied,trendGranularity]);
|
||||
const trendKey=JSON.stringify({query:trendQuery,revision});
|
||||
useEffect(()=>{if(!active||!trendQuery||supportError)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,supportError]);
|
||||
const trendReady=!!trendQuery&&queryReady(trendQuery);
|
||||
useEffect(()=>{if(!active||!trendQuery||!trendReady)return;const controller=new AbortController();setTrendLoading(true);setTrendError('');void api.trend(trendQuery,controller.signal).then(value=>{if(!controller.signal.aborted)setTrendResponse({key:trendKey,value,query:trendQuery});}).catch(error=>{if(!controller.signal.aborted&&!onUnauthorized(error))setTrendError(error instanceof Error?error.message:'趋势读取失败');}).finally(()=>{if(!controller.signal.aborted)setTrendLoading(false);});return()=>controller.abort();},[active,trendKey,trendRevision,onUnauthorized,trendReady]);
|
||||
const visibleTrend=useMemo(()=>fullYearTrend(trendResponse?.key===trendKey?trendResponse.value.trend:[],applied?.year||year,trendGranularity),[trendResponse,trendKey,applied?.year,year,trendGranularity]);
|
||||
const displayedTrend=useRetainedResult(trendResponse?.key===trendKey?visibleTrend:null)||visibleTrend;
|
||||
const selectedTrendPeriod=trendGranularity==='month'&&applied?.month
|
||||
@@ -80,12 +90,14 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
: trendGranularity==='quarter'&&applied?.quarter ? `${applied.year}-Q${applied.quarter}` : undefined;
|
||||
const selectTrendPeriod=(value:number|string)=>{automaticPeriod.current=false;const next=nextTrendPeriod(value,{period,year,month,quarter});setYear(next.year);setMonth(next.month);setQuarter(next.quarter);setPeriod(next.period);};
|
||||
const dirty=!!applied&&JSON.stringify(draft)!==JSON.stringify(applied);
|
||||
const queryBusy=supportLoading||loading||(!error&&!supportError&&(dirty||!current));
|
||||
const queryFailed=!!(error||supportError);
|
||||
const metadataPending=!draftReady;
|
||||
const metadataFailed=!draftReady&&!!supportError;
|
||||
const queryBusy=!metadataFailed&&(metadataPending||loading||(!error&&(dirty||!current)));
|
||||
const queryFailed=!!error||metadataFailed;
|
||||
const submit=()=>{automaticPeriod.current=false;setAppliedSearch(search.trim());setDrilledProduct('');setRevision(v=>v+1);};
|
||||
const setTrendObject=(value?:string)=>{const selected=value||'';setDrilledProduct('');setSearch(selected);setAppliedSearch(selected);};
|
||||
const returnToProducts=()=>{setDrilledProduct('');setSearch(drillParentSearch.current);setAppliedSearch(drillParentSearch.current);if(trendMode!=='defects')setTrendMode('production');};
|
||||
const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:catalog?.releaseId},stage,[]));setRevision(v=>v+1);};
|
||||
const reset=()=>{automaticPeriod.current=false;const previous=defaultQueryMonth(options);const currentYear=new Date().getFullYear();setYear(currentYear);setMonth(previous.month);setQuarter(previousCompleteQuarter().quarter);setPeriod('year');setTrendGranularity('month');setDepartment([]);setGroup([]);setProduct([]);setProjectGroup([]);setSourceFilters([]);setExtraFilters({});setSearch('');setAppliedSearch('');setDrilledProduct('');setApplied(stageProductQuery({year:currentYear,includeUnassigned,view:'product',trendGranularity:'month',releaseId:queryReleaseId},stage,[]));setRevision(v=>v+1);};
|
||||
const groups=(options?.groups||[]).filter(name=>!department.length||department.includes(options?.groupDepartments?.[name]||''));
|
||||
const paths=options?.organizationPaths||[];const products=department.length||group.length?[...new Set(paths.filter(path=>(!department.length||department.includes(path.department||''))&&(!group.length||group.includes(path.group||''))).map(path=>path.product).filter((v):v is string=>!!v))]:options?.products||[];
|
||||
const trendOptions=useMemo(()=>[...products.map(value=>({label:value,value})),...(drilledProduct==='__UNASSIGNED__'?[{label:'未分类',value:drilledProduct}]:[])],[products,drilledProduct]);
|
||||
@@ -124,9 +136,9 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
</div>
|
||||
<div className="query-toolbar-tail"><span className={`query-status ${queryBusy?'busy':queryFailed?'failed':''}`}><i/>{queryBusy?'正在查询':queryFailed?'查询失败':'查询完成'}</span><QuerySnapshotButton title={`${label}数据`} queryKey={JSON.stringify([draft,trendMode,trendObject,trendGranularity,rateBasis,fullFields,defectDisplay,revision])} disabled={queryBusy || !current} onUnauthorized={onUnauthorized} /><Button type="default" onClick={reset}>重置</Button></div>
|
||||
{moreFiltersOpen&&<div className="query-extra-fields" id={`stage-more-filters-${stage}`}>
|
||||
{extraFields.map(field=><SalesOrderTypeSelect key={field.key} field={field} filters={extraFilters} releaseId={draft.releaseId} stage={stage} onUnauthorized={onUnauthorized} onChange={setExtraFilters}/>)}
|
||||
{draft.releaseId&&extraFields.map(field=><SalesOrderTypeSelect key={field.key} field={field} filters={extraFilters} releaseId={draft.releaseId} stage={stage} onUnauthorized={onUnauthorized} onChange={setExtraFilters}/>)}
|
||||
{options?.projectGroupStatus==='READY'&&select('项目组',projectGroup,options.projectGroups||[],setProjectGroup,true)}
|
||||
{catalog&&<StageSourceFilters stage={stage} fields={fields} query={draft} value={sourceFilters} onChange={setSourceFilters} onUnauthorized={onUnauthorized}/>}
|
||||
{catalogMatchesDraft&&<StageSourceFilters stage={stage} fields={fields} query={draft} value={sourceFilters} onChange={setSourceFilters} onUnauthorized={onUnauthorized}/>}
|
||||
</div>}
|
||||
</section>
|
||||
{error&&<Alert showIcon type="error" message={response?"更新失败,当前显示上次结果":"查询失败"} description={error} action={<Button onClick={()=>setRevision(v=>v+1)}>重试</Button>}/>}
|
||||
@@ -150,7 +162,7 @@ export default function StageProductPage({stage,onUnauthorized}:{stage:Stage;onU
|
||||
</div>
|
||||
</QueryTrendPanel>
|
||||
<QueryTablePanel heading={<SectionHeading title={table?.drilledProduct?`产品${table.granularity==='quarter'?'季度':table.granularity==='year'?'年度':'月度'}质量数据`:`${label}产品质量汇总`} subtitle={table?.drilledProduct?`${table.query.year} 年 · ${table.drilledProduct==='__UNASSIGNED__'?'未分类':table.drilledProduct} · 重量 / kg 合格率 / %`:'重量 / kg 合格率 / %'}/>}
|
||||
busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!catalog||!!supportError}
|
||||
busy={queryBusy||!tableReady||defectDrill.tablePending} search={drilledLabel||search} onSearchChange={setSearch} onSearch={submit} searchDisabled={!!drilledProduct} searchUnavailable={!options}
|
||||
fullFields={fullFields} onFullFields={setFullFields} defectDisplay={defectDisplay} onDefectDisplay={setDefectDisplay}
|
||||
returnControls={drilledProduct&&<Button className="query-return-button" icon={<ArrowLeftOutlined/>} onClick={returnToProducts}>返回上一级</Button>}
|
||||
exporting={exporting} exportDisabled={!!defectDrill.category&&(!defectDrill.table||defectDrill.pending||!!defectDrill.error)||!current||loading||!tableReady||!table?.rows.length} onExport={()=>void runExport()}>
|
||||
|
||||
@@ -11,13 +11,15 @@ function SourceValues({field,stage,query,value,onChange,onUnauthorized}:{field:S
|
||||
const [search,setSearch]=useState('');const [choices,setChoices]=useState<string[]>([]);const [busy,setBusy]=useState(false);const [error,setError]=useState('');const [more,setMore]=useState(false);const [revision,setRevision]=useState(0);
|
||||
const {active}=usePageCache();
|
||||
const showBusy=useDelayedIndicator(busy);
|
||||
const contextKey=JSON.stringify(query);const current=useRef(0);
|
||||
const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query;
|
||||
const period={year,...(month?{month}:{}),...(quarter?{quarter}:{})};
|
||||
const filters={...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)};
|
||||
const contextKey=JSON.stringify({releaseId,period,filters,field:field.key,stage});const current=useRef(0);
|
||||
useEffect(()=>{
|
||||
if(!active)return;
|
||||
const generation=++current.current,controller=new AbortController();setBusy(true);setError('');setChoices([]);
|
||||
const timer=window.setTimeout(()=>{
|
||||
const {year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters}=query;
|
||||
void api.sourceColumnValues({releaseId,period:{year,...(month?{month}:{}),...(quarter?{quarter}:{})},filters:{...(department?{department}:{}),...(group?{group}:{}),...(product?{product}:{}),...(projectGroup?{projectGroup}:{}),...(extraFilters?{extraFilters}:{}),includeUnassigned:!!includeUnassigned,columnFilters:(columnFilters||[]).filter(rule=>rule.field!==field.key)},field:field.key,stages:[stage],search,limit:50,offset:0},controller.signal)
|
||||
void api.sourceColumnValues({releaseId,period,filters,field:field.key,stages:[stage],search,limit:50,offset:0,candidateOnly:true},controller.signal)
|
||||
.then(result=>{if(current.current!==generation)return;setChoices(result.values.filter((v):v is string=>typeof v==='string'));setMore(result.hasMore===true||(result.total??0)>50);})
|
||||
.catch(failure=>{if(controller.signal.aborted||current.current!==generation)return;if(!onUnauthorized(failure))setError(failure instanceof Error?failure.message:'选项读取失败');})
|
||||
.finally(()=>{if(current.current===generation)setBusy(false);});
|
||||
|
||||
+1
-1
@@ -239,7 +239,7 @@ export const api = {
|
||||
},
|
||||
activateSalesOrderTypes(versionId:string,expectedVersion:string) { return mutate<{versionId:string}>('/api/sales-order-types/activate',{versionId,expectedVersion}); },
|
||||
sourceColumnCatalog(releaseId?: string, signal?: AbortSignal) { return request<{releaseId:string;sourceFields:SourceColumnField[]}>(`/api/analysis/source/catalog?${toQuery({releaseId})}`, { signal }); },
|
||||
sourceColumnValues(body: Record<string,unknown>, signal?: AbortSignal) { return queryPost<{values:Array<string|null>;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); },
|
||||
sourceColumnValues(body: Record<string,unknown> & {candidateOnly?:true}, signal?: AbortSignal) { return queryPost<{values:Array<string|null>;hasMore?:boolean;total?:number;limit?:number}>('/api/analysis/source/values',body,signal); },
|
||||
workspaceState() { return request<{ authRevision: string; dataRevision: string; currentReleaseId: string | null }>('/api/workspace/state', {}, 8000); },
|
||||
filterFields(manage = false, signal?: AbortSignal) { return request<FilterField[]>(`/api/filter-fields?manage=${manage}`, { signal }); },
|
||||
filterFieldOptions(key: string, search = '', signal?: AbortSignal,releaseId?:string,stage?:string) { return request<{ values: string[]; hasMore: boolean }>(`/api/filter-fields/options?${toQuery({key, search,releaseId,stage})}`, { signal }); },
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import ts from 'typescript';
|
||||
|
||||
const read=name=>fs.readFileSync(new URL(`../src/${name}`,import.meta.url),'utf8');
|
||||
|
||||
test('stage product common queries use a resolved release and do not wait for source catalog when unfiltered',()=>{
|
||||
const source=read('StageProductPage.tsx');
|
||||
const releaseExpression=source.match(/const queryReleaseId=([^;]+);/)[1];
|
||||
const resolvedRelease=new Function('initial','currentReleaseId','catalog',`return ${releaseExpression}`);
|
||||
assert.equal(resolvedRelease({releaseId:'historical'},'current',{releaseId:'catalog'}),'historical');
|
||||
assert.equal(resolvedRelease({},'current',{releaseId:'catalog'}),'current');
|
||||
assert.equal(resolvedRelease({},null,{releaseId:'stale'}),undefined);
|
||||
assert.equal(resolvedRelease({},undefined,{releaseId:'resolved'}),'resolved');
|
||||
const expression=source.match(/const queryReady=\(query:QueryParams\)=>((?:[^;]|\n)+);/)[1];
|
||||
const ready=new Function('query','queryReleaseId','catalog','options',`return ${expression}`);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[]},'r1',null,null),true);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[]},undefined,null,null),false);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',null,null),false);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r1',{releaseId:'r1'},null),true);
|
||||
assert.equal(ready({releaseId:'r1',columnFilters:[{field:'customer'}]},'r2',{releaseId:'r1'},null),false);
|
||||
assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,null),false);
|
||||
assert.equal(ready({releaseId:'r1',productSearch:'P-1'},'r1',null,{}),true);
|
||||
assert.match(source,/if\(active&&draftReady\)setApplied/);
|
||||
assert.doesNotMatch(source,/if\(!active\|\|!applied\|\|supportError\)/);
|
||||
});
|
||||
|
||||
test('source option request context excludes its own selected values and tracks other filters',()=>{
|
||||
const source=read('StageSourceFilters.tsx');
|
||||
const declarations=source.match(/ const \{year,month,quarter,department,group,product,projectGroup,includeUnassigned,columnFilters,releaseId,extraFilters\}=query;\n const period=.*;\n const filters=.*;\n const contextKey=[^;]+;/)[0];
|
||||
const run=new Function('query','field','stage',`${ts.transpileModule(declarations,{compilerOptions:{target:ts.ScriptTarget.ES2022}}).outputText};return {filters,contextKey};`);
|
||||
const base={year:2026,month:4,releaseId:'r1',department:'D1',columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['North']}]};
|
||||
const first=run(base,{key:'customer'},'black');
|
||||
const ownValueChanged=run({...base,columnFilters:[{field:'customer',values:['OTHER']},{field:'region',values:['North']}]},{key:'customer'},'black');
|
||||
const otherValueChanged=run({...base,columnFilters:[{field:'customer',values:['ACME']},{field:'region',values:['South']}]},{key:'customer'},'black');
|
||||
assert.deepEqual(first.filters.columnFilters,[{field:'region',values:['North']}]);
|
||||
assert.equal(first.contextKey,ownValueChanged.contextKey);
|
||||
assert.notEqual(first.contextKey,otherValueChanged.contextKey);
|
||||
});
|
||||
|
||||
test('source value dropdowns request candidate-only results while preserving search and debounce behavior',()=>{
|
||||
const source=read('StageSourceFilters.tsx');
|
||||
assert.match(source,/sourceColumnValues\(\{releaseId,period,filters,field:field\.key,stages:\[stage\],search,limit:50,offset:0,candidateOnly:true\}/);
|
||||
assert.match(source,/search\?300:0/);
|
||||
assert.match(source,/result\.values\.filter\(\(v\):v is string=>typeof v==='string'\)/);
|
||||
assert.match(read('api.ts'),/sourceColumnValues\(body: Record<string,unknown> & \{candidateOnly\?:true\}/);
|
||||
});
|
||||
|
||||
|
||||
test('ordinary product search only depends on product options', () => {
|
||||
assert.match(read('StageProductPage.tsx'), /searchUnavailable=\{!options\}/);
|
||||
});
|
||||
Reference in New Issue
Block a user