优化:统一来源筛选索引与后台构建

This commit is contained in:
2026-11-13 17:56:18 +08:00
parent 5068b8fbc9
commit 7782e33bec
29 changed files with 997 additions and 59 deletions
@@ -369,7 +369,7 @@ public class AnalysisExplorationTools {
SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value);
SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value);
appendDetailExclusions(sql, args, query);
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor);
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor,releaseId);
DataScope.append(sql, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
boolean includeUnassigned = query.includeUnassigned()
@@ -276,7 +276,7 @@ public class AnalysisFineDefectsService {
SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value);
if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor);
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor,releaseId);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
}
@@ -90,7 +90,7 @@ public class AnalysisInvestigationService {
SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x);
SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x);
if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor);
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor,release);
DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
}
if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());}
@@ -43,12 +43,16 @@ public class AnalysisSourceQueryController {
@PostMapping("/values")
public Map<String,Object> values(@RequestBody Map<String,Object> input,@AuthenticationPrincipal AppUserPrincipal actor) {
String scope=captureScope(actor);
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset","candidateOnly").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null));
Map<String,Object> context=context(input);
if(!(input.get("field") instanceof String field)||field.isBlank()||input.get("search")!=null&&!(input.get("search") instanceof String))throw bad("候选字段或搜索值无效");
Map<String,Object> result=new LinkedHashMap<>(columns.values(release,context,field,strings(input.get("stages"),2),Objects.toString(input.get("search"),null),
integer(input.getOrDefault("limit",50),1,200),integer(input.getOrDefault("offset",0),0,100000),actor));
if(input.containsKey("candidateOnly")&&!(input.get("candidateOnly") instanceof Boolean))throw bad("候选分页参数无效");
List<String> stages=strings(input.get("stages"),2);String search=Objects.toString(input.get("search"),null);
int limit=integer(input.getOrDefault("limit",50),1,200),offset=integer(input.getOrDefault("offset",0),0,100000);
Map<String,Object> result=new LinkedHashMap<>(Boolean.TRUE.equals(input.get("candidateOnly"))
? columns.values(release,context,field,stages,search,limit,offset,true,actor)
: columns.values(release,context,field,stages,search,limit,offset,actor));
result.put("period",input.get("period"));result.put("filters",input.get("filters"));
result.put("values",AnalysisProtocolService.list(result.get("rows")).stream().map(row->AnalysisProtocolService.map(row).get("value")).toList());
result.put("candidateOnly",true);reauthorize(actor,scope,false);return result;
@@ -111,7 +111,7 @@ public class AnalyticsController {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前字段授权不包含所选报废类别");
List<Object> args = new ArrayList<>(extra.isEmpty()?List.of(selected, selected):List.of(selected));
StringBuilder where = baseWhere(args, year, month, quarter, department, group, product, projectGroup, includeUnassigned, actor);
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor);
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor,selected);
List<AggregateTop> aggregated;
if (dimension.equals("category")) {
List<Component> selectedComponents = selectedCategory == null ? visibleComponents : List.of(selectedCategory);
@@ -36,9 +36,11 @@ public class FilterFieldsController {
governance.requireUserActivePolicy(actor);
Map<String,List<String>> result=new LinkedHashMap<>();
String release=releases.resolvePublishedRelease(null);
SourceFilterIndex index=new SourceFilterIndex(jdbc);
for(String stage:List.of("black","bright")) {
// Raw source headers are stable within each build. Read a representative row of every current build.
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2 WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
String hint=index.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":"";
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2"+hint+" WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
Set<String> columns=new TreeSet<>();
var json=tools.jackson.databind.json.JsonMapper.builder().build();
Set<String> allowed=governance.allowedFields(actor);
@@ -48,6 +48,9 @@ public class QueryFilterFields {
return Collections.unmodifiableMap(result);
}
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor) {
append(where,args,stage,filters,actor,null);
}
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor,String releaseId) {
if(filters==null || filters.isEmpty())return;
Map<String,Field> fields=new HashMap<>(); visible(actor).forEach(f->fields.put(f.key(),f));
for(var entry:filters.entrySet()) {
@@ -61,6 +64,7 @@ public class QueryFilterFields {
boolean empty=entry.getValue().contains(EMPTY);
if(empty && !f.allowEmpty())throw bad(f.label()+"不允许筛选空值");
String column=stage.equals("black")?f.blackColumn():f.brightColumn();
if(releaseId!=null&&new SourceFilterIndex(jdbc).appendLegacy(where,args,stage,releaseId,column,entry.getValue()))continue;
// Paths and selected values are bound parameters; never interpolate configured identifiers.
String expression="NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs";
List<String> values=entry.getValue().stream().filter(v->!v.equals(EMPTY)).toList();
@@ -90,10 +94,15 @@ public class QueryFilterFields {
StringBuilder scope=new StringBuilder(" WHERE rm.release_id=?");
List<Object> scoped=new ArrayList<>();scoped.add(releaseId);
DataScope.append(scope,scoped,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
var index=new SourceFilterIndex(jdbc);
if(index.ready(releaseId)) {
selects.add(index.candidateSelect(stage,column,scope.toString(),scoped,args).replace("AS candidate_value","AS value"));
continue;
}
selects.add("SELECT NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs AS value FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+scope);
args.add(path(column));args.addAll(scoped);
}
String predicate=" WHERE value IS NOT NULL";
String predicate=" WHERE value IS NOT NULL AND CAST(value AS BINARY)<>CAST('null' AS BINARY)";
if(search!=null && !search.isBlank()) { predicate+=" AND LOCATE(LOWER(?),LOWER(value))>0";args.add(search); }
List<String> found=jdbc.query("SELECT DISTINCT value FROM ("+String.join(" UNION ALL ",selects)+") choices"+predicate+" ORDER BY value LIMIT 201",(rs,n)->rs.getString(1),args.toArray());
return Map.of("values",found.stream().limit(200).toList(),"hasMore",found.size()>200);
@@ -80,11 +80,17 @@ public class SourceColumnQueries {
private final JdbcTemplate jdbc;
private final GovernanceAccessService governance;
private final JsonMapper json;
private SourceFilterIndex filterIndex;
@org.springframework.beans.factory.annotation.Autowired
public void setFilterIndex(SourceFilterIndex index) { this.filterIndex=index; }
public boolean indexed(String release) { return filterIndex.ready(release); }
public SourceColumnQueries(JdbcTemplate jdbc, GovernanceAccessService governance, JsonMapper json) {
this.jdbc = jdbc;
this.governance = governance;
this.json = json;
this.filterIndex = new SourceFilterIndex(jdbc);
}
/**
@@ -110,7 +116,7 @@ public class SourceColumnQueries {
"f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs");
String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) "
+ "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id "
+ "FROM release_member rm JOIN fact_" + stage + " f2" + factIndexHint(stage) + " "
+ "FROM release_member rm JOIN fact_" + stage + " f2" + (filterIndex.available()?" FORCE INDEX (ix_"+stage+"_source_catalog)":factIndexHint(stage)) + " "
+ "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"
+ representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative "
+ "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id";
@@ -249,6 +255,8 @@ public class SourceColumnQueries {
String operator = (String) rule.get("operator");
String expression;
Object column = ((Map<?, ?>) field.get("columns")).get(stage);
@SuppressWarnings("unchecked") List<String> selected = (List<String>) rule.getOrDefault("values",List.of());
if("source_value".equals(field.get("type")) && filterIndex.append(where,args,stage,releaseId,String.valueOf(column),operator,selected))continue;
if ("source_value".equals(field.get("type"))) {
expression = SOURCE_VALUE;
} else {
@@ -383,12 +391,16 @@ public class SourceColumnQueries {
public Map<String, Object> values(String releaseId, Map<String, Object> context, String fieldKey,
List<String> stages, String search, int limit, int offset,
AppUserPrincipal actor) {
return values(releaseId,context,fieldKey,stages,search,limit,offset,false,actor);
}
public Map<String,Object> values(String releaseId,Map<String,Object> context,String fieldKey,List<String> stages,
String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
requireQueryAccess(actor);
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,actor));
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,candidateOnly,actor));
}
private Map<String,Object> valuesSnapshot(String releaseId,Map<String,Object> context,String fieldKey,
List<String> stages,String search,int limit,int offset,AppUserPrincipal actor) {
List<String> stages,String search,int limit,int offset,boolean candidateOnly,AppUserPrincipal actor) {
if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效");
if (fieldKey == null || fieldKey.isBlank()) throw bad("来源列标识无效");
if (stages == null || stages.isEmpty() || stages.size() > STAGES.size()
@@ -419,6 +431,10 @@ public class SourceColumnQueries {
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> whereArgs = new ArrayList<>(List.of(releaseId));
appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor);
if("source_value".equals(field.get("type")) && filterIndex.ready(releaseId)) {
selects.add(filterIndex.candidateSelect(stage,String.valueOf(columns.get(stage)),where.toString(),whereArgs,unionArgs));
continue;
}
String fact = stage.equals("black") ? "fact_black" : "fact_bright";
selects.add("SELECT " + expression + " AS candidate_value "
+ "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id "
@@ -437,6 +453,13 @@ public class SourceColumnQueries {
}
String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition;
String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value";
if(candidateOnly) {
String sql="SELECT candidate_value FROM ("+candidates+") matches GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?";
List<Object> parameters=combine(unionArgs,searchArgs);parameters.add(limit+1);parameters.add(offset);
List<Map<String,Object>> found=pages.get(new QueryKey(sql,parameters,"candidates:"+fieldKey),()->List.copyOf(jdbc.query(sql,(rs,row)->Map.of("value",rs.getString(1)),parameters.toArray())));
boolean more=found.size()>limit;
return Map.of("rows",List.copyOf(found.subList(0,Math.min(limit,found.size()))),"hasMore",more,"limit",limit,"offset",offset,"pageComplete",true,"complete",offset==0&&!more,"releaseId",releaseId,"field",fieldKey,"selectedStages",List.copyOf(stages));
}
long total = total("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", combine(unionArgs,searchArgs));
List<Map<String, Object>> rows = page("SELECT candidate_value FROM (" + candidates + ") matches "
+ "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?",
@@ -563,7 +586,7 @@ public class SourceColumnQueries {
List<String> groups = parseSelections(context.dimensions().get("group"), "group");
if (!context.includeUnassigned() && !groups.contains("__UNASSIGNED__"))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor);
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor,releaseId);
if (!context.columnFilters().isEmpty()) append(where, args, stage, context.columnFilters(), releaseId, actor);
DataScope.append(where, args, actor,
"f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
@@ -0,0 +1,122 @@
package cn.yunsheng.quality.filters;
import java.util.List;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
/** Sparse, build-owned source values. Every consumer still authorizes and scopes the actual facts. */
@Service
public class SourceFilterIndex {
public record BuildAvailable(String buildId) {}
private static final int FORMAT = 1;
private final JdbcTemplate jdbc;
private volatile Boolean available;
public SourceFilterIndex(JdbcTemplate jdbc) { this.jdbc = jdbc; }
public boolean available() {
if (available == null) {
Integer count=jdbc.queryForObject("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('source_filter_build','source_filter_field','source_filter_value','source_filter_member')",Integer.class);
available=count!=null&&count==4;
}
return available;
}
public boolean ready(String release) {
if(release==null||!available())return false;
// Only successful immutable builds are retained; a pending backfill is checked again later.
Integer missing=jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm LEFT JOIN source_filter_build sb ON sb.build_id=rm.transform_build_id AND sb.format_version=? WHERE rm.release_id=? AND sb.build_id IS NULL",Integer.class,FORMAT,release);
if(missing==null||missing!=0)return false;
return true;
}
/** Idempotent and atomic with fact construction, including callers which instantiate this helper. */
public void ensureBuild(String build) {
if(!available())return;
var tx=new TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource()));
tx.executeWithoutResult(status->{
jdbc.queryForObject("SELECT id FROM transform_build WHERE id=? FOR UPDATE",String.class,build);
Integer done=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build WHERE build_id=? AND format_version=?",Integer.class,build,FORMAT);
if(done!=null&&done>0)return;
jdbc.update("DELETE m FROM source_filter_member m JOIN source_filter_field sf ON sf.id=m.field_id WHERE sf.build_id=?",build);
jdbc.update("DELETE v FROM source_filter_value v JOIN source_filter_field sf ON sf.id=v.field_id WHERE sf.build_id=?",build);
jdbc.update("DELETE FROM source_filter_field WHERE build_id=?",build);
jdbc.update("DELETE FROM source_filter_build WHERE build_id=?",build);
// Only staging is bounded to 500 facts. The build transaction and persisted dictionary scale with the source.
jdbc.execute("CREATE TEMPORARY TABLE source_filter_staging(field_id BIGINT NOT NULL,fact_id BIGINT NOT NULL,value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,value_hash BINARY(32) NOT NULL,KEY(field_id,value_hash)) ENGINE=InnoDB");
try {
for(String stage:List.of("black","bright"))buildStage(build,stage);
jdbc.update("INSERT INTO source_filter_build(build_id,format_version) VALUES(?,?)",build,FORMAT);
} finally { jdbc.execute("DROP TEMPORARY TABLE source_filter_staging"); }
});
}
private void buildStage(String build,String stage) {
var bounds=jdbc.queryForMap("SELECT MIN(id) AS first_id,MAX(id) AS last_id FROM fact_"+stage+" FORCE INDEX(uq_fact_"+stage+"_origin) WHERE build_id=?",build);
if(bounds.get("first_id")==null)return;
long after=((Number)bounds.get("first_id")).longValue()-1;
long last=((Number)bounds.get("last_id")).longValue();
while(after<last) {
if(Thread.currentThread().isInterrupted())throw new IllegalStateException("来源筛选索引构建已中断");
List<Long> ids=jdbc.queryForList("SELECT id FROM fact_"+stage+" FORCE INDEX(PRIMARY) WHERE build_id=? AND id>? AND id<=? ORDER BY id LIMIT 500",Long.class,build,after,last);
if(ids.isEmpty())return;
long end=ids.getLast();
String rows=" FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE WHERE f.build_id=? AND f.id>? AND f.id<=?";
jdbc.update("INSERT IGNORE INTO source_filter_field(build_id,stage,column_name,name_hash) SELECT DISTINCT ?,?,k.column_name,UNHEX(SHA2(CAST(k.column_name AS BINARY),256))"+rows,build,stage,build,after,end);
String expression="NULLIF(TRIM(JSON_UNQUOTE(NULLIF(JSON_EXTRACT(f.raw_values,CONCAT('$.',JSON_QUOTE(k.column_name))),CAST('null' AS JSON)))),'')";
String extracted="SELECT sf.id AS field_id,f.id AS fact_id,"+expression+" AS value FROM fact_"+stage+" f FORCE INDEX(PRIMARY) JOIN JSON_TABLE(JSON_KEYS(f.raw_values),'$[*]' COLUMNS(column_name LONGTEXT PATH '$' ERROR ON ERROR)) k ON TRUE JOIN source_filter_field sf ON sf.build_id=f.build_id AND sf.stage=? AND sf.name_hash=UNHEX(SHA2(CAST(k.column_name AS BINARY),256)) AND sf.column_name=k.column_name COLLATE utf8mb4_0900_bin WHERE f.build_id=? AND f.id>? AND f.id<=?";
jdbc.update("DELETE FROM source_filter_staging");
jdbc.update("INSERT INTO source_filter_staging SELECT /*+ NO_MERGE(extracted) */ field_id,fact_id,value,UNHEX(SHA2(CAST(value AS BINARY),256)) FROM ("+extracted+") extracted WHERE value IS NOT NULL",stage,build,after,end);
jdbc.update("INSERT INTO source_filter_value(field_id,value_hash,value) SELECT s.field_id,s.value_hash,ANY_VALUE(s.value) FROM source_filter_staging s LEFT JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE v.id IS NULL GROUP BY s.field_id,s.value_hash");
Integer collisions=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash WHERE CAST(v.value AS BINARY)<>CAST(s.value AS BINARY)",Integer.class);
if(collisions!=null&&collisions>0)throw new IllegalStateException("来源筛选字典校验失败,索引未发布");
jdbc.update("INSERT INTO source_filter_member(field_id,fact_id,value_id) SELECT s.field_id,s.fact_id,v.id FROM source_filter_staging s JOIN source_filter_value v ON v.field_id=s.field_id AND v.value_hash=s.value_hash");
after=end;
}
}
/** Same-row membership is shared by metrics, trends, details and exports; empty values are sparse. */
public boolean append(StringBuilder where,List<Object> args,String stage,String release,String column,
String operator,List<String> values) {
if(!ready(release))return false;
where.append(" AND ");
membership(where,args,stage,release,column,operator.equals("not_in")||operator.equals("is_empty"),
operator.equals("in")||operator.equals("not_in")?values:null,false);
return true;
}
/** Legacy configured filters classify only the byte-exact literal string "null" as empty. */
public boolean appendLegacy(StringBuilder where,List<Object> args,String stage,String release,String column,List<String> selected) {
if(!ready(release))return false;
var values=selected.stream().filter(v->!v.equals(QueryFilterFields.EMPTY)).toList();
where.append(" AND (");
if(values.isEmpty())where.append("FALSE");
else membership(where,args,stage,release,column,false,values,true);
if(selected.contains(QueryFilterFields.EMPTY)) {
where.append(" OR ");membership(where,args,stage,release,column,true,null,true);
}
where.append(')');return true;
}
private void membership(StringBuilder where,List<Object> args,String stage,String release,String column,
boolean negative,List<String> values,boolean legacy) {
where.append("f.id ").append(negative?"NOT IN (":"IN (")
.append("SELECT sm.fact_id FROM source_filter_field sf JOIN release_member srm ON srm.transform_build_id=sf.build_id JOIN source_filter_value sv ON sv.field_id=sf.id JOIN source_filter_member sm ON sm.value_id=sv.id WHERE srm.release_id=? AND sf.stage=? AND sf.column_name=? COLLATE utf8mb4_0900_bin");
args.add(release);args.add(stage);args.add(column);
if(legacy)where.append(" AND CAST(sv.value AS BINARY)<>CAST('null' AS BINARY)");
if(values!=null) {
where.append(" AND sv.value COLLATE utf8mb4_0900_as_cs IN (").append(String.join(",",java.util.Collections.nCopies(values.size(),"?"))).append(')');
args.addAll(values);
}
where.append(')');
}
/** Dictionary-first candidates short-circuit after one authorized matching member per value. */
public String candidateSelect(String stage,String column,String scope,List<Object> scopeArgs,List<Object> args) {
args.add(scopeArgs.getFirst());args.add(column);args.add(stage);args.addAll(scopeArgs);
return "SELECT sv.value AS candidate_value FROM release_member frm JOIN source_filter_field sf ON sf.build_id=frm.transform_build_id JOIN source_filter_value sv ON sv.field_id=sf.id "
+"WHERE frm.release_id=? AND sf.column_name=? COLLATE utf8mb4_0900_bin AND sf.stage=? AND EXISTS (SELECT 1 FROM source_filter_member sm JOIN fact_"+stage+" f ON f.id=sm.fact_id JOIN release_member rm ON rm.transform_build_id=f.build_id AND rm.source_revision_id=f.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id "
+scope+" AND sm.value_id=sv.id AND f.build_id=sf.build_id)";
}
}
@@ -0,0 +1,194 @@
package cn.yunsheng.quality.filters;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.context.event.EventListener;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
/** Asynchronously builds current and newly completed snapshots; unindexed queries retain their raw-value fallback. */
@Component
public class SourceFilterIndexBackfill implements ApplicationRunner {
private static final Logger LOG = LoggerFactory.getLogger(SourceFilterIndexBackfill.class);
private static final int MAX_PENDING_BUILDS = 1024;
private final JdbcTemplate jdbc;
private final SourceFilterIndex index;
private final ExecutorService worker;
private final Object pendingLock = new Object();
private final LinkedHashSet<String> pending = new LinkedHashSet<>();
private boolean drainScheduled;
private String inFlight;
@Autowired
public SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index) {
this(jdbc, index, createWorker());
}
SourceFilterIndexBackfill(JdbcTemplate jdbc, SourceFilterIndex index, ExecutorService worker) {
this.jdbc = jdbc;
this.index = index;
this.worker = worker;
}
@Override
public void run(ApplicationArguments args) {
List<String> currentBuilds;
try {
if (!index.available()) return;
currentBuilds = jdbc.query("""
SELECT DISTINCT b.id
FROM current_release c
JOIN data_release r ON r.id=c.release_id
JOIN release_member rm ON rm.release_id=r.id
JOIN transform_build b ON b.id=rm.transform_build_id
WHERE c.dataset_id='quality'
AND r.status='PUBLISHED'
AND r.complete=TRUE
AND b.status='COMPLETED'
ORDER BY b.id
""", (rs, row) -> rs.getString(1));
} catch (RuntimeException failure) {
LOG.error("Source filter index backfill discovery failed; raw-value fallback remains available", failure);
return;
}
if (currentBuilds.isEmpty()) return;
int accepted = enqueue(currentBuilds);
LOG.info("Source filter index backfill queued: discovered={}, accepted={}", currentBuilds.size(), accepted);
}
@EventListener
public void onBuildAvailable(SourceFilterIndex.BuildAvailable event) {
if (event == null || event.buildId() == null || event.buildId().isBlank()) return;
int accepted = enqueue(List.of(event.buildId()));
if (accepted > 0) LOG.info("Source filter index build queued after commit: build={}", event.buildId());
}
private int enqueue(List<String> builds) {
boolean schedule = false;
int accepted = 0;
int deferred = 0;
synchronized (pendingLock) {
for (String build : builds) {
if (build == null || build.isBlank() || build.equals(inFlight) || pending.contains(build)) continue;
if (pending.size() >= MAX_PENDING_BUILDS) {
deferred++;
continue;
}
pending.add(build);
accepted++;
}
if (!pending.isEmpty() && !drainScheduled) {
drainScheduled = true;
schedule = true;
}
}
if (deferred > 0) {
LOG.warn("Source filter index queue is full; {} builds keep raw-value fallback", deferred);
}
if (schedule) scheduleDrain();
return accepted;
}
private void scheduleDrain() {
try {
worker.execute(this::drainPending);
} catch (RejectedExecutionException failure) {
int queued;
synchronized (pendingLock) {
drainScheduled = false;
queued = pending.size();
}
LOG.error("Source filter index drain could not be scheduled; {} builds keep raw-value fallback", queued, failure);
}
}
private void drainPending() {
long started = System.nanoTime();
int succeeded = 0;
int failed = 0;
while (true) {
String buildId;
synchronized (pendingLock) {
if (pending.isEmpty()) {
drainScheduled = false;
break;
}
var iterator = pending.iterator();
buildId = iterator.next();
iterator.remove();
inFlight = buildId;
}
if (Thread.currentThread().isInterrupted()) {
synchronized (pendingLock) {
pending.add(buildId);
inFlight = null;
drainScheduled = false;
}
LOG.info("Source filter index drain interrupted; {} builds remain queued", pendingCount());
return;
}
long buildStarted = System.nanoTime();
try {
index.ensureBuild(buildId);
succeeded++;
LOG.info("Source filter index build completed: build={}, queuedRemaining={}, elapsedMs={}", buildId,
pendingCount(), (System.nanoTime() - buildStarted) / 1_000_000L);
} catch (RuntimeException failure) {
if (Thread.currentThread().isInterrupted()) {
synchronized (pendingLock) {
pending.add(buildId);
inFlight = null;
drainScheduled = false;
}
LOG.info("Source filter index drain interrupted while building {}; {} builds remain queued", buildId,
pendingCount());
return;
}
failed++;
LOG.error("Source filter index build failed: build={}; raw-value fallback remains available, continuing",
buildId, failure);
} finally {
synchronized (pendingLock) { inFlight = null; }
}
}
LOG.info("Source filter index drain finished: succeeded={}, failed={}, elapsedMs={}", succeeded, failed,
(System.nanoTime() - started) / 1_000_000L);
}
private int pendingCount() {
synchronized (pendingLock) { return pending.size(); }
}
@PreDestroy
public void stopWorker() {
worker.shutdownNow();
try {
if (!worker.awaitTermination(2, TimeUnit.SECONDS)) {
LOG.warn("Source filter index backfill worker did not stop within two seconds");
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
}
}
private static ExecutorService createWorker() {
return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1), runnable -> {
Thread thread = new Thread(runnable, "source-filter-index-backfill");
thread.setDaemon(true);
return thread;
}, new ThreadPoolExecutor.AbortPolicy());
}
}
@@ -1,13 +1,21 @@
package cn.yunsheng.quality.metrics;
import cn.yunsheng.quality.filters.SourceFilterIndex;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
/** Builds immutable, per-transform-build source-component rollups for read-only metric queries. */
@Service
public class MetricsRollupService {
private static final Logger LOG = LoggerFactory.getLogger(MetricsRollupService.class);
static final String DIMENSION_COLLATION = "utf8mb4_0900_as_cs";
private static final List<String> FACT_TABLES = List.of("fact_black", "fact_bright");
private static final String COMPONENTS = "JSON_OBJECT(" +
@@ -22,10 +30,18 @@ public class MetricsRollupService {
private final JdbcTemplate jdbc;
private final cn.yunsheng.quality.analysis.FineDefectRollups fineDefects;
private final SourceFilterIndex sourceFilterIndex;
private ApplicationEventPublisher eventPublisher;
public MetricsRollupService(JdbcTemplate jdbc) {
this.jdbc = jdbc;
this.fineDefects = new cn.yunsheng.quality.analysis.FineDefectRollups(jdbc, tools.jackson.databind.json.JsonMapper.builder().build());
this.sourceFilterIndex = new SourceFilterIndex(jdbc);
}
@Autowired
public void setApplicationEventPublisher(ApplicationEventPublisher eventPublisher) {
this.eventPublisher = eventPublisher;
}
public record GroupKey(String stage,String month,String department,String group,String project,String product) {}
@@ -36,6 +52,7 @@ public class MetricsRollupService {
jdbc.update("INSERT INTO metric_rollup(build_id,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows) SELECT ?,stage,month_key,department,group_name,project_group,product,qualified_weight,scrap_weight,total_weight,components,source_row_count,excluded_blank_rows,unmapped_rows,project_unconfigured_rows FROM metric_rollup WHERE build_id=?",child,parent);
if(jdbc.update("INSERT INTO metric_rollup_build(build_id,source_row_count) SELECT ?,source_row_count FROM metric_rollup_build WHERE build_id=?",child,parent)!=1) throw new IllegalStateException("旧版本缺少已冻结的指标汇总");
fineDefects.copyBuild(parent, child);
announceSourceFilterBuild(child);
}
@Transactional
@@ -74,6 +91,30 @@ public class MetricsRollupService {
""", buildId, buildId, buildId);
fineDefects.refreshBuild(buildId);
new SalesOrderMetricRollups(jdbc).refreshCurrentBuild(buildId);
announceSourceFilterBuild(buildId);
}
void announceSourceFilterBuild(String buildId) {
if (eventPublisher == null) {
// Directly constructed fixtures have no application event bus; retain
// their explicit synchronous behavior (legacy schemas no-op safely).
sourceFilterIndex.ensureBuild(buildId);
return;
}
Runnable publish = () -> {
try {
eventPublisher.publishEvent(new SourceFilterIndex.BuildAvailable(buildId));
} catch (RuntimeException failure) {
LOG.error("Source filter index notification failed for {}; raw-value fallback remains available", buildId, failure);
}
};
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override public void afterCommit() { publish.run(); }
});
} else {
publish.run();
}
}
static String rollupInsertSql(String table) {
@@ -39,7 +39,10 @@ public class MetricsService {
private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128;
private final JdbcTemplate jdbc;
private final GovernanceAccessService governance;
private final SourceColumnQueries sourceColumns;
private SourceColumnQueries sourceColumns;
@Autowired
public void setSourceColumns(SourceColumnQueries sourceColumns) { this.sourceColumns = sourceColumns; }
private final TransactionTemplate readSnapshot;
private final TransactionTemplate governanceSnapshot;
@@ -160,7 +163,7 @@ public class MetricsService {
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if (factFallback) {
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor);
new QueryFilterFields(jdbc, governance).append(where, args, stage, query.extraFilters(), actor,releaseId);
if (!query.columnFilters().isEmpty())
sourceColumns
.append(where, args, stage, query.columnFilters(), releaseId, actor);
@@ -396,7 +399,7 @@ public class MetricsService {
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
@@ -599,7 +602,8 @@ public class MetricsService {
"sourceMonths", sourceMonths);
}
private static String factFilterHint(String stage,boolean factFallback,MetricsQuery query) {
private String factFilterHint(String stage,boolean factFallback,MetricsQuery query) {
if(sourceColumns.indexed(query.releaseId()))return "";
if(!factFallback||query.columnFilters().isEmpty()||!query.extraFilters().isEmpty())return "";
// 仅纠正无组织/产品限制的整年来源筛选,窄范围和销售订单组合保留优化器原选择。
if(query.month()!=null||query.quarter()!=null||query.monthStart()!=null||query.monthEnd()!=null
@@ -622,7 +626,7 @@ public class MetricsService {
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor,releaseId);
if(!query.columnFilters().isEmpty())sourceColumns.append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
@@ -53,6 +53,10 @@ public class RecordController {
private final JsonMapper mapper;
private final GovernanceAccessService governance;
private final RecordsWorkbookExportService workbookExport;
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
@org.springframework.beans.factory.annotation.Autowired
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
public RecordController(JdbcTemplate jdbc, ReleaseService releases, JsonMapper mapper,
GovernanceAccessService governance, RecordsWorkbookExportService workbookExport) {
@@ -61,6 +65,7 @@ public class RecordController {
this.mapper = mapper;
this.governance = governance;
this.workbookExport = workbookExport;
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
}
@@ -156,10 +161,10 @@ public class RecordController {
args.add(search);
args.add(jsonPath(selectedField));
}
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor);
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor,selectedRelease);
var sourceFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters);
for(var rule:sourceFilters)if(!List.of(stage).containsAll((List<?>)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段");
if(!sourceFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,stage,sourceFilters,selectedRelease,actor);
if(!sourceFilters.isEmpty())sourceColumns.append(predicate,args,stage,sourceFilters,selectedRelease,actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
String countSql = """
@@ -52,6 +52,10 @@ public class RecordsWorkbookExportService {
private final ReleaseService releases;
private final GovernanceAccessService governance;
private final JsonMapper mapper;
private cn.yunsheng.quality.filters.SourceColumnQueries sourceColumns;
@org.springframework.beans.factory.annotation.Autowired
public void setSourceColumns(cn.yunsheng.quality.filters.SourceColumnQueries columns) { this.sourceColumns = columns; }
public RecordsWorkbookExportService(JdbcTemplate jdbc, ReleaseService releases,
GovernanceAccessService governance, JsonMapper mapper) {
@@ -59,6 +63,7 @@ public class RecordsWorkbookExportService {
this.releases = releases;
this.governance = governance;
this.mapper = mapper;
this.sourceColumns = new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper);
}
public Path export(Integer year, Integer month, Integer quarter, String department, String group,
@@ -120,8 +125,11 @@ public class RecordsWorkbookExportService {
LinkedHashSet<String> rawFields = new LinkedHashSet<>();
if (releaseId != null) {
QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product,
projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd);
scan(rawScope, rs -> rawFields.addAll(visibleRaw(readJson(rs.getString(1)), allowedFields).keySet()));
projectGroup, includeUnassigned, "CAST(JSON_KEYS(f.raw_values) AS CHAR)", actor, extraFilters, productSearch,columnFilters,monthStart,monthEnd);
scan(rawScope, rs -> {
Object decoded=mapper.readValue(rs.getString(1),Object.class);
if(decoded instanceof List<?> keys)for(Object key:keys)if(key instanceof String name&&(allowedFields.contains("*")||allowedFields.contains(name)))rawFields.add(name);
});
}
boolean showOrderType=SalesOrderTypes.allowed(allowedFields);
if (rawFields.size() + HEADER_COUNT + (showOrderType?1:0) > MAX_COLUMNS)
@@ -167,8 +175,8 @@ public class RecordsWorkbookExportService {
SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value);
if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor);
if(!columnFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor,releaseId);
if(!columnFilters.isEmpty())sourceColumns.append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
String columns = select == null ? "f.raw_values" : select;
@@ -15,6 +15,7 @@ import cn.yunsheng.quality.ingestion.ImportWorker;
import cn.yunsheng.quality.ingestion.UnclassifiedMappingRebuild;
import cn.yunsheng.quality.ingestion.MappingValueCodec;
import cn.yunsheng.quality.metrics.MetricsService;
import cn.yunsheng.quality.metrics.MetricsRollupService;
import cn.yunsheng.quality.release.ReleaseService;
import cn.yunsheng.quality.source.SourceCatalogService;
import cn.yunsheng.quality.source.SourceDeletionService;
@@ -66,6 +67,7 @@ public class DataWorkflowService {
private final TaskExecutor executor;
private final TransactionTemplate transaction;
private final MetricsService metrics;
private MetricsRollupService rollups;
private final AtomicReference<String> runningOperation = new AtomicReference<>();
private final AtomicReference<String> runningLease = new AtomicReference<>();
@@ -108,6 +110,12 @@ public class DataWorkflowService {
this.executor = executor;
this.transaction = new TransactionTemplate(manager);
this.metrics=metrics;
this.rollups=new MetricsRollupService(jdbc);
}
@Autowired
public void setMetricsRollupService(MetricsRollupService rollups) {
this.rollups = rollups;
}
public Map<String, Object> status(AppUserPrincipal actor) {
@@ -376,7 +384,7 @@ public class DataWorkflowService {
sourceAuthorization.requireReleaseSources(operation.expectedCurrentReleaseId(),actor);
updateStage(operation.id(),token,"CLASSIFYING_UNASSIGNED");
var selectedRules=unclassifiedRules;
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,new cn.yunsheng.quality.metrics.MetricsRollupService(jdbc)).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
var candidate=transaction.execute(status->new UnclassifiedMappingRebuild(jdbc,json,importWorker,rollups).create(operation.id(),operation.expectedCurrentReleaseId(),mappingId,ruleId,definition,selectedRules,actor));
jdbc.update("UPDATE data_workflow_operation SET release_id=?,import_run_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",candidate.releaseId(),candidate.runId(),operation.id(),token);
publishWorkflowCandidate(operation.id(),token,actor,mappingId,null,true,false);
jdbc.update("UPDATE data_workflow_operation SET result_payload=JSON_SET(COALESCE(result_payload,JSON_OBJECT()),'$.recalculationScope','UNCLASSIFIED','$.scannedRows',?,'$.classifiedRows',?,'$.excelFilesParsed',0) WHERE id=? AND lease_token=?",candidate.scannedRows(),candidate.classifiedRows(),operation.id(),token);
@@ -0,0 +1,39 @@
CREATE TABLE source_filter_build (
build_id CHAR(36) NOT NULL PRIMARY KEY,
format_version INT NOT NULL,
CONSTRAINT fk_source_filter_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE source_filter_field (
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
build_id CHAR(36) NOT NULL,
stage VARCHAR(16) NOT NULL,
column_name LONGTEXT COLLATE utf8mb4_0900_bin NOT NULL,
name_hash BINARY(32) NOT NULL,
UNIQUE KEY uq_source_filter_field (build_id,stage,name_hash),
CONSTRAINT fk_source_filter_field_build FOREIGN KEY (build_id) REFERENCES transform_build(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE source_filter_value (
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
field_id BIGINT NOT NULL,
value_hash BINARY(32) NOT NULL,
value LONGTEXT COLLATE utf8mb4_0900_as_cs NOT NULL,
UNIQUE KEY uq_source_filter_value_hash (field_id,value_hash),
KEY ix_source_filter_value_text (field_id,value(128)),
CONSTRAINT fk_source_filter_value_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE source_filter_member (
field_id BIGINT NOT NULL,
fact_id BIGINT NOT NULL,
value_id BIGINT NOT NULL,
PRIMARY KEY (field_id,fact_id),
KEY ix_source_filter_member_value (value_id,fact_id),
CONSTRAINT fk_source_filter_member_field FOREIGN KEY (field_id) REFERENCES source_filter_field(id),
CONSTRAINT fk_source_filter_member_value FOREIGN KEY (value_id) REFERENCES source_filter_value(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- Cover representative-row selection without reading every row's large source JSON.
ALTER TABLE fact_black ADD KEY ix_black_source_catalog (build_id,source_revision_id,department,group_name,id);
ALTER TABLE fact_bright ADD KEY ix_bright_source_catalog (build_id,source_revision_id,department,group_name,id);
@@ -265,7 +265,9 @@ class SourceColumnQueriesMySqlTest {
return fields.stream().filter(item -> item.get("label").equals(label)).findFirst().orElse(null);
}
@Test void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters() throws Exception {
@org.junit.jupiter.params.ParameterizedTest
@org.junit.jupiter.params.provider.ValueSource(booleans={false,true})
void stageProductMetricsDetailsAndExcelShareTheSameSourceAndOrganizationFilters(boolean indexed) throws Exception {
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(80),status VARCHAR(24),rule_version VARCHAR(80))");
jdbc.update("INSERT INTO data_release VALUES (?, 'fixture','PUBLISHED',NULL)",RELEASE);
jdbc.execute("CREATE TABLE metric_rollup_build(build_id CHAR(36))");
@@ -275,6 +277,7 @@ class SourceColumnQueriesMySqlTest {
}
jdbc.execute("CREATE TABLE metric_rollup LIKE fact_black");
jdbc.execute("ALTER TABLE metric_rollup ADD stage VARCHAR(10),ADD source_row_count BIGINT,ADD excluded_blank_rows BIGINT,ADD unmapped_rows BIGINT,ADD project_unconfigured_rows BIGINT");
if(indexed){installFilterIndex();var index=new SourceFilterIndex(jdbc);for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);}
var headerReads=new java.util.concurrent.atomic.AtomicInteger();
var capturing=new JdbcTemplate(jdbc.getDataSource()) {
@Override public <T> List<T> query(String sql,org.springframework.jdbc.core.RowMapper<T> mapper,Object... args) {
@@ -417,6 +420,132 @@ class SourceColumnQueriesMySqlTest {
assertThrows(ResponseStatusException.class,()->controller.query(input,actor));
}
@Test void indexedFiltersCandidatesAndTuplesMatchRawSemanticsAndStayAuthorized() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
List<String> samples=List.of("{}","{\"委托单\":null}","{\"委托单\":\"null\"}","{\"委托单\":\" \"}","{\"委托单\":0}","{\"委托单\":\"0007\"}","{\"委托单\":[1,2]}","{\"委托单\":{\"a\":1}}","{\"委托单\":\"é\"}","{\"委托单\":\"é\"}","{\"委托单\":\"A\"}","{\"委托单\":\"a\"}","{\"委托单\":\" padded \"}","{\"委托单\":\"null\"}");
for(String raw:samples)insert("black","build-black",BLACK_REVISION,"部门A","组A",raw);
String longKey="来源".repeat(300);
insert("black","build-black",BLACK_REVISION,"部门A","组A",JsonMapper.builder().build().writeValueAsString(Map.of(longKey,"long-key-value")));
insert("black","build-black",BLACK_REVISION,"部门B","组B","{\"委托单\":\"hidden-only-value\"}");
String key=(String)field(queries.catalog(RELEASE,actor),"委托单").get("key");
List<Map<String,Object>> rules=new java.util.ArrayList<>();
for(String op:List.of("in","not_in","is_empty","not_empty"))for(String value:List.of("null","0007","0","é","A","a","padded","[1, 2]","{\"a\": 1}"))
rules.add(op.endsWith("empty")?Map.of("field",key,"stages",List.of("black"),"operator",op):Map.of("field",key,"stages",List.of("black"),"operator",op,"values",List.of(value)));
var rawCounts=rules.stream().map(rule->filteredCount(queries,rule)).toList();
var rawValues=queries.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor);
var rawMembers=queries.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor);
jdbc.update("INSERT INTO query_filter_field VALUES ('plain','原始委托单','委托单','委托单',TRUE,TRUE,1)");
List<List<String>> legacySelections=List.of(List.of("null"),List.of("0007"),List.of(QueryFilterFields.EMPTY),List.of("0007",QueryFilterFields.EMPTY));
var rawLegacy=legacySelections.stream().map(this::legacyCount).toList();
installFilterIndex();
var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
assertTrue(indexed.indexed(RELEASE));
assertEquals(rawLegacy,legacySelections.stream().map(this::legacyCount).toList());
assertEquals("long-key-value",jdbc.queryForObject("SELECT sv.value FROM source_filter_field sf JOIN source_filter_value sv ON sv.field_id=sf.id WHERE sf.column_name=? AND sf.build_id='build-black'",String.class,longKey));
var combined=List.of(Map.<String,Object>of("field",key,"stages",List.of("black"),"operator","in","values",List.of("0007")),Map.<String,Object>of("field",field(indexed.catalog(RELEASE,actor),"加工单位").get("key"),"stages",List.of("black"),"operator","in","values",List.of("机二")));
assertEquals(0L,indexed.memberPage(RELEASE,Map.of("year",2026,"columnFilters",combined),List.of(key),List.of("black"),50,0,actor).get("totalCount"));
assertEquals(List.of(),indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),"hidden-only-value",50,0,true,actor).get("rows"));
jdbc.update("INSERT INTO transform_build VALUES ('build-child')");
jdbc.update("INSERT INTO fact_black(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT 'build-child',source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no FROM fact_black WHERE build_id='build-black'");
index.ensureBuild("build-child");
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member sm JOIN source_filter_field sf ON sf.id=sm.field_id JOIN fact_black f ON f.id=sm.fact_id WHERE sf.build_id='build-child' AND f.build_id<>'build-child'",Long.class));
assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
assertEquals(rawValues,indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,50,0,actor));
assertEquals(rawMembers,indexed.memberPage(RELEASE,Map.of("year",2026,"includeUnassigned",true),List.of(key),List.of("black"),200,0,actor));
var page=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),key,List.of("black"),null,2,0,true,actor);
assertEquals(2,((List<?>)page.get("rows")).size());assertEquals(true,page.get("hasMore"));assertFalse(page.containsKey("totalCount"));
long members=jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class);
index.ensureBuild("build-black");assertEquals(members,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
when(governance.allowedFields(actor)).thenReturn(Set.of("加工单位"));
assertThrows(ResponseStatusException.class,()->indexed.values(RELEASE,Map.of("year",2026),key,List.of("black"),null,50,0,true,actor));
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
jdbc.update("DELETE FROM source_filter_build WHERE build_id='build-black'");
assertFalse(indexed.indexed(RELEASE));assertEquals(rawCounts,rules.stream().map(rule->filteredCount(indexed,rule)).toList());
}
@Test void interruptedIndexTransactionNeverPublishesPartialMembersAndCanRetry() throws Exception {
installFilterIndex();
var fail=new java.util.concurrent.atomic.AtomicBoolean(true);
var failing=new JdbcTemplate(jdbc.getDataSource()) {
@Override public int update(String sql) {
if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure");
return super.update(sql);
}
@Override public int update(String sql,Object... args) {
if(sql.startsWith("INSERT INTO source_filter_member")&&fail.compareAndSet(true,false))throw new IllegalStateException("synthetic member write failure");
return super.update(sql,args);
}
};
var index=new SourceFilterIndex(failing);
assertThrows(IllegalStateException.class,()->index.ensureBuild("build-black"));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_build",Long.class));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_field",Long.class));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_value",Long.class));
assertEquals(0L,jdbc.queryForObject("SELECT COUNT(*) FROM source_filter_member",Long.class));
assertFalse(index.ready(RELEASE));
for(String build:List.of("build-black","build-bright","build-hidden"))index.ensureBuild(build);
assertTrue(index.ready(RELEASE));
}
@Test
@EnabledIfEnvironmentVariable(named="QUALITY_SOURCE_FILTER_BENCHMARK",matches="true")
void syntheticWideSourceIndexBenchmark() throws Exception {
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));
int size=Integer.getInteger("quality.benchmarkRows",10000);
jdbc.execute("CREATE TABLE benchmark_digit(n INT PRIMARY KEY)");for(int n=0;n<10;n++)jdbc.update("INSERT INTO benchmark_digit VALUES (?)",n);
for(String stage:List.of("black","bright")) {
jdbc.execute("DELETE FROM fact_"+stage);
StringBuilder raw=new StringBuilder("JSON_OBJECT('客户名称',CONCAT('客户',MOD(n,300))");
for(int f=0;f<120;f++)raw.append(",'字段").append(f).append("',CONCAT('value-").append(f).append("-',").append(stage.equals("black")?"MOD(n,50)":"n").append(")");
raw.append(')');
String numbers="SELECT a.n+10*b.n+100*c.n+1000*d.n+10000*e.n AS n FROM benchmark_digit a CROSS JOIN benchmark_digit b CROSS JOIN benchmark_digit c CROSS JOIN benchmark_digit d CROSS JOIN benchmark_digit e LIMIT "+size;
jdbc.update("INSERT INTO fact_"+stage+"(build_id,source_revision_id,month_key,department,group_name,product,project_group,raw_values,sheet_name,physical_row_no) SELECT ?,?,'2026-01','部门A','组A','P-1','项目A',"+raw+",'data',n+1 FROM ("+numbers+") numbers","build-"+stage,stage.equals("black")?BLACK_REVISION:BRIGHT_REVISION);
}
var rawQuery=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
String field=(String)field(rawQuery.catalog(RELEASE,actor),"客户名称").get("key");
var old=new java.util.LinkedHashMap<String,Object>();
for(String stage:List.of("black","bright")) {
long start=System.nanoTime();var result=rawQuery.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
old.put(stage,Map.of("ms",(System.nanoTime()-start)/1_000_000L,"result",result));
}
installFilterIndex();var index=new SourceFilterIndex(jdbc);
for(String build:List.of("build-black","build-bright","build-hidden")) {
long start=System.nanoTime();index.ensureBuild(build);System.out.println("FILTER_BENCH_BUILD "+build+" rows="+size+" ms="+(System.nanoTime()-start)/1_000_000L);
}
var indexed=new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build());
for(String stage:List.of("black","bright")) {
long start=System.nanoTime();var result=indexed.values(RELEASE,Map.of("year",2026,"includeUnassigned",true),field,List.of(stage),"客户2",50,0,true,actor);
long elapsed=(System.nanoTime()-start)/1_000_000L;
assertEquals(((Map<?,?>)old.get(stage)).get("result"),result);
System.out.println("FILTER_BENCH_QUERY "+stage+" rawMs="+((Map<?,?>)old.get(stage)).get("ms")+" indexMs="+elapsed);
}
for(String table:List.of("source_filter_field","source_filter_value","source_filter_member"))jdbc.execute("ANALYZE TABLE "+table);
System.out.println("FILTER_BENCH_SPACE "+jdbc.queryForList("SELECT table_name,table_rows,data_length,index_length FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name LIKE 'source_filter_%'"));
}
private long legacyCount(List<String> selected) {
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
new QueryFilterFields(jdbc,governance).append(where,args,"black",Map.of("plain",selected),actor,RELEASE);
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
}
private long filteredCount(SourceColumnQueries source,Map<String,Object> rule) {
var where=new StringBuilder(" WHERE rm.release_id=?");var args=new java.util.ArrayList<Object>(List.of(RELEASE));
source.append(where,args,"black",List.of(rule),RELEASE,actor);
DataScope.append(where,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
return jdbc.queryForObject("SELECT COUNT(*) FROM release_member rm JOIN fact_black f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"+where,Long.class,args.toArray());
}
private void installFilterIndex() throws Exception {
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
for(String id:List.of("build-black","build-bright","build-hidden"))jdbc.update("INSERT INTO transform_build VALUES (?)",id);
String sql=java.nio.file.Files.readString(java.nio.file.Path.of("src/main/resources/db/migration/V43__source_filter_indexes.sql"));
for(String statement:sql.split(";"))if(!statement.isBlank())jdbc.execute(statement);
}
private void schema() {
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_document_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36),PRIMARY KEY(release_id,source_document_id))");
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY,source_file VARCHAR(190))");
@@ -0,0 +1,117 @@
package cn.yunsheng.quality.filters;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import java.util.ArrayDeque;
import java.util.List;
import java.util.concurrent.AbstractExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
class SourceFilterIndexBackfillTest {
@Test
void skipsWhenIndexTablesAreUnavailable() {
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
ManualExecutor worker = new ManualExecutor();
doReturn(false).when(index).available();
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
runner.run(null);
verify(jdbc, never()).query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
verify(index, never()).ensureBuild(anyString());
assertEquals(0, worker.queued());
runner.stopWorker();
}
@Test
void coalescesEventsDuringDrainAndContinuesAfterAnIndividualFailure() {
JdbcTemplate jdbc = org.mockito.Mockito.mock(JdbcTemplate.class);
SourceFilterIndex index = org.mockito.Mockito.mock(SourceFilterIndex.class);
ManualExecutor worker = new ManualExecutor();
doReturn(true).when(index).available();
doReturn(List.of("failed-build", "ready-build")).when(jdbc)
.query(anyString(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
var runner = new SourceFilterIndexBackfill(jdbc, index, worker);
doAnswer(invocation -> {
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("event-during-drain"));
runner.onBuildAvailable(new SourceFilterIndex.BuildAvailable("failed-build"));
throw new IllegalStateException("synthetic index failure");
}).when(index).ensureBuild("failed-build");
runner.run(null);
assertEquals(1, worker.queued());
verify(index, never()).ensureBuild(anyString());
ArgumentCaptor<String> sql = ArgumentCaptor.forClass(String.class);
verify(jdbc).query(sql.capture(), org.mockito.ArgumentMatchers.<RowMapper<String>>any());
assertTrue(sql.getValue().contains("c.dataset_id='quality'"));
assertTrue(sql.getValue().contains("r.status='PUBLISHED'"));
assertTrue(sql.getValue().contains("r.complete=TRUE"));
assertTrue(sql.getValue().contains("b.status='COMPLETED'"));
worker.runNext();
verify(index).ensureBuild("failed-build");
verify(index).ensureBuild("ready-build");
verify(index).ensureBuild("event-during-drain");
assertEquals(0, worker.queued());
runner.stopWorker();
assertTrue(worker.isShutdown());
}
@Test
void componentHasAnAutowiredProductionConstructorAndCleanShutdown() {
try (var context = new AnnotationConfigApplicationContext()) {
context.registerBean(JdbcTemplate.class, () -> org.mockito.Mockito.mock(JdbcTemplate.class));
context.registerBean(SourceFilterIndex.class, () -> org.mockito.Mockito.mock(SourceFilterIndex.class));
context.register(SourceFilterIndexBackfill.class);
context.refresh();
assertNotNull(context.getBean(SourceFilterIndexBackfill.class));
}
}
private static final class ManualExecutor extends AbstractExecutorService {
private final ArrayDeque<Runnable> tasks = new ArrayDeque<>();
private boolean shutdown;
@Override public void shutdown() { shutdown = true; }
@Override public List<Runnable> shutdownNow() {
shutdown = true;
List<Runnable> pending = List.copyOf(tasks);
tasks.clear();
return pending;
}
@Override public boolean isShutdown() { return shutdown; }
@Override public boolean isTerminated() { return shutdown && tasks.isEmpty(); }
@Override public boolean awaitTermination(long timeout, TimeUnit unit) { return isTerminated(); }
@Override public void execute(Runnable command) {
if (shutdown) throw new RejectedExecutionException("executor is stopped");
tasks.add(command);
}
int queued() { return tasks.size(); }
void runNext() { tasks.removeFirst().run(); }
}
}
@@ -1,8 +1,21 @@
package cn.yunsheng.quality.metrics;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.ArgumentMatchers.contains;
import static org.mockito.ArgumentMatchers.eq;
import cn.yunsheng.quality.filters.SourceFilterIndex;
import org.junit.jupiter.api.Test;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
class MetricsRollupServiceTest {
@Test
@@ -17,4 +30,65 @@ class MetricsRollupServiceTest {
assertTrue(sql.contains("GROUP BY f.build_id, f.month_key " + collation));
assertTrue(sql.contains("f.group_name " + collation + ", f.project_group " + collation));
}
@Test
void publishesFilterBuildAvailableOnlyAfterTheOuterTransactionCommits() {
var service = new MetricsRollupService(mock(JdbcTemplate.class));
var publisher = mock(ApplicationEventPublisher.class);
service.setApplicationEventPublisher(publisher);
String build = "11111111-1111-4111-8111-111111111111";
TransactionSynchronizationManager.initSynchronization();
try {
service.announceSourceFilterBuild(build);
verifyNoInteractions(publisher);
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
assertEquals(1, synchronizations.size());
for (TransactionSynchronization synchronization : synchronizations) synchronization.afterCommit();
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
} finally {
TransactionSynchronizationManager.clearSynchronization();
}
}
@Test
void publishesImmediatelyWhenThereIsNoTransactionSynchronization() {
var service = new MetricsRollupService(mock(JdbcTemplate.class));
var publisher = mock(ApplicationEventPublisher.class);
service.setApplicationEventPublisher(publisher);
String build = "22222222-2222-4222-8222-222222222222";
service.announceSourceFilterBuild(build);
verify(publisher).publishEvent(new SourceFilterIndex.BuildAvailable(build));
}
@Test
void notificationFailureAfterCommitDoesNotEscapeTheTransactionSynchronization() {
var service = new MetricsRollupService(mock(JdbcTemplate.class));
var publisher = mock(ApplicationEventPublisher.class);
service.setApplicationEventPublisher(publisher);
String build = "33333333-3333-4333-8333-333333333333";
var event = new SourceFilterIndex.BuildAvailable(build);
doThrow(new IllegalStateException("synthetic event delivery failure")).when(publisher).publishEvent(event);
TransactionSynchronizationManager.initSynchronization();
try {
service.announceSourceFilterBuild(build);
var synchronizations = TransactionSynchronizationManager.getSynchronizations();
assertDoesNotThrow(() -> synchronizations.get(0).afterCommit());
} finally {
TransactionSynchronizationManager.clearSynchronization();
}
}
@Test
void directFixtureConstructionKeepsTheSynchronousLegacySchemaFallback() {
JdbcTemplate jdbc = mock(JdbcTemplate.class);
var service = new MetricsRollupService(jdbc);
service.announceSourceFilterBuild("44444444-4444-4444-8444-444444444444");
verify(jdbc).queryForObject(contains("information_schema.tables"), eq(Integer.class));
}
}