完善:质量分析平台查询与分析功能

This commit is contained in:
2026-10-22 21:45:29 +08:00
parent 1e8a655adc
commit 5fbeb52b93
54 changed files with 2331 additions and 564 deletions
@@ -62,9 +62,9 @@ public class AnalysisController {
@GetMapping("/sessions")
public Map<String, Object> sessions(@AuthenticationPrincipal AppUserPrincipal actor) {
return Map.of("items", jdbc.query("SELECT id,title,release_id,created_at,updated_at FROM analysis_session WHERE user_id=? AND auth_scope_hash=? ORDER BY updated_at DESC LIMIT 100",
return Map.of("items", jdbc.query("SELECT s.id,s.title,s.release_id,s.created_at,s.updated_at,COALESCE((SELECT MAX(t.status) FROM analysis_turn t WHERE t.session_id=s.id AND t.status IN ('QUEUED','RUNNING')),'') AS active_status FROM analysis_session s WHERE s.user_id=? AND s.auth_scope_hash=? ORDER BY s.updated_at DESC LIMIT 100",
(rs, n) -> Map.of("id", rs.getString("id"), "title", rs.getString("title"), "releaseId", rs.getString("release_id"),
"createdAt", rs.getTimestamp("created_at").toInstant().toString(), "updatedAt", rs.getTimestamp("updated_at").toInstant().toString()),
"createdAt", rs.getTimestamp("created_at").toInstant().toString(), "updatedAt", rs.getTimestamp("updated_at").toInstant().toString(), "activeTurnStatus", rs.getString("active_status")),
actor.getId().toString(), scopeHash(actor)));
}
@@ -93,9 +93,10 @@ public class AnalysisController {
Session session = ownedSession(id, actor);
// A retained version alone never grants access; re-run the scoped query before returning stored evidence.
metrics.calculate(toQuery(session.context(), session.releaseId()), actor);
List<Map<String, Object>> turns = jdbc.query("SELECT id,question,answer_json,created_at FROM analysis_turn WHERE session_id=? AND user_id=? ORDER BY created_at",
List<Map<String, Object>> turns = jdbc.query("SELECT id,question,answer_json,steps_json,plan_json,created_at FROM analysis_turn WHERE session_id=? AND user_id=? ORDER BY created_at",
(rs, n) -> Map.of("id", rs.getString("id"), "question", rs.getString("question"),
"result", visibleAnswer(rs.getString("answer_json"), actor), "createdAt", rs.getTimestamp("created_at").toInstant().toString()),
"result", visibleAnswer(rs.getString("answer_json"), actor), "steps", rs.getString("steps_json") == null ? List.of() : mapper.readValue(rs.getString("steps_json"), List.class),
"plan", rs.getString("plan_json") == null ? Map.of() : read(rs.getString("plan_json")), "createdAt", rs.getTimestamp("created_at").toInstant().toString()),
session.id(), actor.getId().toString());
return Map.of("id", session.id(), "title", session.title(), "releaseId", session.releaseId(), "context", session.context(), "turns", turns);
}
@@ -145,17 +146,18 @@ public class AnalysisController {
if (!List.of("overall", "department", "group", "product", "projectGroup").contains(view == null ? "overall" : view)) view = "overall";
return new MetricsQuery(year, month, quarter, text(values.get("department")), text(values.get("group")),
text(values.get("product")), text(values.get("projectGroup")), view == null ? "overall" : view,
Boolean.TRUE.equals(values.get("includeUnassigned")), releaseId);
Boolean.TRUE.equals(values.get("includeUnassigned")), releaseId).withExtraFilters(values.get("extraFilters"));
}
private static void mergeContext(Map<String, Object> target, Map<String, Object> source) {
if (source == null) return;
if (source.containsKey("quarter") && source.get("quarter") != null) target.remove("month");
if (source.containsKey("month") && source.get("month") != null) target.remove("quarter");
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "view", "includeUnassigned", "releaseId", "stage")) {
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "view", "includeUnassigned", "releaseId", "stage", "extraFilters")) {
if (!source.containsKey(key)) continue;
Object value = source.get(key);
if (value == null || value instanceof String s && s.isBlank()) target.remove(key);
else if (key.equals("extraFilters")) target.put(key, cn.yunsheng.quality.filters.QueryFilterFields.parse(value));
else if (value instanceof String || value instanceof Number || value instanceof Boolean) target.put(key, value);
}
}
@@ -166,6 +168,7 @@ public class AnalysisController {
boolean continuation = relative || question.matches("(?s).*(\\d{1,2}月|季度|黑片|亮片|综合).*" );
if (hasConversation && continuation && intent.warning() == null && "unsupported".equals(values.get("intent")) && !values.containsKey("metricCode")) {
values = new LinkedHashMap<>(AnalysisModelGateway.localIntent(question));
if ("unsupported".equals(values.get("intent"))) values.put("intent", "metric");
intent = new AnalysisModelGateway.Intent(values, "LOCAL_RULES", null);
}
if (relative) {
@@ -2,6 +2,7 @@ package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.dictionary.MetricDictionaryService;
import cn.yunsheng.quality.governance.GovernanceAccessService;
@@ -44,7 +45,7 @@ public class AnalysisExplorationTools {
private static final JsonMapper JSON = new JsonMapper();
private static final Set<String> DIMENSIONS = Set.of("department", "group", "product", "projectGroup");
private static final Set<String> ARGUMENTS = Set.of("year", "month", "quarter", "department", "group", "product",
"projectGroup", "filters", "view", "groupBy", "includeUnassigned", "trendGranularity", "exclusions", "limit", "stage");
"projectGroup", "extraFilters", "filters", "view", "groupBy", "includeUnassigned", "trendGranularity", "exclusions", "limit", "stage");
private static final List<Map<String, String>> DIMENSION_CATALOG = List.of(
dimension("department", "产业部", "组织维度,始终受当前用户组织授权范围约束"),
dimension("group", "业务分组", "业务分组维度,始终受当前用户分组授权范围约束"),
@@ -102,6 +103,7 @@ public class AnalysisExplorationTools {
Map<String, Object> result = new LinkedHashMap<>();
result.put("kind", "analysis_catalog");
result.put("extraFields",new QueryFilterFields(jdbc,governance).visible(actor));
result.put("dataGrain", "正式发布版本 × 阶段 × 月份 × 产业部 × 业务分组 × 产品 × 项目组;汇总以来源映射后的重量分量重聚合");
result.put("periods", Map.of("filters", List.of("year", "month", "quarter"),
"trendGranularity", List.of("month", "quarter", "year")));
@@ -284,6 +286,7 @@ public class AnalysisExplorationTools {
putSelection(range, "projectGroup", parsed.query().projectGroup());
range.put("includeUnassigned", parsed.query().includeUnassigned());
range.put("exclusions", parsed.query().exclusions());
range.put("extraFilters",parsed.query().extraFilters());
range.put("authorizedFields", fields.stream().map(DetailField::authorizationField).toList());
range.put("dataScope", "事实行同时受当前登录用户的产业部和业务组授权范围约束");
@@ -364,6 +367,7 @@ public class AnalysisExplorationTools {
SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value);
SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value);
appendDetailExclusions(sql, args, query);
new QueryFilterFields(jdbc,governance).append(sql,args,stage,query.extraFilters(),actor);
DataScope.append(sql, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
boolean includeUnassigned = query.includeUnassigned()
@@ -494,7 +498,7 @@ public class AnalysisExplorationTools {
integerObject(args.get("quarter"), "quarter"), selection(filters.get("department"), "department"),
selection(filters.get("group"), "group"), selection(filters.get("product"), "product"),
selection(filters.get("projectGroup"), "projectGroup"), view, includeUnassigned,
releaseId, trendGranularity, exclusions);
releaseId, trendGranularity, exclusions).withExtraFilters(args.get("extraFilters"));
return new ParsedQuery(query, limit, stage);
}
@@ -527,7 +531,7 @@ public class AnalysisExplorationTools {
MetricsQuery historyQuery = new MetricsQuery(selectedYear, null, null, current.department(), current.group(),
current.product(), current.projectGroup(), current.view(), current.includeUnassigned(), current.releaseId(),
current.trendGranularity(), current.exclusions());
current.trendGranularity(), current.exclusions()).withExtraFilters(current.extraFilters());
MetricSnapshot history = visibleSnapshot(historyQuery, actor);
List<TrendMetricRow> rows = new ArrayList<>(history.trend() == null ? List.of() : history.trend());
@@ -540,7 +544,7 @@ public class AnalysisExplorationTools {
if (hasOnlyFirstPeriod && selectedYear > 2000) {
MetricsQuery previousYearQuery = new MetricsQuery(selectedYear - 1, null, null, current.department(), current.group(),
current.product(), current.projectGroup(), current.view(), current.includeUnassigned(), current.releaseId(),
current.trendGranularity(), current.exclusions());
current.trendGranularity(), current.exclusions()).withExtraFilters(current.extraFilters());
MetricSnapshot previousYear = visibleSnapshot(previousYearQuery, actor);
String previousPeriod = previousPeriodForYear(current.trendGranularity(), selectedYear);
if (previousYear.trend() != null) previousYear.trend().stream()
@@ -855,6 +859,7 @@ public class AnalysisExplorationTools {
putSelection(result, "projectGroup", query.projectGroup());
result.put("includeUnassigned", query.includeUnassigned());
result.put("exclusions", query.exclusions());
result.put("extraFilters",query.extraFilters());
result.put("limit", parsed.query().limit());
return result;
}
@@ -1,6 +1,7 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.governance.GovernanceAccessService;
@@ -53,7 +54,7 @@ public class AnalysisFineDefectsService {
private static final Map<String, String> UNAVAILABLE_FIELDS = Map.of(
"晶界扩散磁特性不良数量_1", "来源字段为数量,且没有对应的重量列;不计入重量汇总");
private static final Set<String> REQUEST_KEYS = Set.of("releaseId", "mode", "year", "month", "quarter", "department",
"group", "product", "projectGroup", "includeUnassigned", "stage", "dimension", "defect", "category", "limit");
"group", "product", "productSearch", "projectGroup", "extraFilters", "includeUnassigned", "stage", "dimension", "defect", "category", "limit");
private final JdbcTemplate jdbc;
private final ReleaseService releases;
@@ -84,6 +85,10 @@ public class AnalysisFineDefectsService {
}
String table = request.stage().equals("black") ? "fact_black" : "fact_bright";
String factJoin = table + " f";
if (request.year() != null && request.month() != null) {
factJoin += " FORCE INDEX (ix_" + table + "_month_dims)";
}
List<Category> categories = categories(request.stage());
Set<String> componentCodes = new LinkedHashSet<>();
for (Category category : categories) {
@@ -97,9 +102,14 @@ public class AnalysisFineDefectsService {
SelectionValues.appendPredicate(predicate, args, "f.department", request.department(), DepartmentLabels::queryValue);
SelectionValues.appendPredicate(predicate, args, "f.group_name", request.group(), value -> value);
SelectionValues.appendPredicate(predicate, args, "f.product", request.product(), value -> value);
if (request.productSearch() != null) {
predicate.append(" AND LOCATE(LOWER(?), LOWER(COALESCE(f.product,''))) > 0");
args.add(request.productSearch());
}
SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value);
if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc,governance).append(predicate,args,request.stage(),request.extraFilters(),actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
@@ -109,11 +119,11 @@ public class AnalysisFineDefectsService {
FROM release_member rm
JOIN transform_build b ON b.id=rm.transform_build_id AND b.source_revision_id=rm.source_revision_id
JOIN source_revision sr ON sr.id=rm.source_revision_id
JOIN %s f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id
JOIN %s ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id
%s
""".formatted(jsonObject("f.raw_values", visibleFields.stream().map(DefectField::key).toList()),
jsonObject("f.components", new ArrayList<>(componentCodes)),
request.dimension().equals("product") ? ", f.product AS product_key" : "", table, predicate);
request.dimension().equals("product") ? ", f.product AS product_key" : "", factJoin, predicate);
jdbc.query(sql, (RowCallbackHandler) rs -> {
try {
aggregate.add(readObject(rs.getString("raw_fields")), readObject(rs.getString("formal_components")),
@@ -161,12 +171,14 @@ public class AnalysisFineDefectsService {
throw bad("defect 不属于所选 stage 的重量字段目录");
if (mode.equals("top") && stage != null && category != null
&& !categories(stage).stream().anyMatch(item -> item.name().equals(category))) throw bad("category 不属于所选 stage");
String productSearch = optionalText(input, "productSearch");
if (productSearch != null && productSearch.length() > SelectionValues.MAX_VALUE_LENGTH) throw bad("产品搜索文本过长");
boolean includeUnassigned = bool(input, "includeUnassigned", false);
int limit = integer(input, "limit") == null ? 20 : integer(input, "limit");
if (limit < 1 || limit > 100) throw bad("limit 必须在 1 到 100 之间");
return new Request(releaseId, mode, year, month, quarter, optionalText(input, "department"),
optionalText(input, "group"), optionalText(input, "product"), optionalText(input, "projectGroup"),
includeUnassigned, stage, dimension, defect, category, limit);
optionalText(input, "group"), optionalText(input, "product"), productSearch, optionalText(input, "projectGroup"),
includeUnassigned, stage, dimension, defect, category, limit, QueryFilterFields.parse(input.get("extraFilters")));
}
private Map<String, Object> catalog(Request request, Map<String, Object> release, Set<String> allowed) {
@@ -225,10 +237,32 @@ public class AnalysisFineDefectsService {
return rows.getFirst();
}
private static void appendPeriod(StringBuilder sql, List<Object> args, Request request) {
if (request.year() != null) { sql.append(" AND f.month_key LIKE ?"); args.add(String.format("%04d-%%", request.year())); }
if (request.month() != null) { sql.append(" AND RIGHT(f.month_key,2)=?"); args.add(String.format("%02d", request.month())); }
if (request.quarter() != null) {
static void appendPeriod(StringBuilder sql, List<Object> args, Request request) {
if (request.year() != null && request.month() != null) {
sql.append(" AND f.month_key = ?");
args.add(String.format("%04d-%02d", request.year(), request.month()));
return;
}
if (request.year() != null && request.quarter() != null) {
int first = (request.quarter() - 1) * 3 + 1;
int nextQuarterMonth = first + 3;
int endYear = request.year() + (nextQuarterMonth > 12 ? 1 : 0);
int endMonth = nextQuarterMonth > 12 ? nextQuarterMonth - 12 : nextQuarterMonth;
sql.append(" AND f.month_key >= ? AND f.month_key < ?");
args.add(String.format("%04d-%02d", request.year(), first));
args.add(String.format("%04d-%02d", endYear, endMonth));
return;
}
if (request.year() != null) {
sql.append(" AND f.month_key >= ? AND f.month_key < ?");
args.add(String.format("%04d-01", request.year()));
args.add(String.format("%04d-01", request.year() + 1));
return;
}
if (request.month() != null) {
sql.append(" AND RIGHT(f.month_key,2)=?");
args.add(String.format("%02d", request.month()));
} else if (request.quarter() != null) {
int first = (request.quarter() - 1) * 3 + 1;
sql.append(" AND CAST(RIGHT(f.month_key,2) AS UNSIGNED) BETWEEN ? AND ?");
args.add(first); args.add(first + 2);
@@ -264,8 +298,8 @@ public class AnalysisFineDefectsService {
private static Map<String, Object> context(Request request) {
return mapOf("year", request.year(), "month", request.month(), "quarter", request.quarter(),
"department", request.department(), "group", request.group(), "product", request.product(),
"projectGroup", request.projectGroup(), "includeUnassigned", request.includeUnassigned(),
"department", request.department(), "group", request.group(), "product", request.product(), "productSearch", request.productSearch(),
"projectGroup", request.projectGroup(), "extraFilters",request.extraFilters(), "includeUnassigned", request.includeUnassigned(),
"stage", request.stage(), "dimension", request.dimension(), "defect", request.defect(),
"category", request.category(), "mode", request.mode());
}
@@ -314,8 +348,15 @@ public class AnalysisFineDefectsService {
private static ResponseStatusException forbidden(String message) { return new ResponseStatusException(HttpStatus.FORBIDDEN, message); }
record Request(String releaseId, String mode, Integer year, Integer month, Integer quarter, String department,
String group, String product, String projectGroup, boolean includeUnassigned, String stage,
String dimension, String defect, String category, int limit) {}
String group, String product, String productSearch, String projectGroup, boolean includeUnassigned, String stage,
String dimension, String defect, String category, int limit, Map<String,List<String>> extraFilters) {
Request(String releaseId,String mode,Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,boolean includeUnassigned,String stage,String dimension,String defect,String category,int limit,Map<String,List<String>> extraFilters) {
this(releaseId,mode,year,month,quarter,department,group,product,null,projectGroup,includeUnassigned,stage,dimension,defect,category,limit,extraFilters);
}
Request(String releaseId,String mode,Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,boolean includeUnassigned,String stage,String dimension,String defect,String category,int limit) {
this(releaseId,mode,year,month,quarter,department,group,product,null,projectGroup,includeUnassigned,stage,dimension,defect,category,limit,Map.of());
}
}
record DefectField(String key, String label, String category) {}
record Category(String name, String componentCode, String unavailableReason) {
boolean available() { return unavailableReason == null; }
@@ -15,7 +15,7 @@ import java.util.regex.Pattern;
*/
public final class AnalysisTaskState {
private static final Set<String> CONDITION_KEYS = Set.of(
"year", "month", "quarter", "department", "group", "product", "projectGroup", "view",
"year", "month", "quarter", "department", "group", "product", "projectGroup", "view", "extraFilters",
"stage", "metricCode", "metricName", "includeUnassigned", "releaseId", "category",
"defectCategory", "periodGranularity", "focus", "target", "dimension", "dimensionValue");
private static final Pattern YEAR = Pattern.compile("(20\\d{2})年?");
@@ -235,6 +235,9 @@ public class AnalysisTurnService {
Map<String, Object> budget = budget();
Map<String, Object> plan = new LinkedHashMap<>();
plan.put("status", "PENDING_INTENT"); plan.put("steps", List.of()); plan.put("maxPlanRevisions", PLAN_REVISIONS_MAX);
if (naoRunner != null) {
plan.put("status", "NATIVE_NAO_PENDING"); plan.put("engine", "NAO"); plan.put("nativeRuntime", true);
}
// Freeze the complete effective context at submission time. Session context is mutable
// conversational state and must never silently change the meaning of a retried turn.
plan.put("frozenContext", new LinkedHashMap<>(context));
@@ -290,10 +293,16 @@ public class AnalysisTurnService {
return result;
}
@Transactional
public Map<String, Object> retry(String turnId, AppUserPrincipal actor) {
requireAnalysisAccess(actor);
Turn turn = requireTurn(turnId, actor);
jdbc.queryForObject("SELECT id FROM analysis_session WHERE id=? FOR UPDATE", String.class, turn.sessionId());
turn = requireTurn(turnId, actor);
if (!List.of("FAILED", "CANCELLED", "PARTIAL").contains(turn.status())) throw conflict("仅未完成的分析任务可以重试");
Integer active = jdbc.queryForObject("SELECT COUNT(*) FROM analysis_turn WHERE session_id=? AND status IN ('QUEUED','RUNNING') AND id<>?",
Integer.class, turn.sessionId(), turnId);
if (active != null && active > 0) throw conflict("当前会话仍有分析任务,请等待完成或停止后再重试");
Session session = sessionById(turn.sessionId());
Map<String, Object> storedPlan = readNullable(turn.planJson(), Map.of());
if (naoRunner != null && isNativePlan(storedPlan))
@@ -319,8 +328,7 @@ public class AnalysisTurnService {
write(budget()), turnId, actor.getId().toString());
if (updated != 1) throw conflict("任务状态已变化,请刷新后再试");
appendEvent(turnId, "progress", Map.of("turnId", turnId, "status", "QUEUED", "message", "已验证当前授权,将复用先前成功步骤并继续"));
try { submitWorker(turnId); }
catch (RuntimeException rejected) { finishStopped(turnId, "FAILED", "分析任务队列已满;请稍后重试。"); }
submitAfterCommit(turnId);
return Map.of("turn", turnEnvelope(turnId, turn.sessionId(), ""));
}
@@ -341,9 +349,6 @@ public class AnalysisTurnService {
if (snapshot.release() == null || !session.releaseId().equals(Objects.toString(snapshot.release().get("id"), null)))
throw conflict("原分析固定的数据版本已不可读取,不能安全重试");
Integer active = jdbc.queryForObject("SELECT COUNT(*) FROM analysis_turn WHERE user_id=? AND status IN ('QUEUED','RUNNING') AND id<>?",
Integer.class, actor.getId().toString(), turn.id());
if (active != null && active > 0) throw conflict("该用户已有运行中的分析任务,请完成或停止后再重试");
AppUserPrincipal beforeUpdate = currentUser(actor.getId());
assertScope(beforeUpdate, turn.authScopeHash());
@@ -356,8 +361,7 @@ public class AnalysisTurnService {
if (updated != 1) throw conflict("任务状态或授权已变化,请刷新后再试");
appendEvent(turn.id(), "progress", Map.of("turnId", turn.id(), "status", "QUEUED",
"message", "已重新校验原查询条件,将从原装 NAO 重新分析。"));
try { submitWorker(turn.id()); }
catch (RuntimeException rejected) { finishStopped(turn.id(), "FAILED", "分析任务队列已满;请稍后重试。"); }
submitAfterCommit(turn.id());
return Map.of("turn", turnEnvelope(turn.id(), turn.sessionId(), ""));
}
@@ -503,6 +507,15 @@ public class AnalysisTurnService {
executor.execute(() -> run(turnId));
}
private void submitAfterCommit(String turnId) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override public void afterCommit() {
try { submitWorker(turnId); }
catch (RuntimeException rejected) { finishStopped(turnId, "FAILED", "分析任务队列已满;请稍后重试。"); }
}
});
}
private void run(String turnId) {
Turn turn;
try { turn = turnById(turnId); } catch (Exception ignored) { return; }
@@ -767,6 +780,7 @@ public class AnalysisTurnService {
plan.put("nativeRuntime", true);
plan.put("engine", "NAO");
plan.put("releaseId", session.releaseId());
plan.put("startedAt", Instant.now().toString());
plan.put("context", context);
plan.put("frozenContext", context);
plan.put("steps", List.of());
@@ -775,9 +789,18 @@ public class AnalysisTurnService {
try {
NaoAnalysisRunner.Result result = naoRunner.run(turnId, session.id(), actor, turn.question(),
session.releaseId(), context,
progress -> appendEvent(turnId, "progress", Map.of("turnId", turnId, "status", "RUNNING",
"message", Objects.toString(progress.getOrDefault("message", "原装 NAO 正在处理"), "原装 NAO 正在处理"),
"toolCount", progress.getOrDefault("toolCount", 0))),
progress -> {
String message = Objects.toString(progress.getOrDefault("message", "正在分析质量数据…"), "正在分析质量数据…");
if (progress.get("steps") instanceof List<?> steps) plan.put("steps", steps);
plan.put("toolCount", progress.getOrDefault("toolCount", 0));
plan.put("progressMessage", message);
plan.put("updatedAt", Instant.now().toString());
List<Map<String, Object>> steps = mapList(plan.get("steps"));
persistState(turnId, "RUNNING", plan, steps, List.of(), null);
appendEvent(turnId, "progress", Map.of("turnId", turnId, "status", "RUNNING",
"message", message, "toolCount", plan.get("toolCount"), "steps", steps,
"startedAt", plan.get("startedAt"), "updatedAt", plan.get("updatedAt")));
},
answer -> persistNativeAnswer(turnId, UUID.fromString(turn.userId()), turn.authScopeHash(),
session.releaseId(), context, answer),
() -> cancelRequested(turnId));
@@ -826,7 +849,7 @@ public class AnalysisTurnService {
private void persistNativeAnswer(String turnId, UUID userId, String expectedScopeHash,
String releaseId, Map<String, Object> context, String text) {
if (cancelRequested(turnId)) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "NAO turn stopped");
if (cancelRequested(turnId)) return;
assertScope(currentUser(userId), expectedScopeHash);
Map<String, Object> answer = new LinkedHashMap<>();
answer.put("answer", text);
@@ -857,10 +880,10 @@ public class AnalysisTurnService {
return answer;
}
private List<Map<String, Object>> nativeSteps(Object rawToolCalls) {
static List<Map<String, Object>> nativeSteps(Object rawToolCalls) {
if (!(rawToolCalls instanceof List<?> calls)) return List.of();
List<Map<String, Object>> steps = new ArrayList<>();
for (Object value : calls.stream().limit(100).toList()) {
for (Object value : calls.stream().limit(1000).toList()) {
if (!(value instanceof Map<?, ?> call)) continue;
Map<String, Object> step = new LinkedHashMap<>();
step.put("step", steps.size() + 1);
@@ -873,6 +896,26 @@ public class AnalysisTurnService {
if (path != null) step.put("path", cap(path.toString(), 200));
Object state = call.get("status");
step.put("status", state == null ? "SUCCEEDED" : cap(state.toString(), 40));
if (Boolean.TRUE.equals(call.get("recovered"))) step.put("recovered", true);
for (String field : List.of("label", "outputSummary", "startedAt", "finishedAt"))
if (call.get(field) instanceof String text) step.put(field, cap(text, 500));
if (call.get("input") instanceof Map<?, ?> input) {
Map<String, Object> safeInput = new LinkedHashMap<>();
for (String field : List.of("mode", "scope", "period", "comparisonPeriod", "filters", "view", "stage", "dimension", "defect", "category", "limit", "sort")) {
Object item = input.get(field);
if (item instanceof Map<?, ?> nested) {
Map<String, Object> safeNested = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned")) {
Object child = nested.get(key);
if (child instanceof String s) safeNested.put(key, cap(s, 500));
else if (child instanceof Number || child instanceof Boolean) safeNested.put(key, child);
}
safeInput.put(field, safeNested);
} else if (item instanceof String s) safeInput.put(field, cap(s, 500));
else if (item instanceof Number || item instanceof Boolean) safeInput.put(field, item);
}
step.put("input", safeInput);
}
steps.add(step);
}
return List.copyOf(steps);
@@ -882,7 +925,9 @@ public class AnalysisTurnService {
String releaseId, String status, String reason) {
Map<String, Object> answer = nativeAnswer(Map.of(), reason, context, releaseId, status, reason);
plan.put("stopReason", reason);
finish(turnId, status, plan, List.of(), List.of(), reason, answer);
// A cancelled or failed subprocess can still have completed useful calls.
// Retain its persisted operation trace for recovery and inspection.
finish(turnId, status, plan, mapList(plan.get("steps")), List.of(), reason, answer);
}
private AgentOutcome explore(String turnId, AppUserPrincipal submitted, String question, Map<String, Object> task,
@@ -918,6 +963,28 @@ public class AnalysisTurnService {
persistState(turnId, "RUNNING", plan, steps, evidence, null);
continue;
}
boolean impactAlreadyCompleted = steps.stream().anyMatch(step -> "metrics.impact".equals(step.get("tool")));
boolean comparisonAvailable = comparison != null && Boolean.TRUE.equals(mapValue(comparison.get("delta")).get("available"));
if (comparisonAvailable && !impactAlreadyCompleted && impacts.isEmpty()
&& !metric.endsWith("_WITHOUT_HEADQUARTERS_SCRAP") && steps.size() < TOOL_CALLS_MAX) {
String fallbackStage = "all".equals(stage)
? Objects.requireNonNullElse(AnalysisAgentSupport.leadingStage(comparison), "black") : stage;
Map<String, Object> semantic = mapValue(plan.get("semantic"));
String fallbackDimension = Objects.toString(semantic.get("impactDimension"),
impactDimension(Objects.toString(context.get("view"), "overall")));
if (!"overall".equals(fallbackDimension)) {
Map<String, Object> fallbackArguments = Map.of("stage", fallbackStage, "dimension", fallbackDimension);
Map<String, Object> fallbackContext = agentContext(context, fallbackArguments);
Map<String, Object> fallbackPrior = agentContext(prior, fallbackArguments);
Map<String, Object> fallbackResult = executeImpact(turnId, actor.getId(), fallbackContext, fallbackPrior,
releaseId, fallbackStage, fallbackDimension, hash, steps.size() + 1, steps, evidence, results);
if (fallbackResult == null) return new AgentOutcome(false, sent, "用户请求停止", "", checks);
impacts.add(fallbackResult);
if (Boolean.TRUE.equals(fallbackResult.get("available")))
checks.add(Map.of("name", "模型规划失败后的阶段影响兜底", "evidenceRef", results.size(),
"stage", fallbackStage, "dimension", fallbackDimension));
}
}
return new AgentOutcome(false, sent, decision.warning(), "", checks);
}
Map<String, Object> event = new LinkedHashMap<>();
@@ -1144,7 +1211,7 @@ public class AnalysisTurnService {
private Map<String, Object> explorationArgs(Map<String, Object> context, String dimension) {
Map<String, Object> args = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned", "exclusions", "trendGranularity"))
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "exclusions", "trendGranularity"))
if (context.get(key) != null) args.put(key, context.get(key));
args.put("view", dimension); args.put("limit", 20); return args;
}
@@ -1256,7 +1323,7 @@ public class AnalysisTurnService {
} else result = asSecurityActor(actor, () -> analytics.top(stage, dimension, category, releaseId,
integer(context.get("year")), integer(context.get("month")), integer(context.get("quarter")),
text(context.get("department")), text(context.get("group")), text(context.get("product")),
text(context.get("projectGroup")), Boolean.TRUE.equals(context.get("includeUnassigned")), limit, actor));
text(context.get("projectGroup")), Boolean.TRUE.equals(context.get("includeUnassigned")), limit, json.writeValueAsString(context.getOrDefault("extraFilters",Map.of())), actor));
AppUserPrincipal after = currentUser(userId); assertScope(after, turn.authScopeHash());
if (cancelRequested(turnId)) return null;
results.add(result);
@@ -1662,7 +1729,7 @@ public class AnalysisTurnService {
if (year != null && (year < 2000 || year > 2100) || month != null && (month < 1 || month > 12) || quarter != null && (quarter < 1 || quarter > 4) || month != null && quarter != null) throw bad("期间无效");
return new MetricsQuery(year,month,quarter,text(values.get("department")),text(values.get("group")),text(values.get("product")),
text(values.get("projectGroup")),view,Boolean.TRUE.equals(values.get("includeUnassigned")),releaseId,
Objects.toString(values.getOrDefault("trendGranularity", "month")), exclusions(values));
Objects.toString(values.getOrDefault("trendGranularity", "month")), exclusions(values)).withExtraFilters(values.get("extraFilters"));
}
@SuppressWarnings("unchecked")
private Map<String, List<String>> exclusions(Map<String, Object> context) {
@@ -1673,14 +1740,16 @@ public class AnalysisTurnService {
Map<String,Object> map=new LinkedHashMap<>(); map.put("year",query.year()); map.put("month",query.month()); map.put("quarter",query.quarter());
map.put("department",query.department()); map.put("group",query.group()); map.put("product",query.product()); map.put("projectGroup",query.projectGroup());
map.put("view",query.view()); map.put("includeUnassigned",query.includeUnassigned()); map.put("releaseId",query.releaseId());
if (!query.extraFilters().isEmpty()) map.put("extraFilters",query.extraFilters());
if (!query.exclusions().isEmpty()) map.put("exclusions", query.exclusions());
map.put("trendGranularity",query.trendGranularity()); return map;
}
private String validateView(String view) { return view != null && List.of("overall","department","group","product","projectGroup").contains(view) ? view : "overall"; }
private Map<String,Object> sanitizeContext(Map<String,Object> source) {
Map<String,Object> result=new LinkedHashMap<>(); if(source==null)return result;
for(String key:source.keySet()) { if(!List.of("year","month","quarter","department","group","product","projectGroup","view","includeUnassigned","stage","releaseId","resultRef","exclusions","trendGranularity").contains(key)) throw bad("分析条件包含不支持的字段");
for(String key:source.keySet()) { if(!List.of("year","month","quarter","department","group","product","projectGroup","extraFilters","view","includeUnassigned","stage","releaseId","resultRef","exclusions","trendGranularity").contains(key)) throw bad("分析条件包含不支持的字段");
if ("resultRef".equals(key)) continue; // Validated against the server snapshot below, never trusted as query input.
if ("extraFilters".equals(key)) { result.put(key, cn.yunsheng.quality.filters.QueryFilterFields.parse(source.get(key))); continue; }
if ("exclusions".equals(key)) { result.put(key, AnalysisExplorationTools.exclusions(source.get(key))); continue; }
Object value=source.get(key); if(value == null || value instanceof String || value instanceof Number || value instanceof Boolean) result.put(key,value); }
return result;
@@ -1702,7 +1771,7 @@ public class AnalysisTurnService {
if (!referenceRelease.equals(trusted.get("releaseId"))) throw conflict("引用结果的发布快照不一致");
Map<String,Object> claimed = sanitizeContext(input);
Map<String,Object> canonical = referenceContext(trusted, claimed);
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned", "stage")) {
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "stage")) {
context.remove(key);
if (canonical.containsKey(key)) context.put(key, canonical.get(key));
}
@@ -1739,7 +1808,7 @@ public class AnalysisTurnService {
BoundTopSpec spec = boundTopSpec(storedQuery);
if (!referenceRelease.equals(storedQuery.get("releaseId"))) throw conflict("绑定的TOP查询发布版本不一致");
Map<String, Object> expectedContext = referenceContext(storedQuery, Map.of());
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned", "stage")) {
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "stage")) {
if (!sameValue(expectedContext.get(key), context.get(key)))
throw conflict("绑定的TOP筛选条件与轮次冻结条件不一致,请从原排名重新发起");
}
@@ -1754,7 +1823,7 @@ public class AnalysisTurnService {
static BoundTopSpec boundTopSpec(Map<String, Object> query) {
Set<String> allowed = Set.of("releaseId", "stage", "dimension", "category", "year", "month", "quarter",
"department", "group", "product", "projectGroup", "includeUnassigned", "limit");
"department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "limit");
if (query == null || !allowed.containsAll(query.keySet())) throw new IllegalArgumentException("Invalid bound TOP query");
String stage = Objects.toString(query.get("stage"), "");
String dimension = Objects.toString(query.get("dimension"), "");
@@ -1782,7 +1851,7 @@ public class AnalysisTurnService {
static Map<String, Object> boundTopContext(Map<String, Object> context, Map<String, Object> query) {
Map<String, Object> result = new LinkedHashMap<>(context);
Map<String, Object> canonical = referenceContext(query, Map.of());
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned", "stage")) {
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "stage")) {
result.remove(key);
if (canonical.containsKey(key)) result.put(key, canonical.get(key));
}
@@ -1817,17 +1886,17 @@ public class AnalysisTurnService {
static Map<String,Object> referenceContext(Map<String,Object> trusted, Map<String,Object> claimed) {
Map<String,Object> canonical = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned", "stage")) {
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "stage")) {
Object value = trusted.get(key);
if (value != null) canonical.put(key, value);
Object provided = claimed.get(key);
if (provided != null && !String.valueOf(provided).equals(String.valueOf(value)))
if (provided != null && !(key.equals("extraFilters") ? cn.yunsheng.quality.filters.QueryFilterFields.parse(provided).equals(cn.yunsheng.quality.filters.QueryFilterFields.parse(value)) : String.valueOf(provided).equals(String.valueOf(value))))
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "查询条件与可信结果不一致,请从原结果继续分析");
}
return canonical;
}
private void merge(Map<String,Object> to,Map<String,Object> from) {
for(String key:List.of("year","month","quarter","department","group","product","projectGroup","view","includeUnassigned","stage","metricCode","exclusions","trendGranularity")) if(from.containsKey(key)) {
for(String key:List.of("year","month","quarter","department","group","product","projectGroup","extraFilters","view","includeUnassigned","stage","metricCode","exclusions","trendGranularity")) if(from.containsKey(key)) {
Object value = from.get(key); if (value == null || value instanceof String s && s.isBlank()) to.remove(key); else to.put(key, value);
}
if(from.get("quarter") != null)to.remove("month"); if(from.get("month") != null)to.remove("quarter");
@@ -1894,7 +1963,7 @@ public class AnalysisTurnService {
private String clean(String x) { return x==null||x.isBlank()?null:x.strip(); }
private String clean(Object x) { return x==null?null:clean(x.toString()); }
private static String text(Object x) { return x == null || x.toString().isBlank() ? null : x.toString(); }
private String cap(String x,int max) { if(x==null)return ""; if(x.length()>max)throw bad("文本过长"); return x; }
private static String cap(String x,int max) { if(x==null)return ""; if(x.length()>max)throw bad("文本过长"); return x; }
private Integer integer(Object x) { if(x==null)return null;try{return Integer.valueOf(x.toString());}catch(Exception e){throw bad("期间值无效");} }
private static ResponseStatusException bad(String m) { return new ResponseStatusException(HttpStatus.BAD_REQUEST,m); }
private ResponseStatusException conflict(String m) { return new ResponseStatusException(HttpStatus.CONFLICT,m); }
@@ -75,7 +75,9 @@ public class NaoAnalysisRunner {
}
public void requestCancel(String turnId) {
cancellationRequested.add(turnId);
// Queued work is cancelled through its persisted DB flag. Do not leave a
// process-local marker that could poison a later retry of the same ID.
active.computeIfPresent(turnId, (id, process) -> { cancellationRequested.add(id); return process; });
}
@PreDestroy
@@ -166,12 +168,14 @@ public class NaoAnalysisRunner {
}
}
private Map<String, Object> safeProgress(Map<String, Object> message) {
static Map<String, Object> safeProgress(Map<String, Object> message) {
Map<String, Object> safe = new LinkedHashMap<>();
safe.put("message", cap(Objects.toString(message.getOrDefault("message", "原装 NAO 正在处理"), "原装 NAO 正在处理"), 500));
Object count = message.get("toolCount");
if (count instanceof Number number && number.intValue() >= 0 && number.intValue() <= 1000)
safe.put("toolCount", number.intValue());
if (message.get("toolCalls") instanceof List<?>)
safe.put("steps", AnalysisTurnService.nativeSteps(message.get("toolCalls")));
return safe;
}
@@ -252,7 +256,7 @@ public class NaoAnalysisRunner {
}
private String shortId(String id) { return id.substring(0, Math.min(8, id.length())); }
private String cap(String value, int max) { return value.length() <= max ? value : value.substring(0, max); }
private static String cap(String value, int max) { return value.length() <= max ? value : value.substring(0, max); }
private void terminate(Process process) {
process.destroy();
@@ -30,10 +30,10 @@ import jakarta.servlet.http.HttpServletRequest;
@RequestMapping("/api/analysis/runtime")
public class NaoRuntimeQueryController {
private static final Set<String> METRIC_QUERY = Set.of("year", "month", "quarter", "department", "group",
"product", "projectGroup", "view", "includeUnassigned", "releaseId", "trendGranularity",
"product", "projectGroup", "extraFilters", "view", "includeUnassigned", "releaseId", "trendGranularity",
"rowLimit", "rowOffset", "rowOrder");
private static final Set<String> TOP_QUERY = Set.of("stage", "dimension", "category", "releaseId", "year",
"month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned", "limit");
"month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "limit");
private final NaoRuntimeAccessService access;
private final ReleaseController releases;
@@ -116,7 +116,7 @@ public class NaoRuntimeQueryController {
query.get("department"), query.get("group"), query.get("product"), query.get("projectGroup"),
query.getOrDefault("view", "overall"), bool(query, "includeUnassigned", false), releaseId,
query.getOrDefault("trendGranularity", "month"), integer(query, "rowLimit"),
integer(query, "rowOffset") == null ? 0 : integer(query, "rowOffset"), query.get("rowOrder"),
integer(query, "rowOffset") == null ? 0 : integer(query, "rowOffset"), query.get("rowOrder"), query.get("extraFilters"),
delegation.actor());
}
@@ -128,7 +128,7 @@ public class NaoRuntimeQueryController {
query.get("category"), releaseId, integer(query, "year"), integer(query, "month"),
integer(query, "quarter"), query.get("department"), query.get("group"), query.get("product"),
query.get("projectGroup"), bool(query, "includeUnassigned", false),
integer(query, "limit") == null ? 20 : integer(query, "limit"), delegation.actor());
integer(query, "limit") == null ? 20 : integer(query, "limit"), query.get("extraFilters"), delegation.actor());
}
private Map<String, Object> boundPayload(Map<String, Object> payload, String releaseId) {
@@ -1,6 +1,7 @@
package cn.yunsheng.quality.analytics;
import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.governance.GovernanceAccessService;
@@ -60,6 +61,11 @@ public class AnalyticsController {
}
/** Ordinary Top N only. Share is relative to the selected defect-component weights; no cumulative threshold is applied. */
/** Compatibility for internal callers without source-field filters. */
public Map<String, Object> top(String stage, String dimension, String category, String releaseId, Integer year, Integer month, Integer quarter, String department, String group, String product, String projectGroup, boolean includeUnassigned, int limit, AppUserPrincipal actor) {
return top(stage, dimension, category, releaseId, year, month, quarter, department, group, product, projectGroup, includeUnassigned, limit, null, actor);
}
@GetMapping("/top")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST')")
public Map<String, Object> top(@RequestParam String stage,
@@ -75,6 +81,7 @@ public class AnalyticsController {
@RequestParam(required = false) String projectGroup,
@RequestParam(defaultValue = "false") boolean includeUnassigned,
@RequestParam(defaultValue = "20") int limit,
@RequestParam(required=false) String extraFilters,
@AuthenticationPrincipal AppUserPrincipal actor) {
requireQueryDetail(actor);
validatePeriod(year, month, quarter);
@@ -85,7 +92,8 @@ public class AnalyticsController {
if (selected == null) return Map.ofEntries(Map.entry("releaseId", ""), Map.entry("stage", stage), Map.entry("dimension", dimension), Map.entry("category", ""),
Map.entry("denominator", ZERO), Map.entry("rows", List.of()), Map.entry("limit", limit), Map.entry("resultId", ""), Map.entry("resultQuery", Map.of()),
Map.entry("shareBasis", "所选筛选条件下缺陷类别重量总和"), Map.entry("coverage", Map.of()));
String source = topSourceSql(stage);
Map<String,List<String>> extra=QueryFilterFields.parse(extraFilters);
String source = extra.isEmpty()?topSourceSql(stage):"(SELECT f.*,1 AS source_row_count FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id WHERE rm.release_id=?) f";
List<Component> components = components(stage);
Set<String> allowedFields = governance.allowedFields(actor);
boolean allComponentsVisible = allowedFields.contains("*");
@@ -95,8 +103,9 @@ public class AnalyticsController {
.filter(c -> c.name().equals(category)).findFirst().orElseThrow(() -> bad("category 不属于所选阶段"));
if (selectedCategory != null && visibleComponents.stream().noneMatch(component -> component.key().equals(selectedCategory.key())))
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前字段授权不包含所选报废类别");
List<Object> args = new ArrayList<>(List.of(selected, selected));
List<Object> args = new ArrayList<>(extra.isEmpty()?List.of(selected, selected):List.of(selected));
StringBuilder where = baseWhere(args, year, month, quarter, department, group, product, projectGroup, includeUnassigned, actor);
new QueryFilterFields(jdbc,governance).append(where,args,stage,extra,actor);
List<AggregateTop> aggregated;
if (dimension.equals("category")) {
List<Component> selectedComponents = selectedCategory == null ? visibleComponents : List.of(selectedCategory);
@@ -135,9 +144,10 @@ public class AnalyticsController {
if (group != null && !group.isBlank()) query.put("group", group);
if (product != null && !product.isBlank()) query.put("product", product);
if (projectGroup != null && !projectGroup.isBlank()) query.put("projectGroup", projectGroup);
query.put("extraFilters",extra);
query.put("includeUnassigned", includeUnassigned); query.put("limit", limit);
MetricSnapshot coverageSnapshot = metrics.calculate(new MetricsQuery(year, month, quarter, department, group, product,
projectGroup, "overall", includeUnassigned, selected), actor);
projectGroup, "overall", includeUnassigned, selected).withExtraFilters(extra), actor);
Map<String, Object> response = new LinkedHashMap<>();
response.put("releaseId", selected); response.put("stage", stage); response.put("dimension", dimension);
response.put("category", selectedCategory == null ? "" : selectedCategory.name()); response.put("denominator", denominator);
@@ -540,7 +550,7 @@ public class AnalyticsController {
private Map<String, Object> targetFor(MetricSnapshot snapshot, CompareSide side, String metric) {
if (side == null) return Map.of("status", "UNCONFIGURED", "target", Map.of());
return targetFor(snapshot, new CompareSide(side.releaseId(), side.year(), side.month(), side.quarter(), side.department(),
side.group(), side.product(), side.projectGroup(), side.includeUnassigned(), metric));
side.group(), side.product(), side.projectGroup(), side.includeUnassigned(), metric, side.exclusions(),side.extraFilters()));
}
private String normalizeMetric(String value) {
@@ -667,7 +677,8 @@ public class AnalyticsController {
private MetricsQuery query(CompareSide side, String view) {
validatePeriod(side.year(), side.month(), side.quarter());
return new MetricsQuery(side.year(), side.month(), side.quarter(), side.department(), side.group(), side.product(),
side.projectGroup(), view, side.includeUnassigned(), side.releaseId(), "month", side.exclusions());
side.projectGroup(), view, side.includeUnassigned(), side.releaseId(), "month", side.exclusions())
.withExtraFilters(side.extraFilters()).withProductSearch(side.productSearch());
}
private StringBuilder baseWhere(List<Object> args, Integer year, Integer month, Integer quarter,
@@ -735,12 +746,19 @@ public class AnalyticsController {
public record CompareSide(String releaseId, Integer year, Integer month, Integer quarter, String department, String group,
String product, String projectGroup, Boolean includeUnassigned, String metric,
Map<String, List<String>> exclusions) {
Map<String, List<String>> exclusions, Map<String,List<String>> extraFilters, String productSearch) {
public CompareSide(String releaseId,Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,Boolean includeUnassigned,String metric,Map<String,List<String>> exclusions) {
this(releaseId,year,month,quarter,department,group,product,projectGroup,includeUnassigned,metric,exclusions,Map.of(),null);
}
public CompareSide(String releaseId,Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,Boolean includeUnassigned,String metric,Map<String,List<String>> exclusions,Map<String,List<String>> extraFilters) {
this(releaseId,year,month,quarter,department,group,product,projectGroup,includeUnassigned,metric,exclusions,extraFilters,null);
}
public CompareSide(String releaseId, Integer year, Integer month, Integer quarter, String department, String group,
String product, String projectGroup, Boolean includeUnassigned, String metric) {
this(releaseId, year, month, quarter, department, group, product, projectGroup, includeUnassigned, metric, Map.of());
}
public CompareSide {
extraFilters=QueryFilterFields.parse(extraFilters);
// This field is optional in compare requests; normalize an omitted/null JSON value to false.
includeUnassigned = Boolean.TRUE.equals(includeUnassigned);
exclusions = exclusions == null ? Map.of() : Map.copyOf(exclusions);
@@ -164,7 +164,8 @@ public class ChecklistsController {
private Map<String, Object> sanitizeContext(Map<String, Object> context) {
Map<String, Object> out = new LinkedHashMap<>();
if (context != null) for (String key : context.keySet()) {
if (!List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned", "view").contains(key)) throw bad("核查项筛选条件无效");
if (!List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "view").contains(key)) throw bad("核查项筛选条件无效");
if(key.equals("extraFilters")) { out.put(key,cn.yunsheng.quality.filters.QueryFilterFields.parse(context.get(key)));continue; }
Object value = context.get(key);
if (value instanceof String || value instanceof Number || value instanceof Boolean) out.put(key, value);
}
@@ -176,7 +177,7 @@ public class ChecklistsController {
String view = String.valueOf(context.getOrDefault("view", "overall"));
if (!List.of("overall", "department", "group", "product", "projectGroup").contains(view)) view = "overall";
return new MetricsQuery(year, month, quarter, text(context.get("department")), text(context.get("group")), text(context.get("product")),
text(context.get("projectGroup")), view, Boolean.TRUE.equals(context.get("includeUnassigned")), releaseId);
text(context.get("projectGroup")), view, Boolean.TRUE.equals(context.get("includeUnassigned")), releaseId).withExtraFilters(context.get("extraFilters"));
}
private Checklist requireOwned(String id, AppUserPrincipal actor) {
@@ -219,7 +220,8 @@ public class ChecklistsController {
if (!(rawContext instanceof Map<?, ?>)) rawContext = read(rows.getFirst().get("session"));
if (!(rawContext instanceof Map<?, ?> map)) throw new ResponseStatusException(HttpStatus.CONFLICT, "分析轮次缺少可信筛选快照");
Map<String, Object> trusted = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned", "view", "stage")) {
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "view", "stage")) {
if(key.equals("extraFilters")) { trusted.put(key,cn.yunsheng.quality.filters.QueryFilterFields.parse(map.get(key)));continue; }
Object value = map.get(key);
if (value instanceof String || value instanceof Number || value instanceof Boolean) trusted.put(key, value);
}
@@ -235,7 +237,7 @@ public class ChecklistsController {
private Map<String, Object> contextFromTopQuery(Map<String, Object> query) {
Map<String, Object> context = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned", "stage"))
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "stage"))
if (query.containsKey(key)) context.put(key, query.get(key));
return context;
}
@@ -197,7 +197,7 @@ public class ComparisonViewsController {
if (year != null && (year < 2000 || year > 2100) || month != null && (month < 1 || month > 12)
|| quarter != null && (quarter < 1 || quarter > 4) || month != null && quarter != null) throw bad("保存视图期间无效");
return new MetricsQuery(year, month, quarter, text(c.get("department")), text(c.get("group")), text(c.get("product")),
text(c.get("projectGroup")), groupBy, Boolean.TRUE.equals(c.get("includeUnassigned")), releaseId);
text(c.get("projectGroup")), groupBy, Boolean.TRUE.equals(c.get("includeUnassigned")), releaseId).withExtraFilters(c.get("extraFilters"));
}
private Definition normalize(ViewRequest request, AppUserPrincipal actor) {
@@ -213,7 +213,7 @@ public class ComparisonViewsController {
String groupBy = String.valueOf(input.getOrDefault("view", request.groupBy() == null ? "overall" : request.groupBy()));
if (!List.of("overall", "department", "group", "product", "projectGroup").contains(groupBy)) throw bad("groupBy 无效");
Map<String, Object> context = request.context() == null ? Map.of() : new LinkedHashMap<>(request.context());
for (String key : context.keySet()) if (!List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned").contains(key))
for (String key : context.keySet()) if (!List.of("year", "month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned").contains(key))
throw bad("保存视图包含不支持的查询条件");
List<String> allowedColumns = List.of("label", "compositeRate", "black", "bright", "coverage", "department", "group", "product", "projectGroup");
Object columnsInput = input.get("columns");
@@ -282,7 +282,7 @@ public class ComparisonViewsController {
@SuppressWarnings("unchecked") private Map<String, Object> objectMap(Object value) { return value instanceof Map<?, ?> map ? (Map<String, Object>) map : Map.of(); }
private AnalyticsController.CompareSide side(AnalyticsController.CompareSide old, String releaseId, String metric) {
return new AnalyticsController.CompareSide(releaseId, old.year(), old.month(), old.quarter(), old.department(), old.group(),
old.product(), old.projectGroup(), old.includeUnassigned(), metric);
old.product(), old.projectGroup(), old.includeUnassigned(), metric, old.exclusions(), old.extraFilters());
}
private Integer integer(Object x) { return x == null ? null : Integer.valueOf(x.toString()); }
private String text(Object x) { if (x == null) return null; String s = x.toString().strip(); return s.isEmpty() ? null : s; }
@@ -0,0 +1,70 @@
package cn.yunsheng.quality.filters;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import cn.yunsheng.quality.release.ReleaseService;
import java.util.*;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;
@RestController
@RequestMapping("/api/filter-fields")
public class FilterFieldsController {
private final JdbcTemplate jdbc;
private final GovernanceAccessService governance;
private final ReleaseService releases;
public FilterFieldsController(JdbcTemplate jdbc,GovernanceAccessService governance,ReleaseService releases) { this.jdbc=jdbc;this.governance=governance;this.releases=releases; }
@GetMapping
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST','DATA_MAINTAINER')")
public List<QueryFilterFields.Field> list(@RequestParam(defaultValue="false") boolean manage,@AuthenticationPrincipal AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);
if(manage && actor.getRoleCodes().stream().anyMatch(r->Set.of("SYSTEM_ADMIN","DATA_MAINTAINER").contains(r)))return new QueryFilterFields(jdbc,governance).all();
return new QueryFilterFields(jdbc,governance).visible(actor);
}
@GetMapping("/options")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST')")
public Map<String,Object> options(@RequestParam String key,@RequestParam(required=false) String search,@AuthenticationPrincipal AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);governance.requireBoundAction("QUERY","query.aggregate");
return new QueryFilterFields(jdbc,governance).options(key,search,releases.resolvePublishedRelease(null),actor);
}
@GetMapping("/columns")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER')")
public Map<String,List<String>> columns(@AuthenticationPrincipal AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);
Map<String,List<String>> result=new LinkedHashMap<>();
String release=releases.resolvePublishedRelease(null);
for(String stage:List.of("black","bright")) {
// Raw source headers are stable within each build. Read a representative row of every current build.
List<String> raw=jdbc.query("SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) FROM release_member rm JOIN fact_"+stage+" f ON f.id=(SELECT MIN(f2.id) FROM fact_"+stage+" f2 WHERE f2.build_id=rm.transform_build_id) WHERE rm.release_id=?",(rs,n)->rs.getString(1),release);
Set<String> columns=new TreeSet<>();
var json=tools.jackson.databind.json.JsonMapper.builder().build();
Set<String> allowed=governance.allowedFields(actor);
for(String text:raw) if(text!=null)for(Object value:json.readValue(text,List.class))if(allowed.contains("*")||allowed.contains(String.valueOf(value)))columns.add(String.valueOf(value));
result.put(stage,List.copyOf(columns));
}
return result;
}
@PutMapping
@Transactional
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER')")
public List<QueryFilterFields.Field> save(@RequestBody List<QueryFilterFields.Field> fields,@AuthenticationPrincipal AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);
if(fields==null||fields.size()>20)throw QueryFilterFields.bad("最多配置20个筛选字段");
Set<String> keys=new HashSet<>(),labels=new HashSet<>();
Set<String> allowed=governance.allowedFields(actor);
for(var f:fields) {
if(f.key()==null||!f.key().matches("[a-zA-Z][a-zA-Z0-9_]{0,63}")||!keys.add(f.key()))throw QueryFilterFields.bad("字段标识无效或重复");
if(f.label()==null||f.label().isBlank()||f.label().length()>80||!labels.add(f.label()))throw QueryFilterFields.bad("名称为空、过长或重复");
for(String column:List.of(f.blackColumn()==null?"":f.blackColumn(),f.brightColumn()==null?"":f.brightColumn())) {
if(column.isBlank()||column.length()>160)throw QueryFilterFields.bad("请设置黑片和亮片来源列");
if(!allowed.contains("*")&&!allowed.contains(column))throw QueryFilterFields.bad("没有来源列权限");
}
}
jdbc.update("DELETE FROM query_filter_field");
int index=0;for(var f:fields)jdbc.update("INSERT INTO query_filter_field VALUES (?,?,?,?,?,?,?)",f.key(),f.label().trim(),f.blackColumn(),f.brightColumn(),f.enabled(),f.allowEmpty(),index++);
return new QueryFilterFields(jdbc,governance).all();
}
}
@@ -0,0 +1,90 @@
package cn.yunsheng.quality.filters;
import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import java.util.*;
import org.springframework.http.HttpStatus;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.web.server.ResponseStatusException;
import tools.jackson.databind.json.JsonMapper;
/** Shared, parameterized source-field filters. All consumers use the same stage mapping. */
public class QueryFilterFields {
public static final String EMPTY = "__EMPTY__";
private static final JsonMapper JSON = JsonMapper.builder().build();
private final JdbcTemplate jdbc;
private final GovernanceAccessService governance;
public QueryFilterFields(JdbcTemplate jdbc, GovernanceAccessService governance) { this.jdbc=jdbc; this.governance=governance; }
public record Field(String key, String label, String blackColumn, String brightColumn, boolean enabled, boolean allowEmpty, int sortOrder) {}
public List<Field> all() {
return jdbc.query("SELECT field_key,label,black_column,bright_column,enabled,allow_empty,sort_order FROM query_filter_field ORDER BY sort_order,field_key",
(rs,n)->new Field(rs.getString(1),rs.getString(2),rs.getString(3),rs.getString(4),rs.getBoolean(5),rs.getBoolean(6),rs.getInt(7)));
}
public List<Field> visible(AppUserPrincipal actor) {
Set<String> allowed=governance.allowedFields(actor);
return all().stream().filter(Field::enabled).filter(f->allowed.contains("*") || (allowed.contains(f.blackColumn()) && allowed.contains(f.brightColumn()))).toList();
}
public static Map<String,List<String>> parse(Object raw) {
if(raw==null || "".equals(raw))return Map.of();
if(raw instanceof String text) {
if(text.length()>32000)throw bad("筛选条件过长");
try { raw=JSON.readValue(text,Object.class); } catch(Exception e) { throw bad("筛选条件格式无效"); }
}
if(!(raw instanceof Map<?,?> map) || map.size()>20)throw bad("筛选条件必须是字段到值数组的对象,最多20项");
Map<String,List<String>> result=new LinkedHashMap<>();
for(var entry:map.entrySet()) {
if(!(entry.getKey() instanceof String key) || !key.matches("[a-zA-Z][a-zA-Z0-9_]{0,63}") || !(entry.getValue() instanceof List<?> list) || list.size()>50)throw bad("筛选字段或取值无效,每项最多选择50个值");
List<String> values=new ArrayList<>();
for(Object item:list) {
if(!(item instanceof String text) || text.isBlank() || text.length()>256)throw bad("筛选值必须是1到256字的文本");
if(!values.contains(text))values.add(text);
}
if(!values.isEmpty())result.put(key,List.copyOf(values));
}
return Collections.unmodifiableMap(result);
}
public void append(StringBuilder where,List<Object> args,String stage,Map<String,List<String>> filters,AppUserPrincipal actor) {
if(filters==null || filters.isEmpty())return;
Map<String,Field> fields=new HashMap<>(); visible(actor).forEach(f->fields.put(f.key(),f));
for(var entry:filters.entrySet()) {
Field f=fields.get(entry.getKey());
if(f==null)throw bad("筛选字段未启用或没有权限:"+entry.getKey());
boolean empty=entry.getValue().contains(EMPTY);
if(empty && !f.allowEmpty())throw bad(f.label()+"不允许筛选空值");
String column=stage.equals("black")?f.blackColumn():f.brightColumn();
// Paths and selected values are bound parameters; never interpolate configured identifiers.
String expression="NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs";
List<String> values=entry.getValue().stream().filter(v->!v.equals(EMPTY)).toList();
where.append(" AND (");
if(!values.isEmpty()) {
where.append(expression).append(" IN (").append(String.join(",",Collections.nCopies(values.size(),"?"))).append(')');
args.add(path(column));args.addAll(values);
}
if(empty) {
if(!values.isEmpty())where.append(" OR ");
where.append(expression).append(" IS NULL");args.add(path(column));
}
where.append(')');
}
}
public Map<String,Object> options(String key,String search,String releaseId,AppUserPrincipal actor) {
Field field=visible(actor).stream().filter(f->f.key().equals(key)).findFirst().orElseThrow(()->bad("筛选字段未启用或没有权限"));
if(search!=null && search.length()>256)throw bad("搜索文本过长");
List<Object> args=new ArrayList<>();List<String> selects=new ArrayList<>();
for(String stage:List.of("black","bright")) {
String column=stage.equals("black")?field.blackColumn():field.brightColumn();
StringBuilder scope=new StringBuilder(" WHERE rm.release_id=?");
List<Object> scoped=new ArrayList<>();scoped.add(releaseId);
DataScope.append(scope,scoped,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
selects.add("SELECT NULLIF(NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), 'null'), '') COLLATE utf8mb4_0900_as_cs AS value FROM release_member rm JOIN fact_"+stage+" f ON f.build_id=rm.transform_build_id"+scope);
args.add(path(column));args.addAll(scoped);
}
String predicate=" WHERE value IS NOT NULL";
if(search!=null && !search.isBlank()) { predicate+=" AND LOCATE(LOWER(?),LOWER(value))>0";args.add(search); }
List<String> found=jdbc.query("SELECT DISTINCT value FROM ("+String.join(" UNION ALL ",selects)+") choices"+predicate+" ORDER BY value LIMIT 201",(rs,n)->rs.getString(1),args.toArray());
return Map.of("values",found.stream().limit(200).toList(),"hasMore",found.size()>200);
}
public static String path(String column) { return "$."+JSON.writeValueAsString(column); }
public static ResponseStatusException bad(String text) { return new ResponseStatusException(HttpStatus.BAD_REQUEST,text); }
}
@@ -22,6 +22,19 @@ public class MetricsController {
public MetricsController(MetricsService metrics, MetricsResponseAccess responseAccess) { this.metrics = metrics; this.responseAccess = responseAccess; }
/** Compatibility for internal callers without source-field filters. */
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter, String department, String group, String product, String projectGroup, String view, boolean includeUnassigned, String releaseId, String trendGranularity, Integer rowLimit, int rowOffset, String rowOrder, AppUserPrincipal actor) {
return calculate(year, month, quarter, department, group, product, null, projectGroup, view, includeUnassigned, releaseId, trendGranularity, rowLimit, rowOffset, rowOrder, null, actor);
}
/** Compatibility for internal callers that already pass source-field filters. */
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter, String department, String group,
String product, String projectGroup, String view, boolean includeUnassigned,
String releaseId, String trendGranularity, Integer rowLimit, int rowOffset,
String rowOrder, String extraFilters, AppUserPrincipal actor) {
return calculate(year, month, quarter, department, group, product, null, projectGroup, view, includeUnassigned,
releaseId, trendGranularity, rowLimit, rowOffset, rowOrder, extraFilters, actor);
}
@GetMapping
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST')")
public MetricsService.MetricSnapshot calculate(@RequestParam(required = false) Integer year,
@@ -30,6 +43,7 @@ public class MetricsController {
@RequestParam(required = false) String department,
@RequestParam(required = false) String group,
@RequestParam(required = false) String product,
@RequestParam(required = false) String productSearch,
@RequestParam(required = false) String projectGroup,
@RequestParam(defaultValue = "overall") String view,
@RequestParam(defaultValue = "false") boolean includeUnassigned,
@@ -38,10 +52,11 @@ public class MetricsController {
@RequestParam(required = false) Integer rowLimit,
@RequestParam(defaultValue = "0") int rowOffset,
@RequestParam(required = false) String rowOrder,
@RequestParam(required=false) String extraFilters,
@AuthenticationPrincipal AppUserPrincipal actor) {
Integer quarterNumber = parseQuarter(quarter);
MetricsService.MetricSnapshot visible = responseAccess.visible(metrics.calculate(new MetricsService.MetricsQuery(year, month, quarterNumber, department, group,
product, projectGroup, view, includeUnassigned, releaseId, trendGranularity), actor), actor);
product, projectGroup, view, includeUnassigned, releaseId, trendGranularity).withExtraFilters(extraFilters).withProductSearch(productSearch), actor), actor);
if (rowLimit == null) return visible;
if (rowLimit < 1 || rowLimit > 500 || rowOffset < 0) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
if (rowOrder != null && !rowOrder.equals("scrapDesc")) throw new IllegalArgumentException("rowOrder 无效");
@@ -1,6 +1,7 @@
package cn.yunsheng.quality.metrics;
import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.governance.GovernanceAccessService;
@@ -174,9 +175,9 @@ public class MetricsService {
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor, boolean hasUnrolledBuilds) {
AggregateSet result = new AggregateSet();
aggregateStage(result, "black", query.view(), releaseId, query, actor, false);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, false);
if (hasUnrolledBuilds) {
if (query.extraFilters().isEmpty()) aggregateStage(result, "black", query.view(), releaseId, query, actor, false);
if (query.extraFilters().isEmpty()) aggregateStage(result, "bright", query.view(), releaseId, query, actor, false);
if (hasUnrolledBuilds || !query.extraFilters().isEmpty()) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, true);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true);
}
@@ -198,11 +199,12 @@ public class MetricsService {
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
if (factFallback) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else { where.append(" AND f.stage=?"); args.add(stage); }
if (factFallback && query.extraFilters().isEmpty()) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback)new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
@@ -264,6 +266,10 @@ public class MetricsService {
SelectionValues.appendPredicate(sql, args, "f.department", query.department(), DepartmentLabels::queryValue);
SelectionValues.appendPredicate(sql, args, "f.group_name", query.group(), value -> value);
SelectionValues.appendPredicate(sql, args, "f.product", query.product(), value -> value);
if (query.productSearch() != null && !query.productSearch().isBlank()) {
sql.append(" AND LOCATE(LOWER(?), LOWER(COALESCE(f.product,''))) > 0");
args.add(query.productSearch());
}
SelectionValues.appendPredicate(sql, args, "f.project_group", query.projectGroup(), value -> value);
}
@@ -341,9 +347,9 @@ public class MetricsService {
private Map<String, Object> queryCoverage(String releaseId, MetricsQuery query, AppUserPrincipal actor, boolean hasUnrolledBuilds) {
TreeMap<String, CoverageMonth> months = new TreeMap<>();
coverageStage(months, "black", releaseId, query, actor, false);
coverageStage(months, "bright", releaseId, query, actor, false);
if (hasUnrolledBuilds) {
if (query.extraFilters().isEmpty()) coverageStage(months, "black", releaseId, query, actor, false);
if (query.extraFilters().isEmpty()) coverageStage(months, "bright", releaseId, query, actor, false);
if (hasUnrolledBuilds || !query.extraFilters().isEmpty()) {
coverageStage(months, "black", releaseId, query, actor, true);
coverageStage(months, "bright", releaseId, query, actor, true);
}
@@ -385,11 +391,12 @@ public class MetricsService {
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
if (factFallback) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else { where.append(" AND f.stage=?"); args.add(stage); }
if (factFallback && query.extraFilters().isEmpty()) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback)new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
String countExpression = factFallback ? "COUNT(*)" : "SUM(f.source_row_count)";
String excludedExpression = factFallback
@@ -558,10 +565,12 @@ public class MetricsService {
Map<String, Object> result = new LinkedHashMap<>();
result.put("year", query.year()); result.put("month", query.month()); result.put("quarter", query.quarter());
result.put("department", query.department()); result.put("group", query.group()); result.put("product", query.product());
if (query.productSearch() != null) result.put("productSearch", query.productSearch());
result.put("projectGroup", query.projectGroup()); result.put("view", query.view());
result.put("includeUnassigned", includeUnassigned);
result.put("trendGranularity", query.trendGranularity());
result.put("exclusions", query.exclusions());
result.put("extraFilters", query.extraFilters());
return result;
}
@@ -604,6 +613,8 @@ public class MetricsService {
SelectionValues.parse(query.group());
SelectionValues.parse(query.product());
SelectionValues.parse(query.projectGroup());
if (query.productSearch() != null && query.productSearch().length() > SelectionValues.MAX_VALUE_LENGTH)
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "产品搜索文本过长");
if (query.exclusions().size() > 4) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "exclusions 维度数量无效");
Set<String> dimensions = Set.of("department", "group", "product", "projectGroup");
query.exclusions().forEach((dimension, values) -> {
@@ -619,8 +630,11 @@ public class MetricsService {
public record MetricsQuery(Integer year, Integer month, Integer quarter, String department, String group,
String product, String projectGroup, String view, boolean includeUnassigned,
String releaseId, String trendGranularity, Map<String, List<String>> exclusions) {
String releaseId, String trendGranularity, Map<String, List<String>> exclusions, Map<String,List<String>> extraFilters,
String productSearch) {
public MetricsQuery {
extraFilters=QueryFilterFields.parse(extraFilters);
productSearch = productSearch == null || productSearch.isBlank() ? null : productSearch.trim();
if (exclusions == null) exclusions = Map.of();
else {
Map<String, List<String>> copy = new LinkedHashMap<>();
@@ -629,6 +643,23 @@ public class MetricsService {
exclusions = java.util.Collections.unmodifiableMap(copy);
}
}
public MetricsQuery(Integer year, Integer month, Integer quarter, String department, String group,
String product, String projectGroup, String view, boolean includeUnassigned,
String releaseId, String trendGranularity, Map<String, List<String>> exclusions,
Map<String, List<String>> extraFilters) {
this(year, month, quarter, department, group, product, projectGroup, view, includeUnassigned,
releaseId, trendGranularity, exclusions, extraFilters, null);
}
public MetricsQuery(Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Map<String,List<String>> exclusions) {
this(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,Map.of(),null);
}
public MetricsQuery withExtraFilters(Object filters) {
return new MetricsQuery(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,QueryFilterFields.parse(filters),productSearch);
}
public MetricsQuery withProductSearch(String value) {
return new MetricsQuery(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,
trendGranularity,exclusions,extraFilters,value);
}
/** Preserve the historic 10-argument constructor used by service callers. */
public MetricsQuery(Integer year, Integer month, Integer quarter, String department, String group,
String product, String projectGroup, String view, boolean includeUnassigned, String releaseId) {
@@ -1,6 +1,7 @@
package cn.yunsheng.quality.records;
import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.identity.AppUserPrincipal;
@@ -54,6 +55,32 @@ public class RecordController {
this.workbookExport = workbookExport;
}
/** Compatibility for internal callers without source-field filters. */
public Map<String, Object> records(String stage, Integer year, Integer month, Integer quarter, String department, String group, String product, String projectGroup, boolean includeUnassigned, String releaseId, int page, int pageSize, String field, String search, String sort, String direction, List<String> columns, AppUserPrincipal actor) {
return records(stage, year, month, quarter, department, group, product, null, projectGroup, includeUnassigned, releaseId, page, pageSize, field, search, sort, direction, columns, null, actor);
}
/** Compatibility for internal callers that already pass source-field filters. */
public Map<String, Object> records(String stage, Integer year, Integer month, Integer quarter, String department, String group,
String product, String projectGroup, boolean includeUnassigned, String releaseId,
int page, int pageSize, String field, String search, String sort, String direction,
List<String> columns, String extraFilters, AppUserPrincipal actor) {
return records(stage, year, month, quarter, department, group, product, null, projectGroup, includeUnassigned,
releaseId, page, pageSize, field, search, sort, direction, columns, extraFilters, actor);
}
/** Compatibility for internal callers without source-field filters. */
public ResponseEntity<StreamingResponseBody> export(Integer year, Integer month, Integer quarter, String department, String group, String product, String projectGroup, boolean includeUnassigned, String releaseId, AppUserPrincipal actor) {
return export(year, month, quarter, department, group, product, null, projectGroup, includeUnassigned, releaseId, null, actor);
}
/** Compatibility for internal callers that already pass source-field filters. */
public ResponseEntity<StreamingResponseBody> export(Integer year, Integer month, Integer quarter, String department,
String group, String product, String projectGroup,
boolean includeUnassigned, String releaseId, String extraFilters,
AppUserPrincipal actor) {
return export(year, month, quarter, department, group, product, null, projectGroup, includeUnassigned,
releaseId, extraFilters, actor);
}
@GetMapping
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST')")
public Map<String, Object> records(@RequestParam String stage,
@@ -63,6 +90,7 @@ public class RecordController {
@RequestParam(required = false) String department,
@RequestParam(required = false) String group,
@RequestParam(required = false) String product,
@RequestParam(required = false) String productSearch,
@RequestParam(required = false) String projectGroup,
@RequestParam(defaultValue = "false") boolean includeUnassigned,
@RequestParam(required = false) String releaseId,
@@ -73,12 +101,14 @@ public class RecordController {
@RequestParam(required = false) String sort,
@RequestParam(defaultValue = "asc") String direction,
@RequestParam(required = false) List<String> columns,
@RequestParam(required=false) String extraFilters,
@AuthenticationPrincipal AppUserPrincipal actor) {
governance.requireDetail(actor);
governance.requireBoundAction("QUERY", "query.detail");
if (!List.of("black", "bright").contains(stage)) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "stage 必须是 black 或 bright");
validatePeriodFilters(year, month, quarter);
if (page < 1 || pageSize < 1 || pageSize > 200) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "分页参数无效");
if (productSearch != null && productSearch.length() > SelectionValues.MAX_VALUE_LENGTH) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "产品搜索文本过长");
List<String> selectedColumns = validateColumns(columns, actor);
String selectedField = validateField(field, actor);
if (search != null && !search.isBlank()) {
@@ -97,6 +127,10 @@ public class RecordController {
SelectionValues.appendPredicate(predicate, args, "f.department", department, DepartmentLabels::queryValue);
SelectionValues.appendPredicate(predicate, args, "f.group_name", group, value -> value);
SelectionValues.appendPredicate(predicate, args, "f.product", product, value -> value);
if (productSearch != null && !productSearch.isBlank()) {
predicate.append(" AND LOCATE(LOWER(?), LOWER(COALESCE(f.product,''))) > 0");
args.add(productSearch.trim());
}
SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value);
if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
@@ -105,6 +139,7 @@ public class RecordController {
args.add(search);
args.add(jsonPath(selectedField));
}
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
Long total = jdbc.queryForObject("""
@@ -140,14 +175,17 @@ public class RecordController {
@RequestParam(required = false) String department,
@RequestParam(required = false) String group,
@RequestParam(required = false) String product,
@RequestParam(required = false) String productSearch,
@RequestParam(required = false) String projectGroup,
@RequestParam(defaultValue = "false") boolean includeUnassigned,
@RequestParam(required = false) String releaseId,
@AuthenticationPrincipal AppUserPrincipal actor) {
@RequestParam(required=false) String extraFilters,
@AuthenticationPrincipal AppUserPrincipal actor) {
governance.requireDetail(actor);
governance.requireBoundAction("QUERY", "query.detail");
if (productSearch != null && productSearch.length() > SelectionValues.MAX_VALUE_LENGTH) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "产品搜索文本过长");
Path file = workbookExport.export(year, month, quarter, department, group, product, projectGroup,
includeUnassigned, releaseId, actor);
includeUnassigned, releaseId, actor, QueryFilterFields.parse(extraFilters), productSearch);
StreamingResponseBody body = output -> {
try (InputStream input = Files.newInputStream(file)) {
input.transferTo(output);
@@ -1,6 +1,7 @@
package cn.yunsheng.quality.records;
import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.governance.GovernanceAccessService;
@@ -62,7 +63,15 @@ public class RecordsWorkbookExportService {
public Path export(Integer year, Integer month, Integer quarter, String department, String group,
String product, String projectGroup, boolean includeUnassigned, String releaseId,
AppUserPrincipal actor) {
return export(year,month,quarter,department,group,product,projectGroup,includeUnassigned,releaseId,actor,Map.of());
}
public Path export(Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,boolean includeUnassigned,String releaseId,AppUserPrincipal actor,Map<String,List<String>> extraFilters) {
return export(year, month, quarter, department, group, product, projectGroup, includeUnassigned, releaseId, actor, extraFilters, null);
}
public Path export(Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,boolean includeUnassigned,String releaseId,AppUserPrincipal actor,Map<String,List<String>> extraFilters,String productSearch) {
RecordController.validatePeriodFilters(year, month, quarter);
if (productSearch != null && productSearch.length() > SelectionValues.MAX_VALUE_LENGTH)
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "产品搜索文本过长");
String selectedRelease = releases.resolvePublishedRelease(releaseId);
Set<String> allowedFields = governance.allowedFields(actor);
Path output = null;
@@ -73,9 +82,9 @@ public class RecordsWorkbookExportService {
workbook.setCompressTempFiles(true);
CellStyle headerStyle = createHeaderStyle(workbook);
createStageSheet(workbook, headerStyle, "黑片", "fact_black", selectedRelease, year, month, quarter,
department, group, product, projectGroup, includeUnassigned, allowedFields, actor);
department, group, product, projectGroup, includeUnassigned, allowedFields, actor, extraFilters, productSearch);
createStageSheet(workbook, headerStyle, "亮片", "fact_bright", selectedRelease, year, month, quarter,
department, group, product, projectGroup, includeUnassigned, allowedFields, actor);
department, group, product, projectGroup, includeUnassigned, allowedFields, actor, extraFilters, productSearch);
try (var stream = Files.newOutputStream(output)) {
workbook.write(stream);
}
@@ -95,12 +104,13 @@ public class RecordsWorkbookExportService {
private void createStageSheet(Workbook workbook, CellStyle headerStyle, String sheetName, String fact,
String releaseId, Integer year, Integer month, Integer quarter, String department,
String group, String product, String projectGroup, boolean includeUnassigned,
Set<String> allowedFields, AppUserPrincipal actor) throws IOException {
Set<String> allowedFields, AppUserPrincipal actor, Map<String,List<String>> extraFilters,
String productSearch) throws IOException {
Sheet sheet = workbook.createSheet(sheetName);
LinkedHashSet<String> rawFields = new LinkedHashSet<>();
if (releaseId != null) {
QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product,
projectGroup, includeUnassigned, "f.raw_values", actor);
projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch);
scan(rawScope, rs -> rawFields.addAll(visibleRaw(readJson(rs.getString(1)), allowedFields).keySet()));
}
if (rawFields.size() + HEADER_COUNT > MAX_COLUMNS)
@@ -122,14 +132,15 @@ public class RecordsWorkbookExportService {
QueryScope rowScope = queryScope(fact, releaseId, year, month, quarter, department, group, product,
projectGroup, includeUnassigned,
"sr.source_file, f.sheet_name, f.physical_row_no, sr.id AS source_revision_id, f.month_key, " +
"f.department, f.group_name, f.project_group, f.product, f.raw_values", actor);
"f.department, f.group_name, f.project_group, f.product, f.raw_values", actor, extraFilters, productSearch);
int[] rowIndex = {1};
scan(rowScope, rs -> writeRow(sheet, rowIndex, rs, rawFields, allowedFields));
}
private QueryScope queryScope(String fact, String releaseId, Integer year, Integer month, Integer quarter,
String department, String group, String product, String projectGroup,
boolean includeUnassigned, String select, AppUserPrincipal actor) {
boolean includeUnassigned, String select, AppUserPrincipal actor, Map<String,List<String>> extraFilters,
String productSearch) {
StringBuilder predicate = new StringBuilder(" WHERE rm.release_id = ?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
@@ -137,9 +148,14 @@ public class RecordsWorkbookExportService {
SelectionValues.appendPredicate(predicate, args, "f.department", department, DepartmentLabels::queryValue);
SelectionValues.appendPredicate(predicate, args, "f.group_name", group, value -> value);
SelectionValues.appendPredicate(predicate, args, "f.product", product, value -> value);
if (productSearch != null && !productSearch.isBlank()) {
predicate.append(" AND LOCATE(LOWER(?), LOWER(COALESCE(f.product,''))) > 0");
args.add(productSearch.trim());
}
SelectionValues.appendPredicate(predicate, args, "f.project_group", projectGroup, value -> value);
if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
String columns = select == null ? "f.raw_values" : select;
@@ -58,6 +58,13 @@ public class DataWorkflowController {
return service.deleteSource(id, actor);
}
@PostMapping("/sources/delete-batch")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER','DATA_PUBLISHER')")
public Map<String, Object> deleteSources(@RequestBody DeleteSourcesRequest request,
@AuthenticationPrincipal AppUserPrincipal actor) {
return service.deleteSources(request, actor);
}
@PostMapping("/sources/{id}/restore")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','DATA_MAINTAINER','DATA_PUBLISHER')")
public Map<String, Object> restoreSource(@PathVariable String id,
@@ -80,4 +87,5 @@ public class DataWorkflowController {
}
}
public record ImportRequest(java.util.List<String> sourceIds) {}
public record DeleteSourcesRequest(java.util.List<String> sourceIds, String expectedCurrentReleaseId) {}
}
@@ -194,6 +194,27 @@ public class DataWorkflowService {
return enqueue("DELETE_SOURCE", Map.of("sourceId", sourceId), actor, currentReleaseId());
}
public Map<String, Object> deleteSources(DataWorkflowController.DeleteSourcesRequest request, AppUserPrincipal actor) {
requireEnabled();
requireManager(actor);
List<String> ids = validateSourceIds(request == null ? null : request.sourceIds());
String current = currentReleaseId();
if (!Objects.equals(current, request.expectedCurrentReleaseId())) throw conflict("来源数据已变化,请刷新后重新确认待删除文件");
authorizeSourceDeletes(ids, actor, true);
return enqueue("DELETE_SOURCES", Map.of("sourceIds", ids), actor, current);
}
private void authorizeSourceDeletes(List<String> ids, AppUserPrincipal actor, boolean requireVisible) {
for (String id : ids) {
Map<String, Object> preview = deletion.preview(id, actor);
if (requireVisible && Boolean.TRUE.equals(preview.get("hidden"))) throw conflict("所选来源已删除,请刷新来源清单");
if (requireVisible && Set.of("REVIEW_WITHDRAWAL", "BUILD_WITHDRAWAL_CANDIDATE", "RECONCILE_CANDIDATE", "WAIT_FOR_TASK")
.contains(String.valueOf(preview.get("nextAction"))))
throw conflict("所选来源已有未完成操作,请先完成原操作后再统一删除");
if (Boolean.TRUE.equals(preview.get("currentUsed"))) requirePublisher(actor);
}
}
public Map<String, Object> restoreSource(String sourceId, AppUserPrincipal actor) {
requireEnabled();
requireManager(actor);
@@ -287,6 +308,7 @@ public class DataWorkflowService {
case "RESTORE_SOURCE" -> restoreAndRebuild(operation, actor, token);
case "PUBLISH" -> publishExisting(operation, actor, token);
case "DELETE_SOURCE" -> deleteAndPublish(operation, actor, token);
case "DELETE_SOURCES" -> deleteSourcesAndPublish(operation, actor, token);
default -> throw failure("UNKNOWN_OPERATION", "工作流操作类型无效");
}
} catch (WorkflowFailure e) {
@@ -512,8 +534,29 @@ public class DataWorkflowService {
}
private void deleteAndPublish(WorkflowOperation operation, AppUserPrincipal actor, String token) {
String sourceId = String.valueOf(operation.payload().get("sourceId"));
String key = "TEST_DELETE:" + operation.id();
String releaseId = prepareSourceWithdrawal(operation, actor, token,
String.valueOf(operation.payload().get("sourceId")), "TEST_DELETE:" + operation.id(), null);
finishSourceWithdrawals(operation, actor, token, releaseId);
}
private void deleteSourcesAndPublish(WorkflowOperation operation, AppUserPrincipal actor, String token) {
List<String> ids = validateSourceIds(((List<?>) operation.payload().get("sourceIds")).stream().map(String::valueOf).toList());
// All source visibility changes and the final publication commit together.
// A failed item cannot leave earlier items hidden or publish only part of the batch.
transaction.executeWithoutResult(status -> {
authorizeSourceDeletes(ids, actor, true);
String releaseId = null;
for (String id : ids) {
String candidate = prepareSourceWithdrawal(operation, actor, token, id,
"TEST_DELETE:" + operation.id() + ":" + id, releaseId);
if (candidate != null) releaseId = candidate;
}
finishSourceWithdrawals(operation, actor, token, releaseId);
});
}
private String prepareSourceWithdrawal(WorkflowOperation operation, AppUserPrincipal actor, String token,
String sourceId, String key, String baseCandidateId) {
List<Map<String, Object>> existingDeletes = jdbc.queryForList("SELECT change_request_id AS changeRequestId,expected_delete_version AS expectedDeleteVersion FROM source_delete_operation WHERE idempotency_key=?", key);
Map<String, Object> preview = deletion.preview(sourceId, actor);
if (existingDeletes.isEmpty()) {
@@ -526,8 +569,7 @@ public class DataWorkflowService {
List<Map<String, Object>> changeRows = jdbc.queryForList("SELECT change_request_id AS changeRequestId FROM source_delete_operation WHERE idempotency_key=?", key);
Object changeId = changeRows.isEmpty() ? preview.containsKey("changeRequest") && preview.get("changeRequest") instanceof Map<?, ?> change ? change.get("id") : null : changeRows.getFirst().get("changeRequestId");
if (changeId == null || String.valueOf(changeId).isBlank()) {
complete(operation.id(), token, currentReleaseId(), null, Map.of("hiddenOnly", true, "physicalFilesRetained", true));
return;
return null;
}
String requestId = String.valueOf(changeId);
jdbc.update("UPDATE data_workflow_operation SET source_change_request_id=?,updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?", requestId, operation.id(), token);
@@ -542,10 +584,18 @@ public class DataWorkflowService {
if (!"BUILT".equals(request.get("status"))) {
updateStage(operation.id(), token, "REBUILDING");
Map<String, Object> built = sourceGovernance.build(requestId,
new SourceGovernanceService.BuildRequest(operation.expectedCurrentReleaseId(), "TEST_WITHDRAW:" + operation.id(), null), actor);
new SourceGovernanceService.BuildRequest(operation.expectedCurrentReleaseId(), "TEST_WITHDRAW:" + operation.id() + ":" + sourceId, baseCandidateId), actor);
releaseId = Objects.toString(built.get("releaseId"), null);
}
if (releaseId == null) throw failure("WITHDRAWAL_CANDIDATE_MISSING", "撤销申请尚未生成候选版本");
return releaseId;
}
private void finishSourceWithdrawals(WorkflowOperation operation, AppUserPrincipal actor, String token, String releaseId) {
if (releaseId == null) {
complete(operation.id(), token, currentReleaseId(), null, Map.of("hiddenOnly", true, "physicalFilesRetained", true));
return;
}
jdbc.update("UPDATE data_workflow_operation SET release_id=?,stage='PUBLISHING',updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?", releaseId, operation.id(), token);
Integer members = jdbc.queryForObject("SELECT COUNT(*) FROM release_member WHERE release_id=?", Integer.class, releaseId);
boolean allowEmpty = members != null && members == 0;
@@ -794,7 +844,11 @@ public class DataWorkflowService {
private void authorizeOperation(WorkflowOperation operation, AppUserPrincipal actor) {
requireWrite(actor);
if (Set.of("PUBLISH", "APPLY_MAPPING", "APPLY_RULE", "IMPORT", "RECALCULATE", "RESTORE_SOURCE").contains(operation.type())) requirePublisher(actor);
if (Set.of("DELETE_SOURCE", "RESTORE_SOURCE").contains(operation.type())) requireManager(actor);
if (Set.of("DELETE_SOURCE", "DELETE_SOURCES", "RESTORE_SOURCE").contains(operation.type())) requireManager(actor);
if ("DELETE_SOURCES".equals(operation.type())) {
List<String> ids = ((List<?>) operation.payload().get("sourceIds")).stream().map(String::valueOf).toList();
authorizeSourceDeletes(validateSourceIds(ids), actor, false);
}
if ("DELETE_SOURCE".equals(operation.type())) {
String sourceId = String.valueOf(operation.payload().get("sourceId"));
Integer used = currentReleaseId() == null ? 0 : jdbc.queryForObject("SELECT COUNT(*) FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_candidate c ON c.source_slot=d.source_slot WHERE m.release_id=? AND c.id=?", Integer.class, currentReleaseId(), sourceId);
@@ -868,6 +922,7 @@ public class DataWorkflowService {
result.put("createdAt", operation.createdAt());
result.put("updatedAt", operation.updatedAt());
result.put("errors", operation.errors());
if ("DELETE_SOURCES".equals(operation.type())) result.put("sourceIds", operation.payload().get("sourceIds"));
if (operation.result() != null) result.put("result", operation.result());
if (operation.mappingVersionId() != null) result.put("mappingVersionId", operation.mappingVersionId());
if (operation.ruleVersionId() != null) result.put("ruleVersionId", operation.ruleVersionId());
@@ -0,0 +1,12 @@
CREATE TABLE query_filter_field (
field_key VARCHAR(64) PRIMARY KEY,
label VARCHAR(80) NOT NULL,
black_column VARCHAR(160) NOT NULL,
bright_column VARCHAR(160) NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
allow_empty BOOLEAN NOT NULL DEFAULT TRUE,
sort_order INT NOT NULL DEFAULT 0
);
INSERT INTO query_filter_field(field_key,label,black_column,bright_column,sort_order) VALUES
('customerCode','客户代码','客户代码','客户代码',0),
('blankProducer','坯料生产单位','坯料生产单位','坯料生产单位',1);
@@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -50,6 +51,33 @@ class AnalysisFineDefectsTest {
.anyMatch(field -> field.category().equals("外废")));
}
@Test
void periodFiltersUseExactMonthAndIndexableYearRanges() {
var month = AnalysisFineDefectsService.parse(Map.of("releaseId", "release-1", "year", 2026,
"month", 9, "stage", "black"), Set.of("*"));
StringBuilder monthSql = new StringBuilder(" WHERE rm.release_id = ?");
List<Object> monthArgs = new ArrayList<>(List.of("release-1"));
AnalysisFineDefectsService.appendPeriod(monthSql, monthArgs, month);
assertEquals(" WHERE rm.release_id = ? AND f.month_key = ?", monthSql.toString());
assertEquals(List.of("release-1", "2026-09"), monthArgs);
var quarter = AnalysisFineDefectsService.parse(Map.of("releaseId", "release-1", "year", 2026,
"quarter", 4, "stage", "black"), Set.of("*"));
StringBuilder quarterSql = new StringBuilder();
List<Object> quarterArgs = new ArrayList<>();
AnalysisFineDefectsService.appendPeriod(quarterSql, quarterArgs, quarter);
assertEquals(" AND f.month_key >= ? AND f.month_key < ?", quarterSql.toString());
assertEquals(List.of("2026-10", "2027-01"), quarterArgs);
var year = AnalysisFineDefectsService.parse(Map.of("releaseId", "release-1", "year", 2026,
"stage", "black"), Set.of("*"));
StringBuilder yearSql = new StringBuilder();
List<Object> yearArgs = new ArrayList<>();
AnalysisFineDefectsService.appendPeriod(yearSql, yearArgs, year);
assertEquals(" AND f.month_key >= ? AND f.month_key < ?", yearSql.toString());
assertEquals(List.of("2026-01", "2027-01"), yearArgs);
}
@Test
void productDimensionGroupsOnlyByTheSelectedProductKey() {
var request = AnalysisFineDefectsService.parse(Map.of("releaseId", "release-1", "year", 2026,
@@ -0,0 +1,42 @@
package cn.yunsheng.quality.analysis;
import static org.junit.jupiter.api.Assertions.*;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import java.util.UUID;
import org.springframework.jdbc.core.JdbcTemplate;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import tools.jackson.databind.json.JsonMapper;
class NaoAnalysisProgressTest {
@Test void lateAnswerAfterStopDoesNotAbortTheNativeResultCleanup() {
var jdbc = mock(JdbcTemplate.class);
when(jdbc.queryForObject("SELECT cancel_requested FROM analysis_turn WHERE id=?", Boolean.class, "stopped")).thenReturn(true);
var service = new AnalysisTurnService(jdbc, JsonMapper.builder().build(), null, null, null, null, null, null);
assertDoesNotThrow(() -> ReflectionTestUtils.invokeMethod(service, "persistNativeAnswer", "stopped", UUID.randomUUID(), "scope", "release", Map.of(), "late answer"));
verify(jdbc).queryForObject("SELECT cancel_requested FROM analysis_turn WHERE id=?", Boolean.class, "stopped");
verifyNoMoreInteractions(jdbc);
}
@Test void queuedCancellationCannotPoisonTheRetriedProcess() {
var runner = new NaoAnalysisRunner(JsonMapper.builder().build(), mock(NaoRuntimeAccessService.class));
runner.requestCancel("queued");
var markers = (Set<?>)ReflectionTestUtils.getField(runner, "cancellationRequested");
assertTrue(markers.isEmpty());
}
@Test void persistsEveryCallAndItsRecoveryWithoutCredentialsOrReleaseIds() {
var calls = List.of(Map.of("tool", "mcp_call", "status", "failed", "recovered", true,
"label", "比较期间变化", "input", Map.of("mode", "compare", "releaseId", "hidden", "apiKey", "secret",
"period", Map.of("year", 2026), "filters", Map.of("product", "N52", "token", "secret"))));
var safe = NaoAnalysisRunner.safeProgress(Map.of("message", "正在比较", "toolCalls", calls, "toolCount", 1));
String encoded = safe.toString();
assertFalse(encoded.contains("hidden")); assertFalse(encoded.contains("secret"));
assertTrue(encoded.contains("N52")); assertTrue(encoded.contains("recovered=true"));
assertEquals(1, ((List<?>)safe.get("steps")).size());
}
}
@@ -66,6 +66,14 @@ class NaoRuntimeQueryControllerTest {
assertEquals("规格+T", metrics.product);
}
@Test
void delegatedMetricsPreserveSourceFilterJsonAndLeadingZeros() {
RecordingMetricsController metrics=new RecordingMetricsController();Fixture fixture=new Fixture(metrics);
String filters="{\"customerCode\":[\"000123\"],\"blankProducer\":[\"包头料\"]}";
fixture.query("GET","/api/metrics?extraFilters="+encode(filters),null);
assertEquals(filters,metrics.extraFilters);
}
@Test
void refusesQueryBodyAndIgnoredUrlParameters() {
Fixture fixture = new Fixture();
@@ -117,14 +125,16 @@ class NaoRuntimeQueryControllerTest {
private String department;
private String group;
private String product;
private String extraFilters;
private RecordingMetricsController() { super(null, null); }
@Override public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter,
String department, String group, String product, String projectGroup, String view,
boolean includeUnassigned, String releaseId, String trendGranularity, Integer rowLimit,
int rowOffset, String rowOrder, AppUserPrincipal actor) {
int rowOffset, String rowOrder, String extraFilters, AppUserPrincipal actor) {
this.department = department;
this.group = group;
this.product = product;
this.extraFilters = extraFilters;
return null;
}
}
@@ -0,0 +1,38 @@
package cn.yunsheng.quality.filters;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import java.util.*;
import org.junit.jupiter.api.Test;
import org.springframework.web.server.ResponseStatusException;
class QueryFilterFieldsTest {
private final QueryFilterFields.Field field=new QueryFilterFields.Field("customerCode","客户代码","客户代码","客户编号",true,true,0);
@Test void preservesLeadingZerosAndDeduplicatesWithoutCoercingNumbers() {
assertEquals(Map.of("customerCode",List.of("000123","123")),QueryFilterFields.parse("{\"customerCode\":[\"000123\",\"123\",\"000123\"]}"));
assertThrows(ResponseStatusException.class,()->QueryFilterFields.parse(Map.of("customerCode",List.of(123))));
assertThrows(ResponseStatusException.class,()->QueryFilterFields.parse("[]"));
assertThrows(ResponseStatusException.class,()->QueryFilterFields.parse(Map.of("bad'key",List.of("x"))));
}
@Test void bindsStageSpecificPathsAndQuotedValuesAsParameters() {
var filters=spy(new QueryFilterFields(null,null));doReturn(List.of(field)).when(filters).visible(null);
List<Object> args=new ArrayList<>();StringBuilder sql=new StringBuilder(" WHERE 1=1");
filters.append(sql,args,"bright",Map.of("customerCode",List.of("0001","x' OR 1=1 --",QueryFilterFields.EMPTY)),null);
assertFalse(sql.toString().contains("客户编号")); assertFalse(sql.toString().contains("x'"));
assertEquals(List.of("$.\"客户编号\"","0001","x' OR 1=1 --","$.\"客户编号\""),args);
assertTrue(sql.toString().contains(" OR ")); assertTrue(sql.toString().contains(" IS NULL"));
}
@Test void rejectsUnavailableFieldsAndForbiddenEmptySelection() {
var filters=spy(new QueryFilterFields(null,null));doReturn(List.of(new QueryFilterFields.Field("customerCode","客户代码","客户代码","客户代码",true,false,0))).when(filters).visible(null);
assertThrows(ResponseStatusException.class,()->filters.append(new StringBuilder(),new ArrayList<>(),"black",Map.of("unknown",List.of("x")),null));
assertThrows(ResponseStatusException.class,()->filters.append(new StringBuilder(),new ArrayList<>(),"black",Map.of("customerCode",List.of(QueryFilterFields.EMPTY)),null));
}
@Test void optionCatalogRequiresBothStageFieldsToBeAuthorized() {
var governance=mock(GovernanceAccessService.class);var actor=mock(AppUserPrincipal.class);
var filters=spy(new QueryFilterFields(null,governance));doReturn(List.of(field)).when(filters).all();
when(governance.allowedFields(actor)).thenReturn(Set.of("客户代码"));assertEquals(List.of(),filters.visible(actor));
when(governance.allowedFields(actor)).thenReturn(Set.of("客户代码","客户编号"));assertEquals(List.of(field),filters.visible(actor));
}
}
@@ -3,12 +3,31 @@ package cn.yunsheng.quality.metrics;
import static org.junit.jupiter.api.Assertions.assertEquals;
import cn.yunsheng.quality.ingestion.BusinessRowMapper;
import cn.yunsheng.quality.ingestion.ConfigurableRuleEngine;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
class MetricsServiceAggregationTest {
@Test
void singleStageCompositeEqualsTheAvailableStageUnderBothRulePackages() {
MetricsService service = new MetricsService(null, null);
for (var rules : List.of(ConfigurableRuleEngine.baseline(),
ConfigurableRuleEngine.compile(ConfigurableRuleEngine.defaultDefinition()))) {
var black = service.aggregateForGovernance(List.of(
sourceRow("black", "90", "10", "100", Map.of())), rules);
assertEquals(0, new BigDecimal("0.9").compareTo(black.compositeRate()));
assertEquals(0, black.black().rate().compareTo(black.compositeRate()));
assertEquals(0, black.black().rate().compareTo(black.compositeRateWithoutHeadquartersScrap()));
var bright = service.aggregateForGovernance(List.of(
sourceRow("bright", "8", "2", "10", Map.of("bright_headquarters_scrap", BigDecimal.ONE))), rules);
assertEquals(0, new BigDecimal("0.8").compareTo(bright.compositeRate()));
assertEquals(0, bright.bright().rate().compareTo(bright.compositeRate()));
assertEquals(0, bright.bright().rateWithoutHeadquartersScrap().compareTo(bright.compositeRateWithoutHeadquartersScrap()));
}
}
@Test
void governanceRowsUseSummedSourceComponentsForStageAndAdjustedRates() {
MetricsService service = new MetricsService(null, null);
@@ -32,15 +32,17 @@ import tools.jackson.databind.json.JsonMapper;
class DataWorkflowServiceTest {
private JdbcTemplate jdbc;
private GovernanceAccessService access;
private SourceDeletionService deletion;
private DataWorkflowService service;
@BeforeEach
void setUp() {
jdbc = mock(JdbcTemplate.class);
access = mock(GovernanceAccessService.class);
deletion = mock(SourceDeletionService.class);
service = new DataWorkflowService(jdbc, mock(JsonMapper.class), mock(ImportRunService.class),
mock(ImportWorker.class), mock(ReleaseService.class), mock(SourceCatalogService.class),
mock(SourceDeletionService.class), mock(SourceGovernanceService.class),
deletion, mock(SourceGovernanceService.class),
mock(GovernanceDependencyService.class), mock(SourceAuthorizationService.class), access,
mock(JdbcAppUserDetailsService.class), mock(TaskExecutor.class), mock(PlatformTransactionManager.class));
ReflectionTestUtils.setField(service, "testMode", true);
@@ -231,6 +233,40 @@ class DataWorkflowServiceTest {
verifyNoInteractions(jdbc);
}
@Test
void batchDeleteRejectsStaleBaselineBeforeAnySourceMutation() {
when(jdbc.query(eq("SELECT release_id FROM current_release WHERE dataset_id=?"), any(RowMapper.class), eq("quality")))
.thenReturn(List.of("current"));
var error = assertThrows(ResponseStatusException.class, () -> service.deleteSources(
new DataWorkflowController.DeleteSourcesRequest(List.of("a", "b"), "stale"), admin()));
assertEquals(HttpStatus.CONFLICT, error.getStatusCode());
verifyNoInteractions(deletion);
verify(jdbc, never()).update(anyString(), any(Object[].class));
}
@Test
void batchDeletePreflightsAllSourcesAndEnforcesCurrentSourcePublishingPermission() {
when(jdbc.query(eq("SELECT release_id FROM current_release WHERE dataset_id=?"), any(RowMapper.class), eq("quality")))
.thenReturn(List.of("current"));
when(deletion.preview(eq("a"), any())).thenReturn(Map.of("hidden", false, "currentUsed", false));
when(deletion.preview(eq("b"), any())).thenReturn(Map.of("hidden", false, "currentUsed", true));
var error = assertThrows(ResponseStatusException.class, () -> service.deleteSources(
new DataWorkflowController.DeleteSourcesRequest(List.of("a", "b"), "current"), user("DATA_MAINTAINER")));
assertEquals(HttpStatus.FORBIDDEN, error.getStatusCode());
verify(deletion).preview(eq("a"), any());
verify(deletion).preview(eq("b"), any());
verify(deletion, never()).delete(anyString(), any(), any());
verify(jdbc, never()).update(anyString(), any(Object[].class));
}
@Test
void batchDeleteRejectsDuplicateSelectionsBeforeEnqueue() {
var error = assertThrows(ResponseStatusException.class, () -> service.deleteSources(
new DataWorkflowController.DeleteSourcesRequest(List.of("a", "a"), null), admin()));
assertEquals(HttpStatus.BAD_REQUEST, error.getStatusCode());
verifyNoInteractions(deletion, jdbc);
}
@Test
void workflowWriteEndpointsKeepRoleGuards() throws Exception {
assertPreAuthorize("apply", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER");
@@ -238,6 +274,7 @@ class DataWorkflowServiceTest {
assertPreAuthorize("importSources", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER");
assertPreAuthorize("publish", "SYSTEM_ADMIN", "DATA_PUBLISHER");
assertPreAuthorize("deleteSource", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER");
assertPreAuthorize("deleteSources", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER");
assertPreAuthorize("restoreSource", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER");
}