新增:智能分析图表与报告快照,支持环状图和柏拉图

This commit is contained in:
2026-10-23 20:25:12 +08:00
parent 8684ce1a4d
commit 13695da433
23 changed files with 5252 additions and 57 deletions
@@ -18,6 +18,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.Set;
import org.springframework.http.HttpStatus;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.security.access.prepost.PreAuthorize;
@@ -48,6 +49,8 @@ public class AnalysisController {
private final GovernanceAccessService governance;
private final MetricsResponseAccess responseAccess;
private final TransactionTemplate transaction;
@org.springframework.beans.factory.annotation.Value("${quality.analysis.visuals-enabled:true}")
private boolean visualsEnabled = true;
public AnalysisController(MetricsService metrics, AnalysisModelGateway model, JdbcTemplate jdbc, JsonMapper mapper,
PlatformTransactionManager transactionManager, GovernanceAccessService governance,
@@ -109,21 +112,34 @@ public class AnalysisController {
Session session = ownedSession(id, actor);
// A retained version alone never grants access; re-run the scoped query before returning stored evidence.
metrics.calculate(toQuery(session.context(), session.releaseId()), actor);
Set<String> allowedFields = governance.allowedFields(actor);
Set<String> visibleFilterKeys = AnalysisVisualSnapshotAccess.visibleExtraFilterKeys(jdbc, allowedFields);
List<Map<String, Object>> turns = jdbc.query("SELECT id,question,answer_json,steps_json,plan_json,created_at FROM analysis_turn WHERE session_id=? AND user_id=? ORDER BY created_at",
(rs, n) -> Map.of("id", rs.getString("id"), "question", rs.getString("question"),
"result", visibleAnswer(rs.getString("answer_json"), actor), "steps", rs.getString("steps_json") == null ? List.of() : mapper.readValue(rs.getString("steps_json"), List.class),
"plan", rs.getString("plan_json") == null ? Map.of() : read(rs.getString("plan_json")), "createdAt", rs.getTimestamp("created_at").toInstant().toString()),
(rs, n) -> {
String turnId = rs.getString("id");
Map<String, Object> plan = rs.getString("plan_json") == null ? Map.of() : read(rs.getString("plan_json"));
Map<String, Object> expectedScope = new LinkedHashMap<>(AnalysisTurnService.retryContext(plan));
expectedScope.remove("_task");
return Map.of("id", turnId, "question", rs.getString("question"),
"result", visibleAnswer(rs.getString("answer_json"), turnId, expectedScope, allowedFields, visibleFilterKeys),
"steps", rs.getString("steps_json") == null ? List.of() : mapper.readValue(rs.getString("steps_json"), List.class),
"plan", plan, "createdAt", rs.getTimestamp("created_at").toInstant().toString());
},
session.id(), actor.getId().toString());
return Map.of("id", session.id(), "title", session.title(), "releaseId", session.releaseId(), "context", session.context(), "turns", turns);
}
private Map<String, Object> visibleAnswer(String encoded, AppUserPrincipal actor) {
private Map<String, Object> visibleAnswer(String encoded, String turnId, Map<String, Object> expectedScope,
Set<String> allowedFields, Set<String> visibleFilterKeys) {
Map<String, Object> result = new LinkedHashMap<>(read(encoded));
if (result.get("results") instanceof List<?> list) result.put("results", list.stream().map(value -> {
if (!(value instanceof Map<?, ?> map)) return value;
Map<String, Object> item = new LinkedHashMap<>(); map.forEach((key, cell) -> item.put(String.valueOf(key), cell));
return AnalysisResultAccess.filter(item, governance.allowedFields(actor));
return AnalysisResultAccess.filter(item, allowedFields);
}).toList());
result.putAll(visualsEnabled
? AnalysisVisualSnapshotAccess.fromAnswer(result, allowedFields, visibleFilterKeys, turnId, expectedScope).asMap()
: AnalysisVisualSnapshotAccess.empty("not_requested").asMap());
return result;
}
@@ -66,6 +66,8 @@ public class AnalysisTurnService {
private final AnalysisExplorationTools exploration;
private final NaoAnalysisRunner naoRunner;
private final NaoRuntimeAccessService naoAccess;
@org.springframework.beans.factory.annotation.Value("${quality.analysis.visuals-enabled:true}")
private boolean visualsEnabled = true;
private final ThreadPoolExecutor executor = new ThreadPoolExecutor(2, 2, 0, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(100), task -> { Thread t = new Thread(task, "quality-analysis-turn"); t.setDaemon(true); return t; },
new ThreadPoolExecutor.AbortPolicy());
@@ -272,6 +274,21 @@ public class AnalysisTurnService {
Turn turn = requireTurn(turnId, actor);
Map<String, Object> context = queryContext(turn);
metrics.calculate(toQuery(context, validateView(text(context.get("view"))), sessionById(turn.sessionId()).releaseId()), actor);
return visibleTurnPayload(turn, actor, context);
}
/**
* Reauthorizes and returns the same stored, filtered turn payload without rerunning its business query.
* Used by local report/visual exports, which must preserve the original answer snapshot.
*/
public Map<String, Object> getSnapshot(String turnId, AppUserPrincipal submittedActor) {
AppUserPrincipal actor = currentUser(submittedActor.getId());
requireAnalysisAccess(actor);
Turn turn = requireTurn(turnId, actor);
return visibleTurnPayload(turn, actor, queryContext(turn));
}
private Map<String, Object> visibleTurnPayload(Turn turn, AppUserPrincipal actor, Map<String, Object> context) {
Map<String, Object> answer = read(turn.answerJson());
Map<String, Object> budget = readNullable(turn.budgetJson(), Map.of());
int callsUsed = mapList(readAny(turn.stepsJson(), List.of())).size();
@@ -284,14 +301,15 @@ public class AnalysisTurnService {
try { governance.requireEvidence(actor); result.put("evidence", readAny(turn.evidenceJson(), List.of())); }
catch (ResponseStatusException denied) { result.put("evidence", List.of()); result.put("evidenceAvailable", false); }
result.put("answer", answer.getOrDefault("answer", "")); result.put("results", filterResultList(answer.get("results"), actor));
result.putAll(visibleVisuals(answer, turn, actor, context));
for (String field : List.of("engine", "explanationEngine", "explanationWarning", "warning",
"businessResultsSentToModel", "releaseId", "task", "analysisChecks", "agentDecisions", "completion",
"reportMarkdown", "reportHtml", "nativeChatId", "toolCallCount"))
if (answer.containsKey(field)) result.put(field, answer.get(field));
result.put("stopReason", turn.stopReason()); result.put("budget", budget);
result.putAll(jdbc.queryForObject("SELECT UNIX_TIMESTAMP(created_at)*1000,UNIX_TIMESTAMP(started_at)*1000,UNIX_TIMESTAMP(finished_at)*1000 FROM analysis_turn WHERE id=? AND user_id=?",
(rs, n) -> turnTimes(rs.getBigDecimal(1), rs.getBigDecimal(2), rs.getBigDecimal(3)), turnId, actor.getId().toString()));
result.put("feedback", feedback(turnId));
(rs, n) -> turnTimes(rs.getBigDecimal(1), rs.getBigDecimal(2), rs.getBigDecimal(3)), turn.id(), actor.getId().toString()));
result.put("feedback", feedback(turn.id()));
return result;
}
@@ -390,8 +408,12 @@ public class AnalysisTurnService {
Map<String, Object> context = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup",
"view", "includeUnassigned", "stage", "trendGranularity")) {
"view", "includeUnassigned", "extraFilters", "stage", "category", "defect", "trendGranularity")) {
Object value = source.get(key);
if ("extraFilters".equals(key) && value != null) {
context.put(key, cn.yunsheng.quality.filters.QueryFilterFields.parse(value));
continue;
}
if (value instanceof String || value instanceof Number || value instanceof Boolean) context.put(key, value);
}
context.put("releaseId", fixedReleaseId);
@@ -476,12 +498,15 @@ public class AnalysisTurnService {
(rs, n) -> new Event(rs.getLong(1), rs.getString(2), read(rs.getString(3))), turnId, cursor);
for (Event event : events) {
AppUserPrincipal sendAs = currentUser(userId);
requireTurn(turnId, sendAs);
Turn sendTurn = requireTurn(turnId, sendAs);
Object payload = event.payload();
if (payload instanceof Map<?, ?> map && map.get("result") instanceof Map<?, ?> result) {
Map<String, Object> safe = new LinkedHashMap<>(event.payload());
Map<String, Object> safeResult = stringMap(result);
if (safeResult.get("results") instanceof List<?> values) safeResult.put("results", filterResultList(values, sendAs));
if (safeResult.containsKey("visualSchemaVersion") || safeResult.containsKey("visualState") || safeResult.containsKey("visuals")) {
safeResult.putAll(visibleVisuals(safeResult, sendTurn, sendAs, queryContext(sendTurn)));
}
safe.put("result", safeResult);
payload = safe;
}
@@ -833,7 +858,8 @@ public class AnalysisTurnService {
plan.put("toolCount", steps.size());
plan.put("stopReason", stopReason);
if (result.details().get("timeline") instanceof List<?>) plan.put("timeline", NaoAnalysisRunner.safeTimeline(result.details().get("timeline")));
Map<String, Object> answer = nativeAnswer(result.details(), answerText, context, session.releaseId(), status, stopReason);
Map<String, Object> answer = nativeAnswer(result.details(), answerText, context, session.releaseId(), status, stopReason,
turn.id(), finalActor);
persistContext(session.id(), finalActor, context);
assertScope(currentUser(UUID.fromString(turn.userId())), turn.authScopeHash());
finish(turnId, status, plan, steps, List.of(), stopReason, answer);
@@ -856,8 +882,11 @@ public class AnalysisTurnService {
if (frozen.isEmpty()) frozen = mapValue(persistedPlan.get("context"));
Map<String, Object> context = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter", "department", "group", "product", "projectGroup",
"view", "includeUnassigned", "stage", "trendGranularity")) {
if (frozen.containsKey(key)) context.put(key, frozen.get(key));
"view", "includeUnassigned", "extraFilters", "stage", "category", "defect", "trendGranularity")) {
if (!frozen.containsKey(key)) continue;
Object value = frozen.get(key);
if ("extraFilters".equals(key)) context.put(key, cn.yunsheng.quality.filters.QueryFilterFields.parse(value));
else if (value instanceof String || value instanceof Number || value instanceof Boolean) context.put(key, value);
}
context.put("releaseId", releaseId);
return context;
@@ -878,8 +907,8 @@ public class AnalysisTurnService {
appendEvent(turnId, "answer", Map.of("turnId", turnId, "status", "RUNNING", "result", answer));
}
private Map<String, Object> nativeAnswer(Map<String, Object> details, String text, Map<String, Object> context,
String releaseId, String status, String stopReason) {
Map<String, Object> nativeAnswer(Map<String, Object> details, String text, Map<String, Object> context,
String releaseId, String status, String stopReason, String turnId, AppUserPrincipal actor) {
Map<String, Object> answer = new LinkedHashMap<>();
answer.put("answer", text);
answer.put("results", List.of());
@@ -889,6 +918,16 @@ public class AnalysisTurnService {
answer.put("businessResultsSentToModel", true);
answer.put("completion", Map.of("completed", "COMPLETED".equals(status), "reason", Objects.requireNonNullElse(stopReason, "")));
answer.put("stopReason", Objects.requireNonNullElse(stopReason, ""));
if (!visualsEnabled || actor == null || !List.of("COMPLETED", "PARTIAL").contains(status)) {
answer.putAll(AnalysisVisualSnapshotAccess.empty(visualsEnabled ? "failed" : "not_requested").asMap());
} else {
Set<String> allowedFields = governance.allowedFields(actor);
Set<String> filterKeys = AnalysisVisualSnapshotAccess.visibleExtraFilterKeys(jdbc, allowedFields);
AnalysisVisualSnapshotAccess.Snapshot snapshot = AnalysisVisualSnapshotAccess.normalize(
details.get("visualSchemaVersion"), details.get("visualState"), details.get("visuals"),
allowedFields, filterKeys, turnId, cleanContext(context));
answer.putAll(snapshot.asMap());
}
for (String key : List.of("reportMarkdown", "reportHtml", "nativeChatId"))
if (details.get(key) instanceof String value && value.length() <= 2_000_000) answer.put(key, value);
if (details.get("timeline") instanceof List<?>) answer.put("timeline", NaoAnalysisRunner.safeTimeline(details.get("timeline")));
@@ -940,7 +979,7 @@ public class AnalysisTurnService {
private void finishNativeError(String turnId, Map<String, Object> plan, Map<String, Object> context,
String releaseId, String status, String reason) {
Map<String, Object> answer = nativeAnswer(Map.of(), reason, context, releaseId, status, reason);
Map<String, Object> answer = nativeAnswer(Map.of(), reason, context, releaseId, status, reason, turnId, null);
plan.put("stopReason", reason);
// A cancelled or failed subprocess can still have completed useful calls.
// Retain its persisted operation trace for recovery and inspection.
@@ -1971,6 +2010,13 @@ public class AnalysisTurnService {
}
return filtered;
}
private Map<String, Object> visibleVisuals(Map<String, Object> answer, Turn turn, AppUserPrincipal actor,
Map<String, Object> expectedScope) {
if (!visualsEnabled) return AnalysisVisualSnapshotAccess.empty("not_requested").asMap();
Set<String> allowedFields = governance.allowedFields(actor);
Set<String> visibleFilters = AnalysisVisualSnapshotAccess.visibleExtraFilterKeys(jdbc, allowedFields);
return AnalysisVisualSnapshotAccess.fromAnswer(answer, allowedFields, visibleFilters, turn.id(), cleanContext(expectedScope)).asMap();
}
private Map<String,Object> stringMap(Map<?,?> map) { Map<String,Object> out=new LinkedHashMap<>();map.forEach((k,v)->out.put(String.valueOf(k),v));return out; }
private String write(Object value) { return json.writeValueAsString(value); }
private String sha256(String value) { try{return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)));}catch(Exception e){throw new IllegalStateException(e);} }
@@ -44,6 +44,11 @@ public class AnalysisTurnsController {
return turns.getTurn(turnId, actor);
}
@GetMapping("/turns/{turnId}/snapshot")
public Map<String, Object> getSnapshot(@PathVariable String turnId, @AuthenticationPrincipal AppUserPrincipal actor) {
return turns.getSnapshot(turnId, actor);
}
@GetMapping(value = "/turns/{turnId}/events", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public SseEmitter events(@PathVariable String turnId,
@RequestHeader(name = "Last-Event-ID", required = false) String lastEventId,
@@ -0,0 +1,631 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.filters.QueryFilterFields;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import org.springframework.jdbc.core.JdbcTemplate;
import tools.jackson.databind.json.JsonMapper;
/** Strict, permission-aware protocol boundary for persisted analysis chart snapshots. */
public final class AnalysisVisualSnapshotAccess {
public static final int SCHEMA_VERSION = 1;
public static final int MAX_SNAPSHOT_BYTES = 256 * 1024;
private static final int MAX_VISUALS = 3;
private static final int MAX_CATEGORIES = 36;
private static final int MAX_SERIES = 1;
private static final int MAX_TEXT_BYTES = 2048;
private static final Pattern SAFE_KEY = Pattern.compile("[A-Za-z0-9][A-Za-z0-9_.:-]{0,119}");
private static final Pattern SAFE_EVIDENCE_REF = Pattern.compile("[A-Za-z0-9:_-]{1,160}");
private static final Set<String> CHART_KEYS = Set.of("id", "type", "title", "subtitle", "unit", "categories", "series",
"evidenceRefs", "requiredFields", "scope", "period", "comparisonPeriod", "stage", "metric", "coverage", "note", "originTurnId");
private static final Set<String> SCOPE_KEYS = Set.of("year", "month", "quarter", "department", "group", "product",
"projectGroup", "view", "includeUnassigned", "releaseId", "extraFilters", "stage", "category", "defect");
private static final Set<String> PERIOD_KEYS = Set.of("start", "end", "label", "granularity", "year", "month", "quarter");
private static final Set<String> COVERAGE_KEYS = Set.of("totalRows", "returnedRows", "truncated", "shareBasis");
private static final Set<String> CANONICAL_FIELDS = Set.of("black_good_weight", "black_scrap_weight", "black_total_weight",
"black_responsibility_scrap", "black_material_scrap", "black_external_scrap", "bright_good_weight",
"bright_scrap_weight", "bright_total_weight", "bright_headquarters_scrap", "bright_plating_scrap",
"bright_material_scrap", "bright_processing_scrap", "bright_diffusion_scrap");
private static final Set<String> WEIGHT_METRICS = Set.of("black_good_weight", "black_scrap_weight", "black_total_weight",
"black_responsibility_scrap", "black_material_scrap", "black_external_scrap", "bright_good_weight",
"bright_scrap_weight", "bright_total_weight", "bright_headquarters_scrap", "bright_plating_scrap",
"bright_material_scrap", "bright_processing_scrap", "bright_diffusion_scrap");
private static final JsonMapper JSON = JsonMapper.builder().build();
private AnalysisVisualSnapshotAccess() {}
public record Snapshot(int schemaVersion, String state, List<Map<String, Object>> visuals) {
public Map<String, Object> asMap() {
Map<String, Object> result = new LinkedHashMap<>();
result.put("visualSchemaVersion", schemaVersion);
result.put("visualState", state);
result.put("visuals", visuals);
return result;
}
}
/**
* Rebuilds the only chart fields that may cross the runtime boundary. Invalid charts are
* dropped individually; invalid top-level data or an oversized normalized snapshot fails closed.
*/
public static Snapshot normalize(Object schemaVersion, Object state, Object rawVisuals, Set<String> allowedFields,
Set<String> visibleExtraFilterKeys, String turnId, Map<String, Object> expectedScope) {
if (state == null && rawVisuals == null && schemaVersion == null) return empty("not_requested");
if (!integerEquals(schemaVersion, SCHEMA_VERSION)) return empty("failed");
if (!(state instanceof String stateText) || !Set.of("ready", "not_requested", "unavailable", "failed").contains(stateText))
return empty("failed");
if (!"ready".equals(stateText)) {
if (rawVisuals instanceof List<?> values && !values.isEmpty()) return empty("failed");
return empty(stateText);
}
if (!(rawVisuals instanceof List<?> values) || values.isEmpty() || values.size() > MAX_VISUALS) return empty("failed");
Set<String> safeFields = allowedFields == null ? Set.of() : allowedFields;
Set<String> safeFilterKeys = visibleExtraFilterKeys == null ? Set.of() : visibleExtraFilterKeys;
Set<String> ids = new LinkedHashSet<>();
List<Map<String, Object>> visuals = new ArrayList<>();
for (Object value : values) {
try {
Map<String, Object> chart = normalizeVisual(value, safeFields, safeFilterKeys, turnId, expectedScope);
if (chart != null && ids.add((String) chart.get("id"))) visuals.add(chart);
} catch (RuntimeException invalid) {
// A single candidate never changes the business turn's completion state.
}
}
if (visuals.isEmpty()) return empty("failed");
Snapshot snapshot = new Snapshot(SCHEMA_VERSION, "ready", List.copyOf(visuals));
try { if (encodedSize(snapshot.asMap()) > MAX_SNAPSHOT_BYTES) return empty("failed"); }
catch (RuntimeException invalid) { return empty("failed"); }
return snapshot;
}
public static Snapshot fromAnswer(Map<String, Object> answer, Set<String> allowedFields,
Set<String> visibleExtraFilterKeys, String turnId, Map<String, Object> expectedScope) {
if (answer == null) return empty("not_requested");
return normalize(answer.get("visualSchemaVersion"), answer.get("visualState"), answer.get("visuals"),
allowedFields, visibleExtraFilterKeys, turnId, expectedScope);
}
/** Current enabled filter fields whose source columns are both visible to this identity. */
public static Set<String> visibleExtraFilterKeys(JdbcTemplate jdbc, Set<String> allowedFields) {
Set<String> allowed = allowedFields == null ? Set.of() : allowedFields;
try {
return jdbc.query("SELECT field_key,black_column,bright_column FROM query_filter_field WHERE enabled=TRUE",
(rs, row) -> {
String key = rs.getString(1), black = rs.getString(2), bright = rs.getString(3);
return allowed.contains("*") || allowed.contains(black) && allowed.contains(bright) ? key : null;
}).stream().filter(java.util.Objects::nonNull).collect(java.util.stream.Collectors.toUnmodifiableSet());
} catch (RuntimeException unavailable) {
return Set.of();
}
}
public static Snapshot empty(String state) {
return new Snapshot(SCHEMA_VERSION, Set.of("not_requested", "unavailable", "failed").contains(state) ? state : "failed", List.of());
}
private static Map<String, Object> normalizeVisual(Object raw, Set<String> allowedFields, Set<String> visibleExtraFilterKeys,
String turnId, Map<String, Object> expectedScope) {
Map<String, Object> input = stringMap(raw);
if (!CHART_KEYS.containsAll(input.keySet())) return null;
for (String key : List.of("id", "type", "title", "unit", "categories", "series", "evidenceRefs", "requiredFields",
"scope", "period", "metric", "coverage", "originTurnId"))
if (!input.containsKey(key)) return null;
String id = text(input.get("id"), 120, true);
if (!SAFE_KEY.matcher(id).matches()) return null;
String type = enumText(input.get("type"), Set.of("bar", "line", "column", "area", "heatmap", "donut", "pareto"));
String title = text(input.get("title"), 200, false);
String subtitle = input.containsKey("subtitle") ? text(input.get("subtitle"), 400, false) : null;
String unit = enumText(input.get("unit"), Set.of("ratio", "kg", "pp", "count"));
String metric = enumText(input.get("metric"), metricNames());
String origin = text(input.get("originTurnId"), 64, true);
if (turnId == null || !turnId.equals(origin)) return null;
MetricSemantics semantics = semantics(metric, input.get("stage"));
if (semantics == null || !semantics.unit().equals(unit)) return null;
Set<String> requiredFields = new LinkedHashSet<>(semantics.requiredFields());
List<Map<String, Object>> categories = normalizeCategories(input.get("categories"), type, semantics, requiredFields);
if (categories == null) return null;
List<Map<String, Object>> series = normalizeSeries(input.get("series"), categories.size(), metric);
if (series == null || !metric.equals(series.getFirst().get("key"))) return null;
// requiredFields from the runtime is only checked for shape. The persisted list is rebuilt
// from the allowlisted metric and stage (plus source-defect category semantics and share basis).
Set<String> claimedFields = normalizeClaimedFields(input.get("requiredFields"));
if (claimedFields == null) return null;
// Fine-defect charts are tied to the precise fields drawn. This lets the snapshot
// be reauthorized after role changes without trusting runtime-supplied permissions.
if (semantics.sourceDefect() && !claimedFields.equals(requiredFields)) return null;
Map<String, Object> period = normalizePeriod(input.get("period"));
if (period == null) return null;
Map<String, Object> comparisonPeriod = input.containsKey("comparisonPeriod") ? normalizePeriod(input.get("comparisonPeriod")) : null;
if (input.containsKey("comparisonPeriod") && comparisonPeriod == null) return null;
Map<String, Object> coverage = normalizeCoverage(input.get("coverage"), requiredFields, semantics.stage(), type);
if (coverage == null) return null;
if (!validCompositionChart(type, metric, unit, semantics, categories, series, coverage, comparisonPeriod)
|| !validPeriodChart(type, metric, semantics, period, comparisonPeriod, categories, series)) return null;
if (!allowedFields.contains("*") && !allowedFields.containsAll(requiredFields)) return null;
Map<String, Object> scope = normalizeScope(input.get("scope"), visibleExtraFilterKeys, expectedScope);
Object scopedStage = scope == null ? null : scope.get("stage");
if (scope == null || scopedStage != null && !"all".equals(scopedStage)
&& (semantics.stage() == null || !semantics.stage().equals(scopedStage))) return null;
if (semantics.sourceDefect() && !sourceDefectScopeMatches(scope, semantics.stage(), categories)) return null;
String note = input.containsKey("note") ? text(input.get("note"), 500, false) : null;
Map<String, Object> result = new LinkedHashMap<>();
result.put("id", id); result.put("type", type); result.put("title", title);
if (subtitle != null) result.put("subtitle", subtitle);
result.put("unit", unit); result.put("categories", categories); result.put("series", series);
result.put("evidenceRefs", normalizeEvidenceRefs(input.get("evidenceRefs")));
result.put("requiredFields", requiredFields.stream().sorted().toList()); result.put("scope", scope);
result.put("period", period); if (comparisonPeriod != null) result.put("comparisonPeriod", comparisonPeriod);
if (semantics.stage() != null) result.put("stage", semantics.stage());
result.put("metric", metric); result.put("coverage", coverage);
if (note != null) result.put("note", note);
result.put("originTurnId", turnId);
return result;
}
private static List<Map<String, Object>> normalizeCategories(Object raw, String type, MetricSemantics semantics,
Set<String> requiredFields) {
if (!(raw instanceof List<?> values) || values.isEmpty() || values.size() > MAX_CATEGORIES
|| Set.of("bar", "column").contains(type) && values.size() > 10) return null;
List<Map<String, Object>> result = new ArrayList<>();
Set<String> keys = new LinkedHashSet<>();
for (Object value : values) {
Map<String, Object> item = stringMap(value);
if (!Set.of("key", "label").containsAll(item.keySet()) || !item.keySet().containsAll(Set.of("key", "label"))) return null;
String key = text(item.get("key"), 120, true), label = text(item.get("label"), 240, false);
if (!keys.add(key)) return null;
if (semantics.sourceDefect()) {
if (!isSourceDefectCategoryKey(semantics.stage(), key)) return null;
requiredFields.add(key);
}
result.add(Map.of("key", key, "label", label));
}
return result;
}
private static List<Map<String, Object>> normalizeSeries(Object raw, int categoryCount, String metric) {
if (!(raw instanceof List<?> values) || values.isEmpty() || values.size() > MAX_SERIES) return null;
List<Map<String, Object>> result = new ArrayList<>();
Set<String> keys = new LinkedHashSet<>();
for (Object value : values) {
Map<String, Object> item = stringMap(value);
if (!Set.of("key", "label", "values").containsAll(item.keySet()) || !item.keySet().containsAll(Set.of("key", "label", "values"))) return null;
String key = text(item.get("key"), 120, true), label = text(item.get("label"), 240, false);
if (!SAFE_KEY.matcher(key).matches() || !keys.add(key) || !(item.get("values") instanceof List<?> points)
|| points.size() != categoryCount) return null;
List<Object> numbers = new ArrayList<>();
for (Object point : points) {
if (point == null) numbers.add(null);
else {
BigDecimal decimal = decimal(point);
if (decimal == null || decimal.precision() > 64 || Math.abs(decimal.scale()) > 64) return null;
if (Set.of("composite_rate", "black_rate", "bright_rate").contains(metric)
&& (decimal.signum() < 0 || decimal.compareTo(BigDecimal.ONE) > 0)) return null;
if (Set.of("black_good_weight", "black_scrap_weight", "black_total_weight",
"black_responsibility_scrap", "black_material_scrap", "black_external_scrap",
"bright_good_weight", "bright_scrap_weight", "bright_total_weight",
"bright_headquarters_scrap", "bright_plating_scrap", "bright_material_scrap",
"bright_processing_scrap", "bright_diffusion_scrap", "current_scrap_weight",
"source_defect_weight").contains(metric) && decimal.signum() < 0) return null;
if ("rate_drop_pp".equals(metric) && decimal.abs().compareTo(BigDecimal.valueOf(100)) > 0) return null;
numbers.add(decimal);
}
}
result.add(Map.of("key", key, "label", label, "values", Collections.unmodifiableList(numbers)));
}
return result;
}
private static Map<String, Object> normalizeScope(Object raw, Set<String> visibleExtraFilterKeys, Map<String, Object> expectedScope) {
Map<String, Object> input = stringMap(raw);
if (!SCOPE_KEYS.containsAll(input.keySet())) return null;
Map<String, Object> scope = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter")) {
if (input.containsKey(key)) {
Integer value = integer(input.get(key));
if (value == null || key.equals("year") && (value < 2000 || value > 2100)
|| key.equals("month") && (value < 1 || value > 12) || key.equals("quarter") && (value < 1 || value > 4)) return null;
scope.put(key, value);
}
}
if (scope.containsKey("month") && scope.containsKey("quarter")) return null;
for (String key : List.of("department", "group", "product", "projectGroup", "view", "releaseId")) {
if (input.containsKey(key)) {
String value = text(input.get(key), 256, true);
if ("view".equals(key) && !Set.of("overall", "department", "group", "product", "projectGroup").contains(value)) return null;
scope.put(key, value);
}
}
if (input.containsKey("stage")) {
String value = enumText(input.get("stage"), Set.of("all", "black", "bright"));
if (value == null) return null;
scope.put("stage", value);
}
for (String key : List.of("category", "defect")) {
if (input.containsKey(key)) scope.put(key, text(input.get(key), 256, true));
}
if (input.containsKey("includeUnassigned")) {
if (!(input.get("includeUnassigned") instanceof Boolean value)) return null;
scope.put("includeUnassigned", value);
}
if (input.containsKey("extraFilters")) {
Map<String, List<String>> filters;
try { filters = QueryFilterFields.parse(input.get("extraFilters")); }
catch (RuntimeException invalid) { return null; }
if (!visibleExtraFilterKeys.containsAll(filters.keySet())) return null;
scope.put("extraFilters", filters);
}
Map<String, Object> expected = expectedScope == null ? Map.of() : expectedScope;
if (!scope.containsKey("releaseId")) return null;
if (expected.containsKey("releaseId")) {
Object expectedRelease = normalizeExpected("releaseId", expected.get("releaseId"), visibleExtraFilterKeys);
if (expectedRelease == Invalid.VALUE || !sameValue(scope.get("releaseId"), expectedRelease)) return null;
}
// Date, aggregation view, stage and period labels describe each source query. They may
// legitimately differ from the session defaults (for example, a trend over Jan-Sep from
// a session opened on September or a product ranking from an overall session).
for (String key : List.of("department", "group", "product", "projectGroup", "category", "defect")) {
if (!expected.containsKey(key) || expected.get(key) == null) continue;
if (!(expected.get(key) instanceof String)) return null;
if (!activeExpectedSelector(key, expected.get(key))) continue;
Object expectedValue = normalizeExpected(key, expected.get(key), visibleExtraFilterKeys);
if (expectedValue == Invalid.VALUE || !scope.containsKey(key) || !sameValue(scope.get(key), expectedValue)) return null;
}
if (expected.containsKey("stage") && expected.get("stage") != null) {
Object expectedStage = expected.get("stage");
if (!(expectedStage instanceof String stage) || !Set.of("all", "black", "bright").contains(stage)) return null;
if (Set.of("black", "bright").contains(stage)
&& (!scope.containsKey("stage") || !sameValue(scope.get("stage"), stage))) return null;
}
if (expected.containsKey("extraFilters") && activeExpectedSelector("extraFilters", expected.get("extraFilters"))) {
Object expectedFilters = normalizeExpected("extraFilters", expected.get("extraFilters"), visibleExtraFilterKeys);
if (expectedFilters == Invalid.VALUE || !scope.containsKey("extraFilters")
|| !sameValue(scope.get("extraFilters"), expectedFilters)) return null;
}
if (expected.containsKey("includeUnassigned") && !(expected.get("includeUnassigned") instanceof Boolean)) return null;
if (expected.containsKey("includeUnassigned")
&& (!scope.containsKey("includeUnassigned")
|| !sameValue(scope.get("includeUnassigned"), expected.get("includeUnassigned")))) return null;
return scope;
}
private static boolean activeExpectedSelector(String key, Object value) {
if (value == null) return false;
if ("extraFilters".equals(key)) {
try { return !QueryFilterFields.parse(value).isEmpty(); }
catch (RuntimeException invalid) { return true; }
}
return value instanceof String text && !text.isBlank();
}
private static Object normalizeExpected(String key, Object raw, Set<String> visibleExtraFilterKeys) {
if (raw == null) return Invalid.VALUE;
if (List.of("year", "month", "quarter").contains(key)) {
Integer value = integer(raw);
if (value == null || key.equals("year") && (value < 2000 || value > 2100)
|| key.equals("month") && (value < 1 || value > 12) || key.equals("quarter") && (value < 1 || value > 4)) return Invalid.VALUE;
return value;
}
if ("includeUnassigned".equals(key)) return raw instanceof Boolean ? raw : Invalid.VALUE;
if ("stage".equals(key)) return enumText(raw, Set.of("black", "bright")) == null ? Invalid.VALUE : raw;
if ("extraFilters".equals(key)) {
try {
Map<String, List<String>> values = QueryFilterFields.parse(raw);
return visibleExtraFilterKeys.containsAll(values.keySet()) ? values : Invalid.VALUE;
} catch (RuntimeException invalid) { return Invalid.VALUE; }
}
if (!(raw instanceof String value) || value.isBlank() || value.length() > 256
|| value.chars().anyMatch(Character::isISOControl)) return Invalid.VALUE;
if ("view".equals(key) && !Set.of("overall", "department", "group", "product", "projectGroup").contains(value)) return Invalid.VALUE;
return value;
}
private static Map<String, Object> normalizePeriod(Object raw) {
Map<String, Object> input = stringMap(raw);
if (!PERIOD_KEYS.containsAll(input.keySet())) return null;
Map<String, Object> result = new LinkedHashMap<>();
for (String key : List.of("start", "end", "label", "granularity")) {
if (!input.containsKey(key)) continue;
String value = text(input.get(key), 100, true);
if ("granularity".equals(key) && !Set.of("month", "quarter", "year").contains(value)) return null;
result.put(key, value);
}
for (String key : List.of("year", "month", "quarter")) {
if (!input.containsKey(key)) continue;
Integer value = integer(input.get(key));
if (value == null || key.equals("year") && (value < 2000 || value > 2100)
|| key.equals("month") && (value < 1 || value > 12) || key.equals("quarter") && (value < 1 || value > 4)) return null;
result.put(key, value);
}
if (result.containsKey("month") && result.containsKey("quarter")) return null;
return result;
}
private static Map<String, Object> normalizeCoverage(Object raw, Set<String> requiredFields, String stage, String type) {
Map<String, Object> input = stringMap(raw);
if (!COVERAGE_KEYS.containsAll(input.keySet()) || !input.keySet().containsAll(Set.of("totalRows", "returnedRows", "truncated"))) return null;
Long total = longInteger(input.get("totalRows")), returned = longInteger(input.get("returnedRows"));
if (total == null || returned == null || total < 0 || total > 1_000_000_000L || returned < 0 || returned > total
|| returned > 1_000_000_000L || !(input.get("truncated") instanceof Boolean truncated)
|| !truncated && returned != total) return null;
Map<String, Object> result = new LinkedHashMap<>();
result.put("totalRows", total); result.put("returnedRows", returned); result.put("truncated", truncated);
if (input.containsKey("shareBasis")) {
String basis = text(input.get("shareBasis"), 80, true);
if (CANONICAL_FIELDS.contains(basis)) {
if (stage != null && !basis.startsWith(stage + "_")) return null;
requiredFields.add(basis);
} else if (Set.of("donut", "pareto").contains(type) && "displayed_categories".equals(basis)) {
// These are the only visual types for which the renderer may compute
// percentages from the plotted values.
} else if ("displayed_categories".equals(basis)) return null;
else if (!Set.of("complete_population", "query_total", "not_applicable").contains(basis)) return null;
result.put("shareBasis", basis);
}
if (Set.of("donut", "pareto").contains(type)
&& !"displayed_categories".equals(result.get("shareBasis"))) return null;
return result;
}
private static boolean validCompositionChart(String type, String metric, String unit, MetricSemantics semantics,
List<Map<String, Object>> categories,
List<Map<String, Object>> series,
Map<String, Object> coverage,
Map<String, Object> comparisonPeriod) {
if (!Set.of("donut", "pareto").contains(type)) return true;
if (!"source_defect_weight".equals(metric) || !"kg".equals(unit) || !semantics.sourceDefect()
|| semantics.stage() == null || comparisonPeriod != null || categories.size() < 2 || categories.size() > 4
|| series.size() != 1 || !Boolean.FALSE.equals(coverage.get("truncated"))
|| !"displayed_categories".equals(coverage.get("shareBasis"))) return false;
Set<String> formalKeys = AnalysisFineDefectsService.categories(semantics.stage()).stream()
.map(AnalysisFineDefectsService.Category::componentCode).collect(java.util.stream.Collectors.toSet());
if (categories.stream().anyMatch(item -> !formalKeys.contains(item.get("key")))) return false;
Object total = coverage.get("totalRows"), returned = coverage.get("returnedRows");
if (!(total instanceof Number totalNumber) || !(returned instanceof Number returnedNumber)
|| totalNumber.longValue() != categories.size() || returnedNumber.longValue() != categories.size()) return false;
Object valuesRaw = series.getFirst().get("values");
if (!(valuesRaw instanceof List<?> values) || values.size() != categories.size()) return false;
BigDecimal sum = BigDecimal.ZERO;
BigDecimal previous = null;
for (Object raw : values) {
BigDecimal value = decimal(raw);
if (value == null || value.signum() < 0) return false;
if ("pareto".equals(type) && previous != null && value.compareTo(previous) > 0) return false;
previous = value;
sum = sum.add(value);
}
return sum.signum() > 0;
}
private static boolean validPeriodChart(String type, String metric, MetricSemantics semantics, Map<String, Object> period,
Map<String, Object> comparisonPeriod,
List<Map<String, Object>> categories,
List<Map<String, Object>> series) {
if (!Set.of("area", "heatmap").contains(type)) return true;
if (comparisonPeriod != null || categories.size() < 3 || !period.containsKey("start")
|| !period.containsKey("end") || !period.containsKey("granularity")) return false;
if (!Set.of("composite_rate", "black_rate", "bright_rate", "current_scrap_weight",
"black_scrap_weight", "bright_scrap_weight").contains(metric)) return false;
if ("current_scrap_weight".equals(metric) && semantics.stage() == null) return false;
String grain = (String) period.get("granularity");
Long first = periodOrder((String) period.get("start"), grain);
Long last = periodOrder((String) period.get("end"), grain);
if (first == null || last == null || first > last) return false;
Long previous = null;
for (int index = 0; index < categories.size(); index++) {
Map<String, Object> category = categories.get(index);
String key = (String) category.get("key");
if (!key.equals(category.get("label"))) return false;
Long order = periodOrder(key, grain);
if (order == null || previous != null && !periodNext(previous, grain).equals(order)) return false;
previous = order;
}
if (periodOrder((String) categories.getFirst().get("key"), grain).longValue() != first
|| periodOrder((String) categories.getLast().get("key"), grain).longValue() != last) return false;
if ("area".equals(type)) {
for (Object raw : (List<?>) series.getFirst().get("values")) {
if (raw != null) {
BigDecimal value = decimal(raw);
if (value == null || value.signum() < 0) return false;
}
}
}
return true;
}
private static Long periodOrder(String value, String granularity) {
if (value == null) return null;
if ("month".equals(granularity)) {
var match = Pattern.compile("(20\\d{2}|2100)-(0[1-9]|1[0-2])").matcher(value);
return match.matches() ? Long.parseLong(match.group(1)) * 100L + Long.parseLong(match.group(2)) : null;
}
if ("quarter".equals(granularity)) {
var match = Pattern.compile("(20\\d{2}|2100)-Q([1-4])").matcher(value);
return match.matches() ? Long.parseLong(match.group(1)) * 10L + Long.parseLong(match.group(2)) : null;
}
if ("year".equals(granularity) && value.matches("(20\\d{2}|2100)")) return Long.parseLong(value);
return null;
}
private static Long periodNext(long current, String granularity) {
if ("month".equals(granularity)) {
long year = current / 100L, month = current % 100L;
return month == 12 ? (year + 1L) * 100L + 1L : current + 1L;
}
if ("quarter".equals(granularity)) {
long year = current / 10L, quarter = current % 10L;
return quarter == 4 ? (year + 1L) * 10L + 1L : current + 1L;
}
return current + 1L;
}
private static List<String> normalizeEvidenceRefs(Object raw) {
if (!(raw instanceof List<?> values) || values.isEmpty() || values.size() > 20) throw new IllegalArgumentException("invalid evidence refs");
List<String> result = new ArrayList<>();
for (Object value : values) {
String ref = text(value, 160, true);
if (!SAFE_EVIDENCE_REF.matcher(ref).matches() || result.contains(ref)) throw new IllegalArgumentException("invalid evidence ref");
result.add(ref);
}
return List.copyOf(result);
}
private static Set<String> normalizeClaimedFields(Object raw) {
if (!(raw instanceof List<?> values) || values.size() > 32) return null;
Set<String> unique = new LinkedHashSet<>();
for (Object value : values) {
if (!(value instanceof String field) || field.length() > 120 || field.isBlank()
|| field.chars().anyMatch(Character::isISOControl) || !isKnownRequiredField(field)
|| !unique.add(field)) return null;
}
return unique;
}
private static boolean isKnownRequiredField(String field) {
if (CANONICAL_FIELDS.contains(field)) return true;
return AnalysisFineDefectsService.fields("black").stream().anyMatch(item -> item.key().equals(field))
|| AnalysisFineDefectsService.fields("bright").stream().anyMatch(item -> item.key().equals(field));
}
/** Valid category keys are either exact source detail columns or formal category aggregates for this stage. */
private static boolean isSourceDefectCategoryKey(String stage, String key) {
return AnalysisFineDefectsService.fields(stage).stream().anyMatch(item -> item.key().equals(key))
|| AnalysisFineDefectsService.categories(stage).stream().anyMatch(item -> item.componentCode().equals(key));
}
private static boolean sourceDefectScopeMatches(Map<String, Object> scope, String stage,
List<Map<String, Object>> chartCategories) {
Object categorySelector = scope.get("category");
Object defectSelector = scope.get("defect");
if (categorySelector != null && !(categorySelector instanceof String)) return false;
if (defectSelector != null && !(defectSelector instanceof String)) return false;
if (categorySelector instanceof String selectedCategory
&& AnalysisFineDefectsService.categories(stage).stream().noneMatch(item -> item.name().equals(selectedCategory))) return false;
if (defectSelector instanceof String selectedDefect
&& AnalysisFineDefectsService.fields(stage).stream().noneMatch(item -> item.key().equals(selectedDefect))) return false;
for (Map<String, Object> item : chartCategories) {
String key = (String) item.get("key");
String keyCategory = AnalysisFineDefectsService.fields(stage).stream()
.filter(field -> field.key().equals(key)).map(AnalysisFineDefectsService.DefectField::category)
.findFirst().orElseGet(() -> AnalysisFineDefectsService.categories(stage).stream()
.filter(category -> category.componentCode().equals(key))
.map(AnalysisFineDefectsService.Category::name).findFirst().orElse(null));
if (keyCategory == null || categorySelector instanceof String selectedCategory && !selectedCategory.equals(keyCategory)
|| defectSelector instanceof String selectedDefect && !selectedDefect.equals(key)) return false;
}
return true;
}
private static MetricSemantics semantics(String metric, Object stageRaw) {
String stage = stageRaw == null ? null : enumText(stageRaw, Set.of("black", "bright"));
if (stageRaw != null && stage == null) return null;
if (WEIGHT_METRICS.contains(metric)) {
String implied = metric.startsWith("black_") ? "black" : "bright";
if (stage != null && !stage.equals(implied)) return null;
return new MetricSemantics("kg", implied, Set.of(metric), false);
}
return switch (metric) {
case "composite_rate" -> stage == null ? new MetricSemantics("ratio", null,
Set.of("black_good_weight", "black_total_weight", "bright_good_weight", "bright_total_weight"), false) : null;
case "black_rate" -> stage == null || "black".equals(stage) ? new MetricSemantics("ratio", "black",
Set.of("black_good_weight", "black_total_weight"), false) : null;
case "bright_rate" -> stage == null || "bright".equals(stage) ? new MetricSemantics("ratio", "bright",
Set.of("bright_good_weight", "bright_total_weight"), false) : null;
case "scrap_delta" -> stage == null
? new MetricSemantics("kg", null, Set.of("black_scrap_weight", "bright_scrap_weight"), false)
: new MetricSemantics("kg", stage, Set.of(stage + "_scrap_weight"), false);
case "current_scrap_weight" -> stage == null
? new MetricSemantics("kg", null, Set.of("black_scrap_weight", "bright_scrap_weight"), false)
: new MetricSemantics("kg", stage, Set.of(stage + "_scrap_weight"), false);
case "rate_drop_pp" -> stage == null
? new MetricSemantics("pp", null,
Set.of("black_good_weight", "black_total_weight", "bright_good_weight", "bright_total_weight"), false)
: new MetricSemantics("pp", stage,
stage.equals("black") ? Set.of("black_good_weight", "black_total_weight")
: Set.of("bright_good_weight", "bright_total_weight"), false);
case "source_defect_weight", "source_defect_delta_weight" -> stage == null ? null
: new MetricSemantics("kg", stage, Set.of(), true);
default -> null;
};
}
private static Set<String> metricNames() {
Set<String> result = new LinkedHashSet<>(WEIGHT_METRICS);
result.addAll(Set.of("composite_rate", "black_rate", "bright_rate", "scrap_delta", "rate_drop_pp",
"current_scrap_weight", "source_defect_weight", "source_defect_delta_weight"));
return result;
}
private static BigDecimal decimal(Object raw) {
if (!(raw instanceof Number number)) return null;
if (number instanceof Double doubleValue && !Double.isFinite(doubleValue)
|| number instanceof Float floatValue && !Float.isFinite(floatValue)) return null;
try {
String encoded = number.toString();
if (encoded.length() > 128) return null;
return new BigDecimal(encoded);
} catch (NumberFormatException invalid) { return null; }
}
private static Integer integer(Object raw) {
Long value = longInteger(raw);
return value == null || value < Integer.MIN_VALUE || value > Integer.MAX_VALUE ? null : value.intValue();
}
private static Long longInteger(Object raw) {
if (!(raw instanceof Number number) || number instanceof Float || number instanceof Double || number instanceof BigDecimal || number instanceof BigInteger) return null;
try { return Long.valueOf(number.toString()); } catch (NumberFormatException invalid) { return null; }
}
private static boolean integerEquals(Object raw, int expected) {
Integer value = integer(raw);
return value != null && value == expected;
}
private static String enumText(Object raw, Set<String> allowed) {
if (!(raw instanceof String value) || !allowed.contains(value)) return null;
return value;
}
private static String text(Object raw, int maxLength, boolean nonBlank) {
if (!(raw instanceof String value) || value.length() > maxLength || value.getBytes(StandardCharsets.UTF_8).length > MAX_TEXT_BYTES
|| value.chars().anyMatch(Character::isISOControl) || nonBlank && value.isBlank()) throw new IllegalArgumentException("invalid text");
return value;
}
private static Map<String, Object> stringMap(Object raw) {
if (!(raw instanceof Map<?, ?> map)) throw new IllegalArgumentException("expected object");
Map<String, Object> result = new LinkedHashMap<>();
for (var entry : map.entrySet()) {
if (!(entry.getKey() instanceof String key) || result.putIfAbsent(key, entry.getValue()) != null) throw new IllegalArgumentException("invalid object key");
}
return result;
}
private static boolean sameValue(Object left, Object right) {
if (left instanceof Number a && right instanceof Number b) {
try { return new BigDecimal(a.toString()).compareTo(new BigDecimal(b.toString())) == 0; }
catch (NumberFormatException invalid) { return false; }
}
return left == null ? right == null : left.equals(right);
}
private static int encodedSize(Object value) {
return JSON.writeValueAsString(value).getBytes(StandardCharsets.UTF_8).length;
}
private record MetricSemantics(String unit, String stage, Set<String> requiredFields, boolean sourceDefect) {}
private enum Invalid { VALUE }
}
@@ -42,6 +42,8 @@ management:
show-details: never
quality:
analysis:
visuals-enabled: ${QUALITY_ANALYSIS_VISUALS_ENABLED:true}
archive-root: ${QUALITY_ARCHIVE_ROOT:./runtimedata/archive}
allowed-roots: ${QUALITY_ALLOWED_ROOTS:}
max-upload-bytes: ${QUALITY_MAX_UPLOAD_BYTES:268435456}