同步:完善质量分析工作台并更新主题字体

同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。
补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。
统一午夜黑及其他主题的控件细节、中文和业务数字字体。
排除本地业务数据、凭据、预览、发布与验收产物。

验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
This commit is contained in:
2026-10-30 22:40:58 +08:00
parent edef747e3b
commit ff7bbe3e46
393 changed files with 62999 additions and 1783 deletions
@@ -42,6 +42,46 @@ public class ManagementController {
return Map.of("items", items);
}
/** Read-only summaries; execution endpoints and their permissions remain independent. */
@GetMapping("/tasks/overview")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','SYSTEM_OPERATOR','DATA_MAINTAINER','DATA_PUBLISHER')")
public Map<String, Object> taskOverview(@AuthenticationPrincipal AppUserPrincipal actor) {
boolean allTasks=actor.getRoleCodes().contains("SYSTEM_ADMIN")||actor.getRoleCodes().contains("SYSTEM_OPERATOR");
boolean allWorkflows=actor.getRoleCodes().contains("SYSTEM_ADMIN")||actor.getRoleCodes().contains("DATA_PUBLISHER");
String owner=actor.getId().toString();
List<Map<String,Object>> items=new java.util.ArrayList<>(jdbc.query("""
SELECT id,task_type,status,stage,progress,processed,total,payload,error_code,error_message,created_at,updated_at
FROM task WHERE (?=TRUE OR created_by=?)
ORDER BY CASE WHEN status IN ('QUEUED','RUNNING') THEN 0 WHEN status IN ('FAILED','PARTIAL') THEN 1 ELSE 2 END,created_at DESC LIMIT 50
""", (rs,n)->summaryTask(rs,"TASK",rs.getString("task_type")),allTasks,owner));
items.addAll(jdbc.query("""
SELECT id,operation_type,status,stage,errors,created_at,updated_at FROM data_workflow_operation
WHERE (?=TRUE OR created_by=?)
ORDER BY CASE WHEN status IN ('QUEUED','RUNNING') THEN 0 WHEN status IN ('FAILED','PARTIAL') THEN 1 ELSE 2 END,created_at DESC LIMIT 50
""",(rs,n)->{Map<String,Object> item=summaryTask(rs,"WORKFLOW","WORKFLOW_"+rs.getString("operation_type"));
item.put("errors",readJson(rs.getString("errors")));return item;},allWorkflows,owner));
// Analysis sessions are private to their owner, including for system administrators.
items.addAll(jdbc.query("""
SELECT id,session_id,status,stop_reason,created_at,finished_at FROM analysis_turn WHERE user_id=?
ORDER BY CASE WHEN status IN ('QUEUED','RUNNING') THEN 0 WHEN status IN ('FAILED','PARTIAL') THEN 1 ELSE 2 END,created_at DESC LIMIT 50
""",(rs,n)->{Map<String,Object> item=summaryTask(rs,"ANALYSIS","ANALYSIS");
item.put("errorMessage",rs.getString("stop_reason"));item.put("sessionId",rs.getString("session_id"));return item;},owner));
return Map.of("items",items,"limitPerSource",50);
}
private Map<String,Object> summaryTask(ResultSet rs,String source,String type)throws SQLException {
Map<String,Object> item=new LinkedHashMap<>();
item.put("id",rs.getString("id"));item.put("sourceKind",source);item.put("type",type);
item.put("status",rs.getString("status"));item.put("createdAt",rs.getTimestamp("created_at").toInstant().toString());
if(!"ANALYSIS".equals(source)) {
item.put("stage",rs.getString("stage"));item.put("updatedAt",rs.getTimestamp("updated_at").toInstant().toString());
}
if("TASK".equals(source))for(String key:List.of("progress","processed","total"))item.put(key,rs.getObject(key));
if("TASK".equals(source) && readJson(rs.getString("payload")) instanceof Map<?,?> payload && payload.get("importRunId")!=null)item.put("importRunId",payload.get("importRunId"));
if("TASK".equals(source)){item.put("errorCode",rs.getString("error_code"));item.put("errorMessage",rs.getString("error_message"));}
return item;
}
@GetMapping("/rules")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','RULE_MAINTAINER','QUALITY_VIEWER','BUSINESS_ANALYST','AUDIT_VIEWER')")
public Map<String, Object> rules() {
@@ -47,7 +47,7 @@ public class AnalysisApiSettingsService {
}
AnalysisApiSettingsService(JsonMapper json, Path root, String legacyPath) {
this(json, root, legacyPath, Duration.ofSeconds(40));
this(json, root, legacyPath, AnalysisRuntimePolicy.REQUEST_TIMEOUT);
}
AnalysisApiSettingsService(JsonMapper json, Path root, String legacyPath, Duration probeTimeout) {
@@ -78,7 +78,7 @@ public class AnalysisApiSettingsService {
List<Profile> profiles = profiles(current);
Profile selected = selectedProfile(request.profileId(), current, profiles);
Config next = resolve(request, selected.config());
next = new Config(next.baseUrl(), next.apiKey(), next.model(), UUID.randomUUID().toString());
next = new Config(next.baseUrl(), next.apiKey(), next.model(), UUID.randomUUID().toString(), next.thinkingEnabled(), next.thinkingEffort());
List<Profile> updated = new ArrayList<>();
for (Profile profile : profiles) updated.add(profile.id().equals(selected.id())
? new Profile(profile.id(), profile.name(), next) : profile);
@@ -90,10 +90,10 @@ public class AnalysisApiSettingsService {
temp = Files.createTempFile(target.getParent(), ".nao-model-", ".tmp");
Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------"));
List<Map<String,Object>> storedProfiles = updated.stream().map(p -> Map.<String,Object>of(
"id",p.id(),"name",p.name(),"baseUrl",p.config().baseUrl(),"apiKey",p.config().apiKey(),"model",p.config().model())).toList();
Files.writeString(temp, json.writeValueAsString(Map.of("schemaVersion", 2, "baseUrl", next.baseUrl(),
"id",p.id(),"name",p.name(),"baseUrl",p.config().baseUrl(),"apiKey",p.config().apiKey(),"model",p.config().model(),"thinkingEnabled",p.config().thinkingEnabled(),"thinkingEffort",p.config().thinkingEffort())).toList();
Files.writeString(temp, json.writeValueAsString(Map.of("schemaVersion", 3, "baseUrl", next.baseUrl(),
"apiKey", next.apiKey(), "model", next.model(), "revision", next.revision(),
"activeProfileId", selected.id(), "profiles", storedProfiles)), StandardCharsets.UTF_8);
"activeProfileId", selected.id(), "profiles", storedProfiles,"thinkingEnabled",next.thinkingEnabled(),"thinkingEffort",next.thinkingEffort())), StandardCharsets.UTF_8);
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
return view(next, updated);
} catch (IOException failure) {
@@ -111,8 +111,19 @@ public class AnalysisApiSettingsService {
}
long started = System.nanoTime();
try {
String body = json.writeValueAsString(Map.of("model", c.model(), "stream", false, "max_tokens", 32,
"messages", List.of(Map.of("role", "user", "content", "连接测试,请只回复 OK。"))));
Map<String,Object> parameters = new LinkedHashMap<>(Map.of("model",c.model(),"stream",false,
"max_completion_tokens",AnalysisRuntimePolicy.MAX_OUTPUT_TOKENS,"messages",List.of(Map.of("role","user","content","连接测试,请只回复 OK。"))));
String model = c.model().toLowerCase(java.util.Locale.ROOT);
if (model.startsWith("qwen")) {
var template = new LinkedHashMap<String,Object>();
template.put("enable_thinking",c.thinkingEnabled());
if (c.thinkingEnabled() && !"auto".equals(c.thinkingEffort()) && model.startsWith("qwen3.8"))
template.put("reasoning_effort","high".equals(c.thinkingEffort())?"xhigh":c.thinkingEffort());
parameters.put("chat_template_kwargs",template);
} else parameters.put("thinking",Map.of("type",c.thinkingEnabled()?"enabled":"disabled"));
if (c.thinkingEnabled() && !"auto".equals(c.thinkingEffort()) && !model.startsWith("qwen3.8"))
parameters.put("reasoning_effort",c.thinkingEffort());
String body = json.writeValueAsString(parameters);
HttpRequest probe = HttpRequest.newBuilder(URI.create(c.baseUrl() + "/chat/completions"))
.timeout(probeTimeout).header("Content-Type", "application/json")
.header("Authorization", "Bearer " + c.apiKey()).POST(HttpRequest.BodyPublishers.ofString(body)).build();
@@ -128,8 +139,16 @@ public class AnalysisApiSettingsService {
byte[] bytes = response.body();
Map<?, ?> value = json.readValue(bytes, Map.class);
if (!(value.get("choices") instanceof List<?> choices) || choices.isEmpty() || value.containsKey("error")) return failedTest();
return Map.of("success", true, "message", "连接成功,模型可响应请求。", "model", c.model(),
"elapsedMs", (System.nanoTime() - started) / 1_000_000);
Map<?,?> first = choices.getFirst() instanceof Map<?,?> item ? item : Map.of();
Map<?,?> answer = first.get("message") instanceof Map<?,?> item ? item : Map.of();
boolean observed = answer.get("reasoning_content") instanceof String reasoningContent && !reasoningContent.isBlank()
|| answer.get("reasoning") instanceof String reasoningText && !reasoningText.isBlank()
|| answer.get("content") instanceof String content && content.contains("<think>");
String notice = c.thinkingEnabled() ? (observed?"连接成功,已确认模型返回思考内容。":"连接成功,已请求开启思考;本次响应未返回可验证的思考信息。")
: (observed?"接口仍返回思考内容,请检查开关是否受支持。":"连接成功,已按关闭思考的设置响应。");
return Map.of("success",c.thinkingEnabled() || !observed,"message",notice,"model",c.model(),
"thinkingEnabled",c.thinkingEnabled(),"thinkingEffort",c.thinkingEffort(),"thinkingObserved",observed,
"elapsedMs",(System.nanoTime()-started)/1_000_000);
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
return failedTest();
@@ -173,7 +192,10 @@ public class AnalysisApiSettingsService {
if (key.isBlank() || key.length() > 4096 || key.chars().anyMatch(c -> Character.isWhitespace(c) || Character.isISOControl(c)))
throw bad("请填写有效的 API Key");
if (!model.matches("[A-Za-z0-9_./:@+-]{1,160}")) throw bad("请填写有效的模型名称");
return new Config(base, key, model, current.revision());
String effort = request.thinkingEffort() == null ? current.thinkingEffort() : request.thinkingEffort();
if (!List.of("auto","low","medium","high").contains(effort)) throw bad("思考强度必须为默认、低、中或高");
if (model.toLowerCase(java.util.Locale.ROOT).matches("mimo-v2\\.6(?:-.*)?")) effort = "auto";
return new Config(base,key,model,current.revision(),request.thinkingEnabled()==null?current.thinkingEnabled():request.thinkingEnabled(),effort);
}
static String normalizeUrl(String value) {
@@ -196,7 +218,7 @@ public class AnalysisApiSettingsService {
if (Files.size(target) > 32768) throw new IOException();
Map<?, ?> value = json.readValue(Files.readString(target), Map.class);
Config c = new Config((String) value.get("baseUrl"), (String) value.get("apiKey"),
(String) value.get("model"), (String) value.get("revision"));
(String) value.get("model"), (String) value.get("revision"),thinkingFlag(value),thinkingEffort(value));
if (c.revision() == null || c.revision().isBlank()) throw new IOException();
return resolve(new Settings(c.baseUrl(), c.apiKey(), c.model(), c.revision()), c);
}
@@ -226,7 +248,10 @@ public class AnalysisApiSettingsService {
}
}
private Path settingsPath() { return root.resolve(".runtime/nao-model-config.json"); }
private Path settingsPath() {
String selected=System.getenv("QUALITY_NAO_MODEL_CONFIG_FILE");
return selected==null||selected.isBlank()?root.resolve(".runtime/nao-model-config.json"):Path.of(selected).toAbsolutePath().normalize();
}
private List<Profile> profiles(Config current) {
try {
List<Profile> result = new ArrayList<>();
@@ -239,7 +264,7 @@ public class AnalysisApiSettingsService {
|| !id.matches("[A-Za-z0-9_-]{1,64}") || !(p.get("name") instanceof String name)
|| name.isBlank() || name.length() > 80 || result.stream().anyMatch(v->v.id().equals(id))) throw new IOException();
Config config = resolve(new Settings((String)p.get("baseUrl"),(String)p.get("apiKey"),
(String)p.get("model"),current.revision()),new Config("","","",current.revision()));
(String)p.get("model"),current.revision(),null,thinkingFlag(p),thinkingEffort(p)),new Config("","","",current.revision()));
result.add(new Profile(id,name,config));
}
return List.copyOf(result);
@@ -282,19 +307,33 @@ public class AnalysisApiSettingsService {
}
private Map<String, Object> view(Config c,List<Profile> profiles) {
var result = new LinkedHashMap<String,Object>();
result.putAll(Map.of("baseUrl", c.baseUrl(), "model", c.model(), "hasApiKey", !c.apiKey().isBlank(), "revision", c.revision()));
result.putAll(Map.of("baseUrl", c.baseUrl(), "model", c.model(), "hasApiKey", !c.apiKey().isBlank(), "revision", c.revision(),"thinkingEnabled",c.thinkingEnabled(),"thinkingEffort",c.thinkingEffort()));
result.put("maxOutputTokens",AnalysisRuntimePolicy.MAX_OUTPUT_TOKENS);
result.put("activeProfileId",activeProfileId(c,profiles));
result.put("profiles",profiles.stream().map(p->Map.of("id",p.id(),"name",p.name(),"baseUrl",p.config().baseUrl(),
"model",p.config().model(),"hasApiKey",!p.config().apiKey().isBlank())).toList());
"model",p.config().model(),"hasApiKey",!p.config().apiKey().isBlank(),"thinkingEnabled",p.config().thinkingEnabled(),"thinkingEffort",p.config().thinkingEffort())).toList());
return result;
}
private static boolean thinkingFlag(Map<?,?> value) {
if (!value.containsKey("thinkingEnabled")) return true;
if (!(value.get("thinkingEnabled") instanceof Boolean enabled)) throw bad("思考开关无效");
return enabled;
}
private static String thinkingEffort(Map<?,?> value) {
Object effort = value.get("thinkingEffort");
if (effort == null) return "medium";
if (!(effort instanceof String text) || !List.of("auto","low","medium","high").contains(text)) throw bad("思考强度无效");
return text;
}
private static ResponseStatusException bad(String message) { return new ResponseStatusException(HttpStatus.BAD_REQUEST, message); }
public record Settings(String baseUrl, String apiKey, String model, String revision, String profileId) {
public record Settings(String baseUrl, String apiKey, String model, String revision, String profileId, Boolean thinkingEnabled, String thinkingEffort) {
public Settings(String baseUrl,String apiKey,String model,String revision,String profileId) { this(baseUrl,apiKey,model,revision,profileId,null,null); }
public Settings(String baseUrl,String apiKey,String model,String revision) { this(baseUrl,apiKey,model,revision,null); }
@Override public String toString() { return "Settings[credentials redacted]"; }
}
private record Profile(String id,String name,Config config) { }
private record Config(String baseUrl, String apiKey, String model, String revision) {
private record Config(String baseUrl, String apiKey, String model, String revision, boolean thinkingEnabled, String thinkingEffort) {
Config(String baseUrl,String apiKey,String model,String revision) { this(baseUrl,apiKey,model,revision,true,"medium"); }
@Override public String toString() { return "Config[credentials redacted]"; }
}
}
@@ -0,0 +1,28 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import java.util.Map;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/** Release-bound full-population comparison endpoint used by the analysis runtime. */
@RestController
@RequestMapping("/api/analysis")
public class AnalysisChangesController {
private final AnalysisChangesService changes;
public AnalysisChangesController(AnalysisChangesService changes) {
this.changes = changes;
}
@PostMapping("/changes")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST')")
public Map<String, Object> query(@RequestBody Map<String, Object> body,
@AuthenticationPrincipal AppUserPrincipal actor) {
return changes.query(body, actor);
}
}
@@ -0,0 +1,825 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.filters.SourceColumnQueries;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import cn.yunsheng.quality.metrics.MetricsResponseAccess;
import cn.yunsheng.quality.metrics.MetricsService;
import cn.yunsheng.quality.metrics.MetricsService.ComparisonSnapshots;
import cn.yunsheng.quality.metrics.MetricsService.MetricRow;
import cn.yunsheng.quality.metrics.MetricsService.MetricSnapshot;
import cn.yunsheng.quality.metrics.MetricsService.MetricStage;
import cn.yunsheng.quality.metrics.MetricsService.MetricsQuery;
import cn.yunsheng.quality.options.OptionsController;
import java.math.BigDecimal;
import java.math.MathContext;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
/** Builds release-bound, full-population period-change rankings from official MetricsService rows. */
@Service
public class AnalysisChangesService {
private static final MathContext RATE_CONTEXT = MathContext.DECIMAL128;
private static final Set<String> DIMENSIONS = Set.of("group", "department", "product", "projectGroup");
private static final Set<String> SORTS = Set.of("rate_drop", "current_rate", "scrap_delta", "current_scrap");
private static final Set<String> FILTER_KEYS = Set.of("department", "group", "product", "projectGroup",
"includeUnassigned", "extraFilters", "exclusions", "columnFilters");
private static final Set<String> REQUEST_KEYS = Set.of("releaseId", "period", "comparisonPeriod", "filters",
"dimension", "stage", "sort", "limit", "fields");
private final MetricsService metrics;
private final MetricsResponseAccess responseAccess;
private final GovernanceAccessService governance;
private final OptionsController options;
public AnalysisChangesService(MetricsService metrics, MetricsResponseAccess responseAccess,
GovernanceAccessService governance, OptionsController options) {
this.metrics = metrics;
this.responseAccess = responseAccess;
this.governance = governance;
this.options = options;
}
public Map<String, Object> query(Map<String, Object> body, AppUserPrincipal actor) {
requireDetailAccess(actor);
Request request = parse(body);
Set<String> allowedFields = governance.allowedFields(actor);
requireSortFields(request, allowedFields);
requireRequestedFieldAccess(request, allowedFields);
Map<String, String> groupDepartments = Map.of();
Map<String, Object> groupCatalog = Map.of();
if ("group".equals(request.dimension())) {
groupCatalog = options.options(request.releaseId(), actor);
groupDepartments = groupDepartments(groupCatalog.get("groupDepartments"));
validateGroupSelection(request, groupDepartments);
}
MetricsQuery currentQuery = metricsQuery(request.currentPeriod(), request.filters(), request.dimension(), request.releaseId());
MetricsQuery comparisonQuery = metricsQuery(request.comparisonPeriod(), request.filters(), request.dimension(), request.releaseId());
ComparisonSnapshots snapshots = metrics.calculateComparisonBatch(currentQuery, comparisonQuery, groupDepartments, actor);
MetricSnapshot current = responseAccess.visible(snapshots.current(), actor);
MetricSnapshot comparison = responseAccess.visible(snapshots.comparison(), actor);
return compose(request, current, comparison, snapshots.current(), snapshots.comparison(), allowedFields,
groupDepartments, groupCatalog);
}
private Map<String, Object> compose(Request request, MetricSnapshot currentVisible, MetricSnapshot comparisonVisible,
MetricSnapshot currentRaw, MetricSnapshot comparisonRaw,
Set<String> allowedFields, Map<String, String> groupDepartments,
Map<String, Object> groupCatalog) {
Map<String, MetricRow> currentRows = indexRows(currentVisible, request.dimension(), "本期");
Map<String, MetricRow> comparisonRows = indexRows(comparisonVisible, request.dimension(), "对照期");
Map<String, MetricRow> currentRawRows = indexRows(currentRaw, request.dimension(), "本期");
Map<String, MetricRow> comparisonRawRows = indexRows(comparisonRaw, request.dimension(), "对照期");
TreeMap<String, Boolean> keys = new TreeMap<>();
currentRows.keySet().forEach(key -> keys.put(key, Boolean.TRUE));
comparisonRows.keySet().forEach(key -> keys.put(key, Boolean.TRUE));
boolean rateRequested = request.fields().contains("rate");
CoverageStatus currentRateCoverage = rateCoverage(currentRaw, request.currentPeriod(), request.stage());
CoverageStatus comparisonRateCoverage = rateCoverage(comparisonRaw, request.comparisonPeriod(), request.stage());
boolean ratesComparable = rateRequested && currentRateCoverage.available() && comparisonRateCoverage.available();
String rateReason = !rateRequested ? "rate 未包含在本次请求字段中。"
: currentRateCoverage.available() ? comparisonRateCoverage.reason() : currentRateCoverage.reason();
List<ChangeRow> records = new ArrayList<>(keys.size());
for (String key : keys.keySet()) {
MetricRow nowRow = currentRows.get(key);
MetricRow oldRow = comparisonRows.get(key);
MetricRow nowRawRow = currentRawRows.get(key);
MetricRow oldRawRow = comparisonRawRows.get(key);
boolean nowPresent = nowRow != null;
boolean oldPresent = oldRow != null;
String type = !oldPresent ? "new" : !nowPresent ? "exit" : "continuing";
BigDecimal nowRate = rateRequested ? rowRate(nowRawRow, request.stage(), nowPresent && ratesComparable) : null;
BigDecimal oldRate = rateRequested ? rowRate(oldRawRow, request.stage(), oldPresent && ratesComparable) : null;
BigDecimal rateDrop = nowRate == null || oldRate == null ? null : oldRate.subtract(nowRate, RATE_CONTEXT);
BigDecimal nowScrap = request.fields().contains("scrapWeight") ? rowScrap(nowRow, request.stage(), allowedFields) : null;
BigDecimal oldScrap = request.fields().contains("scrapWeight") ? rowScrap(oldRow, request.stage(), allowedFields) : null;
BigDecimal scrapDelta = null;
if (request.fields().contains("scrapWeight") && canReadScrap(request.stage(), allowedFields) && (nowPresent || oldPresent)) {
BigDecimal deltaNow = nowPresent ? nowScrap : BigDecimal.ZERO;
BigDecimal deltaOld = oldPresent ? oldScrap : BigDecimal.ZERO;
if (nowPresent ? nowScrap != null : true) {
if (oldPresent ? oldScrap != null : true) scrapDelta = deltaNow.subtract(deltaOld, RATE_CONTEXT);
}
}
Map<String, Object> entityFilters = entityFilters(request.filters().echo(), request.dimension(), key, groupDepartments);
records.add(new ChangeRow(key, labelFor(nowRow, oldRow, key), entityFilters,
rowSide(nowRow, nowRawRow, request.stage(), request.fields(), allowedFields, currentRateCoverage.available()),
rowSide(oldRow, oldRawRow, request.stage(), request.fields(), allowedFields, comparisonRateCoverage.available()),
rateDrop, scrapDelta, nowScrap, nowPresent, oldPresent, type));
}
Comparator<ChangeRow> ranking = rankingComparator(request.sort());
records.sort(ranking.thenComparing(ChangeRow::key));
List<Map<String, Object>> allRows = new ArrayList<>(records.size());
for (int index = 0; index < records.size(); index++) allRows.add(records.get(index).asMap(index + 1));
WeightCoverage weightCoverage = missingWeightCoverage(request, currentRaw, comparisonRaw);
CoverageStatus currentCompositeCoverage = rateCoverage(currentRaw, request.currentPeriod(), null);
CoverageStatus comparisonCompositeCoverage = rateCoverage(comparisonRaw, request.comparisonPeriod(), null);
Delta delta = rateRequested ? formalDelta(currentRaw, comparisonRaw, currentCompositeCoverage, comparisonCompositeCoverage)
: new Delta(false, null, null, "rate 未包含在本次请求字段中。");
Map<String, Object> decomposition = rateRequested
? stageDecomposition(currentRaw, comparisonRaw, delta, request.currentPeriod(), request.comparisonPeriod())
: unavailableDecomposition("rate 未包含在本次请求字段中。");
int totalRows = allRows.size();
int returned = Math.min(request.limit(), totalRows);
Map<String, Object> result = new LinkedHashMap<>();
result.put("kind", "quality_changes");
result.put("release", currentVisible.release());
result.put("releaseId", request.releaseId());
result.put("currentPeriod", request.currentPeriod().asMap());
result.put("comparisonPeriod", request.comparisonPeriod().asMap());
result.put("filters", request.filters().echo());
result.put("dimension", request.dimension());
result.put("stage", request.stage());
result.put("fields", request.fields());
result.put("sort", request.sort());
result.put("rows", List.copyOf(allRows.subList(0, returned)));
result.put("totalRows", totalRows);
result.put("returnedRows", returned);
result.put("truncated", totalRows > returned);
result.put("completeRanking", weightCoverage.count() == 0);
result.put("missingCoverage", weightCoverage.items());
result.put("missingCoverageCount", weightCoverage.count());
result.put("missingCoverageTruncated", weightCoverage.count() > weightCoverage.items().size());
result.put("current", comparisonSide(currentVisible, currentRaw, request.currentPeriod(), allowedFields, request.fields(), request.stage()));
result.put("comparison", comparisonSide(comparisonVisible, comparisonRaw, request.comparisonPeriod(), allowedFields, request.fields(), request.stage()));
result.put("delta", delta.asMap());
result.put("stageDecomposition", decomposition);
Map<String, Object> coverage = new LinkedHashMap<>();
coverage.put("current", safeCoverage(currentVisible.coverage()));
coverage.put("comparison", safeCoverage(comparisonVisible.coverage()));
coverage.put("rateComparable", ratesComparable);
coverage.put("rateUnavailableReason", rateReason);
result.put("coverage", coverage);
if ("product".equals(request.dimension())) result.put("prioritySignals", prioritySignals(records, allowedFields, request.stage(), request.fields()));
if ("group".equals(request.dimension())) result.put("groupOwnership", Map.of(
"basis", "仅将每个业务分组的发布目录产业部归属作为精确筛选;未配置归属的分组不参与本次排名。",
"unownedGroupsExcluded", unownedGroupCount(groupCatalog.get("groupDepartments"))));
result.put("basis", "变化行由同一发布版本的 MetricsService 正式聚合结果配对;先对完整分组集合排序,再截取 Top。新进或退出分组的不存在期率为 null;重量增量只把不存在对象按零作为描述性基线。");
return result;
}
private Map<String, Object> comparisonSide(MetricSnapshot snapshot, MetricSnapshot validationSnapshot, Period period,
Set<String> allowedFields, List<String> requestedFields,
String selectedStage) {
MetricRow summary = snapshot.summary();
CoverageStatus blackCoverage = rateCoverage(validationSnapshot, period, "black");
CoverageStatus brightCoverage = rateCoverage(validationSnapshot, period, "bright");
CoverageStatus compositeCoverage = rateCoverage(validationSnapshot, period, null);
Map<String, Object> side = new LinkedHashMap<>();
side.put("period", period.asMap());
side.put("compositeRate", requestedFields.contains("rate") && summary != null ? summary.compositeRate() : null);
side.put("black", stageMap(summary == null ? null : summary.black(), blackCoverage.available(), allowedFields,
requestedFields, selectedStage, "black"));
side.put("bright", stageMap(summary == null ? null : summary.bright(), brightCoverage.available(), allowedFields,
requestedFields, selectedStage, "bright"));
if (requestedFields.contains("rate"))
side.put("rateSupport", rateSupport(validationSnapshot.summary(), null, compositeCoverage.available()));
side.put("coverage", safeCoverage(snapshot.coverage()));
return side;
}
private Map<String, Object> stageMap(MetricStage stage, boolean rateMeasured, Set<String> allowedFields,
List<String> requestedFields, String selectedStage, String name) {
if (stage == null) return null;
boolean stageSelected = selectedStage == null || selectedStage.equals(name);
boolean totalRequested = stageSelected && requestedFields.contains("totalWeight");
boolean scrapRequested = stageSelected && requestedFields.contains("scrapWeight");
Map<String, Object> result = new LinkedHashMap<>();
result.put("rate", requestedFields.contains("rate") && rateMeasured ? stage.rate() : null);
result.put("scrapRate", requestedFields.contains("rate") && scrapRequested && totalRequested
&& allowedField(allowedFields, name + "_scrap_weight") && allowedField(allowedFields, name + "_total_weight")
? stage.scrapRate() : null);
result.put("qualifiedWeight", null);
result.put("scrapWeight", scrapRequested && allowedField(allowedFields, name + "_scrap_weight") ? stage.scrapWeight() : null);
result.put("totalWeight", totalRequested && allowedField(allowedFields, name + "_total_weight") ? stage.totalWeight() : null);
result.put("headquartersScrapWeight", null);
result.put("rateWithoutHeadquartersScrap", null);
result.put("defects", List.of());
return result;
}
private Map<String, Object> rowSide(MetricRow row, MetricRow validationRow, String selectedStage,
List<String> requestedFields, Set<String> allowedFields,
boolean periodCoverageAvailable) {
Map<String, Object> result = new LinkedHashMap<>();
BigDecimal rate = !requestedFields.contains("rate") || row == null || validationRow == null || !periodCoverageAvailable
|| rowRate(validationRow, selectedStage, true) == null ? null
: selectedStage == null ? row.compositeRate() : stage(row, selectedStage).rate();
result.put("rate", rate);
result.put("black", stageMap(row == null ? null : row.black(), validationRow != null && validRowStage(validationRow, "black"),
allowedFields, requestedFields, selectedStage, "black"));
result.put("bright", stageMap(row == null ? null : row.bright(), validationRow != null && validRowStage(validationRow, "bright"),
allowedFields, requestedFields, selectedStage, "bright"));
if (requestedFields.contains("rate")) result.put("rateSupport", rateSupport(validationRow, selectedStage, periodCoverageAvailable));
result.put("blackInputWeight", row == null || !requestedFields.contains("totalWeight")
|| (selectedStage != null && !selectedStage.equals("black")) || !allowedField(allowedFields, "black_total_weight")
? null : row.black() == null ? null : row.black().totalWeight());
result.put("brightInputWeight", row == null || !requestedFields.contains("totalWeight")
|| (selectedStage != null && !selectedStage.equals("bright")) || !allowedField(allowedFields, "bright_total_weight")
? null : row.bright() == null ? null : row.bright().totalWeight());
result.put("scrapWeight", row == null || !requestedFields.contains("scrapWeight") ? null : rowScrap(row, selectedStage, allowedFields));
return result;
}
private Map<String, Object> stageDecomposition(MetricSnapshot current, MetricSnapshot comparison, Delta delta,
Period currentPeriod, Period comparisonPeriod) {
MetricRow now = current.summary();
MetricRow old = comparison.summary();
if (!delta.available()) return unavailableDecomposition(delta.reason());
CoverageStatus nowBlack = rateCoverage(current, currentPeriod, "black");
CoverageStatus oldBlack = rateCoverage(comparison, comparisonPeriod, "black");
CoverageStatus nowBright = rateCoverage(current, currentPeriod, "bright");
CoverageStatus oldBright = rateCoverage(comparison, comparisonPeriod, "bright");
if (!nowBlack.available() || !oldBlack.available() || !nowBright.available() || !oldBright.available())
return unavailableDecomposition("阶段来源覆盖不完整;正式综合率仍可引用,但缺少完整两阶段实测率,不能进行对称分解。");
BigDecimal blackNow = value(now == null ? null : now.black() == null ? null : now.black().rate());
BigDecimal brightNow = value(now == null ? null : now.bright() == null ? null : now.bright().rate());
BigDecimal blackOld = value(old == null ? null : old.black() == null ? null : old.black().rate());
BigDecimal brightOld = value(old == null ? null : old.bright() == null ? null : old.bright().rate());
BigDecimal blackCurrentTotal = value(now == null || now.black() == null ? null : now.black().totalWeight());
BigDecimal brightCurrentTotal = value(now == null || now.bright() == null ? null : now.bright().totalWeight());
BigDecimal blackComparisonTotal = value(old == null || old.black() == null ? null : old.black().totalWeight());
BigDecimal brightComparisonTotal = value(old == null || old.bright() == null ? null : old.bright().totalWeight());
if (!validRate(blackNow) || !validRate(brightNow) || !validRate(blackOld) || !validRate(brightOld)
|| !positive(blackCurrentTotal) || !positive(brightCurrentTotal)
|| !positive(blackComparisonTotal) || !positive(brightComparisonTotal))
return unavailableDecomposition("至少一期缺少黑片或亮片的有效分母/正式阶段率;不会把缺失阶段默认值当作实测率。");
BigDecimal blackContribution = blackNow.subtract(blackOld, RATE_CONTEXT)
.multiply(brightNow.add(brightOld, RATE_CONTEXT), RATE_CONTEXT)
.divide(BigDecimal.valueOf(2), RATE_CONTEXT).multiply(BigDecimal.valueOf(100), RATE_CONTEXT);
BigDecimal brightContribution = brightNow.subtract(brightOld, RATE_CONTEXT)
.multiply(blackNow.add(blackOld, RATE_CONTEXT), RATE_CONTEXT)
.divide(BigDecimal.valueOf(2), RATE_CONTEXT).multiply(BigDecimal.valueOf(100), RATE_CONTEXT);
BigDecimal sum = blackContribution.add(brightContribution, RATE_CONTEXT);
if (delta.percentagePoints() == null || sum.subtract(delta.percentagePoints(), RATE_CONTEXT).abs().compareTo(new BigDecimal("0.0000001")) > 0)
return unavailableDecomposition("阶段乘积拆解与正式综合率差不一致;不将差异归因到阶段。");
Map<String, Object> result = new LinkedHashMap<>();
result.put("status", "available");
result.put("blackContributionToCompositePP", blackContribution);
result.put("brightContributionToCompositePP", brightContribution);
result.put("sumPP", sum);
result.put("method", "black_delta*(bright_current+bright_comparison)/2; bright_delta*(black_current+black_comparison)/2");
result.put("meaning", "对正式综合率变化的对称数学分解;不是生产根因。");
return result;
}
private Map<String, Object> unavailableDecomposition(String reason) {
Map<String, Object> result = new LinkedHashMap<>();
result.put("status", "unavailable");
result.put("blackContributionToCompositePP", null);
result.put("brightContributionToCompositePP", null);
result.put("sumPP", null);
result.put("reason", reason);
return result;
}
private Delta formalDelta(MetricSnapshot current, MetricSnapshot comparison,
CoverageStatus currentCoverage, CoverageStatus comparisonCoverage) {
MetricRow now = current.summary();
MetricRow old = comparison.summary();
BigDecimal nowRate = now == null ? null : now.compositeRate();
BigDecimal oldRate = old == null ? null : old.compositeRate();
if (!currentCoverage.available() || !comparisonCoverage.available() || !validRate(nowRate) || !validRate(oldRate)) {
String reason = currentCoverage.available() ? comparisonCoverage.reason() : currentCoverage.reason();
if (reason == null || reason.isBlank()) reason = "正式compositeRate缺失或不合法。";
return new Delta(false, null, null, reason);
}
BigDecimal difference = nowRate.subtract(oldRate, RATE_CONTEXT);
return new Delta(true, difference, difference.multiply(BigDecimal.valueOf(100), RATE_CONTEXT), null);
}
private WeightCoverage missingWeightCoverage(Request request, MetricSnapshot current, MetricSnapshot comparison) {
if (!request.fields().contains("totalWeight") && !request.fields().contains("scrapWeight"))
return new WeightCoverage(List.of(), 0);
List<Map<String, Object>> sample = new ArrayList<>();
int missing = 0;
List<String> stages = request.stage() == null ? List.of("black", "bright") : List.of(request.stage());
for (var side : List.of(Map.entry("current", current), Map.entry("comparison", comparison))) {
String sideName = side.getKey();
MetricSnapshot snapshot = side.getValue();
for (String stageName : stages) {
if (stageHasSource(snapshot, request.currentPeriod(), request.comparisonPeriod(), sideName, stageName)) {
for (String field : List.of("totalWeight", "scrapWeight")) {
if (!request.fields().contains(field)) continue;
if (weight(snapshot.summary() == null ? null : stage(snapshot.summary(), stageName), field) == null)
missing = addMissingWeight(sample, missing, sideName, "overall", stageName, field);
}
}
for (MetricRow row : snapshot.rows()) {
if (stageRows(row, stageName) <= 0) continue;
for (String field : List.of("totalWeight", "scrapWeight")) {
if (!request.fields().contains(field)) continue;
if (weight(stage(row, stageName), field) == null)
missing = addMissingWeight(sample, missing, sideName, row.key(), stageName, field);
}
}
}
}
return new WeightCoverage(List.copyOf(sample), missing);
}
private boolean stageHasSource(MetricSnapshot snapshot, Period currentPeriod, Period comparisonPeriod,
String side, String stage) {
Period period = "current".equals(side) ? currentPeriod : comparisonPeriod;
if (snapshot == null || snapshot.coverage() == null || !(snapshot.coverage().get("sourceMonths") instanceof List<?> months)) return false;
Set<Integer> expected = Set.copyOf(period.months());
for (Object raw : months) {
if (!(raw instanceof Map<?, ?> row) || !Objects.equals(integer(row.get("year")), period.year())) continue;
Integer month = integer(row.get("month"));
Long count = longValue(row.get(stage + "Rows"));
if (month != null && expected.contains(month) && count != null && count > 0) return true;
}
return false;
}
private int addMissingWeight(List<Map<String, Object>> sample, int count, String side,
String key, String stage, String field) {
if (sample.size() < 100) sample.add(Map.of("side", side, "key", key, "stage", stage, "field", field));
return count + 1;
}
private BigDecimal weight(MetricStage stage, String field) {
if (stage == null) return null;
return "totalWeight".equals(field) ? stage.totalWeight() : stage.scrapWeight();
}
private CoverageStatus rateCoverage(MetricSnapshot snapshot, Period period, String stage) {
if (snapshot == null || snapshot.coverage() == null)
return new CoverageStatus(false, "接口未返回分阶段月份覆盖。");
Object rawMonths = snapshot.coverage().get("sourceMonths");
if (!(rawMonths instanceof List<?> months)) return new CoverageStatus(false, "接口未返回分阶段月份覆盖。");
Map<Integer, Map<String, Object>> observed = new LinkedHashMap<>();
for (Object raw : months) {
if (!(raw instanceof Map<?, ?> item)) continue;
Integer year = integer(item.get("year"));
Integer month = integer(item.get("month"));
if (year == null || !year.equals(period.year()) || month == null || month < 1 || month > 12) continue;
if (observed.putIfAbsent(month, stringMap(item)) != null)
return new CoverageStatus(false, period.year() + "-" + month + "存在重复的阶段来源覆盖。");
}
List<Integer> expected = period.months();
for (int month : expected) {
Map<String, Object> value = observed.get(month);
if (value == null) return new CoverageStatus(false, "所选期间缺少来源月份:" + period.year() + "-" + String.format("%02d", month));
Long blackRows = longValue(value.get("blackRows"));
Long brightRows = longValue(value.get("brightRows"));
if (blackRows == null || blackRows < 0 || brightRows == null || brightRows < 0)
return new CoverageStatus(false, period.year() + "-" + month + "阶段来源覆盖未知或不合法。");
if (stage != null) {
long count = "black".equals(stage) ? blackRows : brightRows;
if (count <= 0) return new CoverageStatus(false, "所选期间存在缺失的" + ("black".equals(stage) ? "黑片" : "亮片")
+ "来源月份:" + period.year() + "-" + String.format("%02d", month));
} else if (blackRows == 0 && brightRows == 0) {
return new CoverageStatus(false, period.year() + "-" + month + "两阶段均无有效来源。");
}
}
MetricRow summary = snapshot.summary();
if (stage != null) {
MetricStage selected = summary == null ? null : stage(summary, stage);
if (selected == null || !positive(selected.totalWeight()) || !nonNegative(selected.qualifiedWeight()) || !validRate(selected.rate()))
return new CoverageStatus(false, "所选阶段正式率缺失、分母无效或不合法。");
return new CoverageStatus(true, null);
}
BigDecimal composite = summary == null ? null : summary.compositeRate();
if (!validRate(composite)) return new CoverageStatus(false, "服务端正式compositeRate缺失或不合法。");
boolean positiveDenominator = false;
for (String candidate : List.of("black", "bright")) {
boolean hasSource = expected.stream().anyMatch(month -> {
Map<String, Object> value = observed.get(month);
return longValue(value.get(candidate + "Rows")) != null && longValue(value.get(candidate + "Rows")) > 0;
});
MetricStage metric = summary == null ? null : stage(summary, candidate);
if (hasSource && metric != null && positive(metric.totalWeight())) positiveDenominator = true;
}
if (!positiveDenominator) return new CoverageStatus(false, "没有来源有效且总投入大于0的阶段,不能引用综合率。");
return new CoverageStatus(true, null);
}
private BigDecimal rowRate(MetricRow row, String selectedStage, boolean globalCoverage) {
if (row == null || !globalCoverage) return null;
if (selectedStage != null) {
MetricStage selected = stage(row, selectedStage);
if (stageRows(row, selectedStage) <= 0 || !positive(selected == null ? null : selected.totalWeight())
|| !nonNegative(selected == null ? null : selected.qualifiedWeight())
|| !validRate(selected == null ? null : selected.rate())) return null;
return selected.rate();
}
if (!validRowStage(row, "black") && !validRowStage(row, "bright")) return null;
return validRate(row.compositeRate()) ? row.compositeRate() : null;
}
private Map<String, Object> rateSupport(MetricRow row, String selectedStage, boolean scopeCoverageAvailable) {
boolean blackMeasured = validRowStage(row, "black");
boolean brightMeasured = validRowStage(row, "bright");
List<String> missingStages = new ArrayList<>();
if (!blackMeasured) missingStages.add("black");
if (!brightMeasured) missingStages.add("bright");
Map<String, Object> result = new LinkedHashMap<>();
result.put("basis", selectedStage == null ? "official_composite" : "stage_rate");
result.put("compositeReferenceAvailable", selectedStage == null && scopeCoverageAvailable
&& (blackMeasured || brightMeasured) && row != null && validRate(row.compositeRate()));
result.put("blackMeasured", blackMeasured);
result.put("brightMeasured", brightMeasured);
result.put("missingStages", List.copyOf(missingStages));
result.put("meaning", "正式综合率直接采用服务端指标;缺失阶段保留为未实测,不当作实测100%。");
return result;
}
private boolean validRowStage(MetricRow row, String name) {
MetricStage stage = stage(row, name);
return stageRows(row, name) > 0 && stage != null && positive(stage.totalWeight())
&& nonNegative(stage.qualifiedWeight()) && validRate(stage.rate());
}
private long stageRows(MetricRow row, String name) {
if (row == null || row.coverage() == null) return 0;
Long value = row.coverage().get(name + "Rows");
return value == null ? 0 : value;
}
private BigDecimal rowScrap(MetricRow row, String selectedStage, Set<String> allowedFields) {
if (row == null) return null;
if (selectedStage != null) {
if (!allowedField(allowedFields, selectedStage + "_scrap_weight")) return null;
MetricStage selected = stage(row, selectedStage);
return selected == null ? null : selected.scrapWeight();
}
if (!allowedField(allowedFields, "black_scrap_weight") || !allowedField(allowedFields, "bright_scrap_weight")
|| row.black() == null || row.bright() == null || row.black().scrapWeight() == null || row.bright().scrapWeight() == null)
return null;
return row.black().scrapWeight().add(row.bright().scrapWeight(), RATE_CONTEXT);
}
private boolean canReadScrap(String selectedStage, Set<String> allowedFields) {
return selectedStage == null ? allowedField(allowedFields, "black_scrap_weight") && allowedField(allowedFields, "bright_scrap_weight")
: allowedField(allowedFields, selectedStage + "_scrap_weight");
}
private Map<String, Object> prioritySignals(List<ChangeRow> records, Set<String> allowedFields, String stage,
List<String> requestedFields) {
Map<String, Object> result = new LinkedHashMap<>();
if (requestedFields.contains("scrapWeight") && canReadScrap(stage, allowedFields)) {
result.put("scrapIncrease", signalRows(records.stream().filter(row -> row.scrapDelta() != null && row.scrapDelta().signum() > 0)
.sorted(Comparator.comparing(ChangeRow::scrapDelta).reversed().thenComparing(ChangeRow::key)).toList()));
result.put("currentScrap", signalRows(records.stream().filter(row -> row.currentScrap() != null && row.currentScrap().signum() > 0)
.sorted(Comparator.comparing(ChangeRow::currentScrap).reversed().thenComparing(ChangeRow::key)).toList()));
} else result.put("currentScrap", List.of());
if (!result.containsKey("scrapIncrease")) result.put("scrapIncrease", List.of());
if (requestedFields.contains("rate")) {
result.put("rateDrop", signalRows(records.stream().filter(row -> row.rateDrop() != null && row.rateDrop().signum() > 0)
.sorted(Comparator.comparing(ChangeRow::rateDrop).reversed().thenComparing(ChangeRow::key)).toList()));
} else result.put("rateDrop", List.of());
result.put("basis", "报废增量、本期报废量、可比率下降分别排序,不合成贡献分数;新进或退出对象的率变化为空。");
return result;
}
private List<Map<String, Object>> signalRows(List<ChangeRow> rows) {
return rows.stream().limit(3).map(row -> {
Map<String, Object> item = new LinkedHashMap<>();
item.put("key", row.key());
item.put("label", row.label());
item.put("changeType", row.changeType());
item.put("scrapDelta", row.scrapDelta());
item.put("rateDrop", row.rateDrop());
item.put("current", row.current());
item.put("comparison", row.comparison());
return item;
}).toList();
}
private Comparator<ChangeRow> rankingComparator(String sort) {
return switch (sort) {
case "rate_drop" -> Comparator.comparing(ChangeRow::rateDrop, Comparator.nullsLast(Comparator.reverseOrder()));
case "current_rate" -> Comparator.<ChangeRow, BigDecimal>comparing(
row -> (BigDecimal) row.current().get("rate"), Comparator.nullsLast(Comparator.naturalOrder()));
case "scrap_delta" -> Comparator.comparing(ChangeRow::scrapDelta, Comparator.nullsLast(Comparator.reverseOrder()));
case "current_scrap" -> Comparator.comparing(ChangeRow::currentScrap, Comparator.nullsLast(Comparator.reverseOrder()));
default -> throw bad("sort 只支持 rate_drop、current_rate、scrap_delta、current_scrap");
};
}
private Map<String, Object> entityFilters(Map<String, Object> root, String dimension, String key,
Map<String, String> groupDepartments) {
Map<String, Object> result = new LinkedHashMap<>(root);
result.put(dimension, key);
if ("group".equals(dimension)) {
String owner = groupDepartments == null ? null : groupDepartments.get(key);
if (owner == null || owner.isBlank()) throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "业务分组的产业部归属未确认");
result.put("department", owner);
}
return result;
}
private Map<String, MetricRow> indexRows(MetricSnapshot snapshot, String dimension, String side) {
if (snapshot == null || snapshot.rows() == null) throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, side + "正式rows缺失,不能生成全量变化排名。");
if (snapshot.rows().size() > 100_000) throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, side + "分组行超过安全上限100000,请收窄筛选。");
Map<String, MetricRow> index = new LinkedHashMap<>();
for (MetricRow row : snapshot.rows()) {
if (row == null || row.key() == null || row.key().isBlank())
throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, side + "正式rows存在空key,不能全外连接配对。");
if (index.putIfAbsent(row.key(), row) != null)
throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, side + "正式rows存在重复key,不能全外连接配对。");
}
return index;
}
private String labelFor(MetricRow current, MetricRow comparison, String key) {
MetricRow source = current == null ? comparison : current;
return source == null || source.label() == null ? key : source.label();
}
private MetricStage stage(MetricRow row, String name) {
if (row == null) return null;
return "black".equals(name) ? row.black() : row.bright();
}
private void requireDetailAccess(AppUserPrincipal actor) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "变化排名必须绑定当前有效用户");
governance.requireUserActivePolicy(actor);
if (!Boolean.TRUE.equals(governance.policy(actor.getId().toString()).get("detailAllowed")))
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "当前身份仅允许汇总,不能查看分组变化排名");
governance.requireBoundAction("QUERY", "query.detail");
}
private void requireSortFields(Request request, Set<String> allowedFields) {
boolean rateSort = "rate_drop".equals(request.sort()) || "current_rate".equals(request.sort());
boolean scrapSort = "scrap_delta".equals(request.sort()) || "current_scrap".equals(request.sort());
if (rateSort && !request.fields().contains("rate"))
throw bad("rate_drop/current_rate 排序要求 fields 包含 rate");
if (scrapSort && !request.fields().contains("scrapWeight"))
throw bad("scrap_delta/current_scrap 排序要求 fields 包含 scrapWeight");
}
private void requireRequestedFieldAccess(Request request, Set<String> allowedFields) {
List<String> stages = request.stage() == null ? List.of("black", "bright") : List.of(request.stage());
for (String field : request.fields()) {
String suffix = switch (field) {
case "totalWeight" -> "_total_weight";
case "scrapWeight" -> "_scrap_weight";
default -> null;
};
if (suffix == null) continue;
for (String stage : stages) if (!allowedField(allowedFields, stage + suffix))
throw new ResponseStatusException(HttpStatus.FORBIDDEN,
"当前身份没有读取请求指标 " + field + " 的" + ("black".equals(stage) ? "黑片" : "亮片") + "字段权限");
}
}
private void validateGroupSelection(Request request, Map<String, String> owners) {
for (String group : SelectionValues.parse(request.filters().department("group"))) {
String owner = owners.get(group);
if (owner == null || owner.isBlank()) throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "业务分组的产业部归属未确认");
List<String> departments = SelectionValues.parse(request.filters().department("department"));
if (!departments.isEmpty() && !departments.contains(owner))
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "业务分组不属于筛选中的产业部");
if (request.filters().exclusions().getOrDefault("department", List.of()).stream()
.anyMatch(excluded -> excluded.equals(owner) || excluded.equals(DepartmentLabels.queryValue(owner))))
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "业务分组产业部已被排除");
}
}
private Map<String, String> groupDepartments(Object raw) {
if (!(raw instanceof Map<?, ?> map)) throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "授权目录缺少业务分组产业部归属");
Map<String, String> result = new LinkedHashMap<>();
for (Map.Entry<?, ?> entry : map.entrySet()) {
if (entry.getKey() instanceof String group && entry.getValue() instanceof String owner && !owner.isBlank())
result.put(group, owner);
}
return result;
}
private int unownedGroupCount(Object raw) {
if (!(raw instanceof Map<?, ?> map)) return 0;
return (int) map.values().stream().filter(value -> !(value instanceof String owner) || owner.isBlank()).count();
}
private MetricsQuery metricsQuery(Period period, FilterSet filters, String dimension, String releaseId) {
MetricsQuery query = new MetricsQuery(period.year(), period.month(), period.quarter(),
filters.department("department"), filters.department("group"), filters.department("product"),
filters.department("projectGroup"), dimension, filters.includeUnassigned(), releaseId, "year",
filters.exclusions(), filters.extraFilters(), null)
.withMonthRange(period.monthStart(), period.monthEnd())
.withColumnFilters(filters.columnFilters());
return query;
}
private Request parse(Map<String, Object> body) {
if (body == null || !REQUEST_KEYS.containsAll(body.keySet())) throw bad("变化查询字段无效");
String releaseId = text(body.get("releaseId"), "releaseId");
Period current = period(body.get("period"), "period");
Period comparison = period(body.get("comparisonPeriod"), "comparisonPeriod");
if (current.equals(comparison)) throw bad("本期与对照期必须不同");
if (!(body.get("filters") instanceof Map<?, ?> rawFilters)) throw bad("filters 必须是筛选对象");
FilterSet filters = filters(rawFilters);
String dimension = text(body.get("dimension"), "dimension");
if (!DIMENSIONS.contains(dimension)) throw bad("dimension 只支持 group、department、product、projectGroup");
String stage = optionalText(body.get("stage"));
if (stage != null && !Set.of("black", "bright").contains(stage)) throw bad("stage 只支持 black 或 bright");
String sort = body.containsKey("sort") ? text(body.get("sort"), "sort") : "rate_drop";
if (!SORTS.contains(sort)) throw bad("sort 只支持 rate_drop、current_rate、scrap_delta、current_scrap");
int limit = body.containsKey("limit") ? integer(body.get("limit"), "limit") : 10;
if (limit < 1 || limit > 10) throw bad("limit 必须在 1 到 10 之间");
List<String> fields = fields(body.get("fields"), body.containsKey("fields"));
return new Request(releaseId, current, comparison, filters, dimension, stage, sort, limit, fields);
}
private List<String> fields(Object raw, boolean present) {
if (!present) return List.of("rate");
if (!(raw instanceof List<?> selected) || selected.isEmpty() || selected.size() > 3)
throw bad("fields 必须是包含 1 到 3 个指标名的数组");
List<String> result = new ArrayList<>();
for (Object value : selected) {
if (!(value instanceof String field) || !Set.of("rate", "totalWeight", "scrapWeight").contains(field))
throw bad("fields 只支持 rate、totalWeight、scrapWeight");
if (result.contains(field)) throw bad("fields 不能包含重复指标");
result.add(field);
}
return List.copyOf(result);
}
private FilterSet filters(Map<?, ?> raw) {
for (Object key : raw.keySet()) if (!(key instanceof String text) || !FILTER_KEYS.contains(text)) throw bad("filters 含不支持字段");
Map<String, Object> echo = new LinkedHashMap<>();
Map<String, String> dimensions = new LinkedHashMap<>();
for (String dimension : List.of("department", "group", "product", "projectGroup")) {
if (!raw.containsKey(dimension)) continue;
Object value = raw.get(dimension);
if (!(value instanceof String selected)) throw bad("filters." + dimension + " 必须为目录筛选字符串");
SelectionValues.parse(selected);
dimensions.put(dimension, selected);
echo.put(dimension, selected);
}
boolean includeUnassigned = false;
if (raw.containsKey("includeUnassigned")) {
if (!(raw.get("includeUnassigned") instanceof Boolean value)) throw bad("includeUnassigned 必须为布尔值");
includeUnassigned = value;
}
echo.put("includeUnassigned", includeUnassigned);
Map<String, List<String>> extraFilters = QueryFilterFields.parse(raw.get("extraFilters"));
Map<String, List<String>> exclusions = AnalysisExplorationTools.exclusions(raw.get("exclusions"));
List<Map<String, Object>> columnFilters = SourceColumnQueries.parse(raw.get("columnFilters"));
if (!extraFilters.isEmpty()) echo.put("extraFilters", extraFilters);
if (!exclusions.isEmpty()) echo.put("exclusions", exclusions);
if (!columnFilters.isEmpty()) echo.put("columnFilters", columnFilters);
return new FilterSet(dimensions, includeUnassigned, extraFilters, exclusions, columnFilters, echo);
}
private Period period(Object raw, String name) {
if (!(raw instanceof Map<?, ?> map)) throw bad(name + " 必须是期间对象");
Set<String> allowed = Set.of("year", "month", "quarter", "monthStart", "monthEnd");
for (Object key : map.keySet()) if (!(key instanceof String text) || !allowed.contains(text)) throw bad(name + " 含不支持字段");
Integer year = integer(map.get("year"), name + ".year");
if (year < 2000 || year > 2100) throw bad(name + ".year 必须在 2000 到 2100 之间");
boolean hasMonth = map.containsKey("month"), hasQuarter = map.containsKey("quarter");
boolean hasStart = map.containsKey("monthStart"), hasEnd = map.containsKey("monthEnd");
if (hasStart != hasEnd || (hasMonth && hasQuarter) || ((hasStart || hasEnd) && (hasMonth || hasQuarter)))
throw bad(name + " 的月份、季度或范围字段冲突");
Integer month = hasMonth ? integer(map.get("month"), name + ".month") : null;
Integer quarter = hasQuarter ? integer(map.get("quarter"), name + ".quarter") : null;
Integer start = hasStart ? integer(map.get("monthStart"), name + ".monthStart") : null;
Integer end = hasEnd ? integer(map.get("monthEnd"), name + ".monthEnd") : null;
if (month != null && (month < 1 || month > 12)) throw bad(name + ".month 必须在 1 到 12 之间");
if (quarter != null && (quarter < 1 || quarter > 4)) throw bad(name + ".quarter 必须在 1 到 4 之间");
if (hasStart && (start < 1 || end > 12 || start > end)) throw bad(name + " 月份范围必须满足1<=monthStart<=monthEnd<=12");
return new Period(year, month, quarter, start, end);
}
private Map<String, Object> safeCoverage(Map<String, Object> coverage) {
return coverage == null ? Map.of() : new LinkedHashMap<>(coverage);
}
private Map<String, Object> stringMap(Map<?, ?> source) {
Map<String, Object> result = new LinkedHashMap<>();
source.forEach((key, value) -> { if (key instanceof String text) result.put(text, value); });
return result;
}
private Integer integer(Object value, String name) {
Integer parsed = integer(value);
if (parsed == null) throw bad(name + " 必须是整数");
return parsed;
}
private Integer integer(Object value) {
if (!(value instanceof Number number) || value instanceof Boolean) return null;
long parsed = number.longValue();
if (number.doubleValue() != (double) parsed || parsed < Integer.MIN_VALUE || parsed > Integer.MAX_VALUE) return null;
return (int) parsed;
}
private Long longValue(Object value) {
if (!(value instanceof Number number) || value instanceof Boolean) return null;
long parsed = number.longValue();
return number.doubleValue() == (double) parsed ? parsed : null;
}
private String text(Object value, String name) {
if (!(value instanceof String text) || text.isBlank()) throw bad(name + " 必须是非空文本");
return text.trim();
}
private String optionalText(Object value) {
if (value == null) return null;
if (!(value instanceof String text)) throw bad("stage 必须为 black、bright 或 null");
return text.isBlank() ? null : text.trim();
}
private boolean allowedField(Set<String> fields, String field) {
return fields != null && (fields.contains("*") || fields.contains(field));
}
private String defectField(String stage, String name) {
return switch (stage + ":" + name) {
case "black:责废" -> "black_responsibility_scrap";
case "black:料废" -> "black_material_scrap";
case "black:外废" -> "black_external_scrap";
case "bright:镀废" -> "bright_plating_scrap";
case "bright:材料废" -> "bright_material_scrap";
case "bright:加工废" -> "bright_processing_scrap";
case "bright:扩散废" -> "bright_diffusion_scrap";
default -> "";
};
}
private boolean validRate(BigDecimal value) {
return value != null && value.compareTo(BigDecimal.ZERO) >= 0 && value.compareTo(BigDecimal.ONE) <= 0;
}
private boolean positive(BigDecimal value) { return value != null && value.signum() > 0; }
private boolean nonNegative(BigDecimal value) { return value != null && value.signum() >= 0; }
private BigDecimal value(BigDecimal value) { return value; }
private ResponseStatusException bad(String message) { return new ResponseStatusException(HttpStatus.BAD_REQUEST, message); }
private record Request(String releaseId, Period currentPeriod, Period comparisonPeriod, FilterSet filters,
String dimension, String stage, String sort, int limit, List<String> fields) {}
private record WeightCoverage(List<Map<String, Object>> items, int count) {}
private record FilterSet(Map<String, String> dimensions, boolean includeUnassigned,
Map<String, List<String>> extraFilters, Map<String, List<String>> exclusions,
List<Map<String, Object>> columnFilters, Map<String, Object> echo) {
String department(String key) { return dimensions.get(key); }
}
private record CoverageStatus(boolean available, String reason) {}
private record Delta(boolean available, BigDecimal rate, BigDecimal percentagePoints, String reason) {
Map<String, Object> asMap() {
Map<String, Object> result = new LinkedHashMap<>();
result.put("available", available); result.put("rate", rate); result.put("percentagePoints", percentagePoints); result.put("reason", reason);
return result;
}
}
private record ChangeRow(String key, String label, Map<String, Object> filters, Map<String, Object> current,
Map<String, Object> comparison, BigDecimal rateDrop, BigDecimal scrapDelta,
BigDecimal currentScrap, boolean currentPresent, boolean comparisonPresent,
String changeType) {
Map<String, Object> asMap(int rank) {
Map<String, Object> result = new LinkedHashMap<>();
result.put("rank", rank); result.put("key", key); result.put("label", label); result.put("filters", filters);
result.put("current", current); result.put("comparison", comparison); result.put("rateDrop", rateDrop);
result.put("scrapDelta", scrapDelta);
result.put("present", Map.of("current", currentPresent, "comparison", comparisonPresent));
result.put("changeType", changeType);
return result;
}
}
private record Period(int year, Integer month, Integer quarter, Integer monthStart, Integer monthEnd) {
Map<String, Integer> asMap() {
Map<String, Integer> result = new LinkedHashMap<>();
result.put("year", year);
if (month != null) result.put("month", month);
if (quarter != null) result.put("quarter", quarter);
if (monthStart != null) result.put("monthStart", monthStart);
if (monthEnd != null) result.put("monthEnd", monthEnd);
return result;
}
List<Integer> months() {
if (month != null) return List.of(month);
if (quarter != null) { int first = (quarter - 1) * 3 + 1; return List.of(first, first + 1, first + 2); }
int first = monthStart == null ? 1 : monthStart;
int last = monthEnd == null ? 12 : monthEnd;
List<Integer> result = new ArrayList<>();
for (int selected = first; selected <= last; selected++) result.add(selected);
return List.copyOf(result);
}
}
}
@@ -0,0 +1,33 @@
package cn.yunsheng.quality.analysis;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
/** Bounded user choices; selecting one still goes through the next turn's authorization. */
final class AnalysisClarificationRequest {
private AnalysisClarificationRequest() {}
static Map<String,Object> fromTask(Map<String,Object> task) {
String question=task.get("clarification") instanceof String s?s:"请补充本轮需要的信息。";
List<Map<String,String>> choices=new ArrayList<>();
var seen=new HashSet<String>();
Object candidates=null;
if(task.get("productResolution") instanceof Map<?,?> resolution
&&resolution.get("productMatch") instanceof Map<?,?> match
&&"ambiguous".equals(match.get("status")))candidates=match.get("candidates");
boolean products=candidates instanceof List<?>;
if(!products&&task.get("clarificationRequest") instanceof Map<?,?> request)candidates=request.get("options");
if(candidates instanceof List<?> items)for(Object raw:items.subList(0,Math.min(20,items.size()))) {
if(!(raw instanceof Map<?,?> item))continue;
Object rawValue=item.get(products?"name":"value"),rawLabel=item.get(products?"name":"label");
if(!(rawValue instanceof String value)||value.isBlank()||value.length()>300
||!(rawLabel instanceof String label)||label.isBlank()||label.length()>300||!seen.add(value))continue;
choices.add(Map.of("label",label,"value",value));
}
var result=new LinkedHashMap<String,Object>();result.put("question",question);
result.put("options",choices);result.put("allowCustom",true);return result;
}
}
@@ -7,6 +7,7 @@ import cn.yunsheng.quality.metrics.MetricsService.MetricsQuery;
import cn.yunsheng.quality.metrics.MetricsResponseAccess;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDate;
@@ -49,20 +50,58 @@ public class AnalysisController {
private final GovernanceAccessService governance;
private final MetricsResponseAccess responseAccess;
private final TransactionTemplate transaction;
private final AnalysisApiSettingsService apiSettings;
private AnalysisProtocolService protocol;
@org.springframework.beans.factory.annotation.Autowired
public void setProtocol(AnalysisProtocolService protocol){this.protocol=protocol;}
@org.springframework.beans.factory.annotation.Value("${quality.analysis.visuals-enabled:true}")
private boolean visualsEnabled = true;
@org.springframework.beans.factory.annotation.Autowired
public AnalysisController(MetricsService metrics, AnalysisModelGateway model, JdbcTemplate jdbc, JsonMapper mapper,
PlatformTransactionManager transactionManager, GovernanceAccessService governance,
MetricsResponseAccess responseAccess) {
MetricsResponseAccess responseAccess, AnalysisApiSettingsService apiSettings) {
this.metrics = metrics; this.model = model; this.jdbc = jdbc; this.mapper = mapper;
this.governance = governance;
this.responseAccess = responseAccess;
this.transaction = new TransactionTemplate(transactionManager);
this.apiSettings = apiSettings;
}
/** Retained for existing direct controller tests; production wiring uses the settings-aware constructor. */
public AnalysisController(MetricsService metrics, AnalysisModelGateway model, JdbcTemplate jdbc, JsonMapper mapper,
PlatformTransactionManager transactionManager, GovernanceAccessService governance,
MetricsResponseAccess responseAccess) {
this(metrics, model, jdbc, mapper, transactionManager, governance, responseAccess, null);
}
@GetMapping("/status")
public Map<String, Object> status() { return model.status(); }
public Map<String, Object> status() {
try {
if (apiSettings == null) return unconfiguredModelStatus();
Map<String, Object> metadata = apiSettings.metadata();
if (!Boolean.TRUE.equals(metadata.get("hasApiKey"))) return unconfiguredModelStatus();
Object modelValue = metadata.get("model");
Object baseUrlValue = metadata.get("baseUrl");
if (!(modelValue instanceof String configuredModel) || configuredModel.isBlank()
|| !(baseUrlValue instanceof String baseUrl) || baseUrl.isBlank())
return unconfiguredModelStatus();
String host = URI.create(baseUrl).getHost();
if (host == null || host.isBlank()) return unconfiguredModelStatus();
return Map.of("configured", true, "model", configuredModel, "serviceHost", host,
"mode", "NAO_AGENT_SERVER_TOOLS", "businessResultsSentToModel", true,
"authorizedQueryResultsMayBeSentToModel", true,
"capabilities", List.of("nativeNaoRuntime", "authorizedServerTools", "structuredQuery",
"statistics", "periodicReports"));
} catch (RuntimeException invalidOrUnavailable) {
return unconfiguredModelStatus();
}
}
private static Map<String, Object> unconfiguredModelStatus() {
return Map.of("configured", false, "mode", "NAO_NOT_CONFIGURED",
"businessResultsSentToModel", false, "authorizedQueryResultsMayBeSentToModel", false);
}
@GetMapping("/sessions")
public Map<String, Object> sessions(@AuthenticationPrincipal AppUserPrincipal actor) {
@@ -99,6 +138,7 @@ public class AnalysisController {
if (active != null && active > 0)
throw new ResponseStatusException(HttpStatus.CONFLICT, "会话仍有运行中的分析,请先停止后再删除");
jdbc.update("DELETE FROM analysis_report WHERE session_id=? AND owner_user_id=?", id, actor.getId().toString());
if(protocol!=null&&protocol.ownsSession(id))protocol.deleteArtifacts(id);
jdbc.update("DELETE FROM analysis_turn WHERE session_id=? AND user_id=?", id, actor.getId().toString());
jdbc.update("DELETE FROM analysis_session WHERE id=? AND user_id=?", id, actor.getId().toString());
return Map.of("deleted", true, "sessionId", id);
@@ -111,22 +151,31 @@ public class AnalysisController {
governance.requireBoundAction("QUERY", "query.detail");
Session session = ownedSession(id, actor);
// A retained version alone never grants access; re-run the scoped query before returning stored evidence.
metrics.calculate(toQuery(session.context(), session.releaseId()), actor);
if(protocol!=null&&protocol.ownsSession(id))protocol.authorizeHistory(id,actor);
else metrics.calculate(toQuery(session.context(), session.releaseId()), actor);
Set<String> allowedFields = governance.allowedFields(actor);
Set<String> visibleFilterKeys = AnalysisVisualSnapshotAccess.visibleExtraFilterKeys(jdbc, allowedFields);
List<Map<String, Object>> turns = jdbc.query("SELECT id,question,answer_json,steps_json,plan_json,created_at FROM analysis_turn WHERE session_id=? AND user_id=? ORDER BY created_at",
List<Map<String, Object>> turns = jdbc.query("SELECT id,question,answer_json,steps_json,plan_json,created_at,status FROM analysis_turn WHERE session_id=? AND user_id=? ORDER BY created_at",
(rs, n) -> {
String turnId = rs.getString("id");
Map<String, Object> plan = rs.getString("plan_json") == null ? Map.of() : read(rs.getString("plan_json"));
if(protocol!=null&&protocol.ownsSession(id)&&"report".equals(AnalysisProtocolService.map(plan.get("task")).get("kind"))
&&Set.of("COMPLETED","PARTIAL").contains(Objects.toString(rs.getString("status"),"")))
protocol.authorizeReportSnapshot(turnId,actor);
if(protocol!=null&&protocol.ownsSession(id)&&AnalysisProtocolInspection.KIND.equals(AnalysisProtocolService.map(plan.get("task")).get("kind"))
&&Set.of("COMPLETED","PARTIAL").contains(Objects.toString(rs.getString("status"),"")))protocol.authorizeInspectionSnapshot(turnId,actor);
Map<String, Object> expectedScope = new LinkedHashMap<>(AnalysisTurnService.retryContext(plan));
expectedScope.remove("_task");
var visible=visibleAnswer(rs.getString("answer_json"),turnId,expectedScope,allowedFields,visibleFilterKeys);
if(visualsEnabled&&protocol!=null&&protocol.ownsTurn(turnId)&&Set.of("COMPLETED","PARTIAL").contains(rs.getString("status")))
visible.putAll(protocol.visualRecoveryOverlay(turnId,actor));
return Map.of("id", turnId, "question", rs.getString("question"),
"result", visibleAnswer(rs.getString("answer_json"), turnId, expectedScope, allowedFields, visibleFilterKeys),
"result", visible,
"steps", rs.getString("steps_json") == null ? List.of() : mapper.readValue(rs.getString("steps_json"), List.class),
"plan", plan, "createdAt", rs.getTimestamp("created_at").toInstant().toString());
},
session.id(), actor.getId().toString());
return Map.of("id", session.id(), "title", session.title(), "releaseId", session.releaseId(), "context", session.context(), "turns", turns);
return Map.of("id", session.id(), "title", session.title(), "releaseId", session.releaseId(), "context", session.context(), "turns", turns, "readOnly",protocol!=null&&protocol.legacyReadOnly(id));
}
private Map<String, Object> visibleAnswer(String encoded, String turnId, Map<String, Object> expectedScope,
@@ -0,0 +1,87 @@
package cn.yunsheng.quality.analysis;
import java.security.SecureRandom;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.function.Predicate;
/**
* Execution-scoped credential primitive for the opt-in protocol. It is deliberately not wired
* into the legacy runtime. The supplied validator must recheck current database state and current
* user authorization. Query publication and evidence commits still require their own transactional
* execution CAS; a successful credential check cannot make a later write safe by itself.
*/
final class AnalysisExecutionCredentials {
record Identity(String sessionId, String turnId, long revision, String executionId) {
Identity {
if (sessionId == null || sessionId.isBlank() || turnId == null || turnId.isBlank()
|| revision < 1 || executionId == null || executionId.isBlank())
throw new IllegalArgumentException("complete execution identity is required");
}
}
private record Binding(Identity identity, Instant expiresAt) { }
private final SecureRandom random = new SecureRandom();
private final Clock clock;
private final Duration ttl;
private final Predicate<Identity> currentAndAuthorized;
private final Map<String, Binding> tokens = new HashMap<>();
AnalysisExecutionCredentials(Clock clock, Duration ttl, Predicate<Identity> currentAndAuthorized) {
this.clock = Objects.requireNonNull(clock);
this.ttl = Objects.requireNonNull(ttl);
this.currentAndAuthorized = Objects.requireNonNull(currentAndAuthorized);
if (ttl.isNegative() || ttl.isZero()) throw new IllegalArgumentException("positive TTL required");
}
synchronized String issue(Identity identity) {
Objects.requireNonNull(identity);
reap();
if (!currentAndAuthorized.test(identity)) throw denied();
byte[] bytes = new byte[32];
String token;
do {
random.nextBytes(bytes);
token = Base64.getUrlEncoder().withoutPadding().encodeToString(bytes);
} while (tokens.containsKey(token));
tokens.put(token, new Binding(identity, clock.instant().plus(ttl)));
return token;
}
synchronized Identity authorize(String token) {
if (token == null) throw denied();
Binding binding = tokens.get(token);
if (binding == null) throw denied();
boolean valid;
try {
valid = clock.instant().isBefore(binding.expiresAt()) && currentAndAuthorized.test(binding.identity())
&& clock.instant().isBefore(binding.expiresAt());
} catch (RuntimeException unavailable) {
tokens.remove(token);
throw denied();
}
if (!valid) {
tokens.remove(token);
throw denied();
}
return binding.identity();
}
/** The caller must supply its own identity; no turn-wide or session-wide cleanup is provided. */
synchronized void revoke(Identity identity) {
Objects.requireNonNull(identity);
tokens.entrySet().removeIf(entry -> entry.getValue().identity().equals(identity));
}
private void reap() {
Instant now = clock.instant();
tokens.entrySet().removeIf(entry -> !now.isBefore(entry.getValue().expiresAt()));
}
private static SecurityException denied() { return new SecurityException("execution credential unavailable"); }
}
@@ -0,0 +1,178 @@
package cn.yunsheng.quality.analysis;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
/**
* Opt-in persistence for the new execution protocol; it does not read legacy context.
* Callers must authorize the session and validate task semantics, clarification selections,
* current permissions, publication snapshot and result evidence before invoking these methods.
* JSON payloads are supplied by the caller. No formal-query path uses this store yet.
* Mutations require Spring transaction interception; direct construction is not a transaction boundary.
*/
@Service
public class AnalysisExecutionProtocolStore {
public enum Disposition { PENDING, CLARIFICATION, CONFIRMED }
public enum Outcome { COMPLETED, PARTIAL, FAILED }
/** revision is the latest draft or confirmed revision; confirmedRevision may point to older history. */
public record State(long version, Long revision, Long confirmedRevision, String executionId, String status) { }
public record Execution(String executionId, long revision, long version) { }
public static class IdempotencyConflict extends IllegalStateException {
public IdempotencyConflict() { super("request key already belongs to a different revision or digest"); }
}
public record Revision(long revision, Disposition disposition, String taskJson) { }
public record ExecutionRecord(String executionId, long revision, String status, String resultJson) { }
private final JdbcTemplate jdbc;
public AnalysisExecutionProtocolStore(JdbcTemplate jdbc) { this.jdbc = jdbc; }
/** Explicit opt-in only; never upgrades an existing legacy context. */
@Transactional
public void initialize(String sessionId) {
required(sessionId, "sessionId");
jdbc.update("INSERT INTO analysis_protocol_session (session_id) VALUES (?)", sessionId);
}
public State state(String sessionId) {
List<State> rows = jdbc.query(
"SELECT version, current_revision, confirmed_revision, current_execution_id, status FROM analysis_protocol_session WHERE session_id = ? FOR UPDATE",
(rs, n) -> new State(rs.getLong(1), (Long) rs.getObject(2), (Long) rs.getObject(3), rs.getString(4), rs.getString(5)), sessionId);
if (rows.size() != 1) throw new IllegalStateException("protocol session does not exist");
return rows.getFirst();
}
/** Caller must authorize access to this session before reading its task or history. */
public Revision revision(String sessionId, long revision) {
required(sessionId, "sessionId");
List<Revision> rows = jdbc.query(
"SELECT revision, disposition, task_json FROM analysis_protocol_revision WHERE session_id = ? AND revision = ? FOR SHARE",
(rs, n) -> new Revision(rs.getLong(1), Disposition.valueOf(rs.getString(2)), rs.getString(3)), sessionId, revision);
if (rows.size() != 1) throw new IllegalStateException("protocol revision does not exist in session");
return rows.getFirst();
}
/**
* Read a specific execution and its result, including historical/invalidated records.
* Caller must authorize the session and revalidate current permissions, snapshot and evidence
* before displaying or reusing a result. Existence alone does not make it current or valid.
*/
public ExecutionRecord execution(String sessionId, String executionId) {
required(sessionId, "sessionId"); required(executionId, "executionId");
List<ExecutionRecord> rows = jdbc.query(
"SELECT execution_id, revision, status, result_json FROM analysis_protocol_execution WHERE session_id = ? AND execution_id = ?",
(rs, n) -> new ExecutionRecord(rs.getString(1), rs.getLong(2), rs.getString(3), rs.getString(4)), sessionId, executionId);
if (rows.size() != 1) throw new IllegalStateException("protocol execution does not exist in session");
return rows.getFirst();
}
/** Append an immutable revision; a clarification cannot be confirmed through this method. */
@Transactional
public State appendRevision(String sessionId, long expectedVersion, Disposition disposition, String validatedTaskJson) {
Objects.requireNonNull(disposition, "disposition");
return append(sessionId, expectedVersion, disposition, validatedTaskJson, false);
}
/**
* Append a new confirmed revision after a valid user choice. The caller must verify the choice,
* preserve the original goal, and recheck session ownership, current authorization and snapshot.
* The clarification revision is never modified. This method cannot prove those preconditions.
*/
@Transactional
public State appendConfirmedResolutionAfterCallerValidation(String sessionId, long expectedVersion, String validatedTaskJson) {
return append(sessionId, expectedVersion, Disposition.CONFIRMED, validatedTaskJson, true);
}
private State append(String sessionId, long expectedVersion, Disposition disposition, String json, boolean resolution) {
required(sessionId, "sessionId"); required(json, "validatedTaskJson");
String condition = resolution ? " AND status = 'CLARIFICATION'" :
disposition == Disposition.CONFIRMED ? " AND status <> 'CLARIFICATION'" : "";
String confirmedPointer = disposition == Disposition.CONFIRMED
? "confirmed_revision = COALESCE(current_revision, 0) + 1, " : "";
int changed = jdbc.update("UPDATE analysis_protocol_session SET version = version + 1, " + confirmedPointer +
"current_revision = COALESCE(current_revision, 0) + 1, current_execution_id = NULL, status = ? " +
"WHERE session_id = ? AND version = ?" + condition, disposition.name(), sessionId, expectedVersion);
requireChanged(changed);
jdbc.update("UPDATE analysis_protocol_execution SET status = 'INVALIDATED' WHERE session_id = ? AND status = 'RUNNING'", sessionId);
State next = state(sessionId);
jdbc.update("INSERT INTO analysis_protocol_revision (session_id, revision, disposition, task_json) VALUES (?, ?, ?, ?)",
sessionId, next.revision(), disposition.name(), json);
return next;
}
/**
* Start or retry a confirmed revision. Each accepted request receives a fresh ID.
* A matching request key, revision and SHA-256 digest returns the original execution, even
* after state changes or cancellation; the returned version is the session's current version.
* Reusing a key for different input fails. To retry a cancelled run, use a new key.
* The caller computes the digest from its canonical, validated request representation.
* Caller must revalidate authorization and snapshot immediately before actual querying.
*/
@Transactional
public Execution start(String sessionId, long expectedVersion, long revision, String requestKey, String requestDigest) {
required(sessionId, "sessionId"); required(requestKey, "requestKey");
if (requestDigest == null || !requestDigest.matches("[0-9a-f]{64}"))
throw new IllegalArgumentException("requestDigest must be a lowercase SHA-256 hex digest");
if (requestKey.length() > 128) throw new IllegalArgumentException("requestKey exceeds 128 characters");
// Serialize all starts for this session before checking an existing key. The lock is held
// by the surrounding transaction, including the lookup, state CAS, and execution insert.
List<Long> versions = jdbc.query("SELECT version FROM analysis_protocol_session WHERE session_id = ? FOR UPDATE",
(rs, n) -> rs.getLong(1), sessionId);
if (versions.size() != 1) throw new IllegalStateException("protocol session does not exist");
record Existing(String id, long revision, String digest) { }
// A locking current read is essential when an enclosing REPEATABLE READ transaction has
// already established an older snapshot before another transaction inserted this key.
List<Existing> previous = jdbc.query("SELECT execution_id, revision, request_digest FROM analysis_protocol_execution WHERE session_id = ? AND request_key = ? FOR UPDATE",
(rs, n) -> new Existing(rs.getString(1), rs.getLong(2), rs.getString(3)), sessionId, requestKey);
if (!previous.isEmpty()) {
Existing existing = previous.getFirst();
if (existing.revision() != revision || !existing.digest().equals(requestDigest)) throw new IdempotencyConflict();
return new Execution(existing.id(), existing.revision(), versions.getFirst());
}
String id = UUID.randomUUID().toString();
int changed = jdbc.update("UPDATE analysis_protocol_session s SET version = version + 1, current_execution_id = ?, status = 'RUNNING' " +
"WHERE session_id = ? AND version = ? AND current_revision = ? AND confirmed_revision = current_revision AND status IN ('CONFIRMED','RUNNING','COMPLETED','PARTIAL','FAILED','CANCELLED') " +
"AND EXISTS (SELECT 1 FROM analysis_protocol_revision r WHERE r.session_id = s.session_id AND r.revision = s.current_revision AND r.disposition = 'CONFIRMED')",
id, sessionId, expectedVersion, revision);
requireChanged(changed);
jdbc.update("UPDATE analysis_protocol_execution SET status = 'INVALIDATED' WHERE session_id = ? AND status = 'RUNNING'", sessionId);
jdbc.update("INSERT INTO analysis_protocol_execution (execution_id, session_id, revision, request_key, request_digest, status) VALUES (?, ?, ?, ?, ?, 'RUNNING')",
id, sessionId, revision, requestKey, requestDigest);
return new Execution(id, revision, expectedVersion + 1);
}
/** The caller must validate all result evidence and current authorization before committing. */
@Transactional
public State commit(String sessionId, long expectedVersion, long revision, String executionId, Outcome outcome, String validatedResultJson) {
required(sessionId, "sessionId"); required(executionId, "executionId");
required(validatedResultJson, "validatedResultJson"); Objects.requireNonNull(outcome, "outcome");
int changed = jdbc.update("UPDATE analysis_protocol_session SET version = version + 1, status = ? " +
"WHERE session_id = ? AND version = ? AND current_revision = ? AND current_execution_id = ? AND status = 'RUNNING'",
outcome.name(), sessionId, expectedVersion, revision, executionId);
requireChanged(changed);
requireChanged(jdbc.update("UPDATE analysis_protocol_execution SET status = ?, result_json = ? " +
"WHERE session_id = ? AND revision = ? AND execution_id = ? AND status = 'RUNNING'",
outcome.name(), validatedResultJson, sessionId, revision, executionId));
return state(sessionId);
}
@Transactional
public State cancel(String sessionId, long expectedVersion) {
required(sessionId, "sessionId");
requireChanged(jdbc.update("UPDATE analysis_protocol_session SET version = version + 1, status = 'CANCELLED', current_execution_id = NULL " +
"WHERE session_id = ? AND version = ? AND status IN ('PENDING','CLARIFICATION','CONFIRMED','RUNNING')", sessionId, expectedVersion));
jdbc.update("UPDATE analysis_protocol_execution SET status = 'CANCELLED' WHERE session_id = ? AND status = 'RUNNING'", sessionId);
return state(sessionId);
}
private static void requireChanged(int count) {
if (count != 1) throw new IllegalStateException("stale version or invalid protocol transition");
}
private static void required(String value, String name) {
if (value == null || value.isBlank()) throw new IllegalArgumentException(name + " is required");
}
}
@@ -116,6 +116,8 @@ public class AnalysisExplorationTools {
result.put("metrics", metricItems);
result.put("visibleWeightFields", visibleWeights);
result.put("visibleDefectFields", visibleDefects);
result.put("fineDefects", detailAllowed ? AnalysisProtocolDefects.catalog(allowed)
: Map.of("kind","fine_defect_catalog","grain","source_defect_weight","available",false,"fields",List.of(),"categories",List.of()));
result.put("units", Map.of("rates", "比例小数;展示为百分数时由调用方格式化,原正式指标不改算",
"weights", "来源重量单位;平台不在查询时做 kg/吨等单位换算"));
result.put("scope", "每次查询由当前登录身份执行 MetricsService 的组织、分组授权过滤;调用参数不能指定或扩大身份范围");
@@ -4,6 +4,7 @@ import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.filters.SourceColumnQueries;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import cn.yunsheng.quality.release.ReleaseService;
@@ -53,13 +54,15 @@ public class AnalysisFineDefectsService {
category("加工废", "bright_processing_scrap"), category("扩散废", "bright_diffusion_scrap"));
private static final Map<String, String> UNAVAILABLE_FIELDS = Map.of(
"晶界扩散磁特性不良数量_1", "来源字段为数量,且没有对应的重量列;不计入重量汇总");
private static final Set<String> REQUEST_KEYS = Set.of("releaseId", "mode", "year", "month", "quarter", "department",
"group", "product", "productSearch", "projectGroup", "extraFilters", "includeUnassigned", "stage", "dimension", "defect", "category", "limit");
private static final Set<String> REQUEST_KEYS = Set.of("releaseId", "mode", "year", "month", "quarter", "monthStart", "monthEnd",
"department", "group", "product", "productSearch", "projectGroup", "extraFilters", "columnFilters", "exclusions",
"includeUnassigned", "stage", "dimension", "defect", "category", "limit");
private final JdbcTemplate jdbc;
private final ReleaseService releases;
private final GovernanceAccessService governance;
private final JsonMapper json;
private SourceColumnQueries sourceColumns;
public AnalysisFineDefectsService(JdbcTemplate jdbc, ReleaseService releases,
GovernanceAccessService governance, JsonMapper json) {
@@ -69,19 +72,37 @@ public class AnalysisFineDefectsService {
this.json = json;
}
@org.springframework.beans.factory.annotation.Autowired(required = false)
public void setSourceColumns(SourceColumnQueries sourceColumns) { this.sourceColumns = sourceColumns; }
@Transactional(readOnly = true, isolation = Isolation.REPEATABLE_READ)
public Map<String, Object> query(Map<String, Object> input, AppUserPrincipal actor) {
Set<String> allowed = governance.allowedFields(actor);
Request request = parse(input, allowed);
boolean extendedScope = hasExtendedScope(input);
if (extendedScope && sourceColumns == null) throw bad("扩展缺陷查询范围当前不可用");
Map<String, Object> scopeContext = sourceColumns == null ? null : SourceColumnQueries.normalizedScope(input);
String releaseId = releases.resolvePublishedRelease(request.releaseId());
Map<String, Object> release = releaseContext(releaseId);
if (request.mode().equals("catalog")) return catalog(request, release, allowed);
if (request.mode().equals("catalog")) {
validateCatalogScope(request, releaseId, input, actor);
return withScope(catalog(request, release, allowed), scopeContext);
}
StringBuilder predicate = new StringBuilder(" WHERE rm.release_id = ?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
appendScope(predicate, args, request, input, releaseId, actor);
if (request.productSearch() != null) {
predicate.append(" AND LOCATE(LOWER(?), LOWER(COALESCE(f.product,''))) > 0");
args.add(request.productSearch());
}
List<DefectField> visibleFields = applicableFields(request, allowed);
if (request.dimension().equals("product") && !isAllowed(request.defect(), allowed))
throw forbidden("没有查看该细分缺陷源字段的权限");
if (visibleFields.isEmpty()) {
return unavailableTop(request, release, "当前类别没有可用于重量汇总的已授权细分字段");
return withScope(unavailableTop(request, release, "当前类别没有可用于重量汇总的已授权细分字段"), scopeContext);
}
String table = request.stage().equals("black") ? "fact_black" : "fact_bright";
@@ -95,24 +116,6 @@ public class AnalysisFineDefectsService {
if (visibleFields.stream().anyMatch(field -> field.category().equals(category.name()))
&& isAllowed(category.componentCode(), allowed)) componentCodes.add(category.componentCode());
}
StringBuilder predicate = new StringBuilder(" WHERE rm.release_id = ?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
appendPeriod(predicate, args, request);
SelectionValues.appendPredicate(predicate, args, "f.department", request.department(), DepartmentLabels::queryValue);
SelectionValues.appendPredicate(predicate, args, "f.group_name", request.group(), value -> value);
SelectionValues.appendPredicate(predicate, args, "f.product", request.product(), value -> value);
if (request.productSearch() != null) {
predicate.append(" AND LOCATE(LOWER(?), LOWER(COALESCE(f.product,''))) > 0");
args.add(request.productSearch());
}
SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value);
if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc,governance).append(predicate,args,request.stage(),request.extraFilters(),actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
Aggregator aggregate = new Aggregator(request, visibleFields, categories, componentCodes, json);
String sql = """
SELECT %s AS raw_fields, %s AS formal_components, f.group_name AS business_group%s
@@ -132,12 +135,51 @@ public class AnalysisFineDefectsService {
throw new IllegalStateException(error);
}
}, args.toArray());
return aggregate.response(request, release);
return withScope(aggregate.response(request, release), scopeContext);
}
private static boolean hasExtendedScope(Map<String, Object> input) {
return input != null && List.of("monthStart", "monthEnd", "columnFilters", "exclusions").stream().anyMatch(input::containsKey);
}
private void appendScope(StringBuilder predicate, List<Object> args, Request request,
Map<String, Object> input, String releaseId, AppUserPrincipal actor) {
if (sourceColumns != null) {
sourceColumns.appendAuthorizedScope(predicate, args, request.stage(), releaseId, input, actor);
return;
}
appendPeriod(predicate, args, request);
SelectionValues.appendPredicate(predicate, args, "f.department", request.department(), DepartmentLabels::queryValue);
SelectionValues.appendPredicate(predicate, args, "f.group_name", request.group(), value -> value);
SelectionValues.appendPredicate(predicate, args, "f.product", request.product(), value -> value);
SelectionValues.appendPredicate(predicate, args, "f.project_group", request.projectGroup(), value -> value);
if (!request.includeUnassigned() && !SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc, governance).append(predicate, args, request.stage(), request.extraFilters(), actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
}
private void validateCatalogScope(Request request, String releaseId, Map<String, Object> input, AppUserPrincipal actor) {
if (sourceColumns == null) return;
List<String> stages = request.stage() == null ? List.of("black", "bright") : List.of(request.stage());
for (String stage : stages) sourceColumns.appendAuthorizedScope(new StringBuilder(), new ArrayList<>(), stage, releaseId, input, actor);
}
private static Map<String, Object> withScope(Map<String, Object> response, Map<String, Object> scopeContext) {
if (scopeContext == null) return response;
Map<String, Object> context = new LinkedHashMap<>();
if (response.get("context") instanceof Map<?, ?> existing)
existing.forEach((key, value) -> context.put(String.valueOf(key), value));
context.putAll(scopeContext);
response.put("context", context);
return response;
}
static Request parse(Map<String, Object> input, Set<String> allowedFields) {
if (input == null) throw bad("请求体不能为空");
for (String key : input.keySet()) if (!REQUEST_KEYS.contains(key)) throw bad("未知请求属性:" + key);
if (hasExtendedScope(input)) SourceColumnQueries.normalizedScope(input);
String releaseId = text(input, "releaseId", true);
if (releaseId.length() > 64) throw bad("releaseId 长度无效");
String mode = defaultText(input, "mode", "top");
@@ -4,6 +4,7 @@ import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.filters.SourceColumnQueries;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import cn.yunsheng.quality.ingestion.ConfigurableRuleEngine;
@@ -29,7 +30,8 @@ import tools.jackson.databind.json.JsonMapper;
public class AnalysisInvestigationService {
static final BigDecimal TOLERANCE = new BigDecimal("0.000001");
static final int MAX_SOURCE_ROWS = 100_000;
private static final Set<String> EXTRA_KEYS = Set.of("action","offset","recordFilter","sourceRevisionId","fields");
private static final Set<String> EXTRA_KEYS = Set.of("action","offset","recordFilter","sourceRevisionId","fields",
"monthStart","monthEnd","columnFilters","exclusions");
private static final Map<String,String> RAW_TOTALS = Map.of("black_responsibility_scrap","责废总重量","black_material_scrap","料废重量","black_external_scrap","外废重量","bright_plating_scrap","镀废重量合计","bright_material_scrap","材料废重量合计","bright_processing_scrap","加工废重量合计","bright_diffusion_scrap","晶界扩散报废合计重量");
private final JdbcTemplate jdbc;
private final ReleaseService releases;
@@ -37,15 +39,21 @@ public class AnalysisInvestigationService {
private final JsonMapper json;
private final SourceScanner scanner;
private final XlsxStreamingReader reader;
private SourceColumnQueries sourceColumns;
public AnalysisInvestigationService(JdbcTemplate jdbc, ReleaseService releases, GovernanceAccessService access, JsonMapper json, SourceScanner scanner, XlsxStreamingReader reader) {
this.jdbc=jdbc;this.releases=releases;this.access=access;this.json=json;this.scanner=scanner;this.reader=reader;
}
@org.springframework.beans.factory.annotation.Autowired(required = false)
public void setSourceColumns(SourceColumnQueries sourceColumns) { this.sourceColumns = sourceColumns; }
@Transactional(readOnly=true, isolation=Isolation.REPEATABLE_READ)
public Map<String,Object> query(Map<String,Object> input, AppUserPrincipal actor) {
access.requireEvidence(actor);
if(input==null)throw bad("调查参数不能为空");
boolean extendedScope=List.of("monthStart","monthEnd","columnFilters","exclusions").stream().anyMatch(input::containsKey);
if(extendedScope&&sourceColumns==null)throw bad("扩展调查范围当前不可用");
if(input.keySet().stream().anyMatch(Set.of("mode","dimension","defect")::contains))throw bad("调查接口不接受mode/dimension/defect;请使用category和fields");
String action=text(input.getOrDefault("action","overview"));
if(!Set.of("overview","rows","fields","source").contains(action))throw bad("action 必须是 overview/rows/fields/source");
@@ -59,6 +67,7 @@ public class AnalysisInvestigationService {
Map<String,Object> base=new LinkedHashMap<>(input);EXTRA_KEYS.forEach(base::remove);base.put("mode","top");base.put("dimension","defect");
var request=AnalysisFineDefectsService.parse(base,allowed);
String release=releases.resolvePublishedRelease(request.releaseId());
Map<String,Object> scopeContext=sourceColumns==null?null:SourceColumnQueries.normalizedScope(input);
List<AnalysisFineDefectsService.Category> categories=AnalysisFineDefectsService.categories(request.stage()).stream().filter(c->request.category()==null||request.category().equals(c.name())).toList();
for(var c:categories)if(!permitted(c.componentCode(),allowed))throw forbidden("当前身份未授权调查正式分项:"+c.componentCode());
@@ -72,23 +81,27 @@ public class AnalysisInvestigationService {
if(selectedFields.size()>32)throw bad("每页最多32个字段,请按字段字典分批选择");
State state=new State(request,categories,selectedFields,allowed,action,offset,recordFilter);
StringBuilder predicate=new StringBuilder(" WHERE rm.release_id=?");List<Object> args=new ArrayList<>(List.of(release));
AnalysisFineDefectsService.appendPeriod(predicate,args,request);
SelectionValues.appendPredicate(predicate,args,"f.department",request.department(),DepartmentLabels::queryValue);
SelectionValues.appendPredicate(predicate,args,"f.group_name",request.group(),x->x);
SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x);
if(sourceColumns!=null){
sourceColumns.appendAuthorizedScope(predicate,args,request.stage(),release,input,actor);
}else{
AnalysisFineDefectsService.appendPeriod(predicate,args,request);
SelectionValues.appendPredicate(predicate,args,"f.department",request.department(),DepartmentLabels::queryValue);
SelectionValues.appendPredicate(predicate,args,"f.group_name",request.group(),x->x);
SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x);
SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x);
if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor);
DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
}
if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());}
SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x);
if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
if(revision!=null){predicate.append(" AND f.source_revision_id=?");args.add(revision);}
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor);
DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
String sql="SELECT f.id,f.source_revision_id,f.build_id,f.sheet_name,f.physical_row_no,f.group_name,f.product,f.raw_values,f.components,sr.source_file,sr.sha256,sr.file_size,sr.archive_path,b.parser_version,b.parser_config,b.parser_config_hash,b.rule_version,b.mapping_version,rv.rule_definition FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id AND b.source_revision_id=rm.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id JOIN fact_"+request.stage()+" f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id JOIN rule_version rv ON rv.id=b.rule_version"+predicate+" ORDER BY f.id";
jdbc.query(sql,(RowCallbackHandler)rs->{try{
Map<String,Object> raw=decode(rs.getString("raw_values")),components=decode(rs.getString("components"));
Map<String,Object> meta=map("sourceRevisionId",rs.getString("source_revision_id"),"buildId",rs.getString("build_id"),"sourceFile",rs.getString("source_file"),"sha256",rs.getString("sha256"),"fileSize",rs.getLong("file_size"),"parserVersion",rs.getString("parser_version"),"parserConfigHash",rs.getString("parser_config_hash"),"ruleVersion",rs.getString("rule_version"),"mappingVersion",rs.getString("mapping_version"));
state.add(rs.getLong("id"),rs.getString("sheet_name"),rs.getInt("physical_row_no"),rs.getString("product"),rs.getString("group_name"),raw,components,meta,decode(rs.getString("parser_config")),decode(rs.getString("rule_definition")),rs.getString("archive_path"));
}catch(SQLException e){throw new IllegalStateException(e);}},args.toArray());
Map<String,Object> result=state.response(release,input);
Map<String,Object> result=state.response(release,input,scopeContext);
if(action.equals("source")){
Map<String,Object> proof=verifySources(state);result.put("sourceVerification",proof);
if(!Boolean.TRUE.equals(proof.get("complete"))||!Boolean.TRUE.equals(proof.get("allSelectedValuesMatchStored"))){
@@ -159,8 +172,9 @@ public class AnalysisInvestigationService {
if(different){examples.add(record);if(examples.size()>5)examples.poll();}
if(action.equals("source")&&totalRows<=MAX_SOURCE_ROWS){List<String> verifyFields=new ArrayList<>(fields);for(var c:categories.values()){String col=sourceColumn(RAW_TOTALS.get(c.category.componentCode()),request.stage(),parser,rule);if(!verifyFields.contains(col))verifyFields.add(col);}source.fields.addAll(verifyFields);source.rows.put(sheet+":"+row,unclipped(raw,verifyFields));}
}
Map<String,Object> response(String release,Map<String,Object> input){
Map<String,Object> response(String release,Map<String,Object> input,Map<String,Object> scopeContext){
Map<String,Object> context=map("releaseId",release,"stage",request.stage(),"year",request.year(),"month",request.month(),"quarter",request.quarter(),"department",request.department(),"group",request.group(),"product",request.product(),"projectGroup",request.projectGroup(),"includeUnassigned",request.includeUnassigned(),"extraFilters",request.extraFilters(),"category",request.category(),"productSearch",request.productSearch(),"sourceRevisionId",input.get("sourceRevisionId"));
if(scopeContext!=null)for(String key:List.of("monthStart","monthEnd","columnFilters","exclusions"))if(scopeContext.containsKey(key))context.put(key,scopeContext.get(key));
List<Map<String,Object>> lineage=sources.values().stream().map(s->{Map<String,Object> m=new LinkedHashMap<>(s.meta);m.put("matchedRows",s.count);m.put("sheets",s.sheets);Map<String,Object> mappings=new LinkedHashMap<>();strings(s.parser.get("fieldMap")).forEach((k,v)->{if((action.equals("fields")||fields.contains(k))&&permitted(k,allowed)&&permitted(v,allowed)&&!sensitive(k)&&!sensitive(v))mappings.put(k,v);});m.put("frozenFieldMap",mappings);m.put("componentRules",rulesFor(s.rule,categories.values().stream().map(c->c.category.componentCode()).toList()));m.put("formulaSourceAliases",safeAliases(s.rule,request.stage(),allowed));return m;}).toList();
for(String field:fields)if(input.containsKey("fields")&&totalRows>0&&!dictionary.containsKey(field))throw new ResponseStatusException(HttpStatus.CONFLICT,"所选源字段不在命中行字段字典中:"+field);
Map<String,Object> result=map("kind","quality_investigation","action",action,"context",context,"totalRows",totalRows,"aggregateComplete",true,"aggregateRowLimit",null,"categories",categories.values().stream().map(CategoryCount::view).toList(),"lineage",lineage,"selectedFields",fields,"availableFieldCount",dictionary.size(),"fieldDictionary",dictionary.entrySet().stream().filter(e->action.equals("fields")||fields.contains(e.getKey())).map(e->map("field",e.getKey(),"coverage",e.getValue().view(totalRows))).toList(),"largestDifferenceExamples",examples.stream().sorted(Comparator.comparing((Map<String,Object> r)->((BigDecimal)r.get("differenceMagnitude"))).reversed()).toList(),"exampleLimit",5,"examplesAreNotFullRows",true,"basis","所有统计扫描全部命中事实行;空/非法/负数细分不作有效重量;差额是正式合计减已知非负细分和,缺失覆盖单列,不能把差额认作某缺失字段的值。不是新的正式KPI,也不推断设备、工艺或人员原因。","nextSteps",action.equals("source")?List.of("结合原件核验与冻结规则解释表内差额;未核验部分明确列为卡点"):List.of("存在差额或字段缺失时调用action=source核验全部相关原表行;多来源/超限按sourceRevisionId拆分","action=rows配合recordFilter=differences查具体行;用nextOffset翻页,不能把一页当全量"));
@@ -0,0 +1,61 @@
package cn.yunsheng.quality.analysis;
import java.nio.charset.StandardCharsets;
import java.util.*;
import tools.jackson.databind.json.JsonMapper;
/** Builds the paginated UI payload solely from recorded server query responses. */
final class AnalysisMemberSnapshot {
static final int MAX_BYTES = 8 * 1024 * 1024;
private AnalysisMemberSnapshot() {}
static Map<String,Object> build(String releaseId, Map<String,Object> task,
List<Map<String,Object>> pages, boolean coverageComplete, JsonMapper json) {
List<?> keys = list(task.get("memberFields"));
List<?> stages = list(task.get("stages"));
Map<?,?> labels = map(task.get("sourceFieldLabels"));
List<Map<String,Object>> fields = new ArrayList<>();
for(Object key:keys) {
if(!(key instanceof String s)||!(labels.get(s) instanceof String label)||label.isBlank())
throw new IllegalArgumentException("成员字段名称缺失");
fields.add(Map.of("key",s,"label",label));
}
if(fields.isEmpty()||fields.size()>5)throw new IllegalArgumentException("成员字段无效");
List<Map<String,Object>> rows=new ArrayList<>();
long total=-1,expectedOffset=0;int bytes=0;boolean available=true;
for(Map<String,Object> page:pages) {
if(!Objects.equals(page.get("releaseId"),releaseId)
|| !Objects.equals(page.get("period"),task.get("period"))
|| !Objects.equals(page.get("filters"),task.get("filters"))
|| !Objects.equals(page.get("memberFields"),keys)
|| !Objects.equals(page.get("selectedStages"),stages)
|| !(page.get("totalCount") instanceof Number count)
|| !(page.get("offset") instanceof Number offset)
|| count.longValue()<0 || offset.longValue()!=expectedOffset
|| total>=0&&total!=count.longValue())throw new IllegalArgumentException("成员响应范围不一致");
total=count.longValue();
for(Object raw:list(page.get("rows"))) {
if(!(raw instanceof Map<?,?> row)||!(row.get("values") instanceof Map<?,?> values)
|| !values.keySet().equals(new HashSet<>(keys))
|| values.values().stream().anyMatch(v->v!=null&&!(v instanceof String))
|| !(row.get("stages") instanceof List<?> sources)||sources.isEmpty()
|| new HashSet<>(sources).size()!=sources.size()||!stages.containsAll(sources))
throw new IllegalArgumentException("成员响应行无效");
Map<String,Object> item=new LinkedHashMap<>();item.put("values",values);item.put("stages",sources);
int size=json.writeValueAsString(item).getBytes(StandardCharsets.UTF_8).length+1;
if(available&&rows.size()<4000&&bytes+size<=MAX_BYTES-16_384) {rows.add(item);bytes+=size;}
else available=false;
expectedOffset++;
}
}
if(total<0||expectedOffset>total)throw new IllegalArgumentException("成员响应数量无效");
boolean complete=coverageComplete&&available&&expectedOffset==total&&rows.size()==total;
Map<String,Object> result=new LinkedHashMap<>();
result.put("schemaVersion",1);result.put("releaseId",releaseId);result.put("period",task.get("period"));
result.put("filters",task.get("filters"));result.put("fields",fields);result.put("rows",rows);
result.put("totalCount",total);result.put("rowsReturned",rows.size());result.put("queryComplete",complete);
return result;
}
private static Map<?,?> map(Object value){return value instanceof Map<?,?> m?m:Map.of();}
private static List<?> list(Object value){return value instanceof List<?> v?v:List.of();}
}
@@ -139,7 +139,7 @@ public class AnalysisModelGateway {
input.put("viewCatalog", List.of("overall", "department", "group", "product", "projectGroup"));
input.put("authorizedObjectCatalog", modelObjectCatalog(objectCatalog));
Map<String, Object> payload = Map.of("model", config.model(), "temperature", 0,
"max_tokens", 384, "stream", false,
"max_completion_tokens", AnalysisRuntimePolicy.MAX_OUTPUT_TOKENS, "stream", false,
"messages", List.of(Map.of("role", "system", "content", SYSTEM),
Map.of("role", "user", "content", mapper.writeValueAsString(input))),
"response_format", Map.of("type", "json_object"));
@@ -522,7 +522,7 @@ public class AnalysisModelGateway {
"evidence", observations);
String encoded = mapper.writeValueAsString(input);
if (encoded.length() > 120000) return new Decision(null, Map.of(), null, Map.of(), false, "证据超过规划上限。");
Map<String, Object> payload = Map.of("model", config.model(), "temperature", 0, "max_tokens", 3000,
Map<String, Object> payload = Map.of("model", config.model(), "temperature", 0, "max_completion_tokens", AnalysisRuntimePolicy.MAX_OUTPUT_TOKENS,
"stream", false, "messages", List.of(Map.of("role", "system", "content", AGENT_SYSTEM),
Map.of("role", "user", "content", encoded)), "response_format", Map.of("type", "json_object"));
HttpRequest request = HttpRequest.newBuilder(config.endpoint()).timeout(Duration.ofSeconds(25))
@@ -605,7 +605,7 @@ public class AnalysisModelGateway {
"question", "请解释当期与比较期的质量指标变化,并提出下一步核查方向。",
"evidence", qualitativeEvidence(evidence)));
Map<String, Object> payload = Map.of("model", config.model(), "temperature", 0,
"max_tokens", 640, "stream", false,
"max_completion_tokens", AnalysisRuntimePolicy.MAX_OUTPUT_TOKENS, "stream", false,
"messages", List.of(Map.of("role", "system", "content", EXPLANATION_SYSTEM),
Map.of("role", "user", "content", input)),
"response_format", Map.of("type", "json_object"));
@@ -0,0 +1,163 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.common.SelectionValues;
import java.math.BigDecimal;
import java.util.*;
/** Exact request/response binding for a server-calculated, full-population change ranking. */
final class AnalysisProtocolChanges {
static final String TARGET="POST /api/analysis/changes";
static final Set<String> SORTS=Set.of("rate_drop","current_rate","scrap_delta","current_scrap");
private static final Set<String> FILTERS=Set.of("department","group","product","projectGroup","extraFilters","columnFilters","exclusions","includeUnassigned");
private AnalysisProtocolChanges() { }
static Map<String,Object> request(String release,Map<String,Object> child) {
Map<String,Object> query=map(child.get("changeQuery"));
if(!query.keySet().equals(Set.of("dimension","sort","limit"))||!Set.of("group","department","product").contains(query.get("dimension"))
||!SORTS.contains(query.get("sort"))||count(query.get("limit"),-1)<1||count(query.get("limit"),-1)>10)
throw invalid("变化节点缺少明确的分组、排序或展示上限");
List<?> stages=list(child.get("stages"));
if(stages.size()>1||!Set.of("black","bright").containsAll(stages))throw invalid("变化节点仅支持综合或单个阶段");
List<?> fields=list(child.get("fields"));
if(fields.isEmpty()||fields.size()!=new HashSet<>(fields).size()||!Set.of("rate","totalWeight","scrapWeight").containsAll(fields)||!sortFields(fields,Objects.toString(query.get("sort"))))throw invalid("变化节点缺少已确认的正式字段");
Map<String,Object> filters=map(child.get("filters"));
if(!FILTERS.containsAll(filters.keySet())||!Boolean.TRUE.equals(child.get("comparisonRequired"))
||map(child.get("period")).isEmpty()||map(child.get("comparisonPeriod")).isEmpty())throw invalid("变化节点必须保持完整筛选和两个明确期间");
Map<String,Object> body=new LinkedHashMap<>();body.put("releaseId",release);body.put("period",child.get("period"));
body.put("comparisonPeriod",child.get("comparisonPeriod"));body.put("filters",filters);body.put("fields",List.copyOf(fields));body.putAll(query);
body.put("stage",stages.isEmpty()?null:stages.getFirst());return body;
}
static boolean acceptParameters(Map<String,Object> expected,Map<String,Object> actual) {
return Objects.equals(AnalysisProtocolService.canonical(expected),AnalysisProtocolService.canonical(actual));
}
static Map<String,Object> inspect(Map<String,Object> response,Map<String,Object> expected) {
List<String> gaps=new ArrayList<>();Map<String,Object> coverage=map(response.get("coverage"));
if(!"quality_changes".equals(response.get("kind")))gaps.add("变化响应类型不匹配");
if(!Objects.equals(expected.get("releaseId"),response.get("releaseId")))gaps.add("变化发布版本不匹配");
for(String key:List.of("dimension","stage","sort"))if(!Objects.equals(expected.get(key),response.get(key)))gaps.add("变化参数未匹配:"+key);
if(!same(expected.get("period"),response.get("currentPeriod"))||!same(expected.get("comparisonPeriod"),response.get("comparisonPeriod")))gaps.add("变化期间未完整匹配");
if(!sameFilters(map(expected.get("filters")),map(response.get("filters"))))gaps.add("变化范围未完整匹配");
long total=count(response.get("totalRows"),-1),limit=count(expected.get("limit"),-1);List<?> rows=list(response.get("rows"));
if(!Boolean.TRUE.equals(response.get("completeRanking")))gaps.add("变化查询没有完整排序证明");
if(!(response.get("rows") instanceof List<?>)||total<0||limit<1||rows.size()!=Math.min(total,limit))gaps.add("变化行数或展示上限不一致");
if(!(coverage.get("current") instanceof Map<?,?>)||!(coverage.get("comparison") instanceof Map<?,?>))gaps.add("变化两期来源覆盖缺失");
if(Set.of("rate_drop","current_rate").contains(expected.get("sort"))&&!Boolean.TRUE.equals(coverage.get("rateComparable")))gaps.add("变化率不可比");
if(!(response.get("current") instanceof Map<?,?>)||!(response.get("comparison") instanceof Map<?,?>))gaps.add("变化正式总体基线缺失");
List<?> fields=list(expected.get("fields"));
if(fields.isEmpty()||!Set.of("rate","totalWeight","scrapWeight").containsAll(fields)||!sortFields(fields,Objects.toString(expected.get("sort"))))gaps.add("变化查询缺少已绑定的正式字段");
Set<Object> keys=new HashSet<>();int rank=0;
for(Object raw:rows) {
Map<String,Object> row=map(raw);Object key=row.get("key");
if(!(key instanceof String text)||text.isBlank()||!keys.add(key)||count(row.get("rank"),-1)!=++rank)gaps.add("变化行对象或名次无效");
Map<String,Object> filters=map(row.get("filters")),base=map(expected.get("filters"));
if(!FILTERS.containsAll(filters.keySet())||!Objects.equals(key,filters.get(expected.get("dimension"))))gaps.add("变化行缺少精确对象筛选");
if(!narrowedFilters(base,filters,Objects.toString(expected.get("dimension"))))gaps.add("变化行超出父范围或改变了非选择条件");
for(String side:List.of("current","comparison")) {
Map<String,Object> value=map(row.get(side));if(value.isEmpty())gaps.add("变化行缺少两期正式值");
if(value.get("rate")!=null&&!validRate(value.get("rate")))gaps.add("变化行合格率无效");
if(fields.contains("totalWeight")||fields.contains("scrapWeight"))inspectWeights(row,side,value,fields,expected.get("stage"),gaps);
}
for(String field:List.of("rateDrop","scrapDelta"))if(row.get(field)!=null&&decimal(row.get(field))==null)gaps.add("变化行差值无效");
}
Map<String,Object> result=new LinkedHashMap<>();result.put("changesComplete",gaps.isEmpty());result.put("completeRanking",gaps.isEmpty());
result.put("rowCount",rows.size());result.put("totalRows",total);result.put("missingEvidence",gaps.stream().distinct().toList());return result;
}
static Map<String,Object> artifact(Map<String,Object> response,Map<String,Object> expected) {
if(!Boolean.TRUE.equals(inspect(response,expected).get("changesComplete")))return Map.of();
Map<String,Object> result=new LinkedHashMap<>();result.put("kind","quality_changes_artifact");result.put("releaseId",expected.get("releaseId"));
for(String key:List.of("dimension","sort","stage","filters","period","comparisonPeriod","limit","fields"))result.put(key,expected.get(key));
for(String key:List.of("rows","totalRows","current","comparison","stageDecomposition"))if(response.containsKey(key))result.put(key,response.get(key));
result.put("completeRanking",true);return result;
}
private static void inspectWeights(Map<String,Object> row,String side,Map<String,Object> values,List<?> fields,Object selected,List<String> gaps) {
Object present=map(row.get("present")).get(side);
if(Boolean.FALSE.equals(present))return;
if(!Boolean.TRUE.equals(present)){gaps.add("变化行请求重量缺少对象存在性覆盖");return;}
List<String> stages=selected instanceof String name?List.of(name):List.of("black","bright");int sourceStages=0;
for(String stage:stages) {
Map<String,Object> block=map(values.get(stage)),support=map(values.get("rateSupport"));
boolean sourced=selected!=null||Boolean.TRUE.equals(support.get(stage+"Measured"))
||List.of("rate","qualifiedWeight","scrapWeight","totalWeight").stream().anyMatch(field->decimal(block.get(field))!=null);
if(!sourced)continue;sourceStages++;
for(String field:List.of("totalWeight","scrapWeight"))if(fields.contains(field)) {
BigDecimal value=decimal(block.get(field));if(value==null||value.signum()<0)gaps.add("变化行"+side+"的"+stage+field+"缺失、不可见或非法");
}
}
if(sourceStages==0)gaps.add("变化行请求重量没有可确认的来源阶段");
}
static List<String> selectedStages(Map<String,Object> artifact,String selection,List<String> requested) {
if(selection.equals("overall"))return List.of();
if(Set.of("black","bright").contains(selection))return List.of(selection);
Map<String,Object> decomposition=map(artifact.get("stageDecomposition"));
if(Set.of("available","valid","complete").contains(Objects.toString(decomposition.get("status"),""))) {
BigDecimal black=decimal(decomposition.getOrDefault("blackContributionToCompositePP",decomposition.get("blackPP")));
BigDecimal bright=decimal(decomposition.getOrDefault("brightContributionToCompositePP",decomposition.get("brightPP")));
if(black!=null&&bright!=null&&(black.signum()<0||bright.signum()<0))return List.of(black.compareTo(bright)<=0?"black":"bright");
}
Object stage=artifact.get("stage");if(stage instanceof String text&&Set.of("black","bright").contains(text))return List.of(text);
return requested.isEmpty()?List.of("black","bright"):requested;
}
static boolean noAdverseStage(Map<String,Object> artifact) {
var value=map(artifact.get("stageDecomposition"));
if(!Set.of("available","valid","complete").contains(Objects.toString(value.get("status"),"")))return false;
BigDecimal black=decimal(value.getOrDefault("blackContributionToCompositePP",value.get("blackPP"))),bright=decimal(value.getOrDefault("brightContributionToCompositePP",value.get("brightPP")));
return black!=null&&bright!=null&&black.signum()>=0&&bright.signum()>=0;
}
static boolean adverse(Map<String,Object> row,String sort) {
BigDecimal delta=decimal(row.get(sort.startsWith("scrap")||sort.equals("current_scrap")?"scrapDelta":"rateDrop"));
if(delta!=null&&delta.signum()>0)return true;
BigDecimal scrap=decimal(row.get("scrapDelta"));return sort.equals("current_rate")&&scrap!=null&&scrap.signum()>0;
}
private static boolean sortFields(List<?> fields,String sort){return Set.of("rate_drop","current_rate").contains(sort)?fields.contains("rate")
:Set.of("scrap_delta","current_scrap").contains(sort)&&fields.contains("scrapWeight");}
private static boolean validRate(Object value){BigDecimal rate=decimal(value);return rate!=null&&rate.signum()>=0&&rate.compareTo(BigDecimal.ONE)<=0;}
static BigDecimal decimal(Object value){if(!(value instanceof Number))return null;try{return new BigDecimal(value.toString());}catch(NumberFormatException invalid){return null;}}
private static boolean same(Object left,Object right){return Objects.equals(AnalysisProtocolService.canonical(left),AnalysisProtocolService.canonical(right));}
static boolean sameFilters(Map<String,Object> left,Map<String,Object> right){return FILTERS.containsAll(left.keySet())&&FILTERS.containsAll(right.keySet())
&&FILTERS.stream().allMatch(key->AnalysisProtocolInvestigation.sameFilter(key,left.get(key),right.get(key)));}
static Set<String> visualMetrics(String sort) {
return switch(sort) {
case "rate_drop" -> Set.of("rate_drop_pp","rate_drop","rate_change","composite_rate_change");
case "current_rate" -> Set.of("current_rate","composite_rate","black_rate","bright_rate");
case "scrap_delta" -> Set.of("scrap_delta","scrap_weight_delta");
case "current_scrap" -> Set.of("current_scrap","current_scrap_weight");
default -> Set.of();
};
}
static boolean visualMatches(Map<String,Object> child,Map<String,Object> chart) {
Map<String,Object> query=map(child.get("changeQuery"));String sort=Objects.toString(query.get("sort"),"");List<?> stages=list(child.get("stages"));
if(!Objects.equals(query.get("dimension"),chart.get("dimension"))||!Set.of("bar","column","pareto").contains(Objects.toString(chart.get("type"),""))
||!visualMetrics(sort).contains(chart.get("metric"))||child.get("visualType")!=null&&!Objects.equals(child.get("visualType"),chart.get("type")))return false;
if(stages.isEmpty()?chart.get("stage")!=null&&!"overall".equals(chart.get("stage")):!stages.contains(chart.get("stage")))return false;
if(sort.equals("current_rate")) {
String metric=Objects.toString(chart.get("metric"));
return metric.equals("current_rate")||metric.equals(stages.isEmpty()?"composite_rate":stages.getFirst()+"_rate");
}
return true;
}
/** Only the selected entity and its catalog-owned department may narrow to one existing member. */
static boolean narrowedFilters(Map<String,Object> base,Map<String,Object> actual,String dimension) {
if(!Set.of("department","group","product").contains(dimension)||!FILTERS.containsAll(base.keySet())||!FILTERS.containsAll(actual.keySet()))return false;
for(String key:FILTERS) {
boolean selection=key.equals(dimension)||dimension.equals("group")&&key.equals("department");
if(selection) {
Object raw=actual.get(key);if(!(raw instanceof String wire))return false;
try {
List<String> values=SelectionValues.parse(wire);
if(values.size()!=1||!selectionAllows(base.get(key),values.getFirst()))return false;
if(list(map(base.get("exclusions")).get(key)).contains(values.getFirst()))return false;
} catch(RuntimeException invalid){return false;}
} else if(!AnalysisProtocolInvestigation.sameFilter(key,base.get(key),actual.get(key)))return false;
}
return true;
}
static boolean selectionAllows(Object parent,String member) {
if(parent!=null&&!(parent instanceof String))return false;
try {List<String> values=SelectionValues.parse((String)parent);return values.isEmpty()||values.contains(member);}
catch(RuntimeException invalid){return false;}
}
private static long count(Object value,long fallback){if(!(value instanceof Number n)||!Double.isFinite(n.doubleValue())||n.doubleValue()!=Math.rint(n.doubleValue()))return fallback;return n.longValue();}
@SuppressWarnings("unchecked") private static Map<String,Object> map(Object value){return value instanceof Map<?,?>?(Map<String,Object>)value:Map.of();}
private static List<?> list(Object value){return value instanceof List<?> values?values:List.of();}
private static IllegalArgumentException invalid(String text){return new IllegalArgumentException(text);}
}
@@ -0,0 +1,128 @@
package cn.yunsheng.quality.analysis;
import java.math.BigDecimal;
import java.util.*;
/** Source defect weights have their own scope and coverage, separate from formal KPI rankings. */
final class AnalysisProtocolDefects {
private static final List<String> PERIODS=List.of("year","month","quarter","monthStart","monthEnd");
private static final List<String> FILTERS=List.of("department","group","product","projectGroup","extraFilters","columnFilters","exclusions","includeUnassigned");
private AnalysisProtocolDefects() { }
static Map<String,Object> catalog(Set<String> allowed) {
List<Map<String,Object>> fields=new ArrayList<>(),categories=new ArrayList<>();
for(String stage:List.of("black","bright")) {
for(var field:AnalysisFineDefectsService.fields(stage)) if(allowed.contains("*")||allowed.contains(field.key()))
fields.add(Map.of("stage",stage,"key",field.key(),"label",field.label(),"category",field.category(),
"sourceField",field.key(),"grain","source_defect_weight","available",true));
for(var category:AnalysisFineDefectsService.categories(stage)) {
boolean visible=fields.stream().anyMatch(f->stage.equals(f.get("stage"))&&category.name().equals(f.get("category")));
Map<String,Object> item=new LinkedHashMap<>();item.put("stage",stage);item.put("category",category.name());
item.put("available",category.available()&&visible);
if(!category.available())item.put("reason",category.unavailableReason());
if(allowed.contains("*")||allowed.contains(category.componentCode()))item.put("formalComponent",category.componentCode());
categories.add(item);
}
}
return Map.of("kind","fine_defect_catalog","grain","source_defect_weight","fields",fields,"categories",categories);
}
static Map<String,Object> normalizeNode(Map<String,Object> node,List<String> stages,Map<String,Object> catalog) {
String dimension=Objects.toString(node.getOrDefault("dimension","defect"),"");
if(!Set.of("defect","product").contains(dimension)||stages.isEmpty())throw invalid("缺陷节点需要明确维度和阶段");
Object rawLimit=node.getOrDefault("limit",100);long limit=count(rawLimit,-1);
if(limit<1||limit>100)throw invalid("缺陷展示上限必须是1至100的整数");
Map<String,Object> metadata=map(catalog.get("fineDefects"));
Map<String,Object> query=new LinkedHashMap<>();query.put("dimension",dimension);query.put("limit",(int)limit);
String defect=optional(node,"defect"),category=optional(node,"category");
if(dimension.equals("product")&&defect==null)throw invalid("按产品查看具体缺陷需要明确授权细项字段");
for(String stage:stages) {
if(defect!=null&&list(metadata.get("fields")).stream().map(AnalysisProtocolDefects::map).noneMatch(f->stage.equals(f.get("stage"))
&&defect.equals(f.get("key"))&&Boolean.TRUE.equals(f.get("available"))&&!Boolean.TRUE.equals(f.get("countOnly"))
&&"source_defect_weight".equals(f.get("grain"))&&(category==null||category.equals(f.get("category")))))
throw invalid("细项字段未授权、不是重量或不适用于所选阶段/类别");
if(category!=null&&list(metadata.get("categories")).stream().map(AnalysisProtocolDefects::map).noneMatch(c->stage.equals(c.get("stage"))
&&category.equals(c.get("category"))&&Boolean.TRUE.equals(c.get("available"))))throw invalid("缺陷类别没有授权重量细项或不适用于所选阶段");
}
if(defect!=null)query.put("defect",defect);if(category!=null)query.put("category",category);return query;
}
static List<Map<String,Object>> requests(String release,Map<String,Object> child) {
Map<String,Object> query=map(child.get("defectQuery")),filters=map(child.get("filters"));
if(!Set.of("dimension","limit","defect","category").containsAll(query.keySet())||query.isEmpty()
||!FILTERS.containsAll(filters.keySet()))throw invalid("缺陷子任务缺少明确的查询参数");
List<?> stages=list(child.get("stages"));
if(stages.isEmpty()||stages.size()>2||!Set.of("black","bright").containsAll(stages))throw invalid("缺陷阶段无效");
List<Map<String,Object>> periods=new ArrayList<>();periods.add(map(child.get("period")));
if(Boolean.TRUE.equals(child.get("comparisonRequired")))periods.add(map(child.get("comparisonPeriod")));
List<Map<String,Object>> result=new ArrayList<>();
for(var period:periods)for(Object stage:stages) {
Map<String,Object> body=new LinkedHashMap<>();body.put("releaseId",release);body.put("mode","top");body.put("stage",stage);body.putAll(query);
if("defect".equals(query.get("dimension"))||Boolean.TRUE.equals(child.get("comparisonRequired")))body.put("limit",100);
for(String key:PERIODS)if(period.containsKey(key))body.put(key,period.get(key));
for(String key:FILTERS)if(filters.containsKey(key))body.put(key,filters.get(key));
body.putIfAbsent("includeUnassigned",false);result.add(Collections.unmodifiableMap(body));
}
return List.copyOf(result);
}
static boolean acceptParameters(Map<String,Object> expected,Map<String,Object> actual) {
return Objects.equals(AnalysisProtocolService.canonical(expected),AnalysisProtocolService.canonical(actual));
}
static Map<String,Object> inspect(Map<String,Object> response,Map<String,Object> expected,Set<String> allowed) {
List<String> gaps=new ArrayList<>();Map<String,Object> context=map(response.get("context")),coverage=map(response.get("coverage"));
if(!"fine_defect_top".equals(response.get("kind"))||!"source_defect_weight".equals(response.get("grain")))gaps.add("细分缺陷响应类型不匹配");
if(!(response.get("categoryReconciliation") instanceof List<?>)||!(response.get("notice") instanceof String notice)||notice.isBlank())gaps.add("缺陷来源口径或类别核对信息缺失");
if(!Objects.equals(expected.get("releaseId"),response.get("releaseId"))
||!Objects.equals(expected.get("releaseId"),map(response.get("release")).get("id")))gaps.add("缺陷发布版本不匹配");
for(String key:PERIODS)if(!AnalysisProtocolInvestigation.samePeriod(expected.get(key),context.get(key)))gaps.add("缺陷期间未匹配:"+key);
for(String key:FILTERS)if(!AnalysisProtocolInvestigation.sameFilter(key,expected.get(key),context.get(key)))gaps.add("缺陷范围未匹配:"+key);
for(String key:List.of("stage","mode","dimension","category","defect","productSearch"))
if(!AnalysisProtocolInvestigation.sameFilter(key,expected.get(key),context.get(key)))gaps.add("缺陷参数未匹配:"+key);
long sourceRows=count(response.get("sourceRows"),-1),total=count(response.get("totalGroups"),-1),limit=count(expected.get("limit"),-1);
List<?> rows=list(response.get("rows"));
if(sourceRows<=0||!Boolean.TRUE.equals(coverage.get("available"))||!Boolean.TRUE.equals(coverage.get("hasValidWeights"))
||count(coverage.get("sourceRows"),-1)!=sourceRows)gaps.add("没有完整可用的缺陷来源覆盖");
if(!(response.get("rows") instanceof List<?>)||total<0||limit<1||limit>100||rows.size()!=Math.min(total,limit)
||!Objects.equals(response.get("rowsTruncated"),total>limit))gaps.add("缺陷行数或截断声明不一致");
if(Boolean.TRUE.equals(response.get("rowsTruncated")))gaps.add("缺陷结果仍有未展示的截断项");
Set<Object> keys=new HashSet<>();String stage=Objects.toString(expected.get("stage")),dimension=Objects.toString(expected.get("dimension"));
Set<String> expectedFields=new HashSet<>();
for(var field:AnalysisFineDefectsService.fields(stage))if((allowed.contains("*")||allowed.contains(field.key()))
&&(expected.get("category")==null||Objects.equals(field.category(),expected.get("category")))
&&(expected.get("defect")==null||Objects.equals(field.key(),expected.get("defect"))))expectedFields.add(field.key());
if(expectedFields.isEmpty())gaps.add("当前没有授权的所选缺陷重量细项");
Set<Object> actualFields=new HashSet<>();boolean repeatedFields=false;
for(Object raw:list(response.get("sourceFields")))if(!actualFields.add(map(raw).get("key")))repeatedFields=true;
if(repeatedFields||!expectedFields.equals(actualFields))gaps.add("缺陷字段集合未覆盖当前授权的所选细项");
if(sourceRows>0&&"defect".equals(dimension)&&total!=expectedFields.size())gaps.add("缺陷总项数未匹配当前授权字段目录");
long productSourceRows=0;
for(Object raw:rows) {
Map<String,Object> row=map(raw);Object key=row.get("key");
if(!(key instanceof String s)||s.isBlank()||!keys.add(key))gaps.add("缺陷行键缺失或重复");
if("defect".equals(dimension)&&(!expectedFields.contains(key)||AnalysisFineDefectsService.fields(stage).stream().noneMatch(f->Objects.equals(f.key(),key)
&&Objects.equals(f.category(),row.get("category"))
&&(expected.get("category")==null||Objects.equals(f.category(),expected.get("category")))
&&(expected.get("defect")==null||Objects.equals(f.key(),expected.get("defect"))))))gaps.add("缺陷行不属于所选阶段、字段或类别");
if("product".equals(dimension)&&!Objects.equals(expected.get("defect"),row.get("defect")))gaps.add("规格行未绑定所选细项字段");
long rowSources=count(row.get("sourceRows"),-1);
if("defect".equals(dimension)&&rowSources!=sourceRows)gaps.add("缺陷行没有覆盖全部所选来源行");
if("product".equals(dimension)&&rowSources>0)productSourceRows+=rowSources;
if(rowSources<=0||count(row.get("presentRows"),-1)!=rowSources||count(row.get("missingRows"),-1)!=0
||count(row.get("invalidRows"),-1)!=0||count(row.get("negativeRows"),-1)!=0||!weight(row.get("weight")))gaps.add("细项重量仍有缺失、非法或负值");
}
if("product".equals(dimension)&&!Boolean.TRUE.equals(response.get("rowsTruncated"))&&productSourceRows!=sourceRows)
gaps.add("规格分组来源行数未回加至完整所选范围");
Map<String,Object> result=new LinkedHashMap<>();result.put("defectsComplete",gaps.isEmpty());result.put("missingEvidence",gaps.stream().distinct().toList());
result.put("sourceRows",sourceRows);result.put("rowCount",rows.size());result.put("totalGroups",total);result.put("rowsTruncated",response.get("rowsTruncated"));
return result;
}
private static boolean weight(Object value){try{return value instanceof Number&&new BigDecimal(value.toString()).signum()>=0;}catch(NumberFormatException e){return false;}}
private static long count(Object value,long fallback){if(!(value instanceof Number n)||!Double.isFinite(n.doubleValue())||n.doubleValue()!=Math.rint(n.doubleValue()))return fallback;return n.longValue();}
private static String optional(Map<String,Object> value,String key){if(!value.containsKey(key))return null;if(!(value.get(key) instanceof String s)||s.isBlank()||s.length()>200)throw invalid("缺陷字段或类别格式无效");return s;}
@SuppressWarnings("unchecked") private static Map<String,Object> map(Object value){return value instanceof Map<?,?>?(Map<String,Object>)value:Map.of();}
private static List<?> list(Object value){return value instanceof List<?> list?list:List.of();}
private static IllegalArgumentException invalid(String message){return new IllegalArgumentException(message);}
}
@@ -0,0 +1,73 @@
package cn.yunsheng.quality.analysis;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
/** Lossless storage of the exact canonical response bound by a protocol query's response_hash. */
final class AnalysisProtocolEvidenceSnapshot {
static final String ENCODING = "gzip-json-v1";
static final int MAX_BYTES = 64 * 1024 * 1024;
static final int MAX_COMPRESSED_BYTES = 8 * 1024 * 1024;
private AnalysisProtocolEvidenceSnapshot() { }
record Encoded(byte[] payload, int bytes, String encoding) { }
static Encoded encode(String canonicalResponse) {
return encode(canonicalResponse, MAX_BYTES, MAX_COMPRESSED_BYTES);
}
/** Oversized cache entries are unavailable for history; the underlying formal query still succeeds. */
static Encoded encode(String canonicalResponse, int maxBytes, int maxCompressedBytes) {
if(canonicalResponse==null||canonicalResponse.isEmpty()||canonicalResponse.length()>maxBytes)return null;
byte[] bytes=canonicalResponse.getBytes(StandardCharsets.UTF_8);
if(bytes.length>maxBytes)return null;
try {
var compressed=new ByteArrayOutputStream();
try(var gzip=new GZIPOutputStream(compressed)) { gzip.write(bytes); }
if(compressed.size()>maxCompressedBytes)return null;
return new Encoded(compressed.toByteArray(),bytes.length,ENCODING);
} catch(IOException error) {
throw new IllegalStateException("查询结果快照编码失败",error);
}
}
static String decode(byte[] payload, int expectedBytes, String encoding, String expectedHash) {
if(!ENCODING.equals(encoding)||payload==null||payload.length==0||payload.length>MAX_COMPRESSED_BYTES
||expectedBytes<1||expectedBytes>MAX_BYTES||expectedHash==null||!expectedHash.matches("[a-f0-9]{64}"))
throw invalid();
try {
var decoded=new ByteArrayOutputStream(Math.min(expectedBytes,8192));
try(var gzip=new GZIPInputStream(new ByteArrayInputStream(payload))) {
byte[] buffer=new byte[8192];int count;
while((count=gzip.read(buffer))!=-1) {
if(count>expectedBytes-decoded.size())throw invalid();
decoded.write(buffer,0,count);
}
}
byte[] bytes=decoded.toByteArray();
if(bytes.length!=expectedBytes)throw invalid();
String text=StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(bytes)).toString();
if(!MessageDigest.isEqual(AnalysisProtocolService.hash(text).getBytes(StandardCharsets.US_ASCII),
expectedHash.getBytes(StandardCharsets.US_ASCII)))throw invalid();
return text;
} catch(CharacterCodingException error) {
throw invalid();
} catch(IOException error) {
throw invalid();
}
}
private static IllegalArgumentException invalid() {
return new IllegalArgumentException("历史查询结果快照缺失、损坏或与原查询指纹不符,请重新查询");
}
}
@@ -0,0 +1,74 @@
package cn.yunsheng.quality.analysis;
import java.util.*;
import tools.jackson.databind.json.JsonMapper;
/** Compiles one observed next step without changing the confirmed business task. */
final class AnalysisProtocolExploration {
static boolean nativeAgent(Map<String,Object> task){return "native_agent".equals(task.get("executionMode"));}
static AnalysisProtocolQueryPlan.Resolution compile(JsonMapper json,Map<String,Object> root,Map<String,Object> node,
Map<String,Object> inherited,String execution,String release,Map<String,Object> catalog) {
String purpose=Objects.toString(node.get("purpose"),"");
var selector=AnalysisProtocolService.map(node.getOrDefault("selector",Map.of("kind","base")));
if(!Set.of("base","catalog_members").contains(selector.get("kind")))throw new IllegalArgumentException("自主步骤只能使用基础范围或授权目录中的单个对象");
boolean catalogMember="catalog_members".equals(selector.get("kind"));
if(catalogMember&&(!selector.keySet().equals(Set.of("kind","dimension","names"))||!(selector.get("dimension") instanceof String)||AnalysisProtocolService.list(selector.get("names")).size()!=1||!(AnalysisProtocolService.list(selector.get("names")).getFirst() instanceof String name)||name.isBlank()))throw new IllegalArgumentException("自主步骤每次只能选择一个目录对象");
if(!catalogMember&&!selector.keySet().equals(Set.of("kind")))throw new IllegalArgumentException("基础范围不能携带额外条件");
if("source".equals(purpose)&&!Boolean.TRUE.equals(root.get("verifySource"))||"records".equals(purpose)&&!Boolean.TRUE.equals(root.get("fullDetails")))throw new IllegalArgumentException("根任务没有授权原件读取");
var rootStages=AnalysisProtocolService.list(root.get("stages"));var stages=AnalysisProtocolService.list(node.get("stages"));
boolean bothStages=new HashSet<>(rootStages).equals(Set.of("black","bright"));
boolean combinedRate=stages.isEmpty()&&bothStages&&Set.of("metrics","trend").contains(purpose)&&AnalysisProtocolService.list(node.get("fields")).equals(List.of("rate"));
boolean combinedChanges=stages.isEmpty()&&bothStages&&"changes".equals(purpose);
if(!rootStages.isEmpty()&&(!rootStages.containsAll(stages)||stages.isEmpty()&&!combinedRate&&!combinedChanges))throw new IllegalArgumentException("自主步骤不能扩大根任务阶段");
// A combined rate read returns both stage baselines. Keep their coverage obligations explicit;
// the actual metrics request has no stage parameter. Combined changes retain stage=null.
var next=new LinkedHashMap<>(node);if(combinedRate)next.put("stages",List.of("black","bright"));
// Parent filters have already passed the same compiler and were persisted with immutable scopes.
Map<String,Object> selected=new LinkedHashMap<>(inherited);
if(catalogMember) {
String dimension=(String)selector.get("dimension"),name=(String)AnalysisProtocolService.list(selector.get("names")).getFirst();
if("projectGroup".equals(dimension)) {
if(!AnalysisProtocolService.list(catalog.get("projectGroups")).contains(name)||!AnalysisProtocolChanges.selectionAllows(inherited.get(dimension),name)||AnalysisProtocolService.list(AnalysisProtocolService.map(inherited.get("exclusions")).get(dimension)).contains(name))throw new IllegalArgumentException("项目组不能超出父范围或授权目录");
selected.put(dimension,name);
} else selected=AnalysisProtocolQueryPlan.entityFilters(inherited,dimension,name,catalog);
next.put("selector",Map.of("kind","base"));
}
var task=new LinkedHashMap<>(root);task.put("filters",selected);task.put("queryPlan",List.of(next));
var dimensions=new LinkedHashSet<Object>();
if(Set.of("ranking","changes").contains(purpose))dimensions.add(node.get("dimension"));
if("defects".equals(purpose)){dimensions.add("defect");if("product".equals(node.get("dimension")))dimensions.add("product");}
task.put("dimensions",List.copyOf(dimensions));task.put("fields",Set.of("source","records").contains(purpose)?List.of():node.get("fields"));
task.put("stages",next.get("stages"));task.put("trendRequired","trend".equals(purpose));
boolean compare=Boolean.TRUE.equals(node.get("comparisonRequired"));task.put("comparisonRequired",compare);if(!compare)task.remove("comparisonPeriod");
task.put("verifySource","source".equals(purpose));task.put("fullDetails","records".equals(purpose));
var compiler=new AnalysisProtocolQueryPlan(json);task.put("queryPlan",compiler.normalize(task,catalog,List.of()));
var result=compiler.resolve(task,execution,release,catalog,List.of(),List.of());
for(var scope:result.scopes()) {
scope.put("parentScopeHash",AnalysisProtocolService.hash(json.writeValueAsString(AnalysisProtocolService.canonical(Map.of("releaseId",release,"filters",AnalysisProtocolQueryPlan.identityFilters(inherited))))));
if(catalogMember)scope.put("label",AnalysisProtocolService.list(selector.get("names")).getFirst());
else {var names=new ArrayList<String>();for(String key:List.of("department","group","product","projectGroup"))if(inherited.get(key) instanceof String name&&!name.isBlank())names.add(name);if(!names.isEmpty())scope.put("label",String.join(" / ",names));}
}
return result;
}
static List<String> goalGaps(Map<String,Object> root,List<Map<String,Object>> scopes) {
var gaps=new ArrayList<String>();
var base=scopes.stream().filter(s->sameFilters(root,AnalysisProtocolService.map(s.get("task")))).toList();
var stages=AnalysisProtocolService.list(root.get("stages"));List<?> requiredStages=stages.isEmpty()?List.of("overall"):stages;
for(Object field:AnalysisProtocolService.list(root.get("fields")))for(Object stage:requiredStages) {
var candidates=base.stream().filter(s->Set.of("metrics","trend").contains(s.get("purpose")))
.filter(s->{var child=AnalysisProtocolService.map(s.get("task"));var own=AnalysisProtocolService.list(child.get("stages"));return AnalysisProtocolService.list(child.get("fields")).contains(field)&&("overall".equals(stage)?own.isEmpty():own.contains(stage));}).toList();
if(candidates.isEmpty())gaps.add("根任务指标阶段尚未核实:"+field+"/"+stage);
if((Boolean.TRUE.equals(root.get("comparisonRequired"))||root.get("comparisonPeriod") instanceof Map<?,?>)&&candidates.stream().noneMatch(s->Boolean.TRUE.equals(AnalysisProtocolService.map(s.get("task")).get("comparisonRequired"))))gaps.add("根任务期间比较尚未核实:"+field+"/"+stage);
if(Boolean.TRUE.equals(root.get("trendRequired"))&&candidates.stream().noneMatch(s->"trend".equals(s.get("purpose"))))gaps.add("根任务趋势尚未核实:"+field+"/"+stage);
}
for(Object dimension:AnalysisProtocolService.list(root.get("dimensions"))) {
var matching=scopes.stream().filter(s->AnalysisProtocolService.list(AnalysisProtocolService.map(s.get("task")).get("dimensions")).contains(dimension)).toList();
var explicit=AnalysisProtocolService.list(AnalysisProtocolService.map(root.get("dimensionStages")).get(dimension));
if(explicit.isEmpty()){if(matching.isEmpty())gaps.add("根任务分组目标尚未核实:"+dimension);}
else for(Object stage:explicit)if(matching.stream().noneMatch(scope->AnalysisProtocolService.list(AnalysisProtocolService.map(scope.get("task")).get("stages")).contains(stage)))gaps.add("根任务分组阶段目标尚未核实:"+dimension+"/"+stage);
}
for(String purpose:List.of("source","records"))if(Boolean.TRUE.equals(root.get(purpose.equals("source")?"verifySource":"fullDetails"))&&base.stream().noneMatch(s->purpose.equals(s.get("purpose"))))gaps.add("根任务原件目标尚未核实:"+purpose);
return gaps;
}
private static boolean sameFilters(Map<String,Object> root,Map<String,Object> child){return AnalysisProtocolService.canonical(AnalysisProtocolQueryPlan.identityFilters(AnalysisProtocolService.map(root.get("filters")))).equals(AnalysisProtocolService.canonical(AnalysisProtocolQueryPlan.identityFilters(AnalysisProtocolService.map(child.get("filters")))));}
}
@@ -0,0 +1,304 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import cn.yunsheng.quality.options.OptionsController;
import java.util.*;
import org.springframework.http.HttpStatus;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.web.server.ResponseStatusException;
import tools.jackson.databind.DeserializationFeature;
import tools.jackson.databind.json.JsonMapper;
/** Server-owned import of frozen, same-session business responses into a distinct report execution. */
final class AnalysisProtocolHistory {
@FunctionalInterface interface QueryValidator {
Map<String,Object> validate(Map<String,Object> source,Map<String,Object> query,Map<String,Object> response,AppUserPrincipal actor);
}
private final JdbcTemplate jdbc;
private final JsonMapper json;
private final GovernanceAccessService governance;
private final OptionsController options;
private final JsonMapper exact=JsonMapper.builder().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS).build();
AnalysisProtocolHistory(JdbcTemplate jdbc,JsonMapper json,GovernanceAccessService governance,OptionsController options) {
this.jdbc=jdbc;this.json=json;this.governance=governance;this.options=options;
}
Map<String,Object> freeze(AnalysisProtocolService.Attempt a,Map<String,Object> proposed,AppUserPrincipal actor,boolean autonomous) {
requireAccess(a.releaseId(),actor);
if(!(autonomous?Set.of("all","last","first"):Set.of("all","last")).contains(Objects.toString(proposed.get("historyMode"),""))||!Boolean.FALSE.equals(proposed.get("freshQuery"))
||Boolean.TRUE.equals(proposed.get("verifySource"))||Boolean.TRUE.equals(proposed.get("fullDetails"))
||proposed.containsKey("queryPlan")||proposed.containsKey("directResponse")
||!map(proposed.get("filters")).isEmpty()||!map(proposed.get("period")).isEmpty()||proposed.get("comparisonPeriod")!=null)
throw bad("历史报告只整理已核验的原范围;重新查询或改变范围需要新的分析任务");
boolean first="first".equals(proposed.get("historyMode"));
List<String> ids=jdbc.query("SELECT id,plan_json,status,answer_json FROM analysis_turn WHERE session_id=? AND user_id=? AND id<>? ORDER BY created_at,id",
(rs,n)->Map.of("id",rs.getString(1),"plan",read(rs.getString(2)),"status",Objects.toString(rs.getString(3),""),"answer",first?map(read(rs.getString(4))):Map.of()),a.sessionId(),a.userId(),a.turnId()).stream()
.filter(row->factTurn(map(row.get("plan"))))
.filter(row->!first||Set.of("COMPLETED","PARTIAL").contains(row.get("status"))&&!list(map(row.get("answer")).get("protocolQueryIds")).isEmpty())
.map(row->Objects.toString(row.get("id"))).toList();
if(first&&!ids.isEmpty())ids=ids.subList(0,1);
if("last".equals(proposed.get("historyMode"))&&!ids.isEmpty())ids=ids.subList(ids.size()-1,ids.size());
if(ids.size()>512)throw bad("历史轮次超过本次整理上限,完整报告尚未生成");
if(proposed.containsKey("historySourceTurnIds")&&!Objects.equals(proposed.get("historySourceTurnIds"),ids))
throw denied("历史来源选择与当前会话完整选择不一致");
Map<String,Object> result=new LinkedHashMap<>(proposed);
result.put("filters",Map.of());result.put("period",null);result.put("comparisonPeriod",null);result.remove("comparisonMode");
for(String key:List.of("dimensions","fields","stages","memberFields"))result.put(key,List.of());
result.put("dimensionStages",Map.of());
for(String key:List.of("comparisonRequired","trendRequired","verifySource","fullDetails"))result.put(key,false);
result.put("historySourceTurnIds",List.copyOf(ids));
result.put("historySelectionHash",hash(selection(a,ids)));
return result;
}
Map<String,Object> importSnapshot(AnalysisProtocolService.Attempt a,Map<String,Object> task,AppUserPrincipal actor,QueryValidator validator) {
requireAccess(a.releaseId(),actor);
List<String> ids=sourceIds(task);String selectionHash=hash(selection(a,ids));
if(!Objects.equals(task.get("historySelectionHash"),selectionHash))throw denied("历史来源在确认后已变化,请重新整理");
List<Map<String,Object>> existing=jdbc.query("SELECT * FROM analysis_protocol_history_import WHERE attempt_id=?",
(rs,n)->importRow(rs),a.attemptId());
if(!existing.isEmpty()) {
Map<String,Object> stored=existing.getFirst();checkImportIdentity(a,task,stored);
return visibleImport(a,stored,verifiedPayload(a,task,actor,validator,stored));
}
Map<String,Object> payload=build(a,task,actor,validator);String text=write(payload),digest=AnalysisProtocolService.hash(text);
var encoded=AnalysisProtocolEvidenceSnapshot.encode(text);
if(encoded==null)throw bad("历史整理结果超过快照容量上限,完整报告尚未生成");
String id=UUID.randomUUID().toString();
jdbc.update("INSERT INTO analysis_protocol_history_import(import_id,attempt_id,execution_id,revision,auth_scope_hash,release_id,task_hash,selection_hash,snapshot_hash,snapshot_payload,snapshot_bytes,snapshot_encoding) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
id,a.attemptId(),a.executionId(),a.revision(),a.scopeHash(),a.releaseId(),hash(task),selectionHash,digest,encoded.payload(),encoded.bytes(),encoded.encoding());
Map<String,Object> stored=new LinkedHashMap<>();stored.put("id",id);stored.put("hash",digest);
return visibleImport(a,stored,payload);
}
Map<String,Object> finish(AnalysisProtocolService.Attempt a,Map<String,Object> task,Map<String,Object> details,AppUserPrincipal actor,QueryValidator validator,boolean autonomous) {
if(!list(details.get("protocolEvidence")).isEmpty())throw denied("历史报告不能把旧查询证明提交为本次正式查询");
Map<String,Object> proof=map(details.get("historyEvidence"));
List<Map<String,Object>> rows=jdbc.query("SELECT * FROM analysis_protocol_history_import WHERE import_id=? AND attempt_id=?",
(rs,n)->importRow(rs),proof.get("importId"),a.attemptId());
if(rows.size()!=1)throw denied("缺少当前报告执行的历史导入凭据");
var stored=rows.getFirst();checkImportIdentity(a,task,stored);
if(!write(canonical(proof)).equals(write(canonical(identity(a,stored)))))throw denied("历史导入凭据属于其他报告执行或已失效");
Map<String,Object> payload=verifiedPayload(a,task,actor,validator,stored);
var facts=list(payload.get("facts")).stream().map(AnalysisProtocolHistory::map).toList();
Set<Object> known=new HashSet<>();for(var fact:facts)known.add(fact.get("id"));
List<?> selected=list(details.get("historyHeadlineFactIds"));
if(!(details.get("historyHeadlineFactIds") instanceof List<?>)||selected.size()>5||new HashSet<>(selected).size()!=selected.size()
||selected.stream().anyMatch(id->!(id instanceof String)||!known.contains(id)))throw denied("报告重点引用了未核验的历史事实");
String title=Objects.toString(details.get("reportTitle"),"");
boolean safeWorkerTitle=autonomous&&"verified_history_summary".equals(details.get("titleSource"))&&"历史质量分析汇总报告".equals(title)&&validTitle(title,payload,task);
boolean titled=(Boolean.TRUE.equals(details.get("titleGeneratedByModel"))||safeWorkerTitle)&&validTitle(title,payload,task);
List<String> gaps=new ArrayList<>(strings(map(payload.get("historyCoverage")).get("missingEvidence")));
if(!titled)gaps.add("报告标题尚未完成有效概括,保留已核实数据");
if(!facts.isEmpty()&&selected.isEmpty())gaps.add("报告重点尚未完成选择,保留完整数据明细");
List<Map<String,Object>> projected=list(payload.get("sourceResults")).stream().map(AnalysisProtocolHistory::map).toList();
var rendered=new LinkedHashMap<>(AnalysisProtocolHistoryFacts.render(titled?title:"历史质量分析汇总",projected,strings(selected),gaps));
if(!facts.isEmpty()&&selected.isEmpty())rendered.put("answer","已核验的历史数据已保留,重点结论尚未整理完成,可查看报告中的完整表格。");
if(autonomous) {
var knownIds=new LinkedHashSet<String>();for(var fact:facts)if(fact.get("id") instanceof String id)knownIds.add(id);
var presented=AnalysisProtocolHistorySummary.normalize(details,knownIds,Objects.toString(rendered.get("answer"),""));
gaps.addAll(presented.gaps());rendered.put("answer",presented.answer());rendered.put("readingSummary",presented.summary());rendered.put("queryEvidenceRefs",presented.refs());
rendered.put("titleSource",safeWorkerTitle?"verified_history_summary":"model");rendered.put("titleGeneratedByModel",Boolean.TRUE.equals(details.get("titleGeneratedByModel")));
StringBuilder anchors=new StringBuilder("<details class=\"history-facts\"><summary>引用依据</summary><ul>");
for(var fact:facts) {String id=Objects.toString(fact.get("id"),"");anchors.append("<li id=\"history-fact:").append(escape(id)).append("\" data-history-fact-id=\"").append(escape(id)).append("\">").append(escape(Objects.toString(fact.get("text"),""))).append("</li>");}
anchors.append("</ul></details>");String report=Objects.toString(rendered.get("reportHtml"),"");int end=report.lastIndexOf("</article>");
if(end>=0)rendered.put("reportHtml",report.substring(0,end)+anchors+report.substring(end));
}
rendered.put("historySources",payload.get("historySources"));rendered.put("historyCoverage",Map.of("complete",gaps.isEmpty()&&Boolean.TRUE.equals(map(payload.get("historyCoverage")).get("complete")),
"expectedSources",map(payload.get("historyCoverage")).get("expectedSources"),"importedSources",map(payload.get("historyCoverage")).get("importedSources"),"missingEvidence",gaps.stream().distinct().toList()));
rendered.put("historyEvidence",identity(a,stored));rendered.put("completionMethod","confirmed_history_report");
rendered.put("reportTitle",titled?title:"历史质量分析汇总");return rendered;
}
void authorizeSaved(String turnId,AppUserPrincipal actor,QueryValidator validator) {
var rows=jdbc.query("SELECT a.attempt_id,a.session_id,a.turn_id,a.execution_id,a.revision,t.auth_scope_hash,s.user_id,s.release_id,r.task_json FROM analysis_protocol_history_import h JOIN analysis_protocol_attempt a ON a.attempt_id=h.attempt_id JOIN analysis_session s ON s.id=a.session_id JOIN analysis_turn t ON t.id=a.turn_id AND JSON_UNQUOTE(JSON_EXTRACT(t.plan_json,'$.attemptId'))=a.attempt_id JOIN analysis_protocol_revision r ON r.session_id=a.session_id AND r.revision=a.revision WHERE a.turn_id=?",
(rs,n)->new Object[]{rs.getString(1),rs.getString(2),rs.getString(3),rs.getString(4),rs.getLong(5),rs.getString(6),rs.getString(7),rs.getString(8),read(rs.getString(9))},turnId);
if(rows.size()!=1)throw denied("历史报告缺少可核对的导入来源");var row=rows.getFirst();
if(!actor.getId().toString().equals(row[6])||!Objects.equals(row[5],AuthorizationFingerprint.of(actor,governance)))throw denied("历史报告授权范围已变化");
var a=new AnalysisProtocolService.Attempt((String)row[0],(String)row[1],(String)row[2],"COMPLETED",(Long)row[4],(String)row[3],Map.of(),(String)row[6],(String)row[7],(String)row[5],false,"COMPLETED",(String)row[0]);
var task=map(row[8]);var stored=jdbc.query("SELECT * FROM analysis_protocol_history_import WHERE attempt_id=?",(rs,n)->importRow(rs),a.attemptId()).getFirst();
checkImportIdentity(a,task,stored);verifiedPayload(a,task,actor,validator,stored);
}
private Map<String,Object> build(AnalysisProtocolService.Attempt a,Map<String,Object> task,AppUserPrincipal actor,QueryValidator validator) {
List<Map<String,Object>> sources=new ArrayList<>(),results=new ArrayList<>(),sections=new ArrayList<>(),facts=new ArrayList<>();
List<String> gaps=new ArrayList<>();long budget=0;
for(String id:sourceIds(task)) {
Map<String,Object> source=source(a,id,actor);List<Map<String,Object>> queries=new ArrayList<>();
Map<String,Object> meta=metadata(source);sources.add(meta);
try {
if(!Set.of("COMPLETED","PARTIAL").contains(source.get("status")))throw denied("该轮未形成已核验结果");
List<?> accepted=list(map(source.get("result")).get("protocolQueryIds"));
if(accepted.isEmpty()||new HashSet<>(accepted).size()!=accepted.size())throw denied("该轮缺少最终接受的正式查询证明");
List<Map<String,Object>> all=queryRows(id);source.put("queryRecords",all);
for(Object queryId:accepted) {
var query=all.stream().filter(q->Objects.equals(q.get("queryId"),queryId)).findFirst().orElseThrow(()->denied("历史查询记录已不存在"));
if(!Objects.equals(source.get("attemptId"),query.get("attemptId"))||!Objects.equals(source.get("executionId"),query.get("executionId"))
||!Objects.equals(source.get("revision"),query.get("revision"))||!Objects.equals(a.scopeHash(),query.get("scopeHash")))throw denied("历史查询与源执行绑定不一致");
int size=((Number)query.getOrDefault("bytes",0)).intValue();
if(size<1||budget+size>AnalysisProtocolEvidenceSnapshot.MAX_BYTES)throw denied("历史原始结果快照缺失或本次整理容量不足");
String raw=AnalysisProtocolEvidenceSnapshot.decode((byte[])query.get("payload"),size,Objects.toString(query.get("encoding"),null),Objects.toString(query.get("responseHash"),null));
budget+=size;var response=exact.readValue(raw,Map.class);
queries.add(validator.validate(source,query,response,actor));
}
source.put("queries",queries);
if("PARTIAL".equals(source.get("status"))&&AnalysisProtocolExploration.nativeAgent(map(source.get("task"))))source.put("gaps",strings(map(map(source.get("result")).get("executionCoverage")).get("missingEvidence")));
var result=new LinkedHashMap<>(AnalysisProtocolHistoryFacts.buildSource(source,json));
result.put("sourceTurnId",id);result.put("task",source.get("task"));result.put("metadata",meta);
for(String key:List.of("status","revision","executionId"))result.put(key,source.get(key));
int sectionOffset=sections.size();
var displayFacts=new ArrayList<Map<String,Object>>();
for(Object item:list(result.get("facts"))) {var fact=new LinkedHashMap<>(map(item));if(fact.get("sectionIndex") instanceof Number index)fact.put("sectionIndex",sectionOffset+index.intValue());displayFacts.add(fact);}
result.put("facts",displayFacts);
if(!list(result.get("metricContexts")).isEmpty())meta.put("metricContexts",result.get("metricContexts"));
results.add(result);sections.addAll(list(result.get("sections")).stream().map(AnalysisProtocolHistory::map).toList());
facts.addAll(displayFacts);
meta.put("available",true);meta.put("queryIds",List.copyOf(accepted));
if(!Boolean.TRUE.equals(result.get("complete"))||!"COMPLETED".equals(source.get("status")))gaps.add("历史轮次 "+id+" 尚未完整覆盖其原任务");
gaps.addAll(strings(result.get("gaps")));
} catch(RuntimeException invalid) {
meta.put("available",false);meta.put("reason",safeReason(invalid));gaps.add("历史轮次 "+id+":"+safeReason(invalid));
}
}
if(sources.isEmpty())gaps.add("当前会话没有可复用的历史正式分析");
var payload=new LinkedHashMap<String,Object>();payload.put("historySources",sources);payload.put("sourceResults",results);payload.put("sections",sections);payload.put("facts",facts);
payload.put("historyCoverage",Map.of("complete",gaps.isEmpty(),"expectedSources",sources.size(),"importedSources",results.size(),"missingEvidence",gaps.stream().distinct().toList()));
return payload;
}
/** A presentation recovery reads the original accepted responses, never a model/evidence file. */
Map<String,Object> recoverySource(AnalysisProtocolService.Attempt a,AppUserPrincipal actor,QueryValidator validator) {
requireAccess(a.releaseId(),actor);
var source=source(a,a.turnId(),actor,true);
if(!Set.of("COMPLETED","PARTIAL").contains(source.get("status"))
||!Objects.equals(a.attemptId(),source.get("attemptId"))||!Objects.equals(a.executionId(),source.get("executionId"))
||!Objects.equals(a.revision(),source.get("revision"))
||!Objects.equals(canonical(source.get("answer")),canonical(source.get("result"))))throw denied("原分析结果或执行身份已变化,不能恢复图表");
var plan=map(source.get("plan"));
if(!Objects.equals(a.executionId(),plan.get("executionId"))||!Objects.equals(String.valueOf(a.revision()),String.valueOf(plan.get("revision"))))
throw denied("原分析修订与执行身份不一致");
List<?> accepted=list(map(source.get("result")).get("protocolQueryIds"));
if(accepted.isEmpty()||new HashSet<>(accepted).size()!=accepted.size())throw denied("原分析没有完整的已接受查询证明");
var all=queryRows(a.turnId());source.put("queryRecords",all);
List<Map<String,Object>> checked=new ArrayList<>();long budget=0;
for(Object id:accepted) {
var q=all.stream().filter(row->Objects.equals(id,row.get("queryId"))).findFirst().orElseThrow(()->denied("原始查询已不存在"));
if(!Objects.equals(a.attemptId(),q.get("attemptId"))||!Objects.equals(a.executionId(),q.get("executionId"))
||!Objects.equals(a.revision(),q.get("revision"))||!Objects.equals(a.scopeHash(),q.get("scopeHash")))throw denied("原始查询不属于该执行");
int size=((Number)q.getOrDefault("bytes",0)).intValue();
if(size<1||(budget+=size)>AnalysisProtocolEvidenceSnapshot.MAX_BYTES)throw denied("原始响应缺失或超出图表恢复容量");
String raw=AnalysisProtocolEvidenceSnapshot.decode((byte[])q.get("payload"),size,Objects.toString(q.get("encoding"),null),Objects.toString(q.get("responseHash"),null));
checked.add(validator.validate(source,q,exact.readValue(raw,Map.class),actor));
}
source.put("queries",checked);source.remove("queryRecords");return source;
}
private Map<String,Object> verifiedPayload(AnalysisProtocolService.Attempt a,Map<String,Object> task,AppUserPrincipal actor,QueryValidator validator,Map<String,Object> stored) {
requireAccess(a.releaseId(),actor);
if(!Objects.equals(task.get("historySelectionHash"),hash(selection(a,sourceIds(task)))))throw denied("历史来源在报告确认后已变化");
// Recheck payload integrity, accepted proofs and current source scopes even for a cached import.
var original=decodeImport(stored);var current=build(a,task,actor,validator);
if(!write(canonical(sourceIdentities(original.get("historySources")))).equals(write(canonical(sourceIdentities(current.get("historySources"))))))
throw denied("历史来源的可用性或原始响应完整性已变化");
var before=map(original.get("historyCoverage"));var after=map(current.get("historyCoverage"));
for(String key:List.of("complete","expectedSources","importedSources"))if(!Objects.equals(before.get(key),after.get(key)))
throw denied("历史来源覆盖状态已变化,请重新整理报告");
return original;
}
private List<Map<String,Object>> sourceIdentities(Object raw) {
// Formula descriptions are optional projection metadata. Older immutable imports may
// omit them; source identity, query IDs and response hashes are still revalidated above.
var identities=new ArrayList<Map<String,Object>>();
for(Object item:list(raw)){var identity=new LinkedHashMap<>(map(item));identity.remove("metricContexts");identities.add(identity);}
return identities;
}
private Map<String,Object> source(AnalysisProtocolService.Attempt a,String id,AppUserPrincipal actor) {
return source(a,id,actor,false);
}
private Map<String,Object> source(AnalysisProtocolService.Attempt a,String id,AppUserPrincipal actor,boolean allowSelf) {
var rows=jdbc.query("SELECT t.id,t.status,t.auth_scope_hash,t.plan_json,t.answer_json,s.release_id,a.attempt_id,a.execution_id,a.revision,a.input_json,e.request_digest,e.result_json,r.task_json FROM analysis_turn t JOIN analysis_session s ON s.id=t.session_id LEFT JOIN analysis_protocol_attempt a ON a.attempt_id=JSON_UNQUOTE(JSON_EXTRACT(t.plan_json,'$.attemptId')) LEFT JOIN analysis_protocol_execution e ON e.execution_id=a.execution_id LEFT JOIN analysis_protocol_revision r ON r.session_id=t.session_id AND r.revision=a.revision WHERE t.id=? AND t.session_id=? AND t.user_id=? AND s.user_id=? AND t.id<>?",
(rs,n)->{var m=new LinkedHashMap<String,Object>();m.put("turnId",rs.getString(1));m.put("status",rs.getString(2));m.put("scopeHash",rs.getString(3));m.put("plan",read(rs.getString(4)));m.put("answer",read(rs.getString(5)));m.put("releaseId",rs.getString(6));m.put("attemptId",rs.getString(7));m.put("executionId",rs.getString(8));m.put("revision",rs.getObject(9,Long.class));m.put("input",read(rs.getString(10)));m.put("taskHash",rs.getString(11));m.put("result",read(rs.getString(12)));m.put("task",read(rs.getString(13)));return m;},id,a.sessionId(),a.userId(),a.userId(),allowSelf?"":a.turnId());
if(rows.size()!=1)throw denied("历史来源不属于当前会话或已删除");var source=rows.getFirst();
if(!Objects.equals(a.releaseId(),source.get("releaseId"))||!Objects.equals(a.scopeHash(),source.get("scopeHash"))
||!Objects.equals(a.scopeHash(),AuthorizationFingerprint.of(actor,governance)))throw denied("历史来源发布版本或授权范围已变化");
if(!map(source.get("task")).isEmpty()&&!Objects.equals(source.get("taskHash"),hash(source.get("task"))))throw denied("源任务已变化");
if(!map(source.get("task")).isEmpty()&&!Objects.equals(canonical(map(source.get("plan")).get("task")),canonical(source.get("task"))))throw denied("源轮次与已确认任务不一致");
if(!Objects.equals(canonical(map(source.get("answer")).get("protocolQueryIds")),canonical(map(source.get("result")).get("protocolQueryIds"))))throw denied("源轮次的最终查询清单已变化");
return source;
}
private List<Map<String,Object>> selection(AnalysisProtocolService.Attempt a,List<String> ids) {
List<Map<String,Object>> result=new ArrayList<>();
for(String id:ids) {
var rows=jdbc.query("SELECT status,plan_json,answer_json,auth_scope_hash FROM analysis_turn WHERE id=? AND session_id=? AND user_id=? AND id<>?",
(rs,n)->Map.of("turnId",id,"status",rs.getString(1),"plan",read(rs.getString(2)),"answer",read(rs.getString(3)),"scopeHash",rs.getString(4)),id,a.sessionId(),a.userId(),a.turnId());
if(rows.size()!=1)throw denied("历史来源已删除或不属于本会话");var row=rows.getFirst();var selected=new LinkedHashMap<String,Object>();
for(String key:List.of("turnId","status","scopeHash"))selected.put(key,row.get(key));
var plan=map(row.get("plan"));for(String key:List.of("attemptId","executionId","revision"))selected.put(key,plan.get(key));
selected.put("queryIds",map(row.get("answer")).get("protocolQueryIds"));selected.put("queries",queryRows(id).stream().filter(q->list(selected.get("queryIds")).contains(q.get("queryId")))
.map(q->{var m=new LinkedHashMap<String,Object>();for(String key:List.of("queryId","attemptId","executionId","revision","scopeHash","responseHash","target","parameters","nodeId","scopeId","bytes","encoding"))m.put(key,q.get(key));return m;}).toList());
result.add(selected);
}
return result;
}
private List<Map<String,Object>> queryRows(String turn) {
return jdbc.query("SELECT q.* FROM analysis_protocol_query q JOIN analysis_protocol_attempt a ON a.attempt_id=q.attempt_id WHERE a.turn_id=? ORDER BY q.created_at DESC,q.query_id DESC",(rs,n)->{
var q=new LinkedHashMap<String,Object>();q.put("queryId",rs.getString("query_id"));q.put("attemptId",rs.getString("attempt_id"));q.put("executionId",rs.getString("execution_id"));q.put("revision",rs.getLong("revision"));q.put("scopeHash",rs.getString("auth_scope_hash"));q.put("responseHash",rs.getString("response_hash"));q.put("target",rs.getString("query_target"));q.put("parameters",read(rs.getString("parameters_json")));q.put("coverage",read(rs.getString("coverage_json")));q.put("rankArtifact",read(rs.getString("rank_artifact_json")));q.put("nodeId",rs.getString("protocol_node_id"));q.put("scopeId",rs.getString("protocol_scope_id"));q.put("payload",rs.getBytes("response_snapshot"));q.put("bytes",rs.getInt("response_snapshot_bytes"));q.put("encoding",rs.getString("response_snapshot_encoding"));return q;},turn);
}
private Map<String,Object> importRow(java.sql.ResultSet rs) throws java.sql.SQLException {
var m=new LinkedHashMap<String,Object>();m.put("id",rs.getString("import_id"));m.put("attemptId",rs.getString("attempt_id"));m.put("executionId",rs.getString("execution_id"));m.put("revision",rs.getLong("revision"));m.put("scopeHash",rs.getString("auth_scope_hash"));m.put("releaseId",rs.getString("release_id"));m.put("taskHash",rs.getString("task_hash"));m.put("selectionHash",rs.getString("selection_hash"));m.put("hash",rs.getString("snapshot_hash"));m.put("payload",rs.getBytes("snapshot_payload"));m.put("bytes",rs.getInt("snapshot_bytes"));m.put("encoding",rs.getString("snapshot_encoding"));return m;
}
private void checkImportIdentity(AnalysisProtocolService.Attempt a,Map<String,Object> task,Map<String,Object> row) {
if(!Objects.equals(a.attemptId(),row.get("attemptId"))||!Objects.equals(a.executionId(),row.get("executionId"))||!Objects.equals(a.revision(),row.get("revision"))
||!Objects.equals(a.scopeHash(),row.get("scopeHash"))||!Objects.equals(a.releaseId(),row.get("releaseId"))||!Objects.equals(hash(task),row.get("taskHash"))
||!Objects.equals(task.get("historySelectionHash"),row.get("selectionHash")))throw denied("历史导入快照与当前报告任务不一致");
}
private Map<String,Object> decodeImport(Map<String,Object> row) {return exact.readValue(AnalysisProtocolEvidenceSnapshot.decode((byte[])row.get("payload"),((Number)row.get("bytes")).intValue(),Objects.toString(row.get("encoding"),null),Objects.toString(row.get("hash"),null)),Map.class);}
private Map<String,Object> visibleImport(AnalysisProtocolService.Attempt a,Map<String,Object> stored,Map<String,Object> payload) {var result=new LinkedHashMap<>(payload);result.remove("sourceResults");result.put("kind","protocol_history_snapshot");result.put("version",1);result.put("releaseId",a.releaseId());result.put("historyEvidence",identity(a,stored));return result;}
private Map<String,Object> identity(AnalysisProtocolService.Attempt a,Map<String,Object> stored) {return Map.of("importId",stored.get("id"),"attemptId",a.attemptId(),"executionId",a.executionId(),"revision",a.revision(),"sessionId",a.sessionId(),"turnId",a.turnId(),"snapshotHash",stored.get("hash"));}
private Map<String,Object> metadata(Map<String,Object> source) {var m=new LinkedHashMap<String,Object>();for(String key:List.of("turnId","revision","executionId","status"))m.put(key,source.get(key));m.put("available",false);return m;}
private void requireAccess(String release,AppUserPrincipal actor) {governance.requireDetail(actor);governance.requireBoundAction("QUERY","query.detail");governance.requireBoundAction("QUERY","query.aggregate");options.options(release,actor);}
private boolean factTurn(Map<String,Object> plan) {var task=map(plan.get("task"));return !Boolean.TRUE.equals(task.get("needsClarification"))&&!task.containsKey("directResponse")&&!Set.of("report","explain","restate").contains(Objects.toString(task.get("kind"),""));}
boolean validTitle(String title,Map<String,Object> payload,Map<String,Object> task) {if(title.isBlank()||title.length()>100||title.equals(task.get("question"))||title.chars().anyMatch(c->c<32)||title.matches(".*[<>].*")||mentionsPartialScope(title,payload))return false;var numeric=java.util.regex.Pattern.compile("[0-9]+(?:\\.[0-9]+)?");Set<String> allowed=new HashSet<>();var input=numeric.matcher(write(payload));while(input.find())allowed.add(input.group());var proposed=numeric.matcher(title);while(proposed.find())if(!allowed.contains(proposed.group()))return false;return true;}
private boolean mentionsPartialScope(String title,Map<String,Object> payload) {
List<Set<String>> frames=new ArrayList<>();Set<String> excluded=new HashSet<>();
for(Object section:list(payload.get("sections"))) {
var filters=map(map(section).get("filters"));frames.add(scopeNames(filters));
for(Object values:map(filters.get("exclusions")).values())scopeStrings(values,excluded);
for(Object rule:list(filters.get("columnFilters")))if("not_in".equals(map(rule).get("operator")))scopeStrings(map(rule).get("values"),excluded);
}
for(Object source:list(payload.get("historySources")))if(!Boolean.TRUE.equals(map(source).get("available")))frames.add(Set.of());
if(frames.isEmpty())return false;
Set<String> all=new HashSet<>(),common=new HashSet<>(frames.getFirst());
for(var frame:frames){all.addAll(frame);common.retainAll(frame);}
all.removeAll(common);all.addAll(excluded);
return all.stream().filter(v->!v.isBlank()).anyMatch(title::contains);
}
private Set<String> scopeNames(Map<String,Object> filters) {
Set<String> names=new HashSet<>();
for(String key:List.of("department","group","product","projectGroup"))scopeStrings(filters.get(key),names);
for(Object values:map(filters.get("extraFilters")).values())scopeStrings(values,names);
for(Object rule:list(filters.get("columnFilters")))if(!"not_in".equals(map(rule).get("operator")))scopeStrings(map(rule).get("values"),names);
return names;
}
private void scopeStrings(Object value,Set<String> names) {
if(value instanceof String text){if(!text.isBlank())names.add(text);}
else for(Object item:list(value))scopeStrings(item,names);
}
private List<String> sourceIds(Map<String,Object> task) {var ids=strings(task.get("historySourceTurnIds"));if(ids.size()!=list(task.get("historySourceTurnIds")).size()||new HashSet<>(ids).size()!=ids.size())throw denied("历史来源清单格式无效");return ids;}
private static String escape(String text){return org.springframework.web.util.HtmlUtils.htmlEscape(text);}
private static String safeReason(RuntimeException error) {return error instanceof ResponseStatusException status?Objects.toString(status.getReason(),"历史来源不可用"):error instanceof IllegalArgumentException?"历史响应完整性或范围校验失败":"历史来源尚不可复用";}
private String hash(Object value) {return AnalysisProtocolService.hash(write(canonical(value)));}
private Object canonical(Object value) {return AnalysisProtocolService.canonical(value);}
private String write(Object value) {return json.writeValueAsString(value);}
private Map<String,Object> read(String value) {return value==null?Map.of():json.readValue(value,Map.class);}
private static Map<String,Object> map(Object value) {return AnalysisProtocolService.map(value);}
private static List<?> list(Object value) {return AnalysisProtocolService.list(value);}
private static List<String> strings(Object value) {return list(value).stream().filter(String.class::isInstance).map(String.class::cast).toList();}
private static ResponseStatusException denied(String text) {return new ResponseStatusException(HttpStatus.FORBIDDEN,text);}
private static ResponseStatusException bad(String text) {return new ResponseStatusException(HttpStatus.CONFLICT,text);}
}
@@ -0,0 +1,873 @@
package cn.yunsheng.quality.analysis;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.*;
import tools.jackson.databind.json.JsonMapper;
/** Rebuilds historical report material from validated API response snapshots only. */
final class AnalysisProtocolHistoryFacts {
private static final List<String> PERIOD_KEYS = List.of("year", "month", "quarter", "monthStart", "monthEnd");
private static final List<String> METRIC_FIELDS = List.of("rate", "scrapRate", "totalWeight", "scrapWeight");
private static final Map<String,String> FIELD_LABELS = Map.of(
"rate", "合格率", "scrapRate", "报废率", "totalWeight", "投入重量", "scrapWeight", "报废重量",
"compositeRate", "综合合格率");
private static final JsonMapper HTML_JSON = JsonMapper.builder().build();
private AnalysisProtocolHistoryFacts() { }
static Map<String,Object> buildSource(Map<String,Object> source, JsonMapper json) {
Map<String,Object> frozen = map(source.get("task"));
String turnId = text(source.get("turnId"), "unknown-turn");
List<Map<String,Object>> sections = new ArrayList<>();
List<Map<String,Object>> facts = new ArrayList<>();
List<Map<String,Object>> metricContexts = new ArrayList<>();
List<String> gaps = new ArrayList<>();
addStrings(gaps, source.get("gaps"));
String sourceStatus = text(source.get("status"), "");
if (Set.of("PARTIAL", "FAILED", "CANCELLED", "NEEDS_CLARIFICATION").contains(sourceStatus.toUpperCase(Locale.ROOT)))
gaps.add("该历史轮次状态为 " + sourceStatus + "。");
int[] sequence = {0};
for (Map<String,Object> query : maps(source.get("queries"))) {
String target = text(query.get("target"), "");
Map<String,Object> response = map(query.get("response"));
Map<String,Object> coverage = map(query.get("coverage"));
Map<String,Object> scope = map(query.get("scope"));
Map<String,Object> queryTask = firstMap(query.get("task"), scope.get("task"), frozen);
Map<String,Object> parameters = stringMap(query.get("parameters"));
if (response.isEmpty()) {
gaps.add("历史查询 " + text(query.get("queryId"), "(unknown)") + " 缺少原始 API 响应。");
continue;
}
int sectionStart=sections.size();
switch (target) {
case "GET /api/metrics" -> buildMetrics(turnId, query, queryTask, scope, parameters, response,
coverage, sections, facts, gaps, sequence);
case "POST /api/analysis/changes" -> buildChanges(turnId, queryTask, scope, parameters, response,
coverage, sections, facts, gaps, sequence, json);
case "POST /api/analysis/defects" -> buildDefects(turnId, queryTask, scope, parameters, response,
coverage, sections, facts, gaps, sequence, json);
case "POST /api/analysis/source/members" -> { /* Rebuild all member pages together below. */ }
case "POST /api/analysis/investigate" -> {
Map<String,Object> body = body(parameters, json);
String action = text(body.get("action"), "");
if ("rows".equals(action)) {
// Grouped and rebuilt below after collecting all recorded pages.
} else if ("source".equals(action)) {
buildSourceReceipt(turnId, queryTask, scope, body, response, coverage,
sections, facts, gaps, sequence);
} else gaps.add("调查历史查询缺少有效 action,不能投影事实。");
}
case AnalysisProtocolInspection.TARGET -> { /* Both sealed inspection stages are projected together below. */ }
default -> gaps.add("历史查询目标未接入事实投影:" + target + "。");
}
Map<String,Object> metricContext=map(query.get("nativeMetricContext"));
if(!metricContext.isEmpty()) {
if(!metricContexts.contains(metricContext))metricContexts.add(metricContext);
for(int index=sectionStart;index<sections.size();index++)sections.get(index).put("metricContext",metricContext);
}
}
if(AnalysisProtocolInspection.KIND.equals(frozen.get("kind"))) {
var projected=AnalysisProtocolInspection.projection(turnId,maps(source.get("queries")));
sections.addAll(maps(projected.get("sections")));facts.addAll(maps(projected.get("facts")));addStrings(gaps,projected.get("gaps"));
}
buildMemberLists(turnId, source, sections, facts, gaps, sequence, json);
buildRecordLists(turnId, source, sections, facts, gaps, sequence, json);
if (sections.isEmpty() && facts.isEmpty()) gaps.add("没有可从已校验原始 API 响应重建的历史事实。");
List<String> uniqueGaps = gaps.stream().filter(Objects::nonNull).map(String::trim)
.filter(s -> !s.isEmpty()).distinct().toList();
var result=mapOf("sections", List.copyOf(sections), "facts", List.copyOf(facts),
"complete", uniqueGaps.isEmpty(), "gaps", uniqueGaps);
if(!metricContexts.isEmpty())result.put("metricContexts",List.copyOf(metricContexts));
return result;
}
private static void buildMetrics(String turnId, Map<String,Object> query, Map<String,Object> task,
Map<String,Object> scope, Map<String,Object> parameters, Map<String,Object> response,
Map<String,Object> coverage, List<Map<String,Object>> sections, List<Map<String,Object>> facts,
List<String> gaps, int[] sequence) {
Map<String,Object> artifact = map(query.get("rankArtifact"));
if (!artifact.isEmpty()) {
buildRanking(turnId, query, task, scope, parameters, artifact, coverage, sections, facts, gaps, sequence);
return;
}
String purpose = text(scope.get("purpose"), text(task.get("purpose"), ""));
String view = text(parameters.get("view"), "");
if ("ranking".equals(purpose) || list(task.get("dimensions")).contains(view)) {
gaps.add("排名查询缺少同一原始查询保存的Top快照,不能用模型回答或普通行补造排名。");
return;
}
List<String> fields = strings(task.get("fields"));
List<String> stages = strings(task.get("stages"));
boolean trend = Boolean.TRUE.equals(task.get("trendRequired"));
if (fields.isEmpty() || !METRIC_FIELDS.containsAll(fields) || !Set.of("black", "bright").containsAll(stages)) {
gaps.add("正式指标历史查询缺少已验证的字段或阶段选择。");
return;
}
if (stages.isEmpty() && !fields.equals(List.of("rate"))) {
gaps.add("综合正式指标只允许使用原始综合合格率,不能合并阶段重量或报废率。");
return;
}
if (Boolean.FALSE.equals(coverage.get("metricsComplete"))) addCoverageGaps("正式指标", coverage, gaps);
else if (!coverage.containsKey("metricsComplete")) gaps.add("正式指标历史查询缺少覆盖完整性回执。");
Map<String,Object> period = period(scope, task, parameters);
if (!validPeriod(period)) gaps.add("正式指标历史查询缺少有效的已验证期间。");
Map<String,Object> filters = filters(scope, task);
Map<String,Object> summary = map(response.get("summary"));
if (summary.isEmpty()) gaps.add("原始正式指标响应缺少summary。");
List<Map<String,Object>> summaryRows = new ArrayList<>();
for (String stage : stages.isEmpty() ? List.of("") : stages) {
for (String field : fields) {
Object value = stages.isEmpty() ? summary.get("compositeRate") : map(summary.get(stage)).get(field);
String stageLabel = stages.isEmpty() ? "综合" : stageLabel(stage);
Map<String,Object> row = mapOf("stage", stageLabel, "field", fieldLabel(field), "value", value);
summaryRows.add(row);
addFact(facts, turnId, sequence, scopedText(period, filters, stageLabel + fieldLabel(field)
+ " = " + valueText(value)));
displayFact(facts,sections.size(),summaryRows.size()-1,period,filters,stage,field,value,null);
}
}
if (summaryRows.isEmpty()) gaps.add("正式指标summary中没有本任务要求的字段数据。");
else sections.add(section("正式指标汇总", turnId, period, filters, null,
columns("stage", "阶段", "field", "指标", "value", "正式值"), summaryRows,
"数值直接取自已校验的正式指标响应;本报告不重算、不平均不同范围。"));
List<Map<String,Object>> rawRows = maps(response.get("rows"));
if (Boolean.TRUE.equals(response.get("rowsTruncated")) || Boolean.TRUE.equals(query.get("coverageTruncated")))
gaps.add("正式指标分组行有截断,不能称为完整分组清单。");
if (!rawRows.isEmpty()) {
List<Map<String,Object>> projected = new ArrayList<>();
for (Map<String,Object> raw : rawRows) {
for (String stage : stages.isEmpty() ? List.of("") : stages) for (String field : fields) {
Object value = stages.isEmpty() ? raw.get("compositeRate") : map(raw.get(stage)).get(field);
projected.add(mapOf("key", raw.get("key"), "label", raw.get("label"),
"stage", stages.isEmpty() ? "综合" : stageLabel(stage), "field", fieldLabel(field), "value", value));
addFact(facts, turnId, sequence, scopedText(period, filters,
text(raw.get("label"), text(raw.get("key"), "未命名分组")) + " · "
+ (stages.isEmpty() ? "综合" : stageLabel(stage)) + fieldLabel(field) + " = " + valueText(value)));
displayFact(facts,sections.size(),projected.size()-1,period,filters,stage,field,value,null);
}
}
sections.add(section("正式指标分组行", turnId, period, filters, null,
columns("key", "分组键", "label", "分组", "stage", "阶段", "field", "指标", "value", "正式值"),
projected, "仅展示原响应返回的分组行;如有截断,未返回分组不会补齐。"));
}
if (trend) {
List<Map<String,Object>> trendRows = maps(response.get("trend"));
if (trendRows.isEmpty()) gaps.add("任务要求月度趋势,但原始 API 响应没有趋势行。");
if (Boolean.TRUE.equals(response.get("trendTruncated"))) gaps.add("正式指标趋势被截断。");
List<Map<String,Object>> projected = new ArrayList<>();
for (Map<String,Object> raw : trendRows) for (String stage : stages.isEmpty() ? List.of("") : stages)
for (String field : fields) {
Object value = stages.isEmpty() ? raw.get("compositeRate") : map(raw.get(stage)).get(field);
String month = text(raw.get("month"), text(raw.get("period"), ""));
Map<String,Object> row = mapOf("month", month, "stage", stages.isEmpty() ? "综合" : stageLabel(stage),
"field", fieldLabel(field), "value", value);
projected.add(row);
addFact(facts, turnId, sequence, scopedText(period, filters, month + " · " + row.get("stage")
+ row.get("field") + " = " + valueText(value)));
displayFact(facts,sections.size(),projected.size()-1,period,filters,stage,field,value,null);
}
if (!projected.isEmpty()) sections.add(section("正式指标月度趋势", turnId, period, filters, null,
columns("month", "月份", "stage", "阶段", "field", "指标", "value", "正式值"), projected,
"逐月值直接取自原始趋势响应,没有计算期间均值或跨范围变化。"));
}
buildNativeAggregateComponents(turnId,query,task,scope,parameters,response,sections,facts,sequence);
}
/** Extra aggregate components already present in an authenticated native response; never query or derive a value. */
private static void buildNativeAggregateComponents(String turnId,Map<String,Object> query,Map<String,Object> task,
Map<String,Object> scope,Map<String,Object> parameters,Map<String,Object> response,
List<Map<String,Object>> sections,List<Map<String,Object>> facts,int[] sequence) {
if(!(query.get("nativeAggregateAllowedFields") instanceof List<?> rawAllowed))return;
Set<Object> allowed=new HashSet<>(rawAllowed);Map<String,Object> period=period(scope,task,parameters),filters=filters(scope,task);
var rows=new ArrayList<Map<String,Object>>();var summary=map(response.get("summary"));
for(String stage:List.of("black","bright"))for(String field:List.of("rate","totalWeight","scrapWeight")) {
String permission=stage+("totalWeight".equals(field)?"_total_weight":"_scrap_weight");
if(!"rate".equals(field)&&!allowed.contains("*")&&!allowed.contains(permission))continue;
if(list(task.get("stages")).contains(stage)&&list(task.get("fields")).contains(field))continue;
Object value=map(summary.get(stage)).get(field);if(!(value instanceof Number))continue;
var componentTask=new LinkedHashMap<>(task);componentTask.put("fields",List.of(field));componentTask.put("stages",List.of(stage));componentTask.put("trendRequired",false);componentTask.put("periodCoverage","available");
if(!Boolean.TRUE.equals(AnalysisProtocolMetricCoverage.inspect(response,componentTask,period).get("metricsComplete")))continue;
rows.add(mapOf("stage",stageLabel(stage),"field",fieldLabel(field),"value",value));
addFact(facts,turnId,sequence,scopedText(period,filters,stageLabel(stage)+fieldLabel(field)+" = "+valueText(value)));
displayFact(facts,sections.size(),rows.size()-1,period,filters,stage,field,value,null);facts.getLast().put("sourceKind","native_saved_aggregate");
}
if(!rows.isEmpty())sections.add(section("已保存的阶段正式聚合分量",turnId,period,filters,null,
columns("stage","阶段","field","指标","value","正式值"),rows,"仅补充原认证响应中有实际来源、且当前字段权限允许的阶段聚合值;不重读数据,不计算差值。"));
}
private static void buildRanking(String turnId, Map<String,Object> query, Map<String,Object> task,
Map<String,Object> scope, Map<String,Object> parameters, Map<String,Object> artifact,
Map<String,Object> coverage, List<Map<String,Object>> sections, List<Map<String,Object>> facts,
List<String> gaps, int[] sequence) {
if (!Boolean.TRUE.equals(coverage.get("weightsComplete"))) addCoverageGaps("正式分组排名", coverage, gaps);
Map<String,Object> period = period(scope, task, parameters);
if (!validPeriod(period)) gaps.add("正式分组排名缺少有效的已验证期间。");
Map<String,Object> filters = filters(scope, task);
List<String> allowedStages = strings(task.get("stages")), allowedFields = strings(task.get("fields"));
List<Map<String,Object>> rankings = maps(artifact.get("rankings"));
if (rankings.isEmpty()) { gaps.add("正式分组排名缺少已保存的 Top 快照。"); return; }
for (Map<String,Object> rank : rankings) {
String stage = text(rank.get("stage"), ""), metric = text(rank.get("metric"), "");
if (!allowedStages.contains(stage) || !allowedFields.contains(metric)
|| !Boolean.TRUE.equals(rank.get("completeRanking"))) {
gaps.add("正式分组排名快照超出本次任务授权范围或未声明完整。");
continue;
}
if (rank.get("denominator") == null) gaps.add("正式分组排名缺少同范围正式分母。");
int limit = integer(artifact.get("displayLimit"), 10);
List<Map<String,Object>> top = maps(rank.get("rows"));
if (limit > 0 && top.size() > limit) top = top.subList(0, limit);
List<Map<String,Object>> projected = new ArrayList<>();
int shown = top.size();
for (Map<String,Object> raw : top) {
Map<String,Object> row = mapOf("rank", raw.get("rank"), "key", raw.get("key"),
"label", raw.get("label"), "weight", raw.get("weight"));
projected.add(row);
addFact(facts, turnId, sequence, scopedText(period, filters,
"Top " + rankBoundary(limit, shown) + " " + text(raw.get("label"), text(raw.get("key"), "未命名分组"))
+ " · " + stageLabel(stage) + fieldLabel(metric) + " = " + valueText(raw.get("weight"))
+ ";同范围正式分母 = " + valueText(rank.get("denominator"))));
displayFact(facts,sections.size(),projected.size()-1,period,filters,stage,metric,raw.get("weight"),null);
}
String dimension = text(artifact.get("dimension"), "分组");
sections.add(section("Top " + rankBoundary(limit, shown) + " " + dimension + "排名", turnId, period, filters,
stage, columns("rank", "排名", "key", "分组键", "label", "分组", "weight", "重量"), projected,
"正式分母(" + stageLabel(stage) + fieldLabel(metric) + ")为 " + valueText(rank.get("denominator"))
+ "。此表只表示已保存的 Top " + rankBoundary(limit, shown) + "(本次展示 " + shown
+ " 条),不代表全量分组;重量沿用来源单位。"));
}
}
private static void buildChanges(String turnId, Map<String,Object> task, Map<String,Object> scope,
Map<String,Object> parameters, Map<String,Object> response, Map<String,Object> coverage,
List<Map<String,Object>> sections, List<Map<String,Object>> facts, List<String> gaps, int[] sequence, JsonMapper json) {
Map<String,Object> expected=body(parameters,json);
Map<String,Object> checked=AnalysisProtocolChanges.inspect(response,expected);
if(!Boolean.TRUE.equals(coverage.get("changesComplete"))||!Boolean.TRUE.equals(checked.get("changesComplete"))) {
addCoverageGaps("正式变化排名",checked,gaps);return;
}
Map<String,Object> period=map(expected.get("period")),previous=map(expected.get("comparisonPeriod")),filters=map(expected.get("filters"));
String stage=text(expected.get("stage"),"overall"),sort=text(expected.get("sort"),"");
List<Map<String,Object>> rows=new ArrayList<>();
for(var row:maps(response.get("rows"))) {
var current=map(row.get("current"));var comparison=map(row.get("comparison"));
rows.add(mapOf("rank",row.get("rank"),"label",row.getOrDefault("label",row.get("key")),
"comparisonRate",comparison.get("rate"),"currentRate",current.get("rate"),"rateDrop",row.get("rateDrop"),
"comparisonScrap",comparison.get("scrapWeight"),"currentScrap",current.get("scrapWeight"),"scrapDelta",row.get("scrapDelta"),
"comparisonBlackInput",comparison.get("blackInputWeight"),"currentBlackInput",current.get("blackInputWeight"),
"comparisonBrightInput",comparison.get("brightInputWeight"),"currentBrightInput",current.get("brightInputWeight")));
addFact(facts,turnId,sequence,scopedText(period,filters,text(row.get("label"),text(row.get("key"),"分组"))+" · "+stageLabel(stage)
+"合格率 "+valueText(comparison.get("rate"))+" → "+valueText(current.get("rate"))+",率降幅 "+valueText(row.get("rateDrop"))
+";报废增量 "+valueText(row.get("scrapDelta"))+"。对照期间:"+periodText(previous)+";排序:"+sort+"。分组变化只表示线索,不等于总体贡献或生产根因。"));
String shown=scopedText(period,filters,text(row.get("label"),text(row.get("key"),"分组"))+" · "+stageLabel(stage)
+"合格率 "+displayValue("rate",comparison.get("rate"))+" → "+displayValue("rate",current.get("rate"))+",率降幅 "+displayValue("rateDrop",row.get("rateDrop"))
+";报废增量 "+valueText(row.get("scrapDelta"))+"。对照期间:"+periodText(previous)+";排序:"+sort+"。分组变化只表示线索,不等于总体贡献或生产根因。");
displayFact(facts,sections.size(),rows.size()-1,period,filters,stage,"rateDrop",row.get("rateDrop"),shown);
var fact=facts.getLast();fact.put("comparisonPeriod",previous);var factScope=map(fact.get("scope"));factScope.put("comparisonPeriod",previous);factScope.put("dimension",expected.get("dimension"));fact.put("scope",factScope);fact.put("values",List.of(displayMetric("comparisonRate",comparison.get("rate")),displayMetric("currentRate",current.get("rate")),displayMetric("rateDrop",row.get("rateDrop")),displayMetric("scrapDelta",row.get("scrapDelta"))));
}
sections.add(section("正式分组变化排名",turnId,period,filters,stage.equals("overall")?null:stage,
columns("rank","排名","label","对象","comparisonRate","比较期合格率","currentRate","本期合格率","rateDrop","率降幅",
"comparisonScrap","比较期报废","currentScrap","本期报废","scrapDelta","报废增量",
"comparisonBlackInput","比较期黑片投入","currentBlackInput","本期黑片投入","comparisonBrightInput","比较期亮片投入","currentBrightInput","本期亮片投入"),rows,
"同范围全部对象完成正式排序后展示Top;对照期间:"+periodText(previous)+"。总体和阶段口径沿用原查询;率以原始小数展示。不能将分组变化加总为总体率贡献。"));
}
private static void buildDefects(String turnId, Map<String,Object> task, Map<String,Object> scope,
Map<String,Object> parameters, Map<String,Object> response, Map<String,Object> coverage,
List<Map<String,Object>> sections, List<Map<String,Object>> facts, List<String> gaps, int[] sequence, JsonMapper json) {
if (!Boolean.TRUE.equals(coverage.get("defectsComplete"))) addCoverageGaps("来源细分缺陷", coverage, gaps);
Map<String,Object> body = body(parameters, json);
Map<String,Object> period = period(scope, task, body);
if (!validPeriod(period)) gaps.add("来源细分缺陷查询缺少有效的已验证期间。");
Map<String,Object> filters = filters(scope, task);
String stage = text(body.get("stage"), firstString(task.get("stages"), ""));
if (stage.isBlank()) gaps.add("来源细分缺陷查询缺少明确阶段。");
String grain = text(response.get("grain"), "");
if (grain.isBlank()) gaps.add("来源细分缺陷响应缺少重量口径标记。");
List<Map<String,Object>> rawRows = maps(response.get("rows"));
List<Map<String,Object>> rows = new ArrayList<>();
for (Map<String,Object> raw : rawRows) {
Map<String,Object> row = new LinkedHashMap<>();
for (String key : List.of("key", "label", "category", "defect", "weight", "presentRows", "missingRows",
"invalidRows", "negativeRows", "sourceRows")) if (raw.containsKey(key)) row.put(key, raw.get(key));
rows.add(row);
String value = raw.get("weight") == null ? "重量缺失(null)" : "重量 = " + valueText(raw.get("weight"));
String field = text(raw.get("label"), text(raw.get("defect"), text(raw.get("key"), "细项")));
addFact(facts, turnId, sequence, scopedText(period, filters,
(stage.isBlank() ? "" : stageLabel(stage) + " · ")
+ text(raw.get("label"), text(raw.get("key"), "规格")) + " · " + field + " · " + value
+ ";来源行 present/missing/invalid/negative = " + counts(raw)));
displayFact(facts,sections.size(),rows.size()-1,period,filters,stage,"scrapWeight",raw.get("weight"),null);
}
List<Map<String,Object>> fields = columnsForDefect(rows);
sections.add(section("来源细分缺陷重量", turnId, period, filters, stage.isBlank() ? null : stage,
fields, rows, text(response.get("notice"), "来源细项重量,不是正式 KPI,也不等同于正式贡献。")));
List<Map<String,Object>> reconciliation = maps(response.get("categoryReconciliation"));
if (!reconciliation.isEmpty()) {
for (Map<String,Object> raw : reconciliation) {
String status = text(raw.get("reconciliationStatus"), "未说明");
String explanation = "DIFF".equals(status) ? ";来源分项与正式分类存在差异,不作正式贡献解释"
: "RECONCILED".equals(status) ? ";来源分项与正式分类已核对一致" : "";
addFact(facts, turnId, sequence, scopedText(period, filters,
text(raw.get("category"), "类别") + "分类核对状态 = " + status + ",差值 = "
+ valueText(raw.get("diff")) + explanation));
}
sections.add(section("来源类别核对", turnId, period, filters, stage.isBlank() ? null : stage,
columnsForMapKeys(reconciliation), reconciliation,
"categoryReconciliation 是口径核对;DIFF 保留为差异说明,不视作正式贡献。"));
} else gaps.add("来源细分缺陷响应缺少类别核对信息。");
if (Boolean.TRUE.equals(response.get("rowsTruncated"))) gaps.add("来源细分缺陷分组存在未展示项。");
if (!"source_defect_weight".equals(grain)) gaps.add("来源细分缺陷响应的重量口径不匹配。");
}
private static void buildMemberLists(String turnId, Map<String,Object> source, List<Map<String,Object>> sections,
List<Map<String,Object>> facts, List<String> gaps, int[] sequence, JsonMapper json) {
Map<String,List<Map<String,Object>>> grouped = new LinkedHashMap<>();
Map<String,Map<String,Object>> tasks = new LinkedHashMap<>();
for (Map<String,Object> query : maps(source.get("queries"))) {
if (!"POST /api/analysis/source/members".equals(query.get("target"))) continue;
Map<String,Object> scope = map(query.get("scope"));
Map<String,Object> task = firstMap(query.get("task"), scope.get("task"), source.get("task"));
Map<String,Object> parameters = stringMap(query.get("parameters"));
Map<String,Object> request = body(parameters, json);
String key = text(scope.get("scopeId"), text(scope.get("nodeId"), "")) + "|" + canonical(withoutPage(request));
grouped.computeIfAbsent(key, ignored -> new ArrayList<>()).add(query);
tasks.putIfAbsent(key, task);
}
for (var entry : grouped.entrySet()) {
List<Map<String,Object>> pageQueries = entry.getValue();
pageQueries.sort(Comparator.comparingLong(query -> {
Map<String,Object> request = body(stringMap(query.get("parameters")), json);
return number(request.get("offset"), Long.MAX_VALUE);
}));
List<Map<String,Object>> pages = new ArrayList<>();
boolean pagingComplete = true;
long expectedOffset = 0;
for (Map<String,Object> query : pageQueries) {
Map<String,Object> request = body(stringMap(query.get("parameters")), json);
Map<String,Object> response = map(query.get("response"));
Map<String,Object> coverage = map(query.get("coverage"));
long requestedOffset = number(request.get("offset"), -1), actualOffset = number(response.get("offset"), -1);
if (requestedOffset != expectedOffset || actualOffset != requestedOffset
|| Boolean.FALSE.equals(response.get("pageComplete"))
|| Boolean.FALSE.equals(coverage.get("pageComplete"))) pagingComplete = false;
boolean shouldHaveMore = query != pageQueries.getLast();
if (!Objects.equals(response.get("hasMore"), shouldHaveMore)) pagingComplete = false;
if (coverage.get("hasMore") != null && !Objects.equals(coverage.get("hasMore"), shouldHaveMore)) pagingComplete = false;
long returned = list(response.get("rows")).size();
expectedOffset = Math.max(expectedOffset, requestedOffset + returned);
pages.add(response);
}
Map<String,Object> task = tasks.get(entry.getKey());
Map<String,Object> first = pages.isEmpty() ? Map.of() : pages.getFirst();
String release = firstString(map(source.get("task")).get("releaseId"), first.get("releaseId"));
try {
Map<String,Object> snapshot = AnalysisMemberSnapshot.build(release, task, pages, pagingComplete, json);
if (!Boolean.TRUE.equals(snapshot.get("queryComplete"))) gaps.add("成员清单历史页不完整,不能称为全量。");
List<Map<String,Object>> fieldDefs = maps(snapshot.get("fields"));
List<Map<String,Object>> cols = new ArrayList<>();
for (Map<String,Object> field : fieldDefs) cols.add(column(text(field.get("key"), ""), text(field.get("label"), text(field.get("key"), ""))));
cols.add(column("stages", "阶段"));
List<Map<String,Object>> rows = new ArrayList<>();
for (Map<String,Object> raw : maps(snapshot.get("rows"))) {
Map<String,Object> row = new LinkedHashMap<>(map(raw.get("values")));
row.put("stages", raw.get("stages")); rows.add(row);
}
Map<String,Object> period = map(snapshot.get("period")), filters = map(snapshot.get("filters"));
if (!validPeriod(period)) gaps.add("成员清单缺少有效的已验证期间。");
sections.add(section("完整成员清单", turnId, period, filters, null, cols, rows,
Boolean.TRUE.equals(snapshot.get("queryComplete")) ? "清单由已保存的原始成员页重建,去重总数 "
+ valueText(snapshot.get("totalCount")) + "。" : "清单只含已保存页,当前不完整。"));
addFact(facts, turnId, sequence, scopedText(period, filters, "成员清单共 "
+ valueText(snapshot.get("totalCount")) + " 项,已载入 " + valueText(snapshot.get("rowsReturned"))
+ " 项;" + (Boolean.TRUE.equals(snapshot.get("queryComplete")) ? "已查全" : "尚未查全")));
} catch (RuntimeException invalid) {
gaps.add("成员清单历史页未通过快照重建:" + safeException(invalid));
}
}
}
private static void buildSourceReceipt(String turnId, Map<String,Object> task, Map<String,Object> scope,
Map<String,Object> request, Map<String,Object> response, Map<String,Object> coverage,
List<Map<String,Object>> sections, List<Map<String,Object>> facts, List<String> gaps, int[] sequence) {
Map<String,Object> proof = map(response.get("sourceVerification"));
boolean complete = Boolean.TRUE.equals(coverage.get("sourceVerificationComplete"))
&& Boolean.TRUE.equals(proof.get("complete"))
&& Boolean.TRUE.equals(proof.get("allSelectedValuesMatchStored"));
if (!complete) addCoverageGaps("原件核验历史回执", coverage, gaps);
Map<String,Object> period = period(scope, task, request);
if (!validPeriod(period)) gaps.add("历史原件核验回执缺少有效的已验证期间。");
Map<String,Object> filters = filters(scope, task);
String stage = text(request.get("stage"), firstString(task.get("stages"), ""));
if (stage.isBlank()) gaps.add("历史原件核验回执缺少明确阶段。");
List<Map<String,Object>> rows = List.of(mapOf("totalRows", response.get("totalRows"),
"verifiedRows", proof.get("verifiedRows"), "missingRows", proof.get("missingRows"),
"mismatchRows", proof.get("mismatchRows"), "complete", complete));
sections.add(section("历史原件核验回执", turnId, period, filters, stage.isBlank() ? null : stage,
columns("totalRows", "命中行", "verifiedRows", "已核验", "missingRows", "缺失", "mismatchRows", "不匹配", "complete", "回执通过"), rows,
"这是历史查询保存的核验回执;本报告没有重新打开或复核原件。"));
addFact(facts, turnId, sequence, scopedText(period, filters, "历史原件核验回执:"
+ valueText(proof.get("verifiedRows")) + "/" + valueText(response.get("totalRows"))
+ " 行已核验,缺失 " + valueText(proof.get("missingRows")) + " 行、不匹配 "
+ valueText(proof.get("mismatchRows")) + " 行;" + (complete ? "回执通过" : "回执未通过")
+ ",本轮未重新打开原件。"));
}
private static void buildRecordLists(String turnId, Map<String,Object> source, List<Map<String,Object>> sections,
List<Map<String,Object>> facts, List<String> gaps, int[] sequence, JsonMapper json) {
Map<String,List<Map<String,Object>>> grouped = new LinkedHashMap<>();
Map<String,Map<String,Object>> requests = new LinkedHashMap<>(), scopes = new LinkedHashMap<>();
for (Map<String,Object> query : maps(source.get("queries"))) {
if (!"POST /api/analysis/investigate".equals(query.get("target"))) continue;
Map<String,Object> parameters = stringMap(query.get("parameters"));
Map<String,Object> request = body(parameters, json);
if (!"rows".equals(request.get("action"))) continue;
Map<String,Object> scope = map(query.get("scope"));
Map<String,Object> task = firstMap(query.get("task"), scope.get("task"), source.get("task"));
String key = text(scope.get("scopeId"), text(scope.get("nodeId"), "")) + "|"
+ canonical(withoutPage(request));
grouped.computeIfAbsent(key, ignored -> new ArrayList<>()).add(map(query.get("response")));
requests.putIfAbsent(key, request);
Map<String,Object> safeScope = new LinkedHashMap<>(scope);
safeScope.putIfAbsent("task", task);
scopes.putIfAbsent(key, safeScope);
}
if (grouped.isEmpty()) return;
List<AnalysisRecordSnapshot.Input> inputs = new ArrayList<>();
for (String key : grouped.keySet()) inputs.add(new AnalysisRecordSnapshot.Input(scopes.get(key), requests.get(key), grouped.get(key)));
try {
AnalysisRecordSnapshot.Result result = AnalysisRecordSnapshot.build(
firstString(map(source.get("task")).get("releaseId"),
inputs.isEmpty() ? "" : text(inputs.getFirst().request().get("releaseId"), "")), inputs, json);
gaps.addAll(result.gaps());
Map<String,Object> snapshot = result.snapshot();
if (!Boolean.TRUE.equals(snapshot.get("queryComplete"))) gaps.add("物理记录历史分页未完整读取或展示。");
for (Map<String,Object> set : maps(snapshot.get("sets"))) {
List<String> fields = strings(set.get("fields"));
List<Map<String,Object>> columns = columns("recordId", "记录ID", "sourceRevisionId", "来源修订", "buildId", "构建ID",
"sourceFile", "来源文件", "sheet", "工作表", "rowNumber", "行号", "product", "产品", "group", "分组");
for (String field : fields) columns.add(column("value:" + field, field));
List<Map<String,Object>> rows = new ArrayList<>();
for (Map<String,Object> raw : maps(set.get("rows"))) {
Map<String,Object> row = new LinkedHashMap<>();
for (String field : List.of("recordId", "sourceRevisionId", "buildId", "sourceFile", "sheet", "rowNumber", "product", "group"))
if (raw.containsKey(field)) row.put(field, raw.get(field));
Map<String,Object> values = map(raw.get("values"));
for (String field : fields) row.put("value:" + field, values.get(field));
rows.add(row);
}
Map<String,Object> period = map(set.get("period")), filters = map(set.get("filters"));
String stage = text(set.get("stage"), "");
if (!validPeriod(period)) gaps.add("物理记录缺少有效的已验证期间。");
if (stage.isBlank()) gaps.add("物理记录缺少明确阶段。");
sections.add(section("物理原始记录", turnId, period, filters, stage.isBlank() ? null : stage,
columns, rows, Boolean.TRUE.equals(set.get("complete")) ? "已从原始分页响应重建全部 "
+ valueText(set.get("totalRows")) + " 行。" : "仅展示已保存的物理记录页,不完整。"));
addFact(facts, turnId, sequence, scopedText(period, filters, "物理原始记录共 "
+ valueText(set.get("totalRows")) + " 行,已载入 " + valueText(set.get("loadedRows"))
+ " 行;" + (Boolean.TRUE.equals(set.get("complete")) ? "已查全" : "尚未查全")));
}
} catch (RuntimeException invalid) {
gaps.add("物理记录历史页未通过快照重建:" + safeException(invalid));
}
}
static Map<String,Object> render(String title, List<Map<String,Object>> sourceResults,
List<String> headlineFactIds, List<String> gaps) {
return renderDocument(title,sourceResults,headlineFactIds,gaps,true);
}
static Map<String,Object> renderCurrent(String title,List<Map<String,Object>> sourceResults,List<String> headlineFactIds,List<String> gaps) {
return renderDocument(title,sourceResults,headlineFactIds,gaps,false);
}
private static Map<String,Object> renderDocument(String title,List<Map<String,Object>> sourceResults,List<String> headlineFactIds,List<String> gaps,boolean historical) {
String safeTitle = text(title, "历史分析报告");
List<Map<String,Object>> sources = sourceResults == null ? List.of() : sourceResults;
List<Map<String,Object>> allSections = new ArrayList<>(), allFacts = new ArrayList<>();
List<String> allGaps = new ArrayList<>();
addStrings(allGaps, gaps);
for (Map<String,Object> source : sources) {
String turn = text(source.get("sourceTurnId"), text(source.get("turnId"), ""));
for (Map<String,Object> section : maps(source.get("sections"))) {
Map<String,Object> copy = new LinkedHashMap<>(section);
copy.putIfAbsent("sourceTurnId", turn);
Map<String,Object> metadata = map(source.get("metadata"));
copy.put("sourceRevision", text(source.get("revision"), text(metadata.get("revision"), "")));
copy.put("sourceExecutionId", text(source.get("executionId"), text(metadata.get("executionId"), "")));
copy.put("sourceStatus", text(source.get("status"), ""));
allSections.add(copy);
}
allFacts.addAll(maps(source.get("facts")));
addStrings(allGaps, source.get("gaps"));
if (Boolean.FALSE.equals(source.get("complete"))) allGaps.add("历史来源 " + turn + " 存在未完成范围。");
}
Map<String,Map<String,Object>> factsById = new LinkedHashMap<>();
for (Map<String,Object> fact : allFacts) factsById.put(text(fact.get("id"), ""), fact);
List<Map<String,Object>> headlines = new ArrayList<>();
if (headlineFactIds != null) for (String id : headlineFactIds) {
Map<String,Object> fact = factsById.get(id);
if (fact != null && !text(fact.get("text"), "").isBlank() && !headlines.contains(fact)) headlines.add(fact);
}
String answer;
if (headlines.isEmpty()) {
answer = "没有可引用的已校验历史数值事实。";
} else {
List<String> lines = new ArrayList<>();
for (Map<String,Object> fact : headlines) {
String text = text(fact.get("text"), "");
if (!text.isBlank()) lines.add("- " + md(text(fact.get("sourceTurnId"), "历史来源")) + ":" + md(text));
}
answer = lines.isEmpty() ? "没有可引用的已校验历史数值事实。" : String.join("\n", lines);
}
String markdown = renderMarkdown(safeTitle, headlines, allSections, allGaps, sources, historical);
String html = renderHtml(safeTitle, headlines, allSections, allGaps, sources, historical);
return mapOf("answer", answer, "reportMarkdown", markdown, "reportHtml", html,
"historySections", List.copyOf(allSections));
}
private static String renderMarkdown(String title, List<Map<String,Object>> headlines, List<Map<String,Object>> sections, List<String> gaps,
List<Map<String,Object>> sources, boolean historical) {
StringBuilder out = new StringBuilder("# ").append(md(title)).append("\n\n");
out.append(historical?"本报告按每个历史来源的原查询范围分别列示;不同期间与重叠总体没有合并或求均值。\n\n":"本报告依据本轮受控正式类别查询;范围为当前发布版本全部已发布期间,不含未分类。\n\n");
for (Map<String,Object> source : sources) out.append("- 来源轮次:").append(md(sourceReference(source)))
.append(";状态:").append(md(text(source.get("status"), "未知"))).append("\n");
if (!sources.isEmpty()) out.append('\n');
if (!headlines.isEmpty()) {
out.append("## 重点结论\n\n");
for (Map<String,Object> fact : headlines) out.append("- ").append(md(text(fact.get("text"), ""))).append('\n');
out.append('\n');
}
for (Map<String,Object> section : sections) {
out.append("## ").append(md(text(section.get("title"), "历史数据"))).append("\n\n");
out.append("范围:").append(md(scopeText(map(section.get("period")), map(section.get("filters")), section.get("stage"))))
.append("\n\n来源轮次:").append(md(text(section.get("sourceTurnId"), "未知"))).append("\n\n");
List<Map<String,Object>> columns = maps(section.get("columns"));
if (!columns.isEmpty()) {
out.append('|'); for (Map<String,Object> column : columns) out.append(' ').append(md(text(column.get("label"), text(column.get("key"), "")))).append(" |");
out.append("\n|"); for (int i=0;i<columns.size();i++) out.append(" --- |"); out.append('\n');
for (Map<String,Object> row : maps(section.get("rows"))) {
out.append('|'); for (Map<String,Object> column : columns) out.append(' ').append(md(valueText(row.get(text(column.get("key"), ""))))).append(" |"); out.append('\n');
}
out.append('\n');
}
if (section.get("note") != null) out.append(md(text(section.get("note"), ""))).append("\n\n");
}
if (!gaps.isEmpty()) {
out.append("## 尚未覆盖的范围\n\n");
for (String gap : gaps.stream().distinct().toList()) out.append("- ").append(md(gap)).append('\n');
out.append('\n');
}
return out.toString().trim() + "\n";
}
private static String renderHtml(String title, List<Map<String,Object>> headlines, List<Map<String,Object>> sections, List<String> gaps,
List<Map<String,Object>> sources, boolean historical) {
StringBuilder out = new StringBuilder("<!doctype html><html lang=\"zh-CN\"><head><meta charset=\"utf-8\">")
.append("<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><title>")
.append(html(title)).append("</title><style>")
.append("*{box-sizing:border-box}body{margin:0;background:#eef2f6;color:#243447;font:14px/1.65 -apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif}")
.append("main{max-width:920px;margin:32px auto;padding:0 18px}.paper{background:#fff;border:1px solid #d8e1eb;border-radius:12px;padding:30px;box-shadow:0 8px 24px #23364a12}")
.append("h1{font-size:25px;margin:0 0 8px;color:#1d344d}h2{font-size:18px;margin:26px 0 8px;color:#284563;border-bottom:1px solid #e4eaf0;padding-bottom:6px}")
.append("p{margin:8px 0;color:#53677c}.scope{font-size:13px;color:#63788d}.source{color:#77889a;font-size:12px}")
.append("table{width:100%;border-collapse:collapse;margin:12px 0 18px;table-layout:fixed}th,td{border:1px solid #dbe3eb;padding:7px 9px;text-align:left;vertical-align:top;overflow-wrap:anywhere;word-break:break-word}th{background:#edf3f8;color:#354f68;font-weight:600}")
.append(".note{border-left:3px solid #8ca8c4;padding:7px 11px;background:#f6f9fb;color:#53677c}.highlights{background:#f5f8fb;border:1px solid #dbe5ef;border-radius:8px;padding:4px 14px 10px;margin:18px 0}.highlights li{margin:5px 0}.gaps{background:#fff8ed;border:1px solid #f1d7a8;border-radius:8px;padding:12px 16px;color:#765521}.gaps li{margin:4px 0}")
.append("details{margin:8px 0}summary{cursor:pointer;color:#48627b}.empty{color:#77889a;font-style:italic}")
.append("</style></head><body><main><article class=\"paper\"><h1>").append(html(title)).append("</h1>")
.append(historical?"<p>按每个历史来源的原查询范围分别列示;不同期间与重叠总体没有合并或求均值。</p>":"<p>依据本轮受控正式类别查询;范围为当前发布版本全部已发布期间,不含未分类。</p>");
for (Map<String,Object> source : sources) {
out.append("<p class=\"source\">来源轮次:").append(html(sourceReference(source))).append(";状态:")
.append(html(text(source.get("status"), "未知"))).append("</p>");
}
if (!headlines.isEmpty()) {
out.append("<section class=\"highlights\"><h2>重点结论</h2><ul>");
for (Map<String,Object> fact : headlines) out.append("<li>").append(html(text(fact.get("text"), ""))).append("</li>");
out.append("</ul></section>");
}
for (Map<String,Object> section : sections) {
out.append("<section><h2>").append(html(text(section.get("title"), "历史数据"))).append("</h2><p class=\"scope\">范围:")
.append(html(scopeText(map(section.get("period")), map(section.get("filters")), section.get("stage"))))
.append("</p><p class=\"source\">来源轮次:").append(html(text(section.get("sourceTurnId"), "未知"))).append("</p>");
List<Map<String,Object>> columns = maps(section.get("columns"));
List<Map<String,Object>> rows = maps(section.get("rows"));
if (columns.isEmpty()) out.append("<p class=\"empty\">没有可展示的列。</p>");
else if ("物理原始记录".equals(section.get("title")) && !rows.isEmpty()) appendRecordTables(out, columns, rows);
else appendHtmlTables(out, columns, rows);
if (section.get("note") != null) out.append("<p class=\"note\">").append(html(text(section.get("note"), ""))).append("</p>");
out.append("</section>");
}
if (!gaps.isEmpty()) {
out.append("<details class=\"gaps\"><summary>范围缺口与完整性说明(").append(gaps.stream().distinct().count()).append(")</summary><ul>");
for (String gap : gaps.stream().distinct().toList()) out.append("<li>").append(html(gap)).append("</li>");
out.append("</ul></details>");
}
return out.append("</article></main></body></html>").toString();
}
/** Factor repeated provenance into a visible index; preserve each original row verbatim in HTML. */
private static void appendRecordTables(StringBuilder out, List<Map<String,Object>> columns, List<Map<String,Object>> rows) {
List<String> sourceKeys = List.of("sourceRevisionId", "buildId", "sourceFile", "sheet");
Set<String> keys = new HashSet<>(); for (var column : columns) keys.add(text(column.get("key"), ""));
if (!keys.containsAll(sourceKeys) || !keys.contains("recordId") || !keys.contains("rowNumber")) {
appendHtmlTables(out, columns, rows); return;
}
Map<List<Object>,String> sourceRefs = new LinkedHashMap<>();
List<Map<String,Object>> sources = new ArrayList<>(), displayed = new ArrayList<>();
Set<String> common = new LinkedHashSet<>();
for (String key : List.of("product", "group")) if (keys.contains(key)
&& rows.stream().allMatch(row -> Objects.equals(rows.getFirst().get(key), row.get(key)))) common.add(key);
for (String key : common) {
String label = key.equals("product") ? "产品" : "分组";
out.append("<p class=\"scope\">").append(label).append(":").append(html(valueText(rows.getFirst().get(key)))).append("</p>");
}
for (Map<String,Object> original : rows) {
List<Object> identity = new ArrayList<>(); for (String key : sourceKeys) identity.add(original.get(key));
String ref = sourceRefs.get(identity);
if (ref == null) {
ref = "S" + (sourceRefs.size() + 1); sourceRefs.put(identity, ref);
Map<String,Object> source = new LinkedHashMap<>(); source.put("sourceRef", ref);
for (String key : sourceKeys) source.put(key, original.get(key)); sources.add(source);
}
Map<String,Object> row = new LinkedHashMap<>(original); row.put("sourceRef", ref); displayed.add(row);
}
out.append("<h3>来源索引</h3>");
appendHtmlTables(out, columns("sourceRef", "来源", "sourceFile", "来源文件", "sheet", "工作表",
"sourceRevisionId", "来源修订", "buildId", "构建ID"), sources);
List<Map<String,Object>> compact = new ArrayList<>();
for (Map<String,Object> column : columns) {
String key = text(column.get("key"), "");
if (sourceKeys.contains(key) || common.contains(key)) continue;
compact.add(column); if (key.equals("recordId")) compact.add(column("sourceRef", "来源"));
}
appendHtmlTables(out, compact, displayed, rows);
}
private static void appendHtmlTables(StringBuilder out, List<Map<String,Object>> columns, List<Map<String,Object>> rows) {
appendHtmlTables(out, columns, rows, List.of());
}
private static void appendHtmlTables(StringBuilder out, List<Map<String,Object>> columns, List<Map<String,Object>> rows,
List<Map<String,Object>> provenance) {
List<List<Map<String,Object>>> segments = new ArrayList<>();
if (columns.size() <= 6) segments.add(columns);
else {
segments.add(columns.subList(0, 6));
for (int start = 6; start < columns.size(); start += 5) {
List<Map<String,Object>> segment = new ArrayList<>();
segment.add(columns.getFirst());
segment.addAll(columns.subList(start, Math.min(start + 5, columns.size())));
segments.add(segment);
}
}
for (List<Map<String,Object>> segment : segments) {
out.append("<table><thead><tr>");
for (Map<String,Object> column : segment)
out.append("<th>").append(html(text(column.get("label"), text(column.get("key"), "")))).append("</th>");
out.append("</tr></thead><tbody>");
if (rows.isEmpty()) out.append("<tr><td colspan=\"").append(segment.size()).append("\" class=\"empty\">无返回行</td></tr>");
for (int index = 0; index < rows.size(); index++) {
Map<String,Object> row = rows.get(index);
out.append("<tr");
if (!provenance.isEmpty()) out.append(" data-record-provenance=\"")
.append(html(HTML_JSON.writeValueAsString(provenance.get(index)))).append("\"");
out.append(">");
for (Map<String,Object> column : segment) out.append("<td>")
.append(html(valueText(row.get(text(column.get("key"), ""))))).append("</td>");
out.append("</tr>");
}
out.append("</tbody></table>");
}
}
private static Map<String,Object> section(String title, String turnId, Map<String,Object> period,
Map<String,Object> filters, String stage, List<Map<String,Object>> columns,
List<Map<String,Object>> rows, String note) {
Map<String,Object> section = new LinkedHashMap<>();
section.put("title", title); section.put("sourceTurnId", turnId); section.put("period", period);
section.put("filters", filters); if (stage != null) section.put("stage", stage);
section.put("columns", columns); section.put("rows", rows); section.put("note", note);
return section;
}
private static List<Map<String,Object>> columnsForDefect(List<Map<String,Object>> rows) {
List<String> keys = List.of("key", "label", "category", "defect", "weight", "presentRows", "missingRows", "invalidRows", "negativeRows", "sourceRows");
Map<String,String> labels = Map.of("key", "分组键", "label", "规格/细项", "category", "类别", "defect", "授权细项",
"weight", "来源重量", "presentRows", "有效行", "missingRows", "缺失行", "invalidRows", "非法行", "negativeRows", "负值行", "sourceRows", "来源行");
Set<String> present = new HashSet<>(); for (Map<String,Object> row : rows) present.addAll(row.keySet());
List<Map<String,Object>> out = new ArrayList<>(); for (String key : keys) if (present.contains(key)) out.add(column(key, labels.get(key)));
return out;
}
private static List<Map<String,Object>> columnsForMapKeys(List<Map<String,Object>> rows) {
Set<String> keys = new LinkedHashSet<>();
for (Map<String,Object> row : rows) for (String key : row.keySet()) keys.add(key);
return keys.stream().map(key -> column(key, key)).toList();
}
private static List<Map<String,Object>> columns(String... pairs) {
List<Map<String,Object>> out = new ArrayList<>(); for (int i=0;i+1<pairs.length;i+=2) out.add(column(pairs[i], pairs[i+1])); return out;
}
private static Map<String,Object> column(String key, String label) { return mapOf("key", key, "label", label); }
private static Map<String,Object> period(Map<String,Object> scope, Map<String,Object> task, Map<String,Object> parameters) {
Map<String,Object> requestPeriod = requestedPeriod(parameters);
if (!requestPeriod.isEmpty()) return requestPeriod;
Map<String,Object> period = firstMap(scope.get("period"), task.get("period"));
if (!period.isEmpty()) return pick(period, PERIOD_KEYS);
return Map.of();
}
private static Map<String,Object> requestedPeriod(Map<String,Object> request) {
Map<String,Object> values = new LinkedHashMap<>();
for (String key : PERIOD_KEYS) if (request.containsKey(key)) values.put(key, scalarParameter(request.get(key)));
return values;
}
private static boolean validPeriod(Map<String,Object> period) {
Integer year = integerValue(period.get("year"));
if (year == null || year < 1 || year > 9999) return false;
boolean month = period.containsKey("month"), start = period.containsKey("monthStart"), end = period.containsKey("monthEnd"), quarter = period.containsKey("quarter");
if (month && (start || end || quarter) || quarter && (start || end)) return false;
if (month) { Integer value = integerValue(period.get("month")); return value != null && value >= 1 && value <= 12; }
if (start || end) {
Integer first = integerValue(period.get("monthStart")), last = integerValue(period.get("monthEnd"));
return first != null && last != null && first >= 1 && first <= last && last <= 12;
}
if (quarter) {
Object raw = period.get("quarter");
String value = Objects.toString(raw, "");
if (value.matches("(?i)Q[1-4]")) value = value.substring(1);
Integer parsed = integerValue(value);
return parsed != null && parsed >= 1 && parsed <= 4;
}
return true;
}
private static Integer integerValue(Object value) {
if (value instanceof Number n && Double.isFinite(n.doubleValue()) && n.doubleValue() == Math.rint(n.doubleValue())) return n.intValue();
if (value instanceof String s) try { return Integer.valueOf(s); } catch (NumberFormatException ignored) { return null; }
return null;
}
private static Map<String,Object> filters(Map<String,Object> scope, Map<String,Object> task) {
return firstMap(scope.get("filters"), task.get("filters"));
}
private static Map<String,Object> pick(Map<String,Object> values, List<String> keys) {
Map<String,Object> out = new LinkedHashMap<>(); for (String key : keys) if (values.containsKey(key)) out.put(key, values.get(key)); return out;
}
private static String scopedText(Map<String,Object> period, Map<String,Object> filters, String fact) {
return scopeText(period, filters, null) + ":" + fact;
}
private static String scopeText(Map<String,Object> period, Map<String,Object> filters, Object stage) {
List<String> parts = new ArrayList<>();
String date = periodText(period); if (!date.isBlank()) parts.add(date);
if (stage instanceof String s && !s.isBlank()) parts.add(stageLabel(s));
if (!filters.isEmpty()) parts.add("筛选 " + valueText(filters));
return parts.isEmpty() ? "未提供明确期间筛选" : String.join(";", parts);
}
private static String periodText(Map<String,Object> period) {
if(Boolean.TRUE.equals(period.get("allPublished")))return "当前发布版本全部已发布期间";
if (period.containsKey("monthStart") || period.containsKey("monthEnd"))
return text(period.get("year"), "") + "年" + text(period.get("monthStart"), "?") + "–" + text(period.get("monthEnd"), "?") + "月";
if (period.containsKey("year") && period.containsKey("month")) return text(period.get("year"), "") + "年" + text(period.get("month"), "") + "月";
if (period.containsKey("year") && period.containsKey("quarter")) return text(period.get("year"), "") + "年第" + text(period.get("quarter"), "") + "季度";
if (period.containsKey("year")) return text(period.get("year"), "") + "年";
return "";
}
private static String sourceReference(Map<String,Object> source) {
String turn = text(source.get("sourceTurnId"), text(source.get("turnId"), "未知"));
Map<String,Object> metadata = map(source.get("metadata"));
String revision = text(source.get("revision"), text(metadata.get("revision"), ""));
String execution = text(source.get("executionId"), text(metadata.get("executionId"), ""));
List<String> parts = new ArrayList<>(); parts.add(turn);
if (!revision.isBlank()) parts.add("revision " + revision);
if (!execution.isBlank()) parts.add("execution " + execution);
return String.join(" · ", parts);
}
private static String fieldLabel(String field) { return FIELD_LABELS.getOrDefault(field, field); }
private static String stageLabel(String stage) { return "black".equals(stage) ? "黑片" : "bright".equals(stage) ? "亮片" : stage; }
private static String counts(Map<String,Object> row) { return valueText(row.get("presentRows")) + "/" + valueText(row.get("missingRows")) + "/" + valueText(row.get("invalidRows")) + "/" + valueText(row.get("negativeRows")); }
private static String rankBoundary(int limit, int shown) { return Integer.toString(limit > 0 ? limit : shown); }
private static void addFact(List<Map<String,Object>> facts, String turnId, int[] sequence, String text) {
facts.add(mapOf("id", turnId + ":fact:" + String.format(Locale.ROOT, "%05d", ++sequence[0]), "text", text, "sourceTurnId", turnId));
}
private static void displayFact(List<Map<String,Object>> facts,int sectionIndex,int rowIndex,Map<String,Object> period,
Map<String,Object> filters,String stage,String metric,Object raw,String displayText) {
var fact=facts.getLast();fact.putAll(displayMetric(metric,raw));fact.put("sectionIndex",sectionIndex);fact.put("rowIndex",rowIndex);
var scope=mapOf("period",period,"filters",filters,"stage",stage.isBlank()?"overall":stage);fact.put("scope",scope);
String original=text(fact.get("text"),"");String token=valueText(raw);
fact.put("displayText",displayText!=null?displayText:raw==null?original:original.replace(" = "+token," = "+displayValue(metric,raw)));
}
private static Map<String,Object> displayMetric(String metric,Object raw) {
boolean ratio=Set.of("rate","scrapRate","compositeRate","comparisonRate","currentRate").contains(metric),delta="rateDrop".equals(metric);
return mapOf("metricCode",metric,"rawValue",raw,"rawUnit",ratio||delta?"ratio":"source_weight",
"unit",delta?"percentage_point":ratio?"percent":"source_weight","displayValue",displayValue(metric,raw));
}
private static String displayValue(String metric,Object raw) {
if(raw==null)return "null";
if(Set.of("rate","scrapRate","compositeRate","comparisonRate","currentRate","rateDrop").contains(metric))try {
BigDecimal value=raw instanceof BigDecimal decimal?decimal:new BigDecimal(raw.toString());
return value.multiply(BigDecimal.valueOf(100)).setScale(2,RoundingMode.HALF_UP).toPlainString()+("rateDrop".equals(metric)?"pp":"%");
}catch(NumberFormatException invalid){return valueText(raw);}
return valueText(raw);
}
private static void addCoverageGaps(String label, Map<String,Object> coverage, List<String> gaps) {
List<String> evidence = strings(coverage.get("missingEvidence"));
if (evidence.isEmpty()) gaps.add(label + "覆盖回执未通过。");
else for (String item : evidence) gaps.add(label + ":" + item);
}
private static void addStrings(List<String> out, Object raw) { for (Object item : list(raw)) if (item instanceof String s && !s.isBlank()) out.add(s); }
private static String valueText(Object value) {
if (value == null) return "null";
if (value instanceof BigDecimal decimal) return decimal.toPlainString();
if (value instanceof Number number) return number.toString();
if (value instanceof String s) return s;
if (value instanceof Boolean b) return b.toString();
if (value instanceof Map<?,?> map) {
List<String> items = new ArrayList<>(); for (var entry : map.entrySet()) items.add(valueText(entry.getKey()) + ": " + valueText(entry.getValue()));
return "{" + String.join(", ", items) + "}";
}
if (value instanceof Collection<?> collection) return "[" + String.join(", ", collection.stream().map(AnalysisProtocolHistoryFacts::valueText).toList()) + "]";
return Objects.toString(value);
}
private static String md(String value) { return value.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
.replace("|", "\\|").replace("`", "\\`").replace("\r", " ").replace("\n", "<br>"); }
private static String html(String value) { return value.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;").replace("'", "&#39;"); }
private static String safeException(RuntimeException error) { return error.getMessage() == null ? "校验失败" : error.getMessage(); }
private static Map<String,Object> body(Map<String,Object> parameters, JsonMapper json) {
Object raw = parameters.get("body");
if (raw instanceof Map<?,?> map) return stringMap(map);
if (raw instanceof String value && !value.isBlank()) {
try { Object parsed = json.readValue(value, Object.class); return stringMap(parsed); }
catch (RuntimeException ignored) { return Map.of(); }
}
return Map.of();
}
private static Map<String,Object> withoutPage(Map<String,Object> request) { Map<String,Object> out = new LinkedHashMap<>(request); out.remove("offset"); out.remove("limit"); return out; }
private static String canonical(Map<String,Object> value) { return valueText(new TreeMap<>(value)); }
private static Object scalarParameter(Object raw) {
if (!(raw instanceof String value)) return raw;
if (value.matches("-?\\d+")) try { return Long.valueOf(value); } catch (NumberFormatException ignored) { return value; }
return value;
}
private static int integer(Object value, int fallback) { if (value instanceof Number n && n.doubleValue() == n.intValue()) return n.intValue(); return fallback; }
private static long number(Object value, long fallback) {
if (value instanceof Number n && Double.isFinite(n.doubleValue()) && n.doubleValue() == Math.rint(n.doubleValue())) return n.longValue();
if (value instanceof String s) try { return Long.parseLong(s); } catch (NumberFormatException ignored) { return fallback; }
return fallback;
}
private static String firstString(Object... values) { for (Object value : values) if (value instanceof String s && !s.isBlank()) return s; return ""; }
private static String text(Object value, String fallback) { return value == null ? fallback : Objects.toString(value, fallback); }
private static List<String> strings(Object value) { return list(value).stream().filter(String.class::isInstance).map(String.class::cast).toList(); }
private static List<Map<String,Object>> maps(Object value) { List<Map<String,Object>> out = new ArrayList<>(); for (Object item : list(value)) if (item instanceof Map<?,?> m) out.add(stringMap(m)); return out; }
private static List<?> list(Object value) { return value instanceof List<?> items ? items : List.of(); }
private static Map<String,Object> firstMap(Object... values) { for (Object value : values) if (value instanceof Map<?,?> map && !map.isEmpty()) return stringMap(map); return Map.of(); }
private static Map<String,Object> stringMap(Object value) { if (!(value instanceof Map<?,?> raw)) return Map.of(); Map<String,Object> out = new LinkedHashMap<>(); raw.forEach((k,v)->{if(k instanceof String s)out.put(s,v);}); return out; }
private static Map<String,Object> map(Object value) { return stringMap(value); }
private static Map<String,Object> mapOf(Object... pairs) { Map<String,Object> out = new LinkedHashMap<>(); for (int i=0;i+1<pairs.length;i+=2) out.put((String)pairs[i], pairs[i+1]); return out; }
}
@@ -0,0 +1,56 @@
package cn.yunsheng.quality.analysis;
import java.util.*;
import java.util.regex.Pattern;
/** Plain-text presentation of a verified history import; references never create new facts. */
final class AnalysisProtocolHistorySummary {
private static final Pattern BRACKET=Pattern.compile("\\[([^\\[\\]\\r\\n]{1,240})\\]");
private static final Pattern ACTIVE_HTML=Pattern.compile("(?is)<(script|style|iframe|object|embed)\\b[^>]*>.*?</\\1\\s*>");
record Result(String answer,Map<String,Object> summary,List<String> refs,List<String> gaps){}
private record Text(String value,List<String> refs,boolean filtered){}
static Result normalize(Map<String,Object> details,Set<String> known,String fallback) {
var raw=AnalysisProtocolService.map(details.get("readingSummary"));var gaps=new LinkedHashSet<String>();
boolean shape=Objects.equals(raw.get("version"),1)&&raw.get("lead") instanceof String&&raw.get("evidence") instanceof List<?>&&raw.get("notices") instanceof List<?>&&raw.get("evidenceRefs") instanceof List<?>;
if(!shape)gaps.add("历史解释摘要格式未通过核验,保留完整数据附录。");
var used=new LinkedHashSet<String>();var summaryRefs=new LinkedHashSet<String>();
var lead=clean(shape?raw.get("lead"):fallback,known,12000);summaryRefs.addAll(lead.refs());
var evidence=new ArrayList<String>();var notices=new LinkedHashSet<String>();
if(lead.filtered())gaps.add("历史解释中的未知引用或HTML已过滤,相关结论仍需核验。");
if(shape) {
collect(raw.get("evidence"),evidence,summaryRefs,gaps,known,20);
var noticeList=new ArrayList<String>();collect(raw.get("notices"),noticeList,summaryRefs,gaps,known,40);notices.addAll(noticeList);
var offered=AnalysisProtocolService.list(raw.get("evidenceRefs"));
if(offered.size()>100||offered.stream().anyMatch(id->!(id instanceof String)||!known.contains(id)))gaps.add("历史解释中的未知引用或HTML已过滤,相关结论仍需核验。");
}
used.addAll(summaryRefs);
var paragraphs=new ArrayList<String>();Object rawAnswer=details.get("answer");
if(rawAnswer instanceof String text&&text.length()<=64000)for(String paragraph:text.split("\\n\\s*\\n")) {
var cleaned=clean(paragraph,known,12000);if(!cleaned.value().isBlank()){paragraphs.add(cleaned.value());used.addAll(cleaned.refs());}
if(cleaned.filtered())gaps.add("历史解释中的未知引用或HTML已过滤,相关结论仍需核验。");
}
if(paragraphs.isEmpty())gaps.add("历史解释正文尚未形成有效内容,保留完整数据附录。");
if(!known.isEmpty()&&used.isEmpty())gaps.add("历史解释没有绑定已核验事实,保留完整数据附录。");
if(known.isEmpty())gaps.add("历史快照没有可用于解释的事实,保留完整数据附录。");
notices.addAll(gaps);
var summary=new LinkedHashMap<String,Object>();summary.put("version",1);summary.put("lead",lead.value().isBlank()?fallback:lead.value());summary.put("evidence",List.copyOf(evidence));summary.put("notices",List.copyOf(notices));summary.put("evidenceRefs",List.copyOf(summaryRefs));
return new Result(paragraphs.isEmpty()?fallback:String.join("\n\n",paragraphs),summary,List.copyOf(used),List.copyOf(gaps));
}
private static void collect(Object raw,List<String> target,Set<String> refs,Set<String> gaps,Set<String> known,int max) {
var values=AnalysisProtocolService.list(raw);
if(values.size()>max)gaps.add("历史解释摘要超过展示上限,已保留有限内容与完整附录。");
for(Object value:values.stream().limit(max).toList()) {
var cleaned=clean(value,known,12000);if(!cleaned.value().isBlank()){target.add(cleaned.value());refs.addAll(cleaned.refs());}
if(cleaned.filtered())gaps.add("历史解释中的未知引用或HTML已过滤,相关结论仍需核验。");
}
}
private static Text clean(Object raw,Set<String> known,int limit) {
if(!(raw instanceof String text)||text.length()>limit)return new Text("",List.of(),true);
String safe=ACTIVE_HTML.matcher(text).replaceAll("").replaceAll("(?s)<[^>]*>","").replace("<","").replace(">","").replaceAll("[\\p{Cntrl}&&[^\\n\\t]]","");
boolean filtered=!safe.equals(text);var refs=new LinkedHashSet<String>();var matcher=BRACKET.matcher(safe);boolean unknown=false;
while(matcher.find())if(known.contains(matcher.group(1)))refs.add(matcher.group(1));else unknown=true;
// Do not retain an unsupported assertion after merely hiding its invented citation.
if(unknown)return new Text("",List.of(),true);
return new Text(safe.strip(),List.copyOf(refs),filtered);
}
}
@@ -0,0 +1,168 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.analytics.AnalyticsController;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import java.math.BigDecimal;
import java.math.MathContext;
import java.util.*;
import org.springframework.http.HttpStatus;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.web.server.ResponseStatusException;
import tools.jackson.databind.DeserializationFeature;
import tools.jackson.databind.json.JsonMapper;
/** A sealed server-origin inspection task, using the same revision, execution and proof store. */
final class AnalysisProtocolInspection {
static final String KIND="inspection",TARGET="GET /api/analytics/top";
static final int LIMIT=20;
private final JdbcTemplate jdbc;
private final JsonMapper json;
private final GovernanceAccessService governance;
private final JsonMapper exact=JsonMapper.builder().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS).build();
AnalysisProtocolInspection(JdbcTemplate jdbc,JsonMapper json,GovernanceAccessService governance) {
this.jdbc=jdbc;this.json=json;this.governance=governance;
}
Map<String,Object> seal(String runId,String release,String scope,AppUserPrincipal actor,List<Map<String,Object>> submitted) {
var origin=origin(runId,release,scope,actor);
var refs=references(map(origin.get("result")));
if(refs.isEmpty()||!same(refs,submitted))throw denied("巡检发现与受控任务来源不一致");
return Map.of("runId",runId,"originHash",digest(originIdentity(origin)),"findings",refs);
}
void verify(AnalysisProtocolService.Attempt a,Map<String,Object> task,AppUserPrincipal actor) {
var seal=map(task.get("controlledInspection"));
if(!same(seal,a.input().get("controlledInspection"))||!Objects.equals(task.get("question"),jdbc.queryForObject("SELECT question FROM analysis_turn WHERE id=?",String.class,a.turnId())))throw denied("巡检固定任务缺少服务端来源绑定");
var origin=origin(Objects.toString(seal.get("runId"),""),a.releaseId(),a.scopeHash(),actor);
if(!same(seal.get("findings"),references(map(origin.get("result"))))
||!Objects.equals(seal.get("originHash"),digest(originIdentity(origin)))
||!same(task,task(Objects.toString(task.get("question"),""),seal)))throw denied("巡检来源或固定查询任务已变化");
}
Map<String,Object> task(String question,Map<String,Object> seal) {
var t=new LinkedHashMap<String,Object>();
t.put("version",1);t.put("kind",KIND);t.put("goal",question);t.put("rootGoal",question);t.put("question",question);
t.put("controlledInspection",seal);t.put("period",null);t.put("periodMode","all_published");
t.put("filters",Map.of("includeUnassigned",false));t.put("stages",List.of("black","bright"));
t.put("fields",List.of("scrapWeight"));t.put("dimensions",List.of());t.put("dimensionStages",Map.of());
for(String flag:List.of("needsClarification","trendRequired","comparisonRequired","verifySource","fullDetails"))t.put(flag,false);
t.put("freshQuery",true);t.put("visualPreference","none");return t;
}
static Map<String,String> parameters(String release,String stage) {
if(!Set.of("black","bright").contains(stage))throw denied("巡检阶段无效");
return Map.of("releaseId",release,"stage",stage,"dimension","category","includeUnassigned","false","limit",Integer.toString(LIMIT));
}
static void requireParameters(String release,Map<String,String> actual) {
if(!parameters(release,actual.get("stage")).equals(actual))throw denied("巡检查询偏离已确认的正式类别范围");
}
static Map<String,Object> inspect(Map<String,Object> response,Map<String,String> params,Set<String> allowed) {
String release=params.get("releaseId"),stage=params.get("stage");requireParameters(release,params);
var expected=new LinkedHashMap<String,Object>();expected.put("releaseId",release);expected.put("stage",stage);
expected.put("dimension","category");expected.put("includeUnassigned",false);expected.put("limit",LIMIT);expected.put("extraFilters",Map.of());
if(!Objects.equals(release,response.get("releaseId"))||!stage.equals(response.get("stage"))
||!"category".equals(response.get("dimension"))||!"".equals(response.get("category"))
||!Objects.equals(response.get("limit"),LIMIT)||!sameStatic(expected,map(response.get("resultQuery"))))
throw denied("巡检正式类别响应与查询不一致");
Set<String> visible=new HashSet<>();AnalyticsController.formalCategoryFields(stage).forEach((key,label)->{if(allowed.contains("*")||allowed.contains(key))visible.add(label);});
if(visible.isEmpty())throw denied("当前授权不包含巡检正式报废类别");
var rows=maps(response.get("rows"));Set<String> found=new HashSet<>();List<String> gaps=new ArrayList<>();
BigDecimal sum=BigDecimal.ZERO,denominator=number(response.get("denominator"));
if(denominator==null)throw denied("巡检类别分母格式无效");
long sourceRows=-1;BigDecimal previous=null;String previousKey="";int rank=0;
for(var row:rows) {
String label=Objects.toString(row.get("label"),"");
if(!visible.contains(label)||!found.add(label)||!label.equals(row.get("key"))||!label.equals(row.get("sourceCategory"))
||!Objects.equals(row.get("rank"),++rank)||!Objects.equals(number(row.get("denominator")),denominator))
throw denied("巡检类别、排名或分母不属于当前授权结果");
BigDecimal weight=number(row.get("weight"));long count=count(map(row.get("coverage")).get("rows"));
if(sourceRows!=-1&&sourceRows!=count)throw denied("巡检类别来源行数不一致");sourceRows=count;
if(weight==null)gaps.add(stageLabel(stage)+label+"未提供正式类别重量");
else {
if(previous!=null&&(previous.compareTo(weight)<0||previous.compareTo(weight)==0&&previousKey.compareTo(label)>0))throw denied("巡检类别排序不一致");
sum=sum.add(weight);previous=weight;previousKey=label;
BigDecimal share=number(row.get("share"));
if(denominator.signum()==0?row.get("share")!=null:share==null||share.subtract(weight.divide(denominator,MathContext.DECIMAL128)).abs().compareTo(new BigDecimal("1e-28"))>0)
throw denied("巡检类别占比不属于正式分母");
}
}
if(!found.equals(visible))throw denied("巡检类别结果未保留全部授权类别");
if(sum.compareTo(denominator)!=0)throw denied("巡检类别分母与正式类别总和不一致");
long covered=maps(map(response.get("coverage")).get("sourceMonths")).stream().mapToLong(month->count(month.get(stage.equals("black")?"blackRows":"brightRows"))).sum();
if(sourceRows<1||covered<1)gaps.add(stageLabel(stage)+"缺少有效的已发布来源");
String rowsKey=stage.equals("black")?"blackRows":"brightRows";
var included=map(map(response.get("coverage")).get("includedStageRows"));
if(!included.containsKey(rowsKey)||sourceRows!=count(included.get(rowsKey))||sourceRows>covered)
throw denied("巡检类别与发布来源覆盖不一致");
return Map.of("complete",gaps.isEmpty(),"stage",stage,"sourceRows",sourceRows,"formalCategories",rows.size(),"missingEvidence",gaps);
}
Map<String,Object> finish(AnalysisProtocolService.Attempt a,Map<String,Object> task,List<String> ids,AppUserPrincipal actor) {
verify(a,task,actor);var queries=queries(a,ids,actor);var projected=projection(a.turnId(),queries);
var facts=maps(projected.get("facts"));var source=new LinkedHashMap<>(projected);
source.put("sourceTurnId",a.turnId());source.put("revision",a.revision());source.put("executionId",a.executionId());source.put("status",Boolean.TRUE.equals(projected.get("complete"))?"COMPLETED":"PARTIAL");
var rendered=new LinkedHashMap<>(AnalysisProtocolHistoryFacts.renderCurrent("巡检正式报废类别核查",List.of(source),facts.stream().map(f->f.get("id").toString()).limit(2).toList(),List.of()));
rendered.put("answer",rendered.get("reportMarkdown"));rendered.put("inspectionCoverage",Map.of("complete",projected.get("complete"),"missingEvidence",projected.get("gaps"),"stages",List.of("black","bright")));
rendered.put("controlledInspection",task.get("controlledInspection"));rendered.put("protocolQueryIds",ids);
rendered.put("results",List.of());rendered.put("visuals",List.of());rendered.put("visualSchemaVersion",1);rendered.put("visualState","not_requested");
rendered.put("businessResultsSentToModel",false);rendered.put("engine","CONTROLLED_INSPECTION_PROTOCOL");rendered.put("reportTitle","巡检正式报废类别核查");
rendered.put("stopReason",Boolean.TRUE.equals(projected.get("complete"))?"inspection_complete":"inspection_incomplete");
rendered.put("context",AnalysisTurnService.nativeAnswerContext(AnalysisProtocolService.taskContext(a.releaseId(),task),Map.of("releaseId",a.releaseId()),a.releaseId()));
return rendered;
}
List<Map<String,Object>> queries(AnalysisProtocolService.Attempt a,List<String> ids,AppUserPrincipal actor) {
if(ids.isEmpty()||ids.size()>2||new HashSet<>(ids).size()!=ids.size())throw denied("巡检缺少本次执行的正式查询");
List<Map<String,Object>> out=new ArrayList<>();Set<String> stages=new HashSet<>();
for(String id:ids) {
var rows=jdbc.query("SELECT query_target,parameters_json,response_hash,response_snapshot,response_snapshot_bytes,response_snapshot_encoding,protocol_node_id,protocol_scope_id FROM analysis_protocol_query WHERE query_id=? AND attempt_id=? AND execution_id=? AND revision=? AND auth_scope_hash=?",(rs,n)->{
var m=new LinkedHashMap<String,Object>();m.put("target",rs.getString(1));m.put("parameters",exact.readValue(rs.getString(2),Map.class));
m.put("response",exact.readValue(AnalysisProtocolEvidenceSnapshot.decode(rs.getBytes(4),rs.getInt(5),rs.getString(6),rs.getString(3)),Map.class));
if(rs.getString(7)!=null||rs.getString(8)!=null)throw denied("受控巡检查询不能携带自选节点");return m;
},id,a.attemptId(),a.executionId(),a.revision(),a.scopeHash());
if(rows.size()!=1)throw denied("巡检证据属于其他执行");var q=rows.getFirst();if(!TARGET.equals(q.get("target")))throw denied("巡检证据工具无效");
Map<String,String> params=new LinkedHashMap<>();map(q.get("parameters")).forEach((k,v)->params.put(k,Objects.toString(v)));
requireParameters(a.releaseId(),params);if(!stages.add(params.get("stage")))throw denied("巡检阶段证据重复");
q.put("queryId",id);q.put("coverage",inspect(map(q.get("response")),params,governance.allowedFields(actor)));out.add(q);
}
return out;
}
static Map<String,Object> projection(String turn,List<Map<String,Object>> queries) {
List<Map<String,Object>> sections=new ArrayList<>(),facts=new ArrayList<>();List<String> gaps=new ArrayList<>();Set<String> stages=new HashSet<>();
for(var q:queries) {
var response=map(q.get("response"));String stage=Objects.toString(response.get("stage"));stages.add(stage);var coverage=map(q.get("coverage"));
for(Object gap:list(coverage.get("missingEvidence")))gaps.add(gap.toString());
List<Map<String,Object>> columns=new ArrayList<>();for(var entry:List.of(Map.entry("rank","序号"),Map.entry("label","正式报废类别"),Map.entry("weight","类别重量 kg"),Map.entry("denominator","同范围全部授权类别重量 kg"),Map.entry("share","占比(比例值)")))columns.add(Map.of("key",entry.getKey(),"label",entry.getValue()));
var section=new LinkedHashMap<String,Object>();section.put("title","正式报废类别重量");section.put("sourceTurnId",turn);section.put("period",Map.of("allPublished",true));section.put("filters",Map.of("includeUnassigned",false));section.put("stage",stage);
section.put("columns",columns);section.put("rows",maps(response.get("rows")).stream().map(row->{
var projected=new LinkedHashMap<String,Object>();for(var column:columns)projected.put(column.get("key").toString(),row.get(column.get("key")));return projected;
}).toList());section.put("note","本轮正式类别查询 "+q.get("queryId")+";分母为同范围全部授权正式类别重量,不能当作来源细分缺陷或生产根因。已发布来源行:"+coverage.get("sourceRows"));sections.add(section);
var rows=maps(response.get("rows"));if(!rows.isEmpty()&&rows.getFirst().get("weight")!=null)facts.add(Map.of("id",turn+":inspection:"+stage,"sourceTurnId",turn,"text",stageLabel(stage)+"正式报废类别首位为"+rows.getFirst().get("label")+",重量"+number(rows.getFirst().get("weight")).toPlainString()+" kg;同范围授权类别总重"+number(response.get("denominator")).toPlainString()+" kg。"));
}
for(String stage:List.of("black","bright"))if(!stages.contains(stage))gaps.add(stageLabel(stage)+"尚无本次执行的正式类别查询证据");
return Map.of("sections",sections,"facts",facts,"complete",gaps.isEmpty(),"gaps",gaps);
}
void updateAssociation(AnalysisProtocolService.Attempt a,String status) {
var control=map(a.input().get("controlledInspection"));if(control.isEmpty())return;
jdbc.update("UPDATE quality_inspection_run SET result_json=JSON_SET(result_json,'$.aiDrilldownStatus',?,'$.aiDrilldowns[0].status',?) WHERE id=? AND owner_user_id=? AND auth_scope_hash=? AND release_id=? AND JSON_UNQUOTE(JSON_EXTRACT(result_json,'$.analysisTurnId'))=?",status,status,control.get("runId"),a.userId(),a.scopeHash(),a.releaseId(),a.turnId());
}
private Map<String,Object> origin(String id,String release,String scope,AppUserPrincipal actor) {
if(!Objects.equals(scope,AuthorizationFingerprint.of(actor,governance)))throw denied("巡检授权范围已变化");
governance.requireDetail(actor);governance.requireBoundAction("QUERY","query.detail");
var rows=jdbc.query("SELECT id,strategy_id,strategy_row_version,owner_user_id,auth_scope_hash,release_id,status,result_json FROM quality_inspection_run WHERE id=?",(rs,n)->Map.<String,Object>of("id",rs.getString(1),"strategyId",rs.getString(2),"strategyRevision",rs.getLong(3),"owner",rs.getString(4),"scope",rs.getString(5),"release",rs.getString(6),"status",rs.getString(7),"result",exact.readValue(rs.getString(8),Map.class)),id);
if(rows.size()!=1)throw denied("巡检来源不存在");var origin=rows.getFirst();
if(!actor.getId().toString().equals(origin.get("owner"))||!scope.equals(origin.get("scope"))||!release.equals(origin.get("release"))||!"COMPLETED".equals(origin.get("status")))throw denied("巡检来源不属于当前用户、授权或发布版本");return origin;
}
private Map<String,Object> originIdentity(Map<String,Object> origin) {var identity=new LinkedHashMap<>(origin);identity.remove("result");identity.put("findings",map(origin.get("result")).get("findings"));identity.put("maximum",map(origin.get("result")).getOrDefault("aiMaxFindings",3));return identity;}
private List<Map<String,Object>> references(Map<String,Object> result) {
int max=Math.min(3,Math.max(1,((Number)result.getOrDefault("aiMaxFindings",3)).intValue()));
return maps(result.get("findings")).stream().sorted(Comparator.comparingInt(v->switch(Objects.toString(v.get("severity"),"")){case "CRITICAL"->0;case "HIGH"->1;case "MEDIUM"->2;default->3;})).limit(max)
.map(v->Map.<String,Object>of("type",Objects.toString(v.get("type"),""),"subject",Objects.toString(v.get("subject"),""),"severity",Objects.toString(v.get("severity"),""))).toList();
}
private String digest(Object value){return AnalysisProtocolService.hash(json.writeValueAsString(AnalysisProtocolService.canonical(value)));}
private boolean same(Object a,Object b){return sameStatic(a,b);}
private static boolean sameStatic(Object a,Object b){return Objects.equals(AnalysisProtocolService.canonical(a),AnalysisProtocolService.canonical(b));}
private static BigDecimal number(Object value){if(value==null)return null;try{return new BigDecimal(value.toString());}catch(RuntimeException error){throw denied("巡检正式数值格式无效");}}
private static long count(Object value){if(value==null)return 0;try{return new BigDecimal(value.toString()).longValueExact();}catch(RuntimeException error){throw denied("巡检来源行数无效");}}
private static String stageLabel(String stage){return stage.equals("black")?"黑片":"亮片";}
private static Map<String,Object> map(Object value){return AnalysisProtocolService.map(value);}
private static List<?> list(Object value){return AnalysisProtocolService.list(value);}
private static List<Map<String,Object>> maps(Object value){return list(value).stream().map(AnalysisProtocolInspection::map).toList();}
private static ResponseStatusException denied(String text){return new ResponseStatusException(HttpStatus.FORBIDDEN,text);}
}
@@ -0,0 +1,262 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.filters.SourceColumnQueries;
import java.util.*;
/** Canonical request construction and response coverage checks for protocol investigation nodes. */
final class AnalysisProtocolInvestigation {
private static final List<String> PERIOD_KEYS = List.of("year", "month", "quarter", "monthStart", "monthEnd");
private static final List<String> FILTER_KEYS = List.of("department", "group", "product", "projectGroup",
"extraFilters", "columnFilters", "exclusions", "includeUnassigned");
private AnalysisProtocolInvestigation() { }
static List<Map<String,Object>> requests(String release, Map<String,Object> childTask, String purpose) {
boolean source = "source".equals(purpose), records = "records".equals(purpose);
if (!source && !records) throw invalid("调查节点用途无效");
if (release == null || release.isBlank()) throw invalid("调查发布版本缺失");
if (source && (!"source_verify".equals(childTask.get("kind"))
|| !Boolean.TRUE.equals(childTask.get("verifySource")) || Boolean.TRUE.equals(childTask.get("fullDetails"))))
throw invalid("原件核验子任务标志不匹配");
if (records && (!"details".equals(childTask.get("kind"))
|| !Boolean.TRUE.equals(childTask.get("fullDetails")) || Boolean.TRUE.equals(childTask.get("verifySource"))))
throw invalid("明细子任务标志不匹配");
List<String> stages = strings(childTask.get("stages"));
if (stages.isEmpty() || stages.size() > 2 || !Set.of("black", "bright").containsAll(stages)
|| new HashSet<>(stages).size() != stages.size()) throw invalid("调查阶段无效");
List<Map<String,Object>> periods = new ArrayList<>();
periods.add(map(childTask.get("period")));
if (Boolean.TRUE.equals(childTask.get("comparisonRequired"))) {
Map<String,Object> comparison = map(childTask.get("comparisonPeriod"));
if (comparison.isEmpty()) throw invalid("调查对照期间缺失");
periods.add(comparison);
}
Map<String,Object> filters = map(childTask.get("filters"));
if (!FILTER_KEYS.containsAll(filters.keySet())) throw invalid("调查范围包含未支持筛选条件");
Map<String,Object> investigationFields = map(childTask.get("investigationFields"));
if (!stages.containsAll(investigationFields.keySet())) throw invalid("investigationFields 阶段超出子任务范围");
List<Map<String,Object>> requests = new ArrayList<>();
for (Map<String,Object> period : periods) for (String stage : stages) {
Map<String,Object> request = new LinkedHashMap<>();
request.put("releaseId", release);
request.put("stage", stage);
request.put("action", source ? "source" : "rows");
request.put("limit", 100);
for (String key : PERIOD_KEYS) if (period.containsKey(key)) request.put(key, period.get(key));
for (String key : FILTER_KEYS) if (filters.containsKey(key)) request.put(key, filters.get(key));
request.putIfAbsent("includeUnassigned", false);
if (investigationFields.containsKey(stage)) {
Object rawFields = investigationFields.get(stage);
if (!(rawFields instanceof List<?> selected) || selected.isEmpty() || selected.size() > 32
|| selected.stream().anyMatch(value -> !(value instanceof String field) || field.isBlank())
|| new HashSet<>(selected).size() != selected.size()) throw invalid("investigationFields 格式无效");
request.put("fields", List.copyOf(selected));
}
if (records) {
request.put("offset", 0);
request.put("recordFilter", "all");
}
requests.add(Collections.unmodifiableMap(request));
}
return List.copyOf(requests);
}
/** Exact scope binding; records may vary only their bounded page parameters. */
static boolean acceptParameters(Map<String,Object> expected, Map<String,Object> actual) {
if (expected == null || actual == null) return false;
if (!"rows".equals(expected.get("action")))
return Objects.equals(AnalysisProtocolService.canonical(expected), AnalysisProtocolService.canonical(actual));
Integer offset = integer(actual.get("offset"), 0, 1_000_000);
Integer limit = integer(actual.get("limit"), 1, 100);
if (offset == null || limit == null) return false;
Map<String,Object> expectedFixed = new LinkedHashMap<>(expected);
Map<String,Object> actualFixed = new LinkedHashMap<>(actual);
expectedFixed.remove("offset");
expectedFixed.remove("limit");
actualFixed.remove("offset");
actualFixed.remove("limit");
return Objects.equals(AnalysisProtocolService.canonical(expectedFixed),
AnalysisProtocolService.canonical(actualFixed));
}
static Map<String,Object> inspect(Map<String,Object> response, Map<String,Object> expectedPayload) {
List<String> missing = new ArrayList<>();
if (expectedPayload == null) {
expectedPayload = Map.of();
missing.add("期望调查参数缺失");
}
String action = string(expectedPayload.get("action"));
Map<String,Object> context = map(response == null ? null : response.get("context"));
if (response == null || response.isEmpty()) missing.add("response 缺失");
if (!"quality_investigation".equals(response == null ? null : response.get("kind"))) missing.add("响应 kind 不匹配");
if (!action.equals(response == null ? null : response.get("action"))) missing.add("响应 action 不匹配");
if (!Objects.equals(expectedPayload.get("releaseId"), context.get("releaseId"))) missing.add("context releaseId 不匹配");
if (!Objects.equals(expectedPayload.get("stage"), context.get("stage"))) missing.add("context stage 不匹配");
for (String key : PERIOD_KEYS) {
if (!samePeriod(expectedPayload.get(key), context.get(key))) missing.add("context " + key + " 不匹配");
}
for (String key : FILTER_KEYS) {
if (!sameFilter(key, expectedPayload.get(key), context.get(key))) missing.add("context " + key + " 不匹配");
}
for (String key : List.of("category", "productSearch", "sourceRevisionId")) {
if (!sameFilter(key, expectedPayload.get(key), context.get(key))) missing.add("context " + key + " 不匹配");
}
boolean aggregateComplete = response != null && Boolean.TRUE.equals(response.get("aggregateComplete"));
if (!aggregateComplete) missing.add("aggregateComplete 未通过");
if (expectedPayload.get("fields") instanceof List<?> expectedFields
&& !Objects.equals(AnalysisProtocolService.canonical(expectedFields),
AnalysisProtocolService.canonical(response == null ? null : response.get("selectedFields"))))
missing.add("selectedFields 与显式 fields 不一致");
boolean sourceComplete = false, recordsPageComplete = false;
if ("source".equals(action)) sourceComplete = inspectSource(response, missing);
else if ("rows".equals(action)) recordsPageComplete = inspectRecords(response, expectedPayload, missing);
else missing.add("expected action 不是 source/rows");
Map<String,Object> coverage = new LinkedHashMap<>();
coverage.put("complete", missing.isEmpty());
coverage.put("sourceVerificationComplete", sourceComplete && missing.isEmpty());
coverage.put("recordsPageComplete", recordsPageComplete && missing.isEmpty());
coverage.put("aggregateComplete", aggregateComplete);
coverage.put("missingEvidence", List.copyOf(missing));
for (String key : List.of("totalRows", "matchingRows", "offset", "limit", "returnedRows", "hasMore", "nextOffset", "selectedFields"))
coverage.put(key, response == null ? null : response.get(key));
Map<String,Object> proof = map(response == null ? null : response.get("sourceVerification"));
coverage.put("sourceExpectedRows", proof.get("expectedRows"));
coverage.put("sourceVerifiedRows", proof.get("verifiedRows"));
coverage.put("sourceMissingRows", proof.get("missingRows"));
coverage.put("sourceMismatchRows", proof.get("mismatchRows"));
return Collections.unmodifiableMap(coverage);
}
private static boolean inspectSource(Map<String,Object> response, List<String> missing) {
Map<String,Object> proof = map(response == null ? null : response.get("sourceVerification"));
long total = number(response == null ? null : response.get("totalRows"), -1);
long expected = number(proof.get("expectedRows"), -1);
long verified = number(proof.get("verifiedRows"), -1);
long missingRows = number(proof.get("missingRows"), -1);
long mismatches = number(proof.get("mismatchRows"), -1);
boolean complete = Boolean.TRUE.equals(proof.get("complete"))
&& Boolean.TRUE.equals(proof.get("allSelectedValuesMatchStored"))
&& total > 0 && expected == total && verified == total && missingRows == 0 && mismatches == 0;
if (!complete) missing.add("原件核验未完整匹配全部命中行");
return complete;
}
private static boolean inspectRecords(Map<String,Object> response, Map<String,Object> expected, List<String> missing) {
if (!"all".equals(expected.get("recordFilter"))) missing.add("期望 recordFilter 不是 all");
if (!"all".equals(response == null ? null : response.get("recordFilter"))) missing.add("响应 recordFilter 不是 all");
Integer offset = integer(expected.get("offset"), 0, 1_000_000);
Integer limit = integer(expected.get("limit"), 1, 100);
Integer actualOffset = integer(response == null ? null : response.get("offset"), 0, 1_000_000);
Integer actualLimit = integer(response == null ? null : response.get("limit"), 1, 100);
Integer returned = integer(response == null ? null : response.get("returnedRows"), 0, 100);
Integer matching = integer(response == null ? null : response.get("matchingRows"), 0, Integer.MAX_VALUE);
if (offset == null || limit == null || !Objects.equals(offset, actualOffset) || !Objects.equals(limit, actualLimit))
missing.add("rows 页码或页长不匹配");
Object rawRows = response == null ? null : response.get("rows");
if (!(rawRows instanceof List<?> rows) || returned == null || returned != rows.size()
|| actualLimit == null || rows.size() > actualLimit) {
missing.add("returnedRows 与 rows 数量不匹配");
return false;
}
if (matching == null || actualOffset == null) {
missing.add("matchingRows 或 offset 缺失");
return false;
}
boolean hasMore = actualOffset + rows.size() < matching;
if (!Objects.equals(response.get("hasMore"), hasMore)) missing.add("hasMore 与当前页范围不匹配");
Object nextOffset = response.get("nextOffset");
if (hasMore) {
if (!Objects.equals(integer(nextOffset, 0, Integer.MAX_VALUE), actualOffset + rows.size()))
missing.add("nextOffset 与当前页范围不匹配");
} else if (nextOffset != null) missing.add("末页 nextOffset 应为空");
if (number(response.get("totalRows"), -1) != matching || actualOffset + rows.size() > matching)
missing.add("完整记录页的总数或已读范围无效");
if (hasMore && rows.isEmpty()) missing.add("非末页没有推进读取位置");
return missing.isEmpty();
}
static boolean samePeriod(Object expected, Object actual) {
if (empty(expected) && empty(actual)) return true;
Integer left = integer(expected, 1, 9999), right = integer(actual, 1, 9999);
return left != null && left.equals(right);
}
static boolean sameFilter(String key, Object expected, Object actual) {
try {
Object left = normalizeFilter(key, expected), right = normalizeFilter(key, actual);
return Objects.equals(AnalysisProtocolService.canonical(left), AnalysisProtocolService.canonical(right));
} catch (RuntimeException invalid) {
return false;
}
}
private static Object normalizeFilter(String key, Object value) {
if ("includeUnassigned".equals(key)) return value == null ? false : value;
if (Set.of("department","group","product","projectGroup").contains(key)) return normalizeSelections(value);
if ("extraFilters".equals(key)) return QueryFilterFields.parse(empty(value) ? null : value);
if ("columnFilters".equals(key)) return SourceColumnQueries.parse(empty(value) ? null : value);
if ("exclusions".equals(key)) {
if (empty(value)) return Map.of();
if (!(value instanceof Map<?,?> map)) return value;
Map<String,Object> normalized = new LinkedHashMap<>();
for (var entry : map.entrySet()) {
if (!(entry.getKey() instanceof String dimension)) return value;
List<String> selections = normalizeSelections(entry.getValue());
if (!selections.isEmpty()) normalized.put(dimension, selections);
}
return normalized;
}
if (empty(value)) return null;
return value;
}
private static List<String> normalizeSelections(Object value) {
if(empty(value))return List.of();
if(value instanceof String wire)return SelectionValues.parse(wire).stream().sorted().toList();
if(!(value instanceof List<?> list)||list.size()>SelectionValues.MAX_SELECTIONS)
throw new IllegalArgumentException("筛选集合格式无效");
List<String> values=new ArrayList<>();
for(Object raw:list) {
if(!(raw instanceof String text)||text.isBlank()||text.length()>SelectionValues.MAX_VALUE_LENGTH)
throw new IllegalArgumentException("筛选值格式无效");
values.add(text);
}
return values.stream().distinct().sorted().toList();
}
private static boolean empty(Object value) {
return value == null || value instanceof String text && text.isBlank()
|| value instanceof Map<?,?> map && map.isEmpty()
|| value instanceof Collection<?> collection && collection.isEmpty();
}
private static Integer integer(Object value, int min, int max) {
if (!(value instanceof Number number)) return null;
double numeric = number.doubleValue();
if (!Double.isFinite(numeric) || numeric != Math.rint(numeric) || numeric < min || numeric > max) return null;
return (int) numeric;
}
private static long number(Object value, long fallback) {
if (!(value instanceof Number number)) return fallback;
double numeric = number.doubleValue();
return !Double.isFinite(numeric) || numeric != Math.rint(numeric) ? fallback : (long) numeric;
}
private static List<String> strings(Object value) {
if (!(value instanceof List<?> list)) return List.of();
return list.stream().filter(String.class::isInstance).map(String.class::cast).toList();
}
@SuppressWarnings("unchecked") private static Map<String,Object> map(Object value) {
return value instanceof Map<?,?> ? (Map<String,Object>) value : Map.of();
}
private static String string(Object value) { return value instanceof String text ? text : ""; }
private static IllegalArgumentException invalid(String message) { return new IllegalArgumentException(message); }
}
@@ -0,0 +1,475 @@
package cn.yunsheng.quality.analysis;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Checks formal metrics returned by the metrics API and their published source coverage.
* It never calculates or changes a KPI. The caller is responsible for verifying exact scope.
*/
final class AnalysisProtocolMetricCoverage {
private static final Set<String> FIELDS = Set.of("rate", "scrapRate", "totalWeight", "scrapWeight");
private static final Set<String> STAGES = Set.of("black", "bright");
private static final Pattern MONTH_TEXT = Pattern.compile("(\\d{4})-(0[1-9]|1[0-2])");
private AnalysisProtocolMetricCoverage() {}
static Map<String, Object> inspect(Map<String, Object> response, Map<String, Object> task,
Map<String, Object> queriedPeriod) {
List<String> missing = new ArrayList<>();
Map<String, Object> result = new LinkedHashMap<>();
result.put("metricsComplete", false);
result.put("missingEvidence", missing);
Period period = period(queriedPeriod);
if (period == null) {
missing.add("本次查询期间无效,不能核验正式指标覆盖。");
result.put("availableMonths", List.of());
return result;
}
List<String> fields = strings(task == null ? null : task.get("fields"));
List<String> stages = strings(task == null ? null : task.get("stages"));
Object trendFlag = task == null ? null : task.get("trendRequired");
Object coverageMode = task == null ? null : task.get("periodCoverage");
boolean trendRequired = Boolean.TRUE.equals(trendFlag);
boolean taskValid = trendFlag instanceof Boolean
&& ("available".equals(coverageMode) || "complete".equals(coverageMode))
&& !fields.isEmpty() && FIELDS.containsAll(fields)
&& STAGES.containsAll(stages)
&& fields.stream().distinct().count() == fields.size()
&& stages.stream().distinct().count() == stages.size()
&& (stages.isEmpty() ? fields.equals(List.of("rate")) : true);
if (!taskValid) {
missing.add("临时指标任务字段或期间覆盖模式无效。");
result.put("availableMonths", List.of());
return result;
}
Map<String, Object> safeResponse = map(response);
Map<String, Object> coverage = map(safeResponse.get("coverage"));
MonthCoverage monthCoverage = monthCoverage(coverage.get("sourceMonths"), period, missing);
result.put("availableMonths", monthCoverage.availableLabels());
if (trendRequired) {
inspectTrend(safeResponse, fields, stages, period, monthCoverage,
"complete".equals(coverageMode), missing);
} else {
inspectSummary(safeResponse, fields, stages, period, monthCoverage,
"complete".equals(coverageMode), missing);
}
List<String> uniqueMissing = List.copyOf(new LinkedHashSet<>(missing));
result.put("missingEvidence", uniqueMissing);
result.put("metricsComplete", uniqueMissing.isEmpty());
return result;
}
private static void inspectSummary(Map<String, Object> response, List<String> fields, List<String> stages,
Period period, MonthCoverage coverage, boolean complete,
List<String> missing) {
Map<String, Object> summary = map(response.get("summary"));
if (summary.isEmpty()) {
missing.add("正式指标summary缺失。");
return;
}
for (String field : fields) {
if (stages.isEmpty()) {
List<Integer> months = selectedMonths("", period, coverage, complete, missing);
if (months.isEmpty()) {
missing.add("综合合格率没有可核验的已发布来源月份。");
continue;
}
if (!validCompositeRate(summary, months, coverage))
missing.add("综合合格率正式值或双阶段来源覆盖无效。");
continue;
}
for (String stage : stages) {
List<Integer> months = selectedMonths(stage, period, coverage, complete, missing);
if (months.isEmpty()) {
missing.add(stageLabel(stage) + fieldLabel(field) + "没有可核验的阶段来源月份。");
continue;
}
if (!validStageMetric(summary, stage, field, months, coverage))
missing.add(stageLabel(stage) + fieldLabel(field) + "正式值或阶段来源覆盖无效。");
}
}
}
private static boolean validStageMetric(Map<String, Object> summary, String stage, String field,
List<Integer> months, MonthCoverage coverage) {
if (!validStageSource(stage, months, coverage)) return false;
Map<String, Object> values = map(summary.get(stage));
if (values.isEmpty()) return false;
BigDecimal value = decimal(values.get(field));
if (value == null) return false;
if ("rate".equals(field))
return inRange(value, BigDecimal.ZERO, BigDecimal.ONE)
&& positive(decimal(values.get("totalWeight")));
if ("scrapRate".equals(field))
return positive(decimal(values.get("totalWeight")))
&& nonNegative(decimal(values.get("scrapWeight")))
&& value.signum() >= 0;
if ("totalWeight".equals(field)) return "black".equals(stage) || value.signum() >= 0;
return value.signum() >= 0;
}
private static boolean validCompositeRate(Map<String, Object> summary, List<Integer> months,
MonthCoverage coverage) {
BigDecimal value = decimal(summary.get("compositeRate"));
if (!inRange(value, BigDecimal.ZERO, BigDecimal.ONE)) return false;
boolean blackSource = false, brightSource = false;
for (int month : months) {
Map<String, Object> item = coverage.byMonth().get(month);
if (item == null) return false;
Long black = count(item.get("blackRows"));
Long bright = count(item.get("brightRows"));
if (black == null || bright == null || black < 0 || bright < 0 || black == 0 && bright == 0) return false;
blackSource |= black > 0;
brightSource |= bright > 0;
}
Map<String, Object> black = map(summary.get("black"));
Map<String, Object> bright = map(summary.get("bright"));
boolean denominatorExplicit = black.containsKey("totalWeight") || bright.containsKey("totalWeight");
if (!denominatorExplicit) return true;
return (blackSource && positive(decimal(black.get("totalWeight"))))
|| (brightSource && positive(decimal(bright.get("totalWeight"))));
}
private static void inspectTrend(Map<String, Object> response, List<String> fields, List<String> stages,
Period period, MonthCoverage coverage, boolean complete,
List<String> missing) {
if (period.months().size() < 2) {
missing.add("月度趋势需要至少两个查询月份。");
return;
}
Object rawTrend = response.get("trend");
if (!(rawTrend instanceof List<?> rows)) {
missing.add("正式月度趋势数组缺失。");
return;
}
Object countValue = response.get("trendCount");
if (countValue != null && (!strictIntegerEquals(countValue, rows.size()))) {
missing.add("正式月度趋势返回数量与完整数量不一致。");
return;
}
Object truncated = response.get("trendTruncated");
// The raw GET /api/metrics response returns its complete bounded trend directly. Wrapped
// responses expose explicit count/truncation metadata, which is checked when present.
if (Boolean.TRUE.equals(truncated) || truncated != null && !Boolean.FALSE.equals(truncated)
|| rows.size() > 36) {
missing.add("正式月度趋势数据被截断或超过可核验范围。");
return;
}
Map<Integer, Map<String, Object>> trendByMonth = new LinkedHashMap<>();
for (Object rawRow : rows) {
Map<String, Object> row = map(rawRow);
Integer month = trendMonth(row, period.year());
if (month == null || !period.months().contains(month)) {
missing.add("正式月度趋势包含本次查询期间以外或无法确认的月份。");
return;
}
if (trendByMonth.putIfAbsent(month, row) != null) {
missing.add("正式月度趋势存在重复月份。");
return;
}
}
for (Map.Entry<Integer, Map<String, Object>> entry : trendByMonth.entrySet()) {
Map<String, Object> row = entry.getValue();
Object monthStatus = row.get("monthStatus");
if (Boolean.TRUE.equals(row.get("noData"))
|| "future".equals(monthStatus) || "unavailable".equals(monthStatus)) continue;
if (!coverage.byMonth().containsKey(entry.getKey()) || map(row.get("coverage")).isEmpty()) {
missing.add("月度趋势来源与父sourceMonths覆盖不一致:" + monthLabel(period.year(), entry.getKey()) + "。");
return;
}
if (!stages.isEmpty()) {
Map<String, Object> rowCoverage = map(row.get("coverage"));
Map<String, Object> source = coverage.byMonth().get(entry.getKey());
for (String stage : stages) {
Long rowCount = count(rowCoverage.get(stage + "Rows"));
Long parentCount = count(source.get(stage + "Rows"));
if (rowCount != null && rowCount > 0 && parentCount != null && parentCount == 0) {
missing.add(stageLabel(stage) + "月度行声称有来源,但父sourceMonths明确为零:"
+ monthLabel(period.year(), entry.getKey()) + "。");
return;
}
}
}
}
for (String field : fields) {
List<String> requestedStages = stages.isEmpty() ? List.of("") : stages;
for (String stage : requestedStages) {
List<Integer> months = selectedMonths(stage, period, coverage, complete, missing);
if (months.isEmpty()) {
String label = stage.isEmpty() ? "综合合格率" : stageLabel(stage) + fieldLabel(field);
missing.add(label + "没有可核验的已发布来源月份。");
continue;
}
int validCount = 0;
for (int month : months) {
Map<String, Object> row = trendByMonth.get(month);
if (row != null && validTrendMetric(row, stage, field, month, coverage))
validCount++;
}
int required = complete ? period.months().size() : Math.max(2, months.size());
if (validCount < required || validCount != months.size()) {
String label = stage.isEmpty() ? "综合合格率" : stageLabel(stage) + fieldLabel(field);
missing.add(label + "有效月度值不足(" + validCount + "/" + required + ")。");
}
}
}
}
private static boolean validTrendMetric(Map<String, Object> row, String stage, String field, int month,
MonthCoverage parentCoverage) {
Object monthStatus = row.get("monthStatus");
if (Boolean.TRUE.equals(row.get("noData"))
|| "future".equals(monthStatus) || "unavailable".equals(monthStatus)) return false;
Map<String, Object> rowCoverage = map(row.get("coverage"));
Map<String, Object> source = parentCoverage.byMonth().get(month);
if (rowCoverage.isEmpty() || source == null) return false;
Map<String, Object> effectiveSource = effectiveSourceRow(source, rowCoverage);
if (stage.isEmpty()) {
if (!validCompositeCoverage(effectiveSource)) return false;
Map<String, Object> summary = new LinkedHashMap<>();
summary.put("compositeRate", row.get("compositeRate"));
summary.put("black", row.get("black"));
summary.put("bright", row.get("bright"));
// A single monthly reference is checked with the same two-direction composite rule.
if (!validCompositeRate(summary, List.of(month), new MonthCoverage(Map.of(month, effectiveSource))))
return false;
return true;
}
Long rowCount = count(rowCoverage.get(stage + "Rows"));
Long parentCount = count(source.get(stage + "Rows"));
if (rowCount == null || parentCount == null || rowCount <= 0 || parentCount <= 0) return false;
return validStageMetric(row, stage, field, List.of(month),
new MonthCoverage(Map.of(month, effectiveSource)));
}
private static boolean validCompositeCoverage(Map<String, Object> effective) {
Long black = count(effective.get("blackRows"));
Long bright = count(effective.get("brightRows"));
return black != null && bright != null && black >= 0 && bright >= 0 && (black > 0 || bright > 0);
}
private static Map<String, Object> effectiveSourceRow(Map<String, Object> parent, Map<String, Object> row) {
Map<String, Object> effective = new LinkedHashMap<>();
for (String stage : STAGES) {
String field = stage + "Rows";
Long rowCount = count(row.get(field));
Long parentCount = count(parent.get(field));
Object value;
if (rowCount == null || rowCount < 0)
value = row.get(field);
else if (parentCount != null && parentCount >= 0)
value = Math.min(rowCount, parentCount);
else
value = parent.get(field);
effective.put(field, value);
}
return effective;
}
private static boolean validStageSource(String stage, List<Integer> months, MonthCoverage coverage) {
for (int month : months) {
Map<String, Object> item = coverage.byMonth().get(month);
Long rows = item == null ? null : count(item.get(stage + "Rows"));
if (rows == null || rows <= 0) return false;
}
return !months.isEmpty();
}
private static List<Integer> selectedMonths(String stage, Period period, MonthCoverage coverage,
boolean complete, List<String> missing) {
if (complete) return period.months();
List<Integer> selected = new ArrayList<>();
for (Map.Entry<Integer, Map<String, Object>> entry : coverage.byMonth().entrySet()) {
Map<String, Object> item = entry.getValue();
Long black = count(item.get("blackRows"));
Long bright = count(item.get("brightRows"));
if (stage.isEmpty()) {
if (black == null || bright == null || black < 0 || bright < 0) {
missing.add("综合合格率来源月份计数未知或不合法:"
+ monthLabel(period.year(), entry.getKey()) + "。");
continue;
}
if (black > 0 || bright > 0) selected.add(entry.getKey());
} else {
Long own = count(item.get(stage + "Rows"));
if (own == null || own < 0) {
missing.add(stageLabel(stage) + "来源月份计数未知或不合法:"
+ monthLabel(period.year(), entry.getKey()) + "。");
continue;
}
if (own > 0) selected.add(entry.getKey());
}
}
return List.copyOf(selected);
}
private static MonthCoverage monthCoverage(Object raw, Period period, List<String> missing) {
if (!(raw instanceof List<?> items)) {
missing.add("指标接口未返回按月来源覆盖。");
return new MonthCoverage(Map.of());
}
Map<Integer, Map<String, Object>> byMonth = new LinkedHashMap<>();
for (Object value : items) {
Map<String, Object> item = map(value);
Long year = count(item.get("year"));
Long month = count(item.get("month"));
if (year == null || month == null || year != period.year() || month < 1 || month > 12) continue;
int monthNumber = month.intValue();
if (!period.months().contains(monthNumber)) continue;
if (byMonth.putIfAbsent(monthNumber, item) != null) {
missing.add("来源覆盖存在重复月份:" + monthLabel(period.year(), monthNumber) + "。");
}
}
return new MonthCoverage(byMonth);
}
private static Period period(Map<String, Object> raw) {
if (raw == null || raw.isEmpty() || !Set.of("year", "month", "quarter", "monthStart", "monthEnd").containsAll(raw.keySet()))
return null;
Long yearValue = integer(raw.get("year"));
if (yearValue == null || yearValue < 2000 || yearValue > 2100) return null;
int year = yearValue.intValue();
boolean hasMonth = raw.containsKey("month"), hasQuarter = raw.containsKey("quarter");
boolean hasStart = raw.containsKey("monthStart"), hasEnd = raw.containsKey("monthEnd");
if (hasMonth && (hasQuarter || hasStart || hasEnd) || hasQuarter && (hasStart || hasEnd) || hasStart != hasEnd)
return null;
int start = 1, end = 12;
if (hasMonth) {
Long month = integer(raw.get("month"));
if (month == null || month < 1 || month > 12) return null;
start = end = month.intValue();
} else if (hasQuarter) {
Integer quarter = quarter(raw.get("quarter"));
if (quarter == null || quarter < 1 || quarter > 4) return null;
start = (quarter - 1) * 3 + 1;
end = start + 2;
} else if (hasStart) {
Long first = integer(raw.get("monthStart")), last = integer(raw.get("monthEnd"));
if (first == null || last == null || first < 1 || last > 12 || first > last) return null;
start = first.intValue();
end = last.intValue();
}
List<Integer> months = new ArrayList<>();
for (int month = start; month <= end; month++) months.add(month);
return new Period(year, List.copyOf(months));
}
private static Integer trendMonth(Map<String, Object> row, int year) {
Object raw = row.get("period");
if (!(raw instanceof String)) raw = row.get("month");
if (!(raw instanceof String text)) return null;
Matcher match = MONTH_TEXT.matcher(text);
if (!match.matches()) return null;
try {
int actualYear = Integer.parseInt(match.group(1));
int month = Integer.parseInt(match.group(2));
return actualYear == year ? month : null;
} catch (NumberFormatException invalid) {
return null;
}
}
private static Map<String, Object> map(Object value) {
if (!(value instanceof Map<?, ?> raw)) return Map.of();
Map<String, Object> result = new LinkedHashMap<>();
raw.forEach((key, item) -> { if (key instanceof String text) result.put(text, item); });
return result;
}
private static List<String> strings(Object raw) {
if (!(raw instanceof List<?> values)) return List.of();
List<String> result = new ArrayList<>();
for (Object value : values) {
if (!(value instanceof String text) || text.isBlank()) return List.of();
result.add(text);
}
return result;
}
private static Long count(Object raw) {
if (raw instanceof Byte || raw instanceof Short || raw instanceof Integer || raw instanceof Long)
return ((Number) raw).longValue();
if (raw instanceof BigInteger integer
&& integer.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) >= 0
&& integer.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) return integer.longValue();
return null;
}
private static Long integer(Object raw) { return count(raw); }
private static boolean strictIntegerEquals(Object raw, int expected) {
Long value = count(raw);
return value != null && value == expected;
}
private static BigDecimal decimal(Object raw) {
if (!(raw instanceof Number number)) return null;
try {
return raw instanceof BigDecimal decimal ? decimal : new BigDecimal(number.toString());
} catch (NumberFormatException invalid) {
return null;
}
}
private static boolean inRange(BigDecimal value, BigDecimal minimum, BigDecimal maximum) {
return value != null && value.compareTo(minimum) >= 0 && value.compareTo(maximum) <= 0;
}
private static boolean positive(BigDecimal value) { return value != null && value.signum() > 0; }
private static boolean nonNegative(BigDecimal value) { return value != null && value.signum() >= 0; }
private static Integer quarter(Object raw) {
Long number = integer(raw);
if (number != null && number >= Integer.MIN_VALUE && number <= Integer.MAX_VALUE) return number.intValue();
if (raw instanceof String text) {
String normalized = text.trim().toUpperCase(java.util.Locale.ROOT);
if (normalized.startsWith("Q")) normalized = normalized.substring(1);
try { return Integer.valueOf(normalized); }
catch (NumberFormatException ignored) { return null; }
}
return null;
}
private static String monthLabel(int year, int month) { return String.format("%04d-%02d", year, month); }
private static String stageLabel(String stage) { return "black".equals(stage) ? "黑片" : "亮片"; }
private static String fieldLabel(String field) {
return switch (field) {
case "rate" -> "合格率";
case "scrapRate" -> "报废率";
case "totalWeight" -> "投入重量";
case "scrapWeight" -> "报废重量";
default -> field;
};
}
private record Period(int year, List<Integer> months) {}
private record MonthCoverage(Map<Integer, Map<String, Object>> byMonth) {
private MonthCoverage {
byMonth = Collections.unmodifiableMap(new LinkedHashMap<>(byMonth));
}
private List<Integer> availableMonths() { return List.copyOf(byMonth.keySet()); }
private List<String> availableLabels() {
return byMonth.keySet().stream().sorted().map(month -> monthLabel(byYear(), month)).toList();
}
private int byYear() {
if (byMonth.isEmpty()) return 0;
return integer(byMonth.values().iterator().next().get("year")) == null ? 0
: integer(byMonth.values().iterator().next().get("year")).intValue();
}
}
}
@@ -0,0 +1,45 @@
package cn.yunsheng.quality.analysis;
import java.util.Map;
import java.util.Set;
/**
* Narrow exact-match guard for a single, backend-validated metrics request. This is not wired to
* the legacy bridge. The caller must construct it from the current confirmed revision, not from
* model output or historical session data, and must separately verify execution and permissions.
* Exact matching intentionally rejects equivalent encodings and unsupported subqueries rather
* than guessing whether their scopes are equivalent.
*/
final class AnalysisProtocolMetricScope {
private static final Set<String> SUPPORTED = Set.of("releaseId", "year", "month", "monthStart", "monthEnd", "quarter",
"department", "group", "product", "projectGroup", "extraFilters", "columnFilters", "exclusions",
"view", "includeUnassigned", "trendGranularity");
private static final Set<String> EXPLICIT = Set.of("releaseId", "year", "view", "trendGranularity", "includeUnassigned");
private final Map<String, String> parameters;
AnalysisProtocolMetricScope(Map<String, String> validatedParameters) {
if (validatedParameters == null || !SUPPORTED.containsAll(validatedParameters.keySet())
|| !validatedParameters.keySet().containsAll(EXPLICIT)
|| validatedParameters.values().stream().anyMatch(value -> value == null || value.isBlank()))
throw new IllegalArgumentException("complete supported metric scope required");
boolean hasStart=validatedParameters.containsKey("monthStart"),hasEnd=validatedParameters.containsKey("monthEnd");
if(hasStart!=hasEnd)throw new IllegalArgumentException("monthStart and monthEnd must be paired");
if(hasStart) {
try {
int year=Integer.parseInt(validatedParameters.get("year"));
int start=Integer.parseInt(validatedParameters.get("monthStart"));
int end=Integer.parseInt(validatedParameters.get("monthEnd"));
if(year<2000||year>2100||start<1||start>12||end<1||end>12||start>end
||validatedParameters.containsKey("month")||validatedParameters.containsKey("quarter"))
throw new IllegalArgumentException("invalid month range metric scope");
} catch (NumberFormatException invalid) {
throw new IllegalArgumentException("invalid month range metric scope",invalid);
}
}
parameters = Map.copyOf(validatedParameters);
}
void requireExact(Map<String, String> actualParameters) {
if (!parameters.equals(actualParameters)) throw new SecurityException("metric scope mismatch");
}
}
@@ -0,0 +1,329 @@
package cn.yunsheng.quality.analysis;
import java.util.*;
import java.util.regex.Pattern;
import tools.jackson.databind.json.JsonMapper;
/** Compiles business selectors into finite, exact child frames. No model-supplied child filters. */
final class AnalysisProtocolQueryPlan {
private static final Set<String> FIELDS=Set.of("rate","scrapRate","totalWeight","scrapWeight");
private static final Set<String> WEIGHTS=Set.of("totalWeight","scrapWeight");
private static final Set<String> DIMENSIONS=Set.of("product","group","department");
private static final Pattern NODE_ID=Pattern.compile("[a-z][a-z0-9_-]{0,31}");
private final JsonMapper json;
AnalysisProtocolQueryPlan(JsonMapper json){this.json=json;}
record RankRecord(String queryId,String responseHash,String nodeId,Map<String,Object> artifact){}
record Resolution(List<Map<String,Object>> scopes,List<Map<String,Object>> pending,List<String> gaps,List<Map<String,Object>> skipped) {
Map<String,Object> asMap(){return Map.of("version",1,"scopes",scopes,"pending",pending,"gaps",gaps,"skipped",skipped);}
}
List<Map<String,Object>> normalize(Map<String,Object> task,Map<String,Object> catalog,List<Map<String,Object>> availableRankings) {
Object raw=task.get("queryPlan");
task.putIfAbsent("periodCoverage","available");
if(!Set.of("available","complete").contains(task.get("periodCoverage")))throw invalid("期间覆盖方式无效");
if(!(raw instanceof List<?> nodes)||nodes.isEmpty()||nodes.size()>8)throw invalid("查询计划需要1至8个明确节点");
List<Map<String,Object>> out=new ArrayList<>();Map<String,Map<String,Object>> prior=new LinkedHashMap<>();Map<String,Integer> bounds=new HashMap<>();
int scopeBound=0,queryBound=0;boolean trend=false,comparison=false;
Set<Object> requestedFields=new HashSet<>(),requestedDimensions=new HashSet<>(),requestedStages=new HashSet<>();
for(Object item:nodes) {
Map<String,Object> node=map(item);
if(!(item instanceof Map<?,?>)||!Set.of("nodeId","purpose","dimension","fields","stages","limit","selector","visualType","visualPreference","comparisonRequired","sourceFields","defect","category","sort","optional").containsAll(node.keySet()))throw invalid("查询节点含未支持字段");
String id=string(node.get("nodeId")),purpose=string(node.get("purpose"));
if(!NODE_ID.matcher(id).matches()||prior.containsKey(id))throw invalid("查询节点编号无效或重复");
if(!Set.of("ranking","metrics","trend","source","records","defects","changes").contains(purpose))throw invalid("查询节点用途尚未接入");
boolean rawPurpose=Set.of("source","records").contains(purpose),defectPurpose="defects".equals(purpose),changesPurpose="changes".equals(purpose);
List<String> fields=strings(node.get("fields"),rawPurpose?0:1,rawPurpose?0:4),
stages=strings(node.get("stages"),rawPurpose||defectPurpose?1:0,2);
if(!FIELDS.containsAll(fields)||!Set.of("black","bright").containsAll(stages))throw invalid("节点指标或阶段无效");
if(!defectPurpose&&(node.containsKey("defect")||node.containsKey("category")))throw invalid("细项和类别只能用于缺陷节点");
if(!rawPurpose&&node.containsKey("sourceFields"))throw invalid("sourceFields 只能用于 source/records 节点");
Map<String,List<String>> selectedSourceFields=rawPurpose&&node.containsKey("sourceFields")
?normalizeSourceFields(node.get("sourceFields"),stages,catalog):Map.of();
if(stages.isEmpty()&&!fields.equals(List.of("rate"))&&!changesPurpose)throw invalid("重量和报废率节点必须明确阶段,不能自行合并黑片亮片");
boolean compare=bool(node.getOrDefault("comparisonRequired",false));
if(compare&&(!Set.of("metrics","source","records","defects","changes").contains(purpose)||!(task.get("comparisonPeriod") instanceof Map<?,?>)))throw invalid("期间比较节点需要两个已确认期间");
if("trend".equals(purpose)&&map(task.get("period")).containsKey("month"))throw invalid("月度趋势需要多个月份范围");
String preference=string(node.getOrDefault("visualPreference",task.getOrDefault("visualPreference","auto")));
if(!Set.of("auto","required","none").contains(preference))throw invalid("节点图表偏好无效");
Object visual=node.get("visualType");
if(visual==null) {
Object parent=task.get("visualType");
visual=("ranking".equals(purpose)||defectPurpose)?parent!=null&&Set.of("bar","column","pareto").contains(parent)?parent:"bar":
"trend".equals(purpose)?parent!=null&&Set.of("line","area","heatmap").contains(parent)?parent:"line":null;
}
if(rawPurpose&&(!"none".equals(preference)||visual!=null))throw invalid("原件调查节点不能请求图表");
if(visual!=null&&!(("ranking".equals(purpose)||defectPurpose||changesPurpose)?Set.of("bar","column","pareto"):"trend".equals(purpose)?Set.of("line","area","heatmap"):Set.of("bar","column")).contains(visual))throw invalid("节点图形不符合查询用途");
Map<String,Object> selector=map(node.getOrDefault("selector",Map.of("kind","base")));
String kind=string(selector.get("kind"));int count=1;
Map<String,Object> cleanSelector=new LinkedHashMap<>();cleanSelector.put("kind",kind);
switch(kind) {
case "base" -> {if(!selector.keySet().equals(Set.of("kind")))throw invalid("基础范围选择器不能携带额外条件");}
case "catalog_members" -> {
if(!selector.keySet().equals(Set.of("kind","dimension","names")))throw invalid("目录对象选择器格式无效");
String dimension=string(selector.get("dimension"));if(!DIMENSIONS.contains(dimension))throw invalid("对象维度尚未接入");
List<String> names=strings(selector.get("names"),1,10);
for(String name:names)entityFilters(map(task.get("filters")),dimension,name,catalog);
cleanSelector.put("dimension",dimension);cleanSelector.put("names",names);count=names.size();requestedDimensions.add(dimension);
}
case "ranking_top" -> {
if(!selector.keySet().equals(Set.of("kind","sourceNodeId","stage","metric","count")))throw invalid("排名依赖选择器格式无效");
String source=string(selector.get("sourceNodeId"));Map<String,Object> ranking=prior.get(source);
String stage=string(selector.get("stage")),metric=string(selector.get("metric"));
count=integer(selector.get("count"),1,10);
if(ranking==null||!"ranking".equals(ranking.get("purpose"))||!"product".equals(ranking.get("dimension"))
||!"base".equals(map(ranking.get("selector")).get("kind"))||!list(ranking.get("stages")).contains(stage)
||!list(ranking.get("fields")).contains(metric)||count>((Number)ranking.get("limit")).intValue())throw invalid("规格下钻必须依赖前面已声明的实际产品排名");
requestedDimensions.add("product");cleanSelector.put("sourceNodeId",source);cleanSelector.put("stage",stage);cleanSelector.put("metric",metric);cleanSelector.put("count",count);
}
case "changes_top" -> {
if(!selector.keySet().equals(Set.of("kind","sourceNodeId","stage","count")))throw invalid("变化依赖选择器格式无效");
String source=string(selector.get("sourceNodeId")),stage=string(selector.get("stage"));Map<String,Object> previous=prior.get(source);
int selected=integer(selector.get("count"),1,3);
if(previous==null||!"changes".equals(previous.get("purpose"))||selected>((Number)previous.get("limit")).intValue()
||!Set.of("overall","black","bright","dominant").contains(stage))throw invalid("变化下钻必须依赖前面的完整变化节点");
count=selected*bounds.getOrDefault(source,1);requestedDimensions.add(previous.get("dimension"));
cleanSelector.put("sourceNodeId",source);cleanSelector.put("stage",stage);cleanSelector.put("count",selected);
}
case "previous_ranking" -> {
if(!selector.keySet().equals(Set.of("kind","queryId","stage","metric","ranks")))throw invalid("历史排名选择器格式无效");
String query=string(selector.get("queryId")),stage=string(selector.get("stage")),metric=string(selector.get("metric"));
List<Integer> ranks=integers(selector.get("ranks"),1,10);count=ranks.size();
Map<String,Object> available=availableRankings.stream().filter(row->query.equals(row.get("queryId"))&&stage.equals(row.get("stage"))&&metric.equals(row.get("metric"))&&"product".equals(row.get("dimension"))).findFirst().orElseThrow(()->invalid("旧排名缺少当前授权的真实快照,请重新查询排名"));
for(int rank:ranks)if(list(available.get("rows")).stream().noneMatch(row->Objects.equals(map(row).get("rank"),rank)))throw invalid("选择序号不在上轮实际展示排名中");
requestedDimensions.add("product");cleanSelector.put("queryId",query);cleanSelector.put("stage",stage);cleanSelector.put("metric",metric);cleanSelector.put("ranks",ranks);
}
default -> throw invalid("查询对象来源尚未接入");
}
Map<String,Object> clean=new LinkedHashMap<>();clean.put("nodeId",id);clean.put("purpose",purpose);clean.put("fields",fields);clean.put("stages",stages);
if(node.containsKey("optional")) {
boolean optional=bool(node.get("optional"));if(optional&&!kind.equals("changes_top"))throw invalid("只有依赖真实变化结果的下钻节点可按条件跳过");
clean.put("optional",optional);
}
if(node.containsKey("sort")&&!changesPurpose)throw invalid("sort只能用于变化节点");
if(rawPurpose&&node.containsKey("sourceFields"))clean.put("sourceFields",selectedSourceFields);
clean.put("selector",cleanSelector);clean.put("comparisonRequired",compare);clean.put("visualPreference",preference);clean.put("visualType",visual);
if("ranking".equals(purpose)) {
String dimension=string(node.get("dimension"));
if(!DIMENSIONS.contains(dimension)||!WEIGHTS.containsAll(fields)||stages.isEmpty()||compare||!kind.equals("base"))throw invalid("排名节点需要明确阶段、重量和基础分组");
requestedDimensions.add(dimension);clean.put("dimension",dimension);clean.put("limit",integer(node.getOrDefault("limit",10),1,10));
} else if(changesPurpose) {
String dimension=string(node.get("dimension")),sort=string(node.get("sort"));
if(!DIMENSIONS.contains(dimension)||!AnalysisProtocolChanges.SORTS.contains(sort)||stages.size()>1||!compare
||!Set.of("rate","totalWeight","scrapWeight").containsAll(fields)
||Set.of("rate_drop","current_rate").contains(sort)&&!fields.contains("rate")
||Set.of("scrap_delta","current_scrap").contains(sort)&&!fields.contains("scrapWeight")
||!Set.of("base","changes_top").contains(kind))throw invalid("变化节点需要两个期间、正式率、综合或单阶段及明确排序");
requestedDimensions.add(dimension);clean.put("dimension",dimension);clean.put("sort",sort);clean.put("limit",integer(node.getOrDefault("limit",10),1,10));
} else if(defectPurpose) {
if(!fields.equals(List.of("scrapWeight")))throw invalid("缺陷节点只接受来源细项重量");
Map<String,Object> query=AnalysisProtocolDefects.normalizeNode(node,stages,catalog);
clean.putAll(query);requestedDimensions.add("defect");
if("product".equals(query.get("dimension")))requestedDimensions.add("product");
} else if(node.containsKey("dimension")||node.containsKey("limit"))throw invalid("指标或趋势节点不能伪装为分组排名");
int periodCount=compare?2:1;
int readsPerScope=changesPurpose?1:"ranking".equals(purpose)?fields.size()*stages.size():rawPurpose||defectPurpose?stages.size()*periodCount:periodCount;
scopeBound+=count;queryBound+=count*readsPerScope;
if(scopeBound>20||queryBound>40)throw invalid("本轮查询对象或正式读取超过有界上限,请拆分任务");
trend|="trend".equals(purpose);comparison|=compare;
if(!rawPurpose)requestedFields.addAll(fields);
requestedStages.addAll(stages);out.add(clean);prior.put(id,clean);bounds.put(id,count);
}
if(!Objects.equals(task.get("trendRequired"),trend)||bool(task.getOrDefault("comparisonRequired",false))!=comparison||task.get("comparisonPeriod") instanceof Map<?,?>&&!comparison)throw invalid("查询计划没有覆盖本轮要求的趋势或期间比较");
if(!new HashSet<>(list(task.get("stages"))).equals(requestedStages))throw invalid("根任务与节点阶段要求不一致");
if(!new HashSet<>(list(task.get("dimensions"))).equals(requestedDimensions))throw invalid("根任务与节点对象维度要求不一致");
if(!new HashSet<>(list(task.get("fields"))).equals(requestedFields))throw invalid("根任务与节点指标要求不一致");
if(bool(task.getOrDefault("verifySource",false))!=out.stream().anyMatch(node->"source".equals(node.get("purpose")))
||bool(task.getOrDefault("fullDetails",false))!=out.stream().anyMatch(node->"records".equals(node.get("purpose"))))
throw invalid("根任务的源表核验或逐行记录要求没有被节点完整覆盖");
task.putIfAbsent("verifySource",false);task.putIfAbsent("fullDetails",false);
return out;
}
Resolution resolve(Map<String,Object> task,String execution,String release,Map<String,Object> catalog,
List<RankRecord> current,List<Map<String,Object>> history) {
List<Map<String,Object>> scopes=new ArrayList<>(),pending=new ArrayList<>(),skipped=new ArrayList<>();List<String> gaps=new ArrayList<>();
for(Object raw:list(task.get("queryPlan"))) {
Map<String,Object> node=map(raw),selector=map(node.get("selector"));String kind=string(selector.get("kind"));
List<Map<String,Object>> choices=new ArrayList<>();
if(kind.equals("base"))choices.add(Map.of("label","基础范围"));
else if(kind.equals("catalog_members")) {
for(Object name:list(selector.get("names")))choices.add(Map.of("dimension",selector.get("dimension"),"key",name,"label",name));
} else if(kind.equals("changes_top")) {
boolean found=false;Set<String> seen=new HashSet<>();
for(RankRecord record:current)if(Objects.equals(record.nodeId(),selector.get("sourceNodeId"))
&&"quality_changes_artifact".equals(record.artifact().get("kind"))&&Boolean.TRUE.equals(record.artifact().get("completeRanking"))) {
Map<String,Object> artifact=record.artifact();String sourceIdentity=write(artifact.get("filters"));
if(!seen.add(sourceIdentity))continue;found=true;
if(!Objects.equals(release,artifact.get("releaseId"))||!Objects.equals(AnalysisProtocolService.canonical(task.get("period")),AnalysisProtocolService.canonical(artifact.get("period")))
||!Objects.equals(AnalysisProtocolService.canonical(task.get("comparisonPeriod")),AnalysisProtocolService.canonical(artifact.get("comparisonPeriod")))) {gaps.add("变化下钻来源期间或发布版本不匹配");continue;}
boolean sourceMatches=scopes.stream().filter(scope->Objects.equals(scope.get("nodeId"),selector.get("sourceNodeId"))).anyMatch(scope->{
var child=map(scope.get("task"));var query=map(child.get("changeQuery"));var stages=list(child.get("stages"));
return Objects.equals(query.get("dimension"),artifact.get("dimension"))&&Objects.equals(query.get("sort"),artifact.get("sort"))
&&Objects.equals(AnalysisProtocolService.canonical(child.get("fields")),AnalysisProtocolService.canonical(artifact.get("fields")))
&&AnalysisProtocolChanges.sameFilters(map(child.get("filters")),map(artifact.get("filters")))
&&Objects.equals(stages.isEmpty()?null:stages.getFirst(),artifact.get("stage"));
});
if(!sourceMatches){gaps.add("变化下钻来源不属于前置节点的精确范围");continue;}
if("dominant".equals(selector.get("stage"))&&AnalysisProtocolChanges.noAdverseStage(artifact)) {
if(Boolean.TRUE.equals(node.get("optional")))skipped.add(Map.of("nodeId",node.get("nodeId"),"reason","no_adverse_stage"));
else gaps.add("正式阶段基线未出现不利变化,不能制造阶段归因");continue;
}
var rows=list(artifact.get("rows"));int count=((Number)selector.get("count")).intValue();int accepted=0,notComparable=0;
for(Object rawRow:rows.subList(0,Math.min(count,rows.size()))) {
Map<String,Object> row=map(rawRow);if(!AnalysisProtocolChanges.adverse(row,string(artifact.get("sort"))))continue;
Map<String,Object> present=map(row.get("present"));
if(Boolean.TRUE.equals(node.get("comparisonRequired"))&&(!Boolean.TRUE.equals(present.get("current"))||!Boolean.TRUE.equals(present.get("comparison")))){notComparable++;continue;}
Map<String,Object> filters;
try {
filters=entityFilters(map(artifact.get("filters")),string(artifact.get("dimension")),string(row.get("key")),catalog);
if(!AnalysisProtocolChanges.sameFilters(filters,map(row.get("filters"))))throw invalid("变化行对象未匹配精确归属范围");
// sourceMatches already binds the complete ancestor scope. Only this selected dimension
// (and a group's owner) may narrow further; every other inherited condition is unchanged.
if(!AnalysisProtocolChanges.narrowedFilters(map(artifact.get("filters")),filters,string(artifact.get("dimension"))))throw invalid("变化下钻不能扩大父范围或改变其他条件");
} catch(IllegalArgumentException invalid){gaps.add(invalid.getMessage());continue;}
List<String> requested=strings(node.get("stages"),0,2),selected=AnalysisProtocolChanges.selectedStages(artifact,string(selector.get("stage")),requested);
if(string(selector.get("stage")).equals("overall"))selected=requested;
if("changes".equals(node.get("purpose"))&&(selected.size()>1||requested.isEmpty()))selected=requested;
if(!requested.containsAll(selected)||!requested.isEmpty()&&selected.isEmpty()){gaps.add("变化下钻阶段超出节点授权");continue;}
Map<String,Object> source=new LinkedHashMap<>();source.put("queryId",record.queryId());source.put("responseHash",record.responseHash());source.put("nodeId",record.nodeId());source.put("rank",row.get("rank"));source.put("key",row.get("key"));source.put("sort",artifact.get("sort"));source.put("stages",selected);source.put("filters",filters);
Map<String,Object> choice=new LinkedHashMap<>();choice.put("label",row.getOrDefault("label",row.get("key")));choice.put("resolvedFilters",filters);choice.put("resolvedStages",selected);choice.put("source",source);choices.add(choice);accepted++;
}
if(accepted==0) {
if(Boolean.TRUE.equals(node.get("optional")))skipped.add(Map.of("nodeId",node.get("nodeId"),"reason",notComparable>0?"selected_objects_not_comparable":"no_adverse_ranked_objects"));
else gaps.add(notComparable>0?"变化下钻所选对象在至少一个比较期间不存在,不能完成明确的两期查询":"变化下钻所选名次没有可核实的不利变化对象");
}
}
if(!found) {
boolean parentSkipped=skipped.stream().anyMatch(item->Objects.equals(item.get("nodeId"),selector.get("sourceNodeId")))
&&scopes.stream().noneMatch(item->Objects.equals(item.get("nodeId"),selector.get("sourceNodeId")));
if(parentSkipped&&Boolean.TRUE.equals(node.get("optional")))skipped.add(Map.of("nodeId",node.get("nodeId"),"reason","source_node_skipped"));
else pending.add(Map.of("nodeId",node.get("nodeId"),"reason","changes_evidence_pending"));
}
} else {
String stage=string(selector.get("stage")),metric=string(selector.get("metric"));Map<String,Object> profile=null;Map<String,Object> source=new LinkedHashMap<>();
if(kind.equals("ranking_top")) {
for(RankRecord record:current)if(Objects.equals(record.nodeId(),selector.get("sourceNodeId"))&&"product".equals(record.artifact().get("dimension"))) {
profile=profile(record.artifact(),stage,metric);
if(profile!=null){source.put("queryId",record.queryId());source.put("responseHash",record.responseHash());source.put("period",record.artifact().get("period"));source.put("filters",record.artifact().get("filters"));break;}
}
if(profile==null){pending.add(Map.of("nodeId",node.get("nodeId"),"reason","ranking_evidence_pending"));continue;}
int count=((Number)selector.get("count")).intValue();
var rows=list(profile.get("rows"));if(rows.size()<count)gaps.add("排名实际不足"+count+"个正重量规格,未缩小请求数量");
for(Object row:rows.subList(0,Math.min(count,rows.size())))choices.add(choice(map(row),stage,metric,source));
} else {
for(Map<String,Object> available:history)if(Objects.equals(available.get("queryId"),selector.get("queryId"))&&stage.equals(available.get("stage"))&&metric.equals(available.get("metric"))){profile=available;source.putAll(available);source.remove("rows");break;}
if(profile==null){gaps.add("上轮排名快照已失效或授权变化,请重新查询排名");continue;}
for(Object rank:list(selector.get("ranks"))) {
var selected=list(profile.get("rows")).stream().filter(row->Objects.equals(map(row).get("rank"),rank)).findFirst();
if(selected.isEmpty())gaps.add("上轮排名缺少请求的序号"+rank);else choices.add(choice(map(selected.get()),stage,metric,source));
}
}
}
for(Map<String,Object> choice:choices) {
Map<String,Object> filters=new LinkedHashMap<>(map(choice.getOrDefault("resolvedFilters",task.get("filters"))));
try {if(choice.containsKey("dimension"))filters=entityFilters(filters,string(choice.get("dimension")),string(choice.get("key")),catalog);}
catch(IllegalArgumentException invalid){gaps.add(invalid.getMessage());continue;}
Map<String,Object> resolvedNode=new LinkedHashMap<>(node);if(choice.containsKey("resolvedStages"))resolvedNode.put("stages",choice.get("resolvedStages"));
Map<String,Object> child=childTask(task,resolvedNode,filters,catalog);
Map<String,Object> identity=new LinkedHashMap<>();identity.put("executionId",execution);identity.put("nodeId",node.get("nodeId"));identity.put("filters",filters);identity.put("period",child.get("period"));identity.put("comparisonPeriod",child.get("comparisonPeriod"));if(kind.equals("changes_top")){identity.put("stages",child.get("stages"));identity.put("source",choice.get("source"));}if(child.containsKey("changeQuery"))identity.put("changeQuery",child.get("changeQuery"));if(node.containsKey("sourceFields"))identity.put("sourceFields",node.get("sourceFields"));if(child.containsKey("defectQuery"))identity.put("defectQuery",child.get("defectQuery"));
Map<String,Object> scope=new LinkedHashMap<>();scope.put("nodeId",node.get("nodeId"));scope.put("scopeId",AnalysisProtocolService.hash(write(identity)));scope.put("purpose",node.get("purpose"));
scope.put("parentScopeHash",AnalysisProtocolService.hash(write(Map.of("releaseId",release,"filters",identityFilters(map(task.get("filters")))))));
scope.put("label",choice.get("label"));scope.put("task",child);if(node.containsKey("limit"))scope.put("limit",node.get("limit"));
if(choice.containsKey("source"))scope.put("source",choice.get("source"));scopes.add(scope);
}
}
return new Resolution(List.copyOf(scopes),List.copyOf(pending),gaps.stream().distinct().toList(),List.copyOf(skipped));
}
private Map<String,Object> choice(Map<String,Object> row,String stage,String metric,Map<String,Object> source) {
Map<String,Object> audit=new LinkedHashMap<>(source);audit.put("rank",row.get("rank"));audit.put("stage",stage);audit.put("metric",metric);audit.put("key",row.get("key"));
return Map.of("dimension","product","key",row.get("key"),"label",row.getOrDefault("label",row.get("key")),"source",audit);
}
static Map<String,Object> profile(Map<String,Object> artifact,String stage,String metric) {
return list(artifact.get("rankings")).stream().map(AnalysisProtocolQueryPlan::map).filter(r->stage.equals(r.get("stage"))&&metric.equals(r.get("metric"))&&Boolean.TRUE.equals(r.get("completeRanking"))).findFirst().orElse(null);
}
private Map<String,Object> childTask(Map<String,Object> parent,Map<String,Object> node,Map<String,Object> filters,Map<String,Object> catalog) {
Map<String,Object> child=new LinkedHashMap<>(parent);child.remove("queryPlan");child.remove("comparisonMode");
String purpose=string(node.get("purpose"));boolean ranking="ranking".equals(purpose),changes="changes".equals(purpose),defects="defects".equals(purpose),source="source".equals(purpose),records="records".equals(purpose),compare=Boolean.TRUE.equals(node.get("comparisonRequired"));
child.put("kind",source?"source_verify":records?"details":ranking||defects||changes?"analysis":"metrics");child.put("filters",filters);child.put("fields",node.get("fields"));child.put("stages",node.get("stages"));
child.put("dimensions",ranking?List.of(node.get("dimension")):List.of());child.put("dimensionStages",ranking?Map.of(string(node.get("dimension")),node.get("stages")):Map.of());
child.remove("defectQuery");child.remove("changeQuery");
if(changes){child.put("dimensions",List.of(node.get("dimension")));child.put("dimensionStages",Map.of(string(node.get("dimension")),node.get("stages")));child.put("changeQuery",Map.of("dimension",node.get("dimension"),"sort",node.get("sort"),"limit",node.get("limit")));}
if(defects) {
Map<String,Object> query=AnalysisProtocolDefects.normalizeNode(node,strings(node.get("stages"),1,2),catalog);
List<String> dimensions="product".equals(query.get("dimension"))?List.of("product","defect"):List.of("defect");
Map<String,Object> dimensionStages=new LinkedHashMap<>();for(String dimension:dimensions)dimensionStages.put(dimension,node.get("stages"));
child.put("dimensions",dimensions);child.put("dimensionStages",dimensionStages);child.put("defectQuery",query);
}
child.put("trendRequired","trend".equals(purpose));child.put("comparisonRequired",compare);if(!compare)child.remove("comparisonPeriod");
child.put("visualType",node.get("visualType"));child.put("visualPreference",node.get("visualPreference"));child.put("historyMode","none");child.put("freshQuery",true);child.put("verifySource",source);child.put("fullDetails",records);
child.remove("sourceFields");child.remove("investigationFields");
if(node.containsKey("sourceFields")) {
Map<String,List<String>> selected=normalizeSourceFields(node.get("sourceFields"),strings(node.get("stages"),1,2),catalog);
Map<String,Object> investigationFields=new LinkedHashMap<>();
Map<String,Map<String,Object>> available=sourceFieldCatalog(catalog);
for(var entry:selected.entrySet()) {
String stage=entry.getKey();List<String> columns=new ArrayList<>();
for(String key:entry.getValue()) {
Map<String,Object> field=available.get(key);
Object column=map(field.get("columns")).get(stage);
if(column instanceof String name&&!name.isBlank()&&!columns.contains(name))columns.add(name);
}
investigationFields.put(stage,List.copyOf(columns));
}
child.put("sourceFields",selected);
child.put("investigationFields",investigationFields);
}
return child;
}
private static Map<String,List<String>> normalizeSourceFields(Object raw,List<String> stages,Map<String,Object> catalog) {
if(!(raw instanceof Map<?,?> selected)||selected.isEmpty()||selected.size()>stages.size())throw invalid("sourceFields 必须按阶段选择授权源字段");
Map<String,Map<String,Object>> available=sourceFieldCatalog(catalog);Map<String,List<String>> result=new LinkedHashMap<>();
for(var entry:selected.entrySet()) {
if(!(entry.getKey() instanceof String stage)||!stages.contains(stage))throw invalid("sourceFields 阶段不在节点范围内");
List<String> keys=strings(entry.getValue(),1,32);
for(String key:keys) {
Map<String,Object> field=available.get(key);
if(field==null||!list(field.get("stages")).contains(stage)
||!(map(field.get("columns")).get(stage) instanceof String column)||column.isBlank())
throw invalid("sourceFields 包含未授权或不支持所选阶段的字段");
}
result.put(stage,keys);
}
return Collections.unmodifiableMap(result);
}
private static Map<String,Map<String,Object>> sourceFieldCatalog(Map<String,Object> catalog) {
Map<String,Map<String,Object>> result=new LinkedHashMap<>();
for(Object item:list(catalog.get("sourceFields"))) {
Map<String,Object> field=map(item);Object rawKey=field.get("key");
if(rawKey instanceof String key&&!key.isBlank()) {
if(result.putIfAbsent(key,field)!=null)throw invalid("当前授权来源目录存在重复字段键");
}
}
return result;
}
static Map<String,Object> entityFilters(Map<String,Object> base,String dimension,String name,Map<String,Object> catalog) {
String plural=switch(dimension){case "product"->"products";case "group"->"groups";case "department"->"departments";default->throw invalid("子查询对象维度无效");};
if(!list(catalog.get(plural)).contains(name))throw invalid("子查询对象不在当前授权目录");
if(!AnalysisProtocolChanges.selectionAllows(base.get(dimension),name))throw invalid("子查询不能选择父范围以外的对象");
if(list(map(base.get("exclusions")).get(dimension)).contains(name))throw invalid("子查询对象已被父范围排除");
Map<String,Object> filters=new LinkedHashMap<>(base);filters.put(dimension,name);
if(dimension.equals("group")) {
Object owner=map(catalog.get("groupDepartments")).get(name);
if(!(owner instanceof String department)||department.isBlank())throw invalid("业务分组的产业部归属未确认");
if(!AnalysisProtocolChanges.selectionAllows(base.get("department"),department))throw invalid("业务分组不属于父产业部");
if(list(map(base.get("exclusions")).get("department")).contains(department))throw invalid("业务分组产业部已被排除");
filters.put("department",department);
}
return filters;
}
static Map<String,Object> identityFilters(Map<String,Object> filters){Map<String,Object> value=new LinkedHashMap<>(filters);value.putIfAbsent("includeUnassigned",false);return value;}
private String write(Object value){return json.writeValueAsString(AnalysisProtocolService.canonical(value));}
private static String string(Object value){return value instanceof String s?s:"";}
private static boolean bool(Object value){if(!(value instanceof Boolean b))throw invalid("比较要求必须为布尔值");return b;}
private static int integer(Object value,int min,int max){if(!(value instanceof Number n)||n.doubleValue()!=n.intValue()||n.intValue()<min||n.intValue()>max)throw invalid("查询数量或序号超出范围");return n.intValue();}
private static List<String> strings(Object value,int min,int max){if(!(value instanceof List<?> values)||values.size()<min||values.size()>max||values.stream().anyMatch(v->!(v instanceof String s)||s.isBlank())||new HashSet<>(values).size()!=values.size())throw invalid("查询数组格式无效或含重复项");return values.stream().map(String.class::cast).toList();}
private static List<Integer> integers(Object value,int min,int max){if(!(value instanceof List<?> values)||values.isEmpty()||values.size()>10)throw invalid("排名序号数组无效");List<Integer> result=values.stream().map(v->integer(v,min,max)).toList();if(new HashSet<>(result).size()!=result.size())throw invalid("排名序号不能重复");return result;}
@SuppressWarnings("unchecked") private static Map<String,Object> map(Object value){return value instanceof Map<?,?>?(Map<String,Object>)value:Map.of();}
private static List<?> list(Object value){return value instanceof List<?> values?values:List.of();}
private static IllegalArgumentException invalid(String message){return new IllegalArgumentException(message);}
}
@@ -0,0 +1,39 @@
package cn.yunsheng.quality.analysis;
import java.math.BigDecimal;
import java.util.*;
/** Creates a bounded selector catalogue from the actual authorized formal response, never model rows. */
final class AnalysisProtocolRankArtifact {
private AnalysisProtocolRankArtifact(){}
static Map<String,Object> build(Map<String,Object> response,Map<String,Object> task,String dimension,String release) {
List<Map<String,Object>> rankings=new ArrayList<>();
Map<String,Object> perDimension=AnalysisProtocolService.map(task.get("dimensionStages"));
List<?> stages=AnalysisProtocolService.list(perDimension.getOrDefault(dimension,task.get("stages")));
for(Object stage:stages)for(Object field:AnalysisProtocolService.list(task.get("fields"))) {
Map<String,Object> one=new LinkedHashMap<>(task);one.put("stages",List.of(stage));one.put("fields",List.of(field));one.put("dimensionStages",Map.of(dimension,List.of(stage)));
if(!AnalysisProtocolRankingCoverage.complete(response,one,dimension))continue;
List<Map<String,Object>> positive=new ArrayList<>();BigDecimal negative=BigDecimal.ZERO;int negativeCount=0;
for(Object raw:AnalysisProtocolService.list(response.get("rows"))) {
Map<String,Object> row=AnalysisProtocolService.map(raw);Object value=AnalysisProtocolService.map(row.get(stage)).get(field);
if(!(value instanceof Number))continue;
BigDecimal weight=new BigDecimal(value.toString());
if(weight.signum()<0){negative=negative.add(weight);negativeCount++;}
if(weight.signum()>0)positive.add(Map.of("key",row.get("key"),"label",Objects.toString(row.get("label"),row.get("key").toString()),"weight",weight));
}
positive.sort((a,b)->{int weight=((BigDecimal)b.get("weight")).compareTo((BigDecimal)a.get("weight"));return weight!=0?weight:codePointCompare(a.get("key").toString(),b.get("key").toString());});
List<Map<String,Object>> rows=new ArrayList<>();
for(int i=0;i<Math.min(10,positive.size());i++){Map<String,Object> row=new LinkedHashMap<>(positive.get(i));row.put("rank",i+1);rows.add(row);}
Map<String,Object> rank=new LinkedHashMap<>();rank.put("stage",stage);rank.put("metric",field);rank.put("rows",rows);rank.put("totalRows",positive.size());
rank.put("denominator",AnalysisProtocolService.map(AnalysisProtocolService.map(response.get("summary")).get(stage)).get(field));
rank.put("negativeGroups",negativeCount);rank.put("negativeWeight",negative);rank.put("orderPolicy","weight_desc_key_codepoint_asc");rank.put("completeRanking",true);rankings.add(rank);
}
if(rankings.isEmpty())return Map.of();
return Map.of("version",1,"releaseId",release,"dimension",dimension,"period",task.get("period"),"filters",task.get("filters"),"rankings",rankings);
}
private static int codePointCompare(String a,String b) {
int i=0,j=0;
while(i<a.length()&&j<b.length()){int x=a.codePointAt(i),y=b.codePointAt(j);if(x!=y)return Integer.compare(x,y);i+=Character.charCount(x);j+=Character.charCount(y);}
return Boolean.compare(i<a.length(),j<b.length());
}
}
@@ -0,0 +1,63 @@
package cn.yunsheng.quality.analysis;
import java.math.BigDecimal;
import java.util.*;
/** Verifies a full formal grouping against its own authorized summary, without changing KPI formulas. */
final class AnalysisProtocolRankingCoverage {
private AnalysisProtocolRankingCoverage() {}
static boolean complete(Map<String,Object> response,Map<String,Object> task,String dimension) {
List<?> rows=AnalysisProtocolService.list(response.get("rows"));
Object raw=AnalysisProtocolService.map(response.get("context")).getOrDefault("rowCount",rows.size());
if(!(raw instanceof Number n)||n.doubleValue()!=n.longValue()||n.longValue()!=rows.size()||rows.size()>100000)return false;
Map<String,Object> perDimension=AnalysisProtocolService.map(task.get("dimensionStages"));
List<?> stages=AnalysisProtocolService.list(perDimension.getOrDefault(dimension,task.get("stages")));
if(stages.isEmpty())return false;
Set<Object> keys=new HashSet<>();
for(Object value:rows) {
Object key=AnalysisProtocolService.map(value).get("key");
if(!(key instanceof String text)||text.isBlank()||!keys.add(key))return false;
}
for(Object stage:stages)for(Object field:AnalysisProtocolService.list(task.get("fields"))) {
BigDecimal total=number(AnalysisProtocolService.map(AnalysisProtocolService.map(response.get("summary")).get(stage)).get(field));
if(total==null||total.signum()<=0)return false;
BigDecimal sum=BigDecimal.ZERO;
for(Object value:rows) {
Map<String,Object> row=AnalysisProtocolService.map(value);
BigDecimal weight=number(AnalysisProtocolService.map(row.get(stage)).get(field));
if(weight==null) {
Object count=AnalysisProtocolService.map(row.get("coverage")).get(stage+"Rows");
if(count instanceof Number c&&c.doubleValue()==0)continue;
return false;
}
// Black formal total already includes the existing signed utilization correction.
if(weight.signum()<0&&!("black".equals(stage)&&"totalWeight".equals(field)))return false;
sum=sum.add(weight);
}
if(sum.subtract(total).abs().compareTo(new BigDecimal("0.00001").max(total.multiply(new BigDecimal("1e-10"))))>0)return false;
}
return true;
}
static boolean visualsComplete(Map<String,Object> answer,Map<String,Object> task) {
if(!"ready".equals(answer.get("visualState")))return false;
String type=Objects.toString(task.get("visualType"),"bar");
Map<String,Object> own=AnalysisProtocolService.map(task.get("dimensionStages"));
for(Object dimension:AnalysisProtocolService.list(task.get("dimensions")))
for(Object stage:AnalysisProtocolService.list(own.getOrDefault(dimension,task.get("stages"))))
for(Object field:AnalysisProtocolService.list(task.get("fields"))) {
String metric=stage+("totalWeight".equals(field)?"_total_weight":"_scrap_weight");
boolean found=AnalysisProtocolService.list(answer.get("visuals")).stream().map(AnalysisProtocolService::map)
.anyMatch(chart->Objects.equals(chart.get("dimension"),dimension)&&Objects.equals(chart.get("stage"),stage)
&&metric.equals(chart.get("metric"))&&type.equals(chart.get("type"))
&&"all_scope_weight".equals(AnalysisProtocolService.map(chart.get("coverage")).get("shareBasis")));
if(!found)return false;
}
return true;
}
private static BigDecimal number(Object raw) {
if(!(raw instanceof Number))return null;
try {return new BigDecimal(raw.toString());}
catch(NumberFormatException invalid){return null;}
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,87 @@
package cn.yunsheng.quality.analysis;
import java.util.List;
import java.util.Objects;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
/**
* Opt-in binding primitive. No controller or worker calls it yet. The caller must authorize the
* actor and fixed release before calling, and coordinate turn state and events in this transaction.
* Existing unbound executions cannot be adopted. Direct construction does not establish a transaction.
*/
@Service
public class AnalysisProtocolTurnBinding {
public record CurrentExecution(String taskJson, String releaseId, String authScopeHash) { }
private final JdbcTemplate jdbc;
private final AnalysisExecutionProtocolStore store;
public AnalysisProtocolTurnBinding(JdbcTemplate jdbc, AnalysisExecutionProtocolStore store) {
this.jdbc = jdbc;
this.store = store;
}
@Transactional
public AnalysisExecutionProtocolStore.Execution start(String sessionId, String turnId, String userId,
long expectedVersion, long revision, String requestKey, String requestDigest) {
required(sessionId); required(turnId); required(userId); required(requestKey);
// Match the lock order used by the store, so retries cannot race with another start.
List<String> sessions = jdbc.query("SELECT session_id FROM analysis_protocol_session WHERE session_id=? FOR UPDATE",
(rs, n) -> rs.getString(1), sessionId);
if (sessions.size() != 1) throw denied();
List<String> owners=jdbc.query("SELECT user_id FROM analysis_session WHERE id=?",(rs,n)->rs.getString(1),sessionId);
if(owners.size()!=1||!userId.equals(owners.getFirst()))throw denied();
List<String> turns = jdbc.query("SELECT id FROM analysis_turn WHERE id=? AND session_id=? AND user_id=? FOR UPDATE",
(rs, n) -> rs.getString(1), turnId, sessionId, userId);
if (turns.size() != 1) throw denied();
List<String> existing = jdbc.query("SELECT execution_id FROM analysis_protocol_execution WHERE session_id=? AND request_key=? FOR UPDATE",
(rs, n) -> rs.getString(1), sessionId, requestKey);
if (!existing.isEmpty()) {
List<String> bindings = jdbc.query("SELECT turn_id FROM analysis_protocol_execution_turn WHERE execution_id=? FOR UPDATE",
(rs, n) -> rs.getString(1), existing.getFirst());
if (bindings.size() != 1 || !Objects.equals(turnId, bindings.getFirst())) throw denied();
}
AnalysisExecutionProtocolStore.Execution result = store.start(sessionId, expectedVersion, revision, requestKey, requestDigest);
if (existing.isEmpty()) jdbc.update("INSERT INTO analysis_protocol_execution_turn(execution_id,turn_id) VALUES (?,?)",
result.executionId(), turnId);
return result;
}
/**
* Current database identity check only. Caller must revalidate the current principal and scope.
* The result is a point-in-time observation; query results and events require a separate
* transactional CAS before publication. A locking read avoids an enclosing old snapshot.
*/
@Transactional
public CurrentExecution current(String sessionId, String turnId, String userId, long revision, String executionId) {
required(sessionId); required(turnId); required(userId); required(executionId);
if (revision < 1) throw denied();
// Session is the common first lock; do not let a JOIN lock an attempt/turn first.
List<String> sessions=jdbc.query("SELECT session_id FROM analysis_protocol_session WHERE session_id=? FOR UPDATE",(rs,n)->rs.getString(1),sessionId);
if(sessions.size()!=1)throw denied();
List<String> owners=jdbc.query("SELECT release_id FROM analysis_session WHERE id=? AND user_id=?",(rs,n)->rs.getString(1),sessionId,userId);
if(owners.size()!=1)throw denied();
List<CurrentExecution> result = jdbc.query("""
SELECT r.task_json, t.auth_scope_hash
FROM analysis_protocol_session p
JOIN analysis_protocol_execution e ON e.execution_id=p.current_execution_id AND e.session_id=p.session_id
JOIN analysis_protocol_execution_turn b ON b.execution_id=e.execution_id
JOIN analysis_protocol_revision r ON r.session_id=p.session_id AND r.revision=p.current_revision
JOIN analysis_turn t ON t.id=b.turn_id AND t.session_id=p.session_id
WHERE p.session_id=? AND t.id=? AND t.user_id=? AND p.current_revision=?
AND p.confirmed_revision=p.current_revision AND p.current_execution_id=? AND p.status='RUNNING'
AND e.revision=p.current_revision AND e.status='RUNNING' AND r.disposition='CONFIRMED'
AND t.status='RUNNING' AND t.cancel_requested=FALSE
FOR UPDATE
""", (rs, n) -> new CurrentExecution(rs.getString(1), owners.getFirst(), rs.getString(2)),
sessionId, turnId, userId, revision, executionId);
if (result.size() != 1) throw denied();
return result.getFirst();
}
private static void required(String value) {
if (value == null || value.isBlank()) throw new IllegalArgumentException("binding identity required");
}
private static IllegalStateException denied() { return new IllegalStateException("invalid execution turn binding"); }
}
@@ -0,0 +1,59 @@
package cn.yunsheng.quality.analysis;
import java.util.*;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.http.HttpStatus;
import org.springframework.web.server.ResponseStatusException;
import tools.jackson.databind.json.JsonMapper;
/** A bounded presentation artifact; never updates an original turn, execution or query. */
final class AnalysisProtocolVisualRecovery {
private final JdbcTemplate jdbc;
private final JsonMapper json;
AnalysisProtocolVisualRecovery(JdbcTemplate jdbc,JsonMapper json){this.jdbc=jdbc;this.json=json;}
Map<String,Object> begin(Map<String,Object> input) {
var rows=jdbc.query("SELECT recovery_id,status,claim_token,TIMESTAMPDIFF(SECOND,updated_at,CURRENT_TIMESTAMP(6)) FROM analysis_protocol_visual_recovery WHERE turn_id=? AND attempt_id=? AND source_hash=? FOR UPDATE",
(rs,n)->Map.<String,Object>of("id",rs.getString(1),"status",rs.getString(2),"token",rs.getString(3),"age",rs.getLong(4)),input.get("turnId"),input.get("attemptId"),input.get("sourceHash"));
String id=rows.isEmpty()?UUID.randomUUID().toString():rows.getFirst().get("id").toString();
if(!rows.isEmpty()&&"COMPLETED".equals(rows.getFirst().get("status")))return Map.of("id",id,"claimed",false);
if(!rows.isEmpty()&&"RUNNING".equals(rows.getFirst().get("status"))&&((Number)rows.getFirst().get("age")).longValue()<90)
throw new ResponseStatusException(HttpStatus.CONFLICT,"图表正在恢复,请稍后查看");
String token=UUID.randomUUID().toString();
if(rows.isEmpty())jdbc.update("INSERT INTO analysis_protocol_visual_recovery(recovery_id,turn_id,attempt_id,execution_id,revision,release_id,auth_scope_hash,source_hash,claim_token,status) VALUES(?,?,?,?,?,?,?,?,?,'RUNNING')",
id,input.get("turnId"),input.get("attemptId"),input.get("executionId"),input.get("revision"),input.get("releaseId"),input.get("scopeHash"),input.get("sourceHash"),token);
else jdbc.update("UPDATE analysis_protocol_visual_recovery SET status='RUNNING',claim_token=?,updated_at=CURRENT_TIMESTAMP(6) WHERE recovery_id=?",token,id);
return Map.of("id",id,"token",token,"claimed",true);
}
Map<String,Object> identity(String id) {
var rows=jdbc.query("SELECT turn_id,attempt_id,execution_id,revision,release_id,auth_scope_hash,source_hash,claim_token,status FROM analysis_protocol_visual_recovery WHERE recovery_id=?",
(rs,n)->Map.<String,Object>of("turnId",rs.getString(1),"attemptId",rs.getString(2),"executionId",rs.getString(3),"revision",rs.getLong(4),"releaseId",rs.getString(5),"scopeHash",rs.getString(6),"sourceHash",rs.getString(7),"token",rs.getString(8),"status",rs.getString(9)),id);
if(rows.size()!=1)throw denied("图表恢复记录不存在");return rows.getFirst();
}
void commit(String id,String token,Map<String,Object> input,Map<String,Object> result) {
var stored=identity(id);requireIdentity(stored,input);
if(!token.equals(stored.get("token"))||!"RUNNING".equals(stored.get("status")))throw denied("该图表恢复请求已失效");
String raw=json.writeValueAsString(result);if(raw.getBytes(java.nio.charset.StandardCharsets.UTF_8).length>512*1024)throw denied("图表恢复结果超出容量");
var encoded=AnalysisProtocolEvidenceSnapshot.encode(raw);if(encoded==null)throw denied("图表恢复结果无法保存");
String status="ready".equals(result.get("visualState"))?(Boolean.TRUE.equals(result.get("chartsComplete"))?"COMPLETED":"PARTIAL"):"FAILED";
if("FAILED".equals(status)) {fail(id,token);return;}
if(jdbc.update("UPDATE analysis_protocol_visual_recovery SET status=?,result_payload=?,result_hash=?,result_bytes=?,result_encoding=? WHERE recovery_id=? AND claim_token=? AND status='RUNNING'",
status,encoded.payload(),AnalysisProtocolService.hash(raw),encoded.bytes(),encoded.encoding(),id,token)!=1)throw denied("该图表恢复请求已被替代");
}
void fail(String id,String token){jdbc.update("UPDATE analysis_protocol_visual_recovery SET status='FAILED' WHERE recovery_id=? AND claim_token=? AND status='RUNNING'",id,token);}
boolean hasArtifact(String turnId) {return Boolean.TRUE.equals(jdbc.queryForObject("SELECT EXISTS(SELECT 1 FROM analysis_protocol_visual_recovery v JOIN analysis_turn t ON t.id=v.turn_id AND JSON_UNQUOTE(JSON_EXTRACT(t.plan_json,'$.attemptId'))=v.attempt_id WHERE v.turn_id=? AND v.result_payload IS NOT NULL)",Boolean.class,turnId));}
Map<String,Object> read(Map<String,Object> input) {
var rows=jdbc.query("SELECT recovery_id,result_payload,result_bytes,result_encoding,result_hash FROM analysis_protocol_visual_recovery WHERE turn_id=? AND attempt_id=? AND source_hash=? AND auth_scope_hash=? AND result_payload IS NOT NULL",
(rs,n)->{
String raw=AnalysisProtocolEvidenceSnapshot.decode(rs.getBytes(2),rs.getInt(3),rs.getString(4),rs.getString(5));
var result=new LinkedHashMap<String,Object>(json.readValue(raw,Map.class));result.put("recoveryId",rs.getString(1));return result;
},input.get("turnId"),input.get("attemptId"),input.get("sourceHash"),input.get("scopeHash"));
return rows.isEmpty()?Map.of():rows.getFirst();
}
static void requireIdentity(Map<String,Object> stored,Map<String,Object> input) {
for(String key:List.of("turnId","attemptId","executionId","revision","releaseId","scopeHash","sourceHash"))
if(!Objects.equals(String.valueOf(stored.get(key)),String.valueOf(input.get(key))))throw denied("原分析或访问权限已变化,未保存迟到图表");
}
private static ResponseStatusException denied(String message){return new ResponseStatusException(HttpStatus.FORBIDDEN,message);}
}
@@ -0,0 +1,84 @@
package cn.yunsheng.quality.analysis;
import java.nio.charset.StandardCharsets;
import java.util.*;
import tools.jackson.databind.json.JsonMapper;
/** Physical source rows reconstructed from the current execution's recorded responses only. */
final class AnalysisRecordSnapshot {
static final int MAX_ROWS=4000,MAX_BYTES=4*1024*1024;
record Input(Map<String,Object> scope,Map<String,Object> request,List<Map<String,Object>> pages) {}
record Result(Map<String,Object> snapshot,List<String> gaps) {}
static Result build(String release,List<Input> inputs,JsonMapper json) {
List<Map<String,Object>> sets=new ArrayList<>();List<String> gaps=new ArrayList<>();
long total=0;int loaded=0,bytes=16_384;boolean complete=true;
for(Input input:inputs) {
Map<String,Object> scope=input.scope(),request=input.request();String label=Objects.toString(scope.get("label"),"所选范围");
TreeMap<Long,Map<String,Object>> pages=new TreeMap<>();
for(Map<String,Object> page:input.pages()) {
Map<String,Object> actual=new LinkedHashMap<>(request);actual.put("offset",page.get("offset"));actual.put("limit",page.get("limit"));
if(!Boolean.TRUE.equals(AnalysisProtocolInvestigation.inspect(page,actual).get("recordsPageComplete"))) {
gaps.add(label+"原始记录响应范围或分页结构未通过校验");complete=false;continue;
}
long offset=count(page.get("offset"));Map<String,Object> previous=pages.putIfAbsent(offset,page);
if(previous!=null&&!Objects.equals(AnalysisProtocolService.canonical(previous),AnalysisProtocolService.canonical(page)))
throw new IllegalArgumentException("同一原始记录页出现不同内容");
}
if(!pages.containsKey(0L)) {gaps.add(label+"缺少原始记录首页,不能确认总数");complete=false;continue;}
Map<String,Object> first=pages.get(0L);long expected=count(first.get("totalRows")),position=0;
List<?> fields=list(first.get("selectedFields"));
if(fields.isEmpty()||fields.size()>32||fields.stream().anyMatch(v->!(v instanceof String s)||s.isBlank())||new HashSet<>(fields).size()!=fields.size())
throw new IllegalArgumentException("原始记录列无效");
List<Map<String,Object>> rows=new ArrayList<>();Set<Object> rowIds=new HashSet<>();Set<List<?>> physicalRows=new HashSet<>();
boolean setComplete=true,clipped=false;
for(var entry:pages.entrySet()) {
Map<String,Object> page=entry.getValue();
if(count(page.get("totalRows"))!=expected||count(page.get("matchingRows"))!=expected||!fields.equals(page.get("selectedFields")))
throw new IllegalArgumentException("原始记录分页总数或列发生变化");
if(entry.getKey()!=position) {setComplete=false;break;}
for(Object raw:list(page.get("rows"))) {
Map<String,Object> row=map(raw),values=map(row.get("values"));
Object id=row.get("recordId");
if(!(id instanceof String s&&!s.isBlank()||id instanceof Number n&&n.doubleValue()==n.longValue()&&n.longValue()>0)
||!(row.get("rowNumber") instanceof Number number)||number.doubleValue()!=number.intValue()||number.intValue()<1
||!nonblank(row.get("sourceRevisionId"))||!nonblank(row.get("buildId"))||!nonblank(row.get("sourceFile"))||!nonblank(row.get("sheet"))
||!values.keySet().equals(new HashSet<>(fields))||values.values().stream().anyMatch(v->v!=null&&!(v instanceof String||v instanceof Number||v instanceof Boolean))
||!rowIds.add(id.toString())||!physicalRows.add(List.of(row.get("sourceRevisionId"),row.get("buildId"),row.get("sheet"),number.intValue())))
throw new IllegalArgumentException("原始物理记录格式无效或重复");
Map<String,Object> item=new LinkedHashMap<>();
for(String key:List.of("recordId","sourceRevisionId","buildId","sourceFile","sheet","rowNumber","product","group","values"))item.put(key,row.get(key));
item.put("recordId",id.toString());
if(values.values().stream().anyMatch(v->v instanceof String s&&(s.endsWith("(内容截断)")||s.equals("(非标量内容未展示)")))) {clipped=true;setComplete=false;}
int size=json.writeValueAsString(item).getBytes(StandardCharsets.UTF_8).length+1;
if(loaded<MAX_ROWS&&bytes+size<MAX_BYTES-16_384) {rows.add(item);loaded++;bytes+=size;}
else setComplete=false;
position++;
}
}
setComplete&=position==expected&&rows.size()==expected;
if(!setComplete)gaps.add(label+(clipped?"部分原始字段内容未完整展示":"原始记录尚未全部读取或展示"));
complete&=setComplete;total=Math.addExact(total,expected);
Map<String,Object> set=new LinkedHashMap<>();set.put("id","records_"+sets.size());
for(String key:List.of("nodeId","scopeId","label"))set.put(key,scope.get(key));
set.put("stage",request.get("stage"));Map<String,Object> period=new LinkedHashMap<>();
for(String key:List.of("year","month","quarter","monthStart","monthEnd"))if(request.containsKey(key))period.put(key,request.get(key));
set.put("period",period);set.put("filters",map(map(scope.get("task")).get("filters")));set.put("fields",fields);
set.put("totalRows",expected);set.put("loadedRows",rows.size());set.put("complete",setComplete);set.put("rows",rows);sets.add(set);
}
Map<String,Object> result=new LinkedHashMap<>();result.put("schemaVersion",1);result.put("kind","source_record_list");result.put("releaseId",release);
result.put("queryComplete",complete&&!inputs.isEmpty());result.put("totalRows",total);result.put("loadedRows",loaded);result.put("sets",sets);
// Account for metadata too; clipping can only make completion weaker.
while(json.writeValueAsString(result).getBytes(StandardCharsets.UTF_8).length>MAX_BYTES) {
Map<String,Object> largest=sets.stream().max(Comparator.comparingInt(set->list(set.get("rows")).size())).orElseThrow();
List<?> rows=list(largest.get("rows"));if(rows.isEmpty())throw new IllegalArgumentException("原始记录范围元数据超出展示上限");
int kept=rows.size()/2;largest.put("rows",new ArrayList<>(rows.subList(0,kept)));largest.put("loadedRows",kept);largest.put("complete",false);
loaded-=rows.size()-kept;result.put("loadedRows",loaded);result.put("queryComplete",false);gaps.add("原始记录展示大小达到上限,尚未完整展示");
}
return new Result(result,gaps.stream().distinct().toList());
}
private static long count(Object value){if(!(value instanceof Number n)||n.doubleValue()!=n.longValue()||n.longValue()<0)throw new IllegalArgumentException("原始记录数量无效");return n.longValue();}
private static boolean nonblank(Object value){return value instanceof String s&&!s.isBlank();}
private static List<?> list(Object value){return value instanceof List<?> values?values:List.of();}
private static Map<String,Object> map(Object value){return AnalysisProtocolService.map(value);}
}
@@ -33,6 +33,7 @@ public final class AnalysisResultAccess {
}
private static boolean permitted(String key, String stage, Set<String> allowed) {
if (stage == null) return true;
if ("scrapRate".equals(key)) return allowed.contains(stage+"_scrap_weight") && allowed.contains(stage+"_total_weight");
String field = switch (key) {
case "qualifiedWeight" -> stage + "_good_weight";
case "scrapWeight", "currentScrapWeight", "comparisonScrapWeight", "scrapDelta" -> stage + "_scrap_weight";
@@ -0,0 +1,37 @@
package cn.yunsheng.quality.analysis;
import java.io.IOException;
import java.time.Duration;
import java.util.Map;
import tools.jackson.databind.json.JsonMapper;
/** Same bundled policy is read by the Python bridge; model profiles cannot override it. */
final class AnalysisRuntimePolicy {
private static final Map<?,?> POLICY = load();
static final int MAX_OUTPUT_TOKENS = number("maxOutputTokens");
static final Duration TURN_TIMEOUT = Duration.ofSeconds(number("turnTimeoutSeconds"));
static final Duration REQUEST_TIMEOUT = Duration.ofSeconds(number("requestTimeoutSeconds"));
static final Duration WATCHDOG_TIMEOUT = TURN_TIMEOUT.plusSeconds(60);
static final Duration DELEGATION_TTL = WATCHDOG_TIMEOUT.plusSeconds(60);
private static Map<?,?> load() {
try (var stream = AnalysisRuntimePolicy.class.getResourceAsStream("/analysis-runtime-policy.json")) {
if (stream == null) throw new IOException("Missing analysis runtime policy");
Map<?,?> value = JsonMapper.builder().build().readValue(stream, Map.class);
if (!Integer.valueOf(1).equals(value.get("version"))) throw new IOException("Invalid policy version");
int output = integer(value, "maxOutputTokens"), turn = integer(value, "turnTimeoutSeconds"),
request = integer(value, "requestTimeoutSeconds"), reserve = integer(value, "finalizationReserveSeconds");
if (output < 8192 || output > 32768 || reserve < 60 || reserve >= request || request >= turn || turn > 3600)
throw new IOException("Invalid analysis runtime limits");
return value;
} catch (IOException failure) {
throw new ExceptionInInitializerError(failure);
}
}
private static int integer(Map<?,?> values, String key) throws IOException {
if (!(values.get(key) instanceof Integer value)) throw new IOException("Invalid policy field: " + key);
return value;
}
private static int number(String key) { return (Integer) POLICY.get(key); }
private AnalysisRuntimePolicy() {}
}
@@ -0,0 +1,97 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.filters.SourceColumnQueries;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import cn.yunsheng.quality.release.ReleaseService;
import java.util.*;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.*;
/** Authorized source schema and projected member lists; never accepts SQL. */
@RestController
@RequestMapping("/api/analysis/source")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','BUSINESS_ANALYST','QUALITY_VIEWER')")
public class AnalysisSourceQueryController {
private final SourceColumnQueries columns;
private final ReleaseService releases;
private cn.yunsheng.quality.governance.GovernanceAccessService governance;
private cn.yunsheng.quality.identity.JdbcAppUserDetailsService users;
@org.springframework.beans.factory.annotation.Autowired(required=false)
public void setReauthorization(cn.yunsheng.quality.governance.GovernanceAccessService governance,
cn.yunsheng.quality.identity.JdbcAppUserDetailsService users){this.governance=governance;this.users=users;}
public AnalysisSourceQueryController(SourceColumnQueries columns,ReleaseService releases) {
this.columns=columns;this.releases=releases;
}
@GetMapping("/catalog")
public Map<String,Object> catalog(@RequestParam(required=false) String releaseId,@AuthenticationPrincipal AppUserPrincipal actor) {
String scope=captureScope(actor);
String release=releases.resolvePublishedRelease(releaseId);
Map<String,Object> result=Map.of("releaseId",release,"sourceFields",columns.catalog(release,actor),"joinSemantics","independent_stages");
reauthorize(actor,scope,false);return result;
}
@PostMapping("/members")
public Map<String,Object> members(@RequestBody Map<String,Object> input,@AuthenticationPrincipal AppUserPrincipal actor) {
String scope=captureScope(actor);
if(input==null||!Set.of("releaseId","period","filters","memberFields","stages","limit","offset").containsAll(input.keySet()))throw bad("清单请求包含未知字段");
String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null));
Map<String,Object> context=context(input);
Map<String,Object> result=columns.memberPage(release,context,strings(input.get("memberFields"),5),strings(input.get("stages"),2),
integer(input.getOrDefault("limit",200),1,200),integer(input.getOrDefault("offset",0),0,100000),actor);
reauthorize(actor,scope,true);return result;
}
@PostMapping("/values")
public Map<String,Object> values(@RequestBody Map<String,Object> input,@AuthenticationPrincipal AppUserPrincipal actor) {
String scope=captureScope(actor);
if(input==null||!Set.of("releaseId","period","filters","field","stages","search","limit","offset").containsAll(input.keySet()))throw bad("候选值请求包含未知字段");
String release=releases.resolvePublishedRelease(Objects.toString(input.get("releaseId"),null));
Map<String,Object> context=context(input);
if(!(input.get("field") instanceof String field)||field.isBlank()||input.get("search")!=null&&!(input.get("search") instanceof String))throw bad("候选字段或搜索值无效");
Map<String,Object> result=new LinkedHashMap<>(columns.values(release,context,field,strings(input.get("stages"),2),Objects.toString(input.get("search"),null),
integer(input.getOrDefault("limit",50),1,200),integer(input.getOrDefault("offset",0),0,100000),actor));
result.put("period",input.get("period"));result.put("filters",input.get("filters"));
result.put("values",AnalysisProtocolService.list(result.get("rows")).stream().map(row->AnalysisProtocolService.map(row).get("value")).toList());
result.put("candidateOnly",true);reauthorize(actor,scope,false);return result;
}
private String captureScope(AppUserPrincipal actor) {
return governance==null?null:AuthorizationFingerprint.of(actor,governance);
}
private void reauthorize(AppUserPrincipal actor,String scope,boolean detail) {
if(scope==null)return;
AppUserPrincipal fresh=(AppUserPrincipal)users.loadUserByUsername(actor.getUsername());
if(!fresh.isEnabled()||fresh.isForcePasswordChange())throw denied();
fresh=governance.resolvePrincipal(fresh);governance.requireUserActivePolicy(fresh);
governance.requireBoundAction("QUERY","query.aggregate");
if(detail){governance.requireDetail(fresh);governance.requireBoundAction("QUERY","query.detail");}
if(!scope.equals(AuthorizationFingerprint.of(fresh,governance)))throw denied();
}
private org.springframework.web.server.ResponseStatusException denied() {
return new org.springframework.web.server.ResponseStatusException(org.springframework.http.HttpStatus.FORBIDDEN,"查询期间授权状态已变化,请按当前权限重新查询");
}
private List<String> strings(Object raw,int max) {
if(!(raw instanceof List<?> values)||values.isEmpty()||values.size()>max||values.stream().anyMatch(v->!(v instanceof String s)||s.isBlank())||new HashSet<>(values).size()!=values.size())throw bad("清单字段或阶段格式无效");
return values.stream().map(String.class::cast).toList();
}
private Map<String,Object> context(Map<String,Object> input) {
if(!(input.get("period") instanceof Map<?,?>)||!(input.get("filters") instanceof Map<?,?>))throw bad("必须明确期间和筛选对象");
Map<String,Object> period=AnalysisProtocolService.map(input.get("period")),filters=AnalysisProtocolService.map(input.get("filters"));
if(!Set.of("year","month","quarter","monthStart","monthEnd").containsAll(period.keySet())||!period.containsKey("year")||period.containsKey("month")&&period.containsKey("quarter"))throw bad("期间字段无效");
for(String key:period.keySet())integer(period.get(key),key.equals("year")?2000:1,key.equals("year")?2100:key.equals("quarter")?4:12);
boolean hasStart=period.containsKey("monthStart"),hasEnd=period.containsKey("monthEnd");
if(hasStart!=hasEnd||hasStart&&(period.containsKey("month")||period.containsKey("quarter")
||integer(period.get("monthStart"),1,12)>integer(period.get("monthEnd"),1,12)))throw bad("月份范围无效");
if(!Set.of("department","group","product","projectGroup","extraFilters","columnFilters","exclusions","includeUnassigned").containsAll(filters.keySet()))throw bad("包含未支持的筛选字段");
if(filters.containsKey("includeUnassigned")&&!(filters.get("includeUnassigned") instanceof Boolean))throw bad("未分类条件必须为布尔值");
cn.yunsheng.quality.filters.QueryFilterFields.parse(filters.get("extraFilters"));
cn.yunsheng.quality.filters.SourceColumnQueries.parse(filters.get("columnFilters"));
AnalysisExplorationTools.exclusions(filters.get("exclusions"));
Map<String,Object> result=new LinkedHashMap<>(filters);result.putAll(period);return result;
}
private int integer(Object raw,int min,int max) {
if(!(raw instanceof Number number)||number.doubleValue()!=number.intValue()||number.intValue()<min||number.intValue()>max)throw bad("清单分页参数无效");
return number.intValue();
}
private org.springframework.web.server.ResponseStatusException bad(String text) {
return new org.springframework.web.server.ResponseStatusException(org.springframework.http.HttpStatus.BAD_REQUEST,text);
}
}
@@ -0,0 +1,57 @@
package cn.yunsheng.quality.analysis;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/** Provider-returned text only. Never reconstruct a thought from an answer or tool event. */
final class AnalysisThinking {
static Map<String,Object> safe(Object raw) {
if (!(raw instanceof Map<?,?> value)) return Map.of();
String id=Objects.toString(value.get("id"),""),phase=Objects.toString(value.get("phase"),""),
status=Objects.toString(value.get("status"),"");
if (!id.matches("api-[a-f0-9]{8}-[0-9]{1,6}") || !List.of("planning","analysis","repair","finalizing").contains(phase)
|| !List.of("running","completed","not_returned","interrupted").contains(status)
|| !"api".equals(value.get("source")) || !(value.get("version") instanceof Number version)
|| version.longValue()<1 || !(value.get("text") instanceof String text) || text.length()>1_200_000) return Map.of();
var safe=new LinkedHashMap<String,Object>();
safe.put("id",id);safe.put("phase",phase);safe.put("status",status);safe.put("source","api");
safe.put("version",version.longValue());safe.put("text",text
.replaceAll("(?i)Bearer\\s+[A-Za-z0-9._~+/-]+","Bearer [REDACTED]")
.replaceAll("(?i)(api[_-]?key|password|token|secret|authorization)(\\s*[=::]\\s*)[^\\s,,;;]+","$1$2[REDACTED]")
.replaceAll("\\bsk-[A-Za-z0-9_-]+","[REDACTED]"));
safe.put("model",Objects.toString(value.get("model"),"").substring(0,Math.min(160,Objects.toString(value.get("model"),"").length())));
safe.put("streaming",Boolean.TRUE.equals(value.get("streaming")));
safe.put("truncated",Boolean.TRUE.equals(value.get("truncated")));
for (String field : List.of("at", "startedAt", "finishedAt")) {
if (value.get(field) instanceof String stamp && stamp.length() <= 40) {
try { java.time.Instant.parse(stamp); safe.put(field, stamp); }
catch (java.time.format.DateTimeParseException ignored) { /* Old records need no invented timestamp. */ }
}
}
String finish=Objects.toString(value.get("finishReason"),"");
if(List.of("stop","length","tool_calls","content_filter").contains(finish))safe.put("finishReason",finish);
return safe;
}
static Map<String,Object> merge(Map<String,Object> plan, Map<String,Object> next, boolean checkpoint) {
var rows=new ArrayList<Map<String,Object>>();
if (plan.get("modelThinking") instanceof List<?> saved)
for (Object row:saved) {var clean=safe(row);if(!clean.isEmpty())rows.add(clean);}
int found=-1;
for(int i=0;i<rows.size();i++)if(rows.get(i).get("id").equals(next.get("id"))){found=i;break;}
Map<String,Object> previous=found<0?Map.of():rows.get(found);
long oldVersion=previous.get("version") instanceof Number number?number.longValue():0;
if (((Number)next.get("version")).longValue()<=oldVersion) return Map.of();
if(found<0)rows.add(next);else rows.set(found,next);
plan.put("modelThinking",List.copyOf(rows));
String prior=Objects.toString(previous.get("text"),""),text=Objects.toString(next.get("text"),"");
var event=new LinkedHashMap<String,Object>(next);event.put("baseVersion",oldVersion);
if(rows.size()==1 && found<0)event.put("reset",true);
if(!checkpoint && text.startsWith(prior)) {event.remove("text");event.put("delta",text.substring(prior.length()));}
return event;
}
private AnalysisThinking() {}
}
File diff suppressed because it is too large Load Diff
@@ -78,5 +78,10 @@ public class AnalysisTurnsController {
return turns.feedback(turnId, request, actor);
}
@PostMapping("/turns/{turnId}/visual-recovery")
public Map<String,Object> recoverVisuals(@PathVariable String turnId,@AuthenticationPrincipal AppUserPrincipal actor) {
return turns.recoverVisuals(turnId,actor);
}
public record CancelRequest(String reason) {}
}
@@ -10,6 +10,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
import org.springframework.jdbc.core.JdbcTemplate;
@@ -28,10 +29,10 @@ public final class AnalysisVisualSnapshotAccess {
private static final Set<String> CHART_KEYS = Set.of("id", "type", "title", "subtitle", "unit", "categories", "series",
"evidenceRefs", "requiredFields", "scope", "period", "comparisonPeriod", "stage", "metric", "coverage", "note",
"originTurnId", "sectionKey", "dimension");
private static final Set<String> SCOPE_KEYS = Set.of("year", "month", "quarter", "department", "group", "product",
"projectGroup", "view", "includeUnassigned", "releaseId", "extraFilters", "stage", "category", "defect");
private static final Set<String> PERIOD_KEYS = Set.of("start", "end", "label", "granularity", "year", "month", "quarter");
private static final Set<String> COVERAGE_KEYS = Set.of("totalRows", "returnedRows", "truncated", "shareBasis");
private static final Set<String> SCOPE_KEYS = Set.of("columnFilters", "columnLabels", "year", "month", "monthStart", "monthEnd", "quarter", "department", "group", "product",
"projectGroup", "view", "includeUnassigned", "releaseId", "extraFilters", "exclusions", "stage", "category", "defect");
private static final Set<String> PERIOD_KEYS = Set.of("start", "end", "label", "granularity", "year", "month", "monthStart", "monthEnd", "quarter");
private static final Set<String> COVERAGE_KEYS = Set.of("totalRows", "returnedRows", "truncated", "shareBasis", "denominator");
private static final Set<String> CANONICAL_FIELDS = Set.of("black_good_weight", "black_scrap_weight", "black_total_weight",
"black_responsibility_scrap", "black_material_scrap", "black_external_scrap", "bright_good_weight",
"bright_scrap_weight", "bright_total_weight", "bright_headquarters_scrap", "bright_plating_scrap",
@@ -139,10 +140,18 @@ public final class AnalysisVisualSnapshotAccess {
MetricSemantics semantics = semantics(metric, input.get("stage"));
if (semantics == null || !semantics.unit().equals(unit)) return null;
boolean productSourceDefect = productSourceDefectChart(type, sectionKey, dimension, metric, semantics);
Set<String> requiredFields = new LinkedHashSet<>(semantics.requiredFields());
List<Map<String, Object>> categories = normalizeCategories(input.get("categories"), type, semantics, requiredFields);
AnalysisFineDefectsService.DefectField selectedProductDefect = null;
if (productSourceDefect) {
selectedProductDefect = productSourceDefectField(input.get("scope"), semantics.stage());
if (selectedProductDefect == null || !productSourceDefectCategoryMatches(input.get("scope"), selectedProductDefect)) return null;
requiredFields.add(selectedProductDefect.key());
}
List<Map<String, Object>> categories = normalizeCategories(input.get("categories"), type, semantics, requiredFields,
productSourceDefect);
if (categories == null) return null;
List<Map<String, Object>> series = normalizeSeries(input.get("series"), categories.size(), metric);
List<Map<String, Object>> series = normalizeSeries(input.get("series"), categories.size(), metric, type);
if (series == null || !metric.equals(series.getFirst().get("key"))) return null;
// requiredFields from the runtime is only checked for shape. The persisted list is rebuilt
@@ -156,18 +165,26 @@ public final class AnalysisVisualSnapshotAccess {
if (period == null) return null;
Map<String, Object> comparisonPeriod = input.containsKey("comparisonPeriod") ? normalizePeriod(input.get("comparisonPeriod")) : null;
if (input.containsKey("comparisonPeriod") && comparisonPeriod == null) return null;
if (productSourceDefect && !validProductSourceDefectComparison(metric, period, comparisonPeriod)) return null;
Map<String, Object> coverage = normalizeCoverage(input.get("coverage"), requiredFields, semantics.stage(), type);
if (coverage == null) return null;
if (productSourceDefect && !requiredFields.equals(Set.of(selectedProductDefect.key()))) return null;
if (!validCompositionChart(type, metric, unit, semantics, categories, series, coverage, comparisonPeriod)
|| !validPeriodChart(type, metric, semantics, period, comparisonPeriod, categories, series)
|| !validWaterfallChart(type, metric, unit, semantics, categories, series, coverage, period, comparisonPeriod)) return null;
if ("all_scope_weight".equals(coverage.get("shareBasis"))
&& (!Set.of("product","department","group").contains(dimension)
|| !Set.of("black_total_weight","bright_total_weight","black_scrap_weight","bright_scrap_weight").contains(metric))) return null;
if (!allowedFields.contains("*") && !allowedFields.containsAll(requiredFields)) return null;
Map<String, Object> scope = normalizeScope(input.get("scope"), visibleExtraFilterKeys, expectedScope);
Object scopedStage = scope == null ? null : scope.get("stage");
if (scope == null || scopedStage != null && !"all".equals(scopedStage)
&& (semantics.stage() == null || !semantics.stage().equals(scopedStage))) return null;
&& (semantics.stage() == null || !semantics.stage().equals(scopedStage))
|| productSourceDefect && !semantics.stage().equals(scopedStage)) return null;
if (!validChapterMetadata(sectionKey, dimension, type, metric, semantics, scope, period)) return null;
if (semantics.sourceDefect() && !sourceDefectScopeMatches(scope, semantics.stage(), categories)) return null;
if (semantics.sourceDefect() && (productSourceDefect
? !productSourceDefectScopeMatches(scope, semantics.stage(), selectedProductDefect)
: !sourceDefectScopeMatches(scope, semantics.stage(), categories))) return null;
String note = input.containsKey("note") ? text(input.get("note"), 500, false) : null;
Map<String, Object> result = new LinkedHashMap<>();
@@ -187,7 +204,7 @@ public final class AnalysisVisualSnapshotAccess {
}
private static List<Map<String, Object>> normalizeCategories(Object raw, String type, MetricSemantics semantics,
Set<String> requiredFields) {
Set<String> requiredFields, boolean productLabels) {
if (!(raw instanceof List<?> values) || values.isEmpty() || values.size() > MAX_CATEGORIES
|| Set.of("bar", "column").contains(type) && values.size() > 10) return null;
List<Map<String, Object>> result = new ArrayList<>();
@@ -197,7 +214,7 @@ public final class AnalysisVisualSnapshotAccess {
if (!Set.of("key", "label").containsAll(item.keySet()) || !item.keySet().containsAll(Set.of("key", "label"))) return null;
String key = text(item.get("key"), 120, true), label = text(item.get("label"), 240, false);
if (!keys.add(key)) return null;
if (semantics.sourceDefect()) {
if (semantics.sourceDefect() && !productLabels) {
if (!isSourceDefectCategoryKey(semantics.stage(), key)) return null;
requiredFields.add(key);
}
@@ -206,7 +223,7 @@ public final class AnalysisVisualSnapshotAccess {
return result;
}
private static List<Map<String, Object>> normalizeSeries(Object raw, int categoryCount, String metric) {
private static List<Map<String, Object>> normalizeSeries(Object raw, int categoryCount, String metric, String type) {
if (!(raw instanceof List<?> values) || values.isEmpty() || values.size() > MAX_SERIES) return null;
List<Map<String, Object>> result = new ArrayList<>();
Set<String> keys = new LinkedHashSet<>();
@@ -224,12 +241,14 @@ public final class AnalysisVisualSnapshotAccess {
if (decimal == null || decimal.precision() > 64 || Math.abs(decimal.scale()) > 64) return null;
if (Set.of("composite_rate", "black_rate", "bright_rate").contains(metric)
&& (decimal.signum() < 0 || decimal.compareTo(BigDecimal.ONE) > 0)) return null;
if (Set.of("black_scrap_rate", "bright_scrap_rate").contains(metric) && decimal.signum() < 0) return null;
if (Set.of("black_good_weight", "black_scrap_weight", "black_total_weight",
"black_responsibility_scrap", "black_material_scrap", "black_external_scrap",
"bright_good_weight", "bright_scrap_weight", "bright_total_weight",
"bright_headquarters_scrap", "bright_plating_scrap", "bright_material_scrap",
"bright_processing_scrap", "bright_diffusion_scrap", "current_scrap_weight",
"source_defect_weight").contains(metric) && decimal.signum() < 0) return null;
"source_defect_weight").contains(metric) && decimal.signum() < 0
&& !("line".equals(type)&&"black_total_weight".equals(metric))) return null;
if (Set.of("rate_drop_pp", "composite_impact_pp").contains(metric)
&& decimal.abs().compareTo(BigDecimal.valueOf(100)) > 0) return null;
numbers.add(decimal);
@@ -243,16 +262,21 @@ public final class AnalysisVisualSnapshotAccess {
private static Map<String, Object> normalizeScope(Object raw, Set<String> visibleExtraFilterKeys, Map<String, Object> expectedScope) {
Map<String, Object> input = stringMap(raw);
if (!SCOPE_KEYS.containsAll(input.keySet())) return null;
Map<String, Object> expected = expectedScope == null ? Map.of() : expectedScope;
Map<String, Object> scope = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter")) {
for (String key : List.of("year", "month", "quarter", "monthStart", "monthEnd")) {
if (input.containsKey(key)) {
Integer value = integer(input.get(key));
if (value == null || key.equals("year") && (value < 2000 || value > 2100)
|| key.equals("month") && (value < 1 || value > 12) || key.equals("quarter") && (value < 1 || value > 4)) return null;
|| Set.of("month", "monthStart", "monthEnd").contains(key) && (value < 1 || value > 12)
|| key.equals("quarter") && (value < 1 || value > 4)) return null;
scope.put(key, value);
}
}
if (scope.containsKey("month") && scope.containsKey("quarter")) return null;
if (scope.containsKey("month") && scope.containsKey("quarter")
|| scope.containsKey("monthStart") != scope.containsKey("monthEnd")
|| scope.containsKey("monthStart") && (scope.containsKey("month") || scope.containsKey("quarter")
|| !scope.containsKey("year") || (Integer)scope.get("monthStart") > (Integer)scope.get("monthEnd"))) return null;
for (String key : List.of("department", "group", "product", "projectGroup", "view", "releaseId")) {
if (input.containsKey(key)) {
String value = text(input.get(key), 256, true);
@@ -279,7 +303,39 @@ public final class AnalysisVisualSnapshotAccess {
if (!visibleExtraFilterKeys.containsAll(filters.keySet())) return null;
scope.put("extraFilters", filters);
}
Map<String, Object> expected = expectedScope == null ? Map.of() : expectedScope;
// Exclusions are part of the formal query identity. Require both the presence and the
// canonical parsed values to match the persisted query; otherwise a chart can silently
// represent a broader or narrower population than the metrics result.
if (input.containsKey("exclusions") != expected.containsKey("exclusions")) return null;
if (input.containsKey("exclusions")) {
Object actualExclusions = normalizeExpected("exclusions", input.get("exclusions"), visibleExtraFilterKeys);
Object expectedExclusions = normalizeExpected("exclusions", expected.get("exclusions"), visibleExtraFilterKeys);
if (actualExclusions == Invalid.VALUE || expectedExclusions == Invalid.VALUE
|| !Objects.equals(actualExclusions, expectedExclusions)) return null;
scope.put("exclusions", actualExclusions);
}
if(expected.containsKey("monthStart")||expected.containsKey("monthEnd")) {
Object expectedYear=normalizeExpected("year",expected.get("year"),visibleExtraFilterKeys);
Object expectedStart=normalizeExpected("monthStart",expected.get("monthStart"),visibleExtraFilterKeys);
Object expectedEnd=normalizeExpected("monthEnd",expected.get("monthEnd"),visibleExtraFilterKeys);
if(expectedYear==Invalid.VALUE||expectedStart==Invalid.VALUE||expectedEnd==Invalid.VALUE
||!Objects.equals(scope.get("year"),expectedYear)||!Objects.equals(scope.get("monthStart"),expectedStart)
||!Objects.equals(scope.get("monthEnd"),expectedEnd))return null;
}
// Source-column scope is accepted only when it exactly matches the
// already authorized, persisted task. Never infer it from chart prose.
if(input.containsKey("columnFilters")||expected.containsKey("columnFilters")) {
try {
var actual=cn.yunsheng.quality.filters.SourceColumnQueries.parse(input.get("columnFilters"));
var bound=cn.yunsheng.quality.filters.SourceColumnQueries.parse(expected.get("columnFilters"));
if(!actual.equals(bound))return null;
if(!actual.isEmpty())scope.put("columnFilters",actual);
}catch(RuntimeException invalid){return null;}
}
if(input.containsKey("columnLabels")) {
if(!Objects.equals(input.get("columnLabels"),expected.get("columnLabels")))return null;
scope.put("columnLabels",input.get("columnLabels"));
} else if(expected.containsKey("columnLabels"))scope.put("columnLabels",expected.get("columnLabels"));
if (!scope.containsKey("releaseId")) return null;
if (expected.containsKey("releaseId")) {
Object expectedRelease = normalizeExpected("releaseId", expected.get("releaseId"), visibleExtraFilterKeys);
@@ -323,11 +379,16 @@ public final class AnalysisVisualSnapshotAccess {
}
private static Object normalizeExpected(String key, Object raw, Set<String> visibleExtraFilterKeys) {
if ("exclusions".equals(key)) {
try { return AnalysisExplorationTools.exclusions(raw); }
catch (RuntimeException invalid) { return Invalid.VALUE; }
}
if (raw == null) return Invalid.VALUE;
if (List.of("year", "month", "quarter").contains(key)) {
if (List.of("year", "month", "quarter", "monthStart", "monthEnd").contains(key)) {
Integer value = integer(raw);
if (value == null || key.equals("year") && (value < 2000 || value > 2100)
|| key.equals("month") && (value < 1 || value > 12) || key.equals("quarter") && (value < 1 || value > 4)) return Invalid.VALUE;
|| Set.of("month", "monthStart", "monthEnd").contains(key) && (value < 1 || value > 12)
|| key.equals("quarter") && (value < 1 || value > 4)) return Invalid.VALUE;
return value;
}
if ("includeUnassigned".equals(key)) return raw instanceof Boolean ? raw : Invalid.VALUE;
@@ -354,14 +415,18 @@ public final class AnalysisVisualSnapshotAccess {
if ("granularity".equals(key) && !Set.of("month", "quarter", "year").contains(value)) return null;
result.put(key, value);
}
for (String key : List.of("year", "month", "quarter")) {
for (String key : List.of("year", "month", "quarter", "monthStart", "monthEnd")) {
if (!input.containsKey(key)) continue;
Integer value = integer(input.get(key));
if (value == null || key.equals("year") && (value < 2000 || value > 2100)
|| key.equals("month") && (value < 1 || value > 12) || key.equals("quarter") && (value < 1 || value > 4)) return null;
|| Set.of("month", "monthStart", "monthEnd").contains(key) && (value < 1 || value > 12)
|| key.equals("quarter") && (value < 1 || value > 4)) return null;
result.put(key, value);
}
if (result.containsKey("month") && result.containsKey("quarter")) return null;
if (result.containsKey("month") && result.containsKey("quarter")
|| result.containsKey("monthStart") != result.containsKey("monthEnd")
|| result.containsKey("monthStart") && (result.containsKey("month") || result.containsKey("quarter")
|| !result.containsKey("year") || (Integer)result.get("monthStart") > (Integer)result.get("monthEnd"))) return null;
return result;
}
@@ -383,11 +448,18 @@ public final class AnalysisVisualSnapshotAccess {
// These are the only visual types for which the renderer may compute
// percentages from the plotted values.
} else if ("displayed_categories".equals(basis)) return null;
else if (!Set.of("complete_population", "query_total", "not_applicable").contains(basis)) return null;
else if (!Set.of("complete_population", "query_total", "not_applicable", "all_scope_weight").contains(basis)) return null;
result.put("shareBasis", basis);
}
if (input.containsKey("denominator")) {
BigDecimal denominator = decimal(input.get("denominator"));
if (!"all_scope_weight".equals(result.get("shareBasis")) || denominator == null || denominator.signum() <= 0) return null;
result.put("denominator", denominator);
}
if ("all_scope_weight".equals(result.get("shareBasis")) && !result.containsKey("denominator")) return null;
if (Set.of("donut", "pareto").contains(type)
&& !"displayed_categories".equals(result.get("shareBasis"))) return null;
&& !"displayed_categories".equals(result.get("shareBasis"))
&& !("pareto".equals(type) && "all_scope_weight".equals(result.get("shareBasis")))) return null;
return result;
}
@@ -397,6 +469,20 @@ public final class AnalysisVisualSnapshotAccess {
Map<String, Object> coverage,
Map<String, Object> comparisonPeriod) {
if (!Set.of("donut", "pareto").contains(type)) return true;
if ("pareto".equals(type) && "all_scope_weight".equals(coverage.get("shareBasis"))) {
if (!Set.of("black_total_weight","bright_total_weight","black_scrap_weight","bright_scrap_weight").contains(metric)
|| !"kg".equals(unit) || semantics.stage() == null || comparisonPeriod != null
|| categories.isEmpty() || categories.size() > 10 || series.size() != 1
|| !integerEquals(coverage.get("returnedRows"),categories.size())) return false;
BigDecimal denominator = decimal(coverage.get("denominator")), sum = BigDecimal.ZERO, previous = null;
if (denominator == null || denominator.signum() <= 0) return false;
for (Object raw : (List<?>) series.getFirst().get("values")) {
BigDecimal value = decimal(raw);
if (value == null || value.signum() < 0 || previous != null && value.compareTo(previous) > 0) return false;
sum = sum.add(value); previous = value;
}
return sum.signum() > 0 && sum.compareTo(denominator.add(new BigDecimal("0.00001"))) <= 0;
}
if (!"source_defect_weight".equals(metric) || !"kg".equals(unit) || !semantics.sourceDefect()
|| semantics.stage() == null || comparisonPeriod != null || categories.size() < 2 || categories.size() > 4
|| series.size() != 1 || !Boolean.FALSE.equals(coverage.get("truncated"))
@@ -457,6 +543,8 @@ public final class AnalysisVisualSnapshotAccess {
}
return java.util.Objects.equals(period.get("year"), comparisonPeriod.get("year"))
&& java.util.Objects.equals(period.get("month"), comparisonPeriod.get("month"))
&& java.util.Objects.equals(period.get("monthStart"), comparisonPeriod.get("monthStart"))
&& java.util.Objects.equals(period.get("monthEnd"), comparisonPeriod.get("monthEnd"))
&& java.util.Objects.equals(period.get("quarter"), comparisonPeriod.get("quarter"));
}
@@ -464,13 +552,17 @@ public final class AnalysisVisualSnapshotAccess {
MetricSemantics semantics, Map<String, Object> scope,
Map<String, Object> period) {
if (section == null && dimension == null) return true; // schema-1 snapshots created before chapter metadata
if (Set.of("black_total_weight","bright_total_weight","black_scrap_weight","bright_scrap_weight").contains(metric)
&& Set.of("bar","column","pareto").contains(type)
&& dimension != null && Set.of("department","group","product").contains(dimension))
return section == null || section.equals(dimension);
boolean grouped = Set.of("bar", "column").contains(type);
boolean sourceMetric = Set.of("source_defect_weight", "source_defect_delta_weight").contains(metric)
&& semantics.sourceDefect();
if (dimension != null && section == null) {
return switch (dimension) {
case "department", "group", "product" -> grouped
&& Set.of("current_scrap_weight", "scrap_delta", "rate_drop_pp").contains(metric);
&& Set.of("current_scrap_weight", "scrap_delta", "rate_drop_pp", "composite_rate", "black_rate", "bright_rate").contains(metric);
case "category" -> sourceMetric && (grouped || Set.of("donut", "pareto").contains(type));
case "defect" -> sourceMetric && grouped;
default -> false;
@@ -479,21 +571,22 @@ public final class AnalysisVisualSnapshotAccess {
return switch (section) {
case "overview" -> dimension == null && (("composite_rate".equals(metric) && Set.of("bar", "column").contains(type))
|| "composite_impact_pp".equals(metric) && "waterfall".equals(type));
case "stage" -> dimension == null && Set.of("black_rate", "bright_rate").contains(metric)
case "stage" -> dimension == null && Set.of("black_rate", "bright_rate", "black_scrap_rate", "bright_scrap_rate", "black_total_weight", "bright_total_weight", "black_scrap_weight", "bright_scrap_weight").contains(metric)
&& semantics.stage() != null && semantics.stage().equals(metric.substring(0, metric.indexOf('_')))
&& grouped;
case "department" -> "department".equals(dimension) && grouped
&& Set.of("current_scrap_weight", "scrap_delta", "rate_drop_pp").contains(metric);
&& Set.of("current_scrap_weight", "scrap_delta", "rate_drop_pp", "composite_rate", "black_rate", "bright_rate").contains(metric);
case "group" -> "group".equals(dimension) && grouped
&& Set.of("current_scrap_weight", "scrap_delta", "rate_drop_pp").contains(metric);
&& Set.of("current_scrap_weight", "scrap_delta", "rate_drop_pp", "composite_rate", "black_rate", "bright_rate").contains(metric);
case "product" -> "product".equals(dimension) && grouped
&& Set.of("current_scrap_weight", "scrap_delta", "rate_drop_pp").contains(metric);
&& (Set.of("current_scrap_weight", "scrap_delta", "rate_drop_pp", "composite_rate", "black_rate", "bright_rate").contains(metric)
|| sourceMetric && semantics.sourceDefect());
case "category_black", "category_bright" -> "category".equals(dimension) && grouped && sourceMetric
&& semantics.stage().equals("category_black".equals(section) ? "black" : "bright");
case "defect_black", "defect_bright" -> "defect".equals(dimension) && grouped && sourceMetric
&& semantics.stage().equals("defect_black".equals(section) ? "black" : "bright");
case "trend" -> dimension == null && Set.of("line", "area", "heatmap").contains(type)
&& Set.of("composite_rate", "black_rate", "bright_rate", "current_scrap_weight").contains(metric)
&& Set.of("composite_rate", "black_rate", "bright_rate", "black_scrap_rate", "bright_scrap_rate", "current_scrap_weight", "black_total_weight", "bright_total_weight", "black_scrap_weight", "bright_scrap_weight").contains(metric)
&& period.containsKey("start") && period.containsKey("end") && period.containsKey("granularity");
case "composition" -> "category".equals(dimension) && Set.of("donut", "pareto").contains(type)
&& "source_defect_weight".equals(metric) && semantics.sourceDefect();
@@ -508,8 +601,8 @@ public final class AnalysisVisualSnapshotAccess {
if (!Set.of("area", "heatmap").contains(type)) return true;
if (comparisonPeriod != null || categories.size() < 3 || !period.containsKey("start")
|| !period.containsKey("end") || !period.containsKey("granularity")) return false;
if (!Set.of("composite_rate", "black_rate", "bright_rate", "current_scrap_weight",
"black_scrap_weight", "bright_scrap_weight").contains(metric)) return false;
if (!Set.of("composite_rate", "black_rate", "bright_rate", "black_scrap_rate", "bright_scrap_rate", "current_scrap_weight",
"black_scrap_weight", "bright_scrap_weight", "black_total_weight", "bright_total_weight").contains(metric)) return false;
if ("current_scrap_weight".equals(metric) && semantics.stage() == null) return false;
String grain = (String) period.get("granularity");
Long first = periodOrder((String) period.get("start"), grain);
@@ -597,6 +690,53 @@ public final class AnalysisVisualSnapshotAccess {
|| AnalysisFineDefectsService.categories(stage).stream().anyMatch(item -> item.componentCode().equals(key));
}
private static boolean productSourceDefectChart(String type, String section, String dimension, String metric,
MetricSemantics semantics) {
return Set.of("bar", "column").contains(type) && "product".equals(section) && "product".equals(dimension)
&& Set.of("source_defect_weight", "source_defect_delta_weight").contains(metric)
&& semantics.sourceDefect() && semantics.stage() != null;
}
private static AnalysisFineDefectsService.DefectField productSourceDefectField(Object rawScope, String stage) {
if (!(rawScope instanceof Map<?, ?> scope) || !(scope.get("defect") instanceof String defect)
|| defect.isBlank()) return null;
return AnalysisFineDefectsService.fields(stage).stream().filter(field -> field.key().equals(defect))
.findFirst().orElse(null);
}
private static boolean productSourceDefectCategoryMatches(Object rawScope,
AnalysisFineDefectsService.DefectField defect) {
if (!(rawScope instanceof Map<?, ?> scope)) return false;
Object category = scope.get("category");
return category == null || category instanceof String value && value.equals(defect.category());
}
private static boolean productSourceDefectScopeMatches(Map<String, Object> scope, String stage,
AnalysisFineDefectsService.DefectField defect) {
return stage.equals(scope.get("stage")) && defect.key().equals(scope.get("defect"))
&& (scope.get("category") == null || defect.category().equals(scope.get("category")));
}
private static boolean validProductSourceDefectComparison(String metric, Map<String, Object> period,
Map<String, Object> comparisonPeriod) {
if ("source_defect_delta_weight".equals(metric) && comparisonPeriod == null) return false;
return comparisonPeriod == null || !sameProductSourcePeriod(period, comparisonPeriod);
}
private static boolean sameProductSourcePeriod(Map<String, Object> period, Map<String, Object> comparisonPeriod) {
if (period.containsKey("start") || period.containsKey("end")
|| comparisonPeriod.containsKey("start") || comparisonPeriod.containsKey("end")) {
return period.get("start") != null && Objects.equals(period.get("start"), comparisonPeriod.get("start"))
&& Objects.equals(period.get("end"), comparisonPeriod.get("end"))
&& Objects.equals(period.get("granularity"), comparisonPeriod.get("granularity"));
}
return Objects.equals(period.get("year"), comparisonPeriod.get("year"))
&& Objects.equals(period.get("month"), comparisonPeriod.get("month"))
&& Objects.equals(period.get("monthStart"), comparisonPeriod.get("monthStart"))
&& Objects.equals(period.get("monthEnd"), comparisonPeriod.get("monthEnd"))
&& Objects.equals(period.get("quarter"), comparisonPeriod.get("quarter"));
}
private static boolean sourceDefectScopeMatches(Map<String, Object> scope, String stage,
List<Map<String, Object>> chartCategories) {
Object categorySelector = scope.get("category");
@@ -635,6 +775,10 @@ public final class AnalysisVisualSnapshotAccess {
Set.of("black_good_weight", "black_total_weight"), false) : null;
case "bright_rate" -> stage == null || "bright".equals(stage) ? new MetricSemantics("ratio", "bright",
Set.of("bright_good_weight", "bright_total_weight"), false) : null;
case "black_scrap_rate" -> stage == null || "black".equals(stage) ? new MetricSemantics("ratio", "black",
Set.of("black_scrap_weight", "black_total_weight"), false) : null;
case "bright_scrap_rate" -> stage == null || "bright".equals(stage) ? new MetricSemantics("ratio", "bright",
Set.of("bright_scrap_weight", "bright_total_weight"), false) : null;
case "scrap_delta" -> stage == null
? new MetricSemantics("kg", null, Set.of("black_scrap_weight", "bright_scrap_weight"), false)
: new MetricSemantics("kg", stage, Set.of(stage + "_scrap_weight"), false);
@@ -657,7 +801,7 @@ public final class AnalysisVisualSnapshotAccess {
private static Set<String> metricNames() {
Set<String> result = new LinkedHashSet<>(WEIGHT_METRICS);
result.addAll(Set.of("composite_rate", "black_rate", "bright_rate", "scrap_delta", "rate_drop_pp",
result.addAll(Set.of("composite_rate", "black_rate", "bright_rate", "black_scrap_rate", "bright_scrap_rate", "scrap_delta", "rate_drop_pp",
"current_scrap_weight", "source_defect_weight", "source_defect_delta_weight", "composite_impact_pp"));
return result;
}
@@ -26,7 +26,7 @@ import jakarta.annotation.PreDestroy;
/** Starts the local original NAO bridge and consumes its strict NDJSON protocol. */
@Component
public class NaoAnalysisRunner {
private static final long WATCHDOG_MILLIS = Duration.ofSeconds(650).toMillis();
private static final long WATCHDOG_MILLIS = AnalysisRuntimePolicy.WATCHDOG_TIMEOUT.toMillis();
private static final long CANCEL_GRACE_MILLIS = Duration.ofSeconds(8).toMillis();
private static final String STDOUT_EOF = "\u0000<NAO_STDOUT_EOF>";
@@ -34,6 +34,7 @@ public class NaoAnalysisRunner {
private final NaoRuntimeAccessService access;
private final Map<String, Process> active = new ConcurrentHashMap<>();
private final java.util.Set<String> cancellationRequested = ConcurrentHashMap.newKeySet();
private final java.util.concurrent.Semaphore visualRecoverySlots=new java.util.concurrent.Semaphore(2);
public NaoAnalysisRunner(JsonMapper json, NaoRuntimeAccessService access) {
this.json = json;
@@ -44,18 +45,22 @@ public class NaoAnalysisRunner {
String releaseId, Map<String, Object> context,
Consumer<Map<String, Object>> onProgress, Consumer<String> onAnswer,
BooleanSupplier cancellationRequested) throws IOException, InterruptedException {
String token = access.issue(actor, releaseId, turnId);
String attemptId=null;
if(context!=null&&context.get("_protocol") instanceof Map<?,?> p&&p.get("attemptId") instanceof String id)attemptId=id;
String processKey=attemptId==null?turnId:attemptId;
String token = attemptId==null?access.issue(actor, releaseId, turnId):access.issueProtocol(actor,releaseId,turnId,attemptId);
Process process = null;
try {
RuntimeLocation location = runtimeLocation();
ProcessBuilder builder = new ProcessBuilder(command(location));
builder.directory(location.root().toFile());
process = builder.start();
active.put(turnId, process);
active.put(processKey, process);
try (BufferedWriter stdin = process.outputWriter(StandardCharsets.UTF_8)) {
Map<String, Object> request = new LinkedHashMap<>();
request.put("turnId", turnId);
request.put("sessionId", sessionId);
// Each protocol attempt owns a separate runtime session, credential file and evidence directory.
request.put("sessionId", attemptId==null?sessionId:attemptId);
request.put("actorId", actor.getId().toString());
request.put("question", question);
request.put("releaseId", releaseId);
@@ -64,12 +69,12 @@ public class NaoAnalysisRunner {
stdin.write(json.writeValueAsString(request));
stdin.newLine();
stdin.flush();
return readProtocol(turnId, process, stdin, onProgress, onAnswer, cancellationRequested);
return readProtocol(processKey, token, process, stdin, onProgress, onAnswer, cancellationRequested,attemptId);
}
} finally {
access.revokeTurn(turnId);
active.remove(turnId);
this.cancellationRequested.remove(turnId);
if(attemptId==null)access.revokeTurn(turnId);else access.revokeAttempt(attemptId);
if(process!=null)active.remove(processKey,process);
this.cancellationRequested.remove(processKey);
if (process != null && process.isAlive()) terminate(process);
}
}
@@ -80,6 +85,37 @@ public class NaoAnalysisRunner {
active.computeIfPresent(turnId, (id, process) -> { cancellationRequested.add(id); return process; });
}
/** Deterministic local chart conversion: no delegation credential and no model process. */
public Map<String,Object> rebuildVisuals(String recoveryId,Map<String,Object> input) throws IOException,InterruptedException {
byte[] payload=json.writeValueAsBytes(input);
if(payload.length>16*1024*1024)throw new IOException("原始结果超过图表恢复容量");
if(!visualRecoverySlots.tryAcquire())throw new IOException("图表恢复繁忙,请稍后重试");
Process process=null;String key="visual-recovery:"+recoveryId;
try {
Path root=projectRoot(),script=root.resolve("scripts/nao_visual_recovery.py");
if(!Files.isRegularFile(script)||!Files.isReadable(script))throw new IOException("图表恢复组件不可用");
process=new ProcessBuilder("python3",script.toString()).directory(root.toFile()).redirectError(ProcessBuilder.Redirect.DISCARD).start();
active.put(key,process);Process running=process;
var output=new java.util.concurrent.CompletableFuture<byte[]>();
Thread.ofVirtual().start(()->{try{output.complete(running.getInputStream().readNBytes(512*1024+1));}catch(IOException error){output.completeExceptionally(error);}});
long deadline=System.nanoTime()+TimeUnit.SECONDS.toNanos(20);
Thread.ofVirtual().start(()->{try(var stdin=running.getOutputStream()){stdin.write(payload);}catch(IOException error){output.completeExceptionally(error);}});
byte[] raw;
try {raw=output.get(Math.max(1,deadline-System.nanoTime()),TimeUnit.NANOSECONDS);}
catch(java.util.concurrent.TimeoutException|java.util.concurrent.ExecutionException error){throw new IOException("图表恢复未在时限内完成");}
if(raw.length>512*1024||!process.waitFor(Math.max(1,deadline-System.nanoTime()),TimeUnit.NANOSECONDS)||process.exitValue()!=0)
throw new IOException("原始结果暂不能恢复图表,已保留原分析");
try {
Map<String,Object> result=json.readValue(raw,Map.class);
if(result==null)throw new IOException("图表恢复组件未返回有效结果");
return result;
} catch(RuntimeException malformed) {throw new IOException("图表恢复组件未返回有效结果");}
} finally {
if(process!=null){active.remove(key,process);if(process.isAlive())terminate(process);}
visualRecoverySlots.release();
}
}
@PreDestroy
public void shutdown() {
active.values().forEach(this::terminate);
@@ -87,9 +123,9 @@ public class NaoAnalysisRunner {
cancellationRequested.clear();
}
private Result readProtocol(String turnId, Process process, BufferedWriter stdin,
private Result readProtocol(String turnId, String token, Process process, BufferedWriter stdin,
Consumer<Map<String, Object>> onProgress, Consumer<String> onAnswer,
BooleanSupplier databaseCancellation) throws IOException, InterruptedException {
BooleanSupplier databaseCancellation,String attemptId) throws IOException, InterruptedException {
BlockingQueue<String> lines = new ArrayBlockingQueue<>(128);
Thread stdoutReader = new Thread(() -> pumpStdout(process, lines), "nao-runtime-stdout-" + shortId(turnId));
Thread stderrReader = new Thread(() -> drainStderr(process), "nao-runtime-stderr-" + shortId(turnId));
@@ -101,6 +137,8 @@ public class NaoAnalysisRunner {
long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(WATCHDOG_MILLIS);
long cancelAt = 0;
boolean cancelSent = false;
boolean modelRevoked = false;
long nextModelCheck = 0;
String lastAnswer = "";
while (true) {
if (!cancelSent && (databaseCancellation.getAsBoolean() || cancellationRequested.contains(turnId))) {
@@ -108,6 +146,19 @@ public class NaoAnalysisRunner {
cancelSent = true;
cancelAt = System.nanoTime();
}
if (!cancelSent && System.nanoTime() >= nextModelCheck) {
try {
access.authorizeModel(token);
nextModelCheck = System.nanoTime() + TimeUnit.SECONDS.toNanos(1);
}
catch (RuntimeException revokedOrUnavailable) {
if(attemptId==null)access.revokeTurn(turnId);else access.revokeAttempt(attemptId);
writeCancel(stdin);
modelRevoked = true;
cancelSent = true;
cancelAt = System.nanoTime();
}
}
if (!cancelSent && System.nanoTime() >= deadline) throw new IOException("NAO runtime watchdog expired");
if (cancelSent && System.nanoTime() - cancelAt > TimeUnit.MILLISECONDS.toNanos(CANCEL_GRACE_MILLIS))
throw new IOException("NAO runtime did not stop after cancellation");
@@ -132,6 +183,7 @@ public class NaoAnalysisRunner {
onAnswer.accept(lastAnswer);
int exitCode = waitAfterResult(process);
Result result = Result.from(message, lastAnswer, cancelSent || databaseCancellation.getAsBoolean());
if (modelRevoked) return new Result("FAILED", result.answer(), result.details(), "模型服务授权已失效,本轮分析已停止");
return enforceExitStatus(result, exitCode);
}
default -> throw new IOException("NAO runtime emitted an unsupported message type");
@@ -169,6 +221,10 @@ public class NaoAnalysisRunner {
}
static Map<String, Object> safeProgress(Map<String, Object> message) {
if (message.containsKey("thinkingUpdate")) {
var thinking=AnalysisThinking.safe(message.get("thinkingUpdate"));
return thinking.isEmpty()?Map.of():Map.of("thinkingUpdate",thinking);
}
Map<String, Object> safe = new LinkedHashMap<>();
safe.put("message", cap(Objects.toString(message.getOrDefault("message", "原装 NAO 正在处理"), "原装 NAO 正在处理"), 500));
Object count = message.get("toolCount");
@@ -232,14 +288,18 @@ public class NaoAnalysisRunner {
}
private RuntimeLocation runtimeLocation() throws IOException {
String configuredRoot = firstNonBlank(System.getProperty("quality.project.root"), System.getenv("QUALITY_PROJECT_ROOT"));
Path root = configuredRoot == null ? findProjectRoot(Path.of("").toAbsolutePath()) : Path.of(configuredRoot).toAbsolutePath().normalize();
if (root == null || !Files.isDirectory(root)) throw new IOException("NAO project root was not found");
Path root=projectRoot();
String configuredScript = firstNonBlank(System.getProperty("quality.analysis.nao.script"), System.getenv("QUALITY_ANALYSIS_NAO_SCRIPT"));
Path script = configuredScript == null ? root.resolve("scripts/nao_quality_runtime.py") : Path.of(configuredScript).toAbsolutePath().normalize();
if (!Files.isRegularFile(script) || !Files.isReadable(script)) throw new IOException("NAO runtime script is missing or unreadable");
return new RuntimeLocation(root, script.toAbsolutePath().normalize());
}
private Path projectRoot() throws IOException {
String configuredRoot = firstNonBlank(System.getProperty("quality.project.root"), System.getenv("QUALITY_PROJECT_ROOT"));
Path root = configuredRoot == null ? findProjectRoot(Path.of("").toAbsolutePath()) : Path.of(configuredRoot).toAbsolutePath().normalize();
if (root == null || !Files.isDirectory(root)) throw new IOException("NAO project root was not found");
return root;
}
private Path findProjectRoot(Path start) {
Path candidate = start;
@@ -315,11 +375,54 @@ public class NaoAnalysisRunner {
if (cancellationRequested || "CANCELLED".equals(status)) {
status = "CANCELLED";
reason = "用户请求停止";
} else if ("NEEDS_CLARIFICATION".equals(status)) {
// A clarification is a complete protocol result, but it has not
// completed the business analysis. Do not reuse streamed prose
// or accept another finish reason as a clarification prompt.
if (!"result".equals(message.get("type"))
|| !"needs_clarification".equals(reason)
|| !(message.get("answer") instanceof String prompt) || prompt.isBlank()) {
status = "FAILED";
reason = "原装 NAO 返回了无效澄清请求";
}
} else if ("COMPLETED".equals(status) && !"stop".equalsIgnoreCase(reason)
&& !("evidence_finalized".equals(reason)
&& "bounded_evidence_finalization".equals(message.get("completionMethod"))
&& "stop".equals(message.get("finishReason"))
&& !Boolean.FALSE.equals(message.get("nativeStopSucceeded")))) {
&& !Boolean.FALSE.equals(message.get("nativeStopSucceeded")))
&& !("evidence_finalized".equals(reason)
&& "bounded_scoped_query_plan".equals(message.get("completionMethod"))
&& "stop".equals(message.get("finishReason"))
&& !Boolean.FALSE.equals(message.get("nativeStopSucceeded"))
&& message.get("agentTask") instanceof Map<?,?> scopedTask
&& scopedTask.get("queryPlan") instanceof List<?> nodes&&!nodes.isEmpty()&&nodes.size()<=8
&& message.get("protocol") instanceof Map<?,?> scopedIdentity&&scopedIdentity.get("executionId") instanceof String
&& message.get("protocolEvidence") instanceof List<?> scopedProofs&&!scopedProofs.isEmpty()
&& scopedProofs.stream().allMatch(raw->raw instanceof Map<?,?> proof&&proof.get("nodeId") instanceof String&&proof.get("scopeId") instanceof String)
&& message.get("evidenceCompletion") instanceof Map<?,?> scopedCoverage&&Boolean.TRUE.equals(scopedCoverage.get("serverCoverageSufficient")))
&& !("evidence_finalized".equals(reason)
&& "native_autonomous_analysis".equals(message.get("completionMethod"))
&& "stop".equals(message.get("finishReason"))
&& !Boolean.FALSE.equals(message.get("nativeStopSucceeded"))
&& ("stop".equals(message.get("nativeStopReason"))||"stop".equals(message.get("nativeFinishReason")))
&& message.get("agentTask") instanceof Map<?,?> autonomousTask&&"native_agent".equals(autonomousTask.get("executionMode"))
&& message.get("protocol") instanceof Map<?,?> autonomousIdentity&&autonomousIdentity.get("executionId") instanceof String
&& message.get("protocolEvidence") instanceof List<?> autonomousProofs&&!autonomousProofs.isEmpty()
&& autonomousProofs.stream().allMatch(raw->raw instanceof Map<?,?> proof&&proof.get("nodeId") instanceof String&&proof.get("scopeId") instanceof String)
&& message.get("evidenceCompletion") instanceof Map<?,?> autonomousCoverage&&Boolean.TRUE.equals(autonomousCoverage.get("serverCoverageSufficient")))
&& !("history_report_complete".equals(reason)
&& "confirmed_history_report".equals(message.get("completionMethod"))
&& "stop".equals(message.get("finishReason"))
&& message.get("agentTask") instanceof Map<?,?> reportTask&&"report".equals(reportTask.get("kind"))
&& message.get("historyEvidence") instanceof Map<?,?> importProof&&importProof.get("importId") instanceof String
&& message.get("historyCoverage") instanceof Map<?,?> historyCoverage&&Boolean.TRUE.equals(historyCoverage.get("complete")))
&& !("members_list_complete".equals(reason)
&& "bounded_authorized_member_list".equals(message.get("completionMethod"))
&& "stop".equals(message.get("finishReason"))
&& message.get("agentTask") instanceof Map<?,?> task&&"members".equals(task.get("kind"))
&& message.get("protocol") instanceof Map<?,?> identity&&identity.get("executionId") instanceof String
&& message.get("protocolEvidence") instanceof List<?> proofs&&!proofs.isEmpty()
&& message.get("evidenceCompletion") instanceof Map<?,?> coverage&&Boolean.TRUE.equals(coverage.get("serverCoverageSufficient")))) {
status = "PARTIAL";
if (reason.isBlank()) reason = "原装 NAO 未以 stop 原因正常结束";
} else if (!List.of("COMPLETED", "PARTIAL", "FAILED").contains(status)) {
@@ -22,7 +22,7 @@ import org.springframework.web.server.ResponseStatusException;
/** Short-lived, in-memory delegation for one native NAO turn. */
@Service
public class NaoRuntimeAccessService {
private static final Duration TOKEN_TTL = Duration.ofMinutes(12);
private static final Duration TOKEN_TTL = AnalysisRuntimePolicy.DELEGATION_TTL;
private static final SecureRandom RANDOM = new SecureRandom();
private final Map<String, Binding> tokens = new ConcurrentHashMap<>();
@@ -30,6 +30,10 @@ public class NaoRuntimeAccessService {
private final JdbcAppUserDetailsService users;
private final GovernanceAccessService governance;
private final Clock clock;
private AnalysisProtocolService protocol;
@Autowired
public void setProtocol(AnalysisProtocolService protocol) { this.protocol=protocol; }
@Autowired
public NaoRuntimeAccessService(JdbcTemplate jdbc, JdbcAppUserDetailsService users,
@@ -52,21 +56,42 @@ public class NaoRuntimeAccessService {
String token = Base64.getUrlEncoder().withoutPadding().encodeToString(bytes);
AppUserPrincipal current = freshPrincipal(actor.getId());
requireAnalysisAccess(current);
requireModelAccess(current);
String fingerprint = AuthorizationFingerprint.of(current, governance);
TurnState turn = turnState(turnId, current.getId());
if (!"RUNNING".equals(turn.status()) || turn.cancelRequested() || !fingerprint.equals(turn.authFingerprint()))
throw denied();
tokens.put(token, new Binding(current.getId(), fingerprint, releaseId, turnId, Instant.now(clock).plus(TOKEN_TTL)));
tokens.put(token, new Binding(current.getId(), fingerprint, releaseId, turnId, null, Instant.now(clock).plus(TOKEN_TTL)));
return token;
}
public String issueProtocol(AppUserPrincipal actor,String releaseId,String turnId,String attemptId) {
AppUserPrincipal current=freshPrincipal(actor.getId());requireAnalysisAccess(current);requireModelAccess(current);
var a=protocol.require(attemptId,current,false);
if(!a.turnId().equals(turnId)||!a.releaseId().equals(releaseId))throw denied();
byte[] bytes=new byte[32];RANDOM.nextBytes(bytes);
String token=Base64.getUrlEncoder().withoutPadding().encodeToString(bytes);
tokens.put(token,new Binding(current.getId(),AuthorizationFingerprint.of(current,governance),releaseId,turnId,attemptId,Instant.now(clock).plus(TOKEN_TTL)));
return token;
}
public Delegation authorize(String token) {
if (token == null || token.length() < 40 || token.length() > 80) throw denied();
Binding binding = tokens.get(token);
if (binding == null || binding.expiresAt().isBefore(Instant.now(clock))) {
if (binding == null || !Instant.now(clock).isBefore(binding.expiresAt())) {
tokens.remove(token);
throw denied();
}
if(binding.attemptId()!=null) {
try {
AppUserPrincipal actor=freshPrincipal(binding.actorId());requireAnalysisAccess(actor);
var a=protocol.require(binding.attemptId(),actor,false);
if(!binding.authFingerprint().equals(AuthorizationFingerprint.of(actor,governance))
||!a.turnId().equals(binding.turnId())||!a.releaseId().equals(binding.releaseId())
||!Instant.now(clock).isBefore(binding.expiresAt()))throw denied();
return new Delegation(actor,binding.releaseId(),binding.turnId(),binding.attemptId());
} catch(RuntimeException invalid) { tokens.remove(token);throw denied(); }
}
TurnState turn = turnState(binding.turnId(), binding.actorId());
if (!"RUNNING".equals(turn.status()) || turn.cancelRequested()
|| !binding.authFingerprint().equals(turn.authFingerprint())) {
@@ -83,6 +108,17 @@ public class NaoRuntimeAccessService {
return new Delegation(current, binding.releaseId(), binding.turnId());
}
/** Used by the native model gate before every provider request and by the worker watchdog. */
public void authorizeModel(String token) {
Delegation delegation = authorize(token);
requireModelAccess(delegation.actor());
}
public void requireModelAccess(AppUserPrincipal actor) {
if (actor.isForcePasswordChange()) throw denied();
governance.requireBoundAction("MODEL", "model.intent.parse");
}
private TurnState turnState(String turnId, UUID actorId) {
return jdbc.query("SELECT status,cancel_requested,auth_scope_hash FROM analysis_turn WHERE id=? AND user_id=?",
(rs, row) -> new TurnState(rs.getString(1), rs.getBoolean(2), rs.getString(3)), turnId, actorId.toString())
@@ -94,6 +130,10 @@ public class NaoRuntimeAccessService {
tokens.entrySet().removeIf(entry -> entry.getValue().turnId().equals(turnId));
}
public void revokeAttempt(String attemptId) {
if(attemptId!=null)tokens.entrySet().removeIf(e->attemptId.equals(e.getValue().attemptId()));
}
public void revokeAll() { tokens.clear(); }
private AppUserPrincipal freshPrincipal(UUID actorId) {
@@ -111,6 +151,7 @@ public class NaoRuntimeAccessService {
}
private void requireAnalysisAccess(AppUserPrincipal actor) {
if (actor.isForcePasswordChange()) throw denied();
governance.requireDetail(actor);
governance.requireBoundAction("QUERY", "query.detail");
}
@@ -124,7 +165,9 @@ public class NaoRuntimeAccessService {
return new ResponseStatusException(HttpStatus.FORBIDDEN, "NAO 查询授权已失效");
}
public record Delegation(AppUserPrincipal actor, String releaseId, String turnId) {}
private record Binding(UUID actorId, String authFingerprint, String releaseId, String turnId, Instant expiresAt) {}
public record Delegation(AppUserPrincipal actor, String releaseId, String turnId, String attemptId) {
public Delegation(AppUserPrincipal actor,String releaseId,String turnId){this(actor,releaseId,turnId,null);}
}
private record Binding(UUID actorId, String authFingerprint, String releaseId, String turnId, String attemptId, Instant expiresAt) {}
private record TurnState(String status, boolean cancelRequested, String authFingerprint) {}
}
@@ -12,6 +12,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import tools.jackson.databind.json.JsonMapper;
import org.springframework.http.HttpStatus;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
@@ -29,8 +30,8 @@ import jakarta.servlet.http.HttpServletRequest;
@RestController
@RequestMapping("/api/analysis/runtime")
public class NaoRuntimeQueryController {
private static final Set<String> METRIC_QUERY = Set.of("year", "month", "quarter", "department", "group",
"product", "projectGroup", "extraFilters", "view", "includeUnassigned", "releaseId", "trendGranularity",
private static final Set<String> METRIC_QUERY = Set.of("year", "month", "monthStart", "monthEnd", "quarter", "department", "group",
"product", "projectGroup", "extraFilters", "columnFilters", "exclusions", "view", "includeUnassigned", "releaseId", "trendGranularity",
"rowLimit", "rowOffset", "rowOrder");
private static final Set<String> TOP_QUERY = Set.of("stage", "dimension", "category", "releaseId", "year",
"month", "quarter", "department", "group", "product", "projectGroup", "extraFilters", "includeUnassigned", "limit");
@@ -43,6 +44,17 @@ public class NaoRuntimeQueryController {
private final AnalyticsController analytics;
private final AnalysisFineDefectsController defects;
private final AnalysisInvestigationController investigation;
private AnalysisProtocolService protocol;
private AnalysisSourceQueryController source;
private AnalysisChangesController changes;
private final JsonMapper protocolJson=JsonMapper.builder().build();
@org.springframework.beans.factory.annotation.Autowired
public void setProtocol(AnalysisProtocolService protocol){this.protocol=protocol;}
@org.springframework.beans.factory.annotation.Autowired(required=false)
public void setSource(AnalysisSourceQueryController source){this.source=source;}
@org.springframework.beans.factory.annotation.Autowired(required=false)
public void setChanges(AnalysisChangesController changes){this.changes=changes;}
public NaoRuntimeQueryController(NaoRuntimeAccessService access, ReleaseController releases,
OptionsController options, MetricsController metrics,
@@ -66,13 +78,48 @@ public class NaoRuntimeQueryController {
ParsedTarget target = parseTarget(request.method(), request.path());
if ("GET".equals(target.method()) && request.payload() != null) throw bad("GET 查询不能包含请求体");
if ("POST".equals(target.method()) && !target.query().isEmpty()) throw bad("POST 查询不能包含 URL 参数");
return asActor(delegation.actor(), () -> dispatch(target, request.payload(), delegation));
return asActor(delegation.actor(), () -> {
Map<String,String> parameters=target.method().equals("POST")&&Set.of("/api/analysis/source/members","/api/analysis/investigate","/api/analysis/defects","/api/analysis/changes").contains(target.path())
?Map.of("body",protocolJson.writeValueAsString(AnalysisProtocolService.canonical(request.payload()))):target.query();
if(delegation.attemptId()!=null)protocol.checkTarget(delegation.attemptId(),target.method(),target.path(),parameters,delegation.actor(),request.nodeId(),request.scopeId());
Object result=dispatch(target,request.payload(),delegation);
if(delegation.attemptId()!=null&&(target.method().equals("GET")&&target.path().equals("/api/metrics")
||target.method().equals("POST")&&Set.of("/api/analysis/source/members","/api/analysis/investigate","/api/analysis/defects","/api/analysis/changes").contains(target.path()))) {
Map<String,Object> proof=protocol.recordQuery(delegation.attemptId(),target.method(),target.path(),parameters,result,delegation.actor(),request.nodeId(),request.scopeId());
Map<String,Object> visible=new LinkedHashMap<>(protocolJson.readValue(protocolJson.writeValueAsString(result),Map.class));
visible.put("protocolEvidence",proof);return visible;
}
// Authorization is checked again after a potentially long catalog query.
if(delegation.attemptId()!=null&&!target.path().equals("/api/analysis/protocol/task"))access.authorize(request.token());
return result;
});
}
private Object dispatch(ParsedTarget target, Map<String, Object> payload,
NaoRuntimeAccessService.Delegation delegation) {
String releaseId = delegation.releaseId();
return switch (target.method() + " " + target.path()) {
case "POST /api/analysis/protocol/history" -> {
if(delegation.attemptId()==null||protocol==null||payload==null||!payload.isEmpty())throw bad("历史来源导入需要当前执行和空请求对象");
yield protocol.history(delegation.attemptId(),delegation.actor());
}
case "POST /api/analysis/protocol/exploration/steps" -> {
if(delegation.attemptId()==null||protocol==null)throw bad("自主步骤需要当前执行");
yield protocol.explorationStep(delegation.attemptId(),payload,delegation.actor());
}
case "POST /api/analysis/protocol/queries" -> {
if(delegation.attemptId()==null||protocol==null||payload==null||!payload.isEmpty())throw bad("查询节点解析需要当前执行和空请求对象");
yield protocol.queries(delegation.attemptId(),delegation.actor());
}
case "POST /api/analysis/protocol/task" -> {
if(delegation.attemptId()==null||protocol==null)throw bad("当前会话未启用统一协议");
yield protocol.confirm(delegation.attemptId(),payload,delegation.actor());
}
case "GET /api/analysis/model-authorization" -> {
rejectUnknown(target.query(), Set.of());
access.requireModelAccess(delegation.actor());
yield Map.of("authorized", true);
}
case "GET /api/releases" -> {
rejectUnknown(target.query(), Set.of());
yield releaseList(releaseId);
@@ -85,7 +132,22 @@ public class NaoRuntimeQueryController {
case "GET /api/metrics" -> metricQuery(target.query(), releaseId, delegation);
case "GET /api/analysis/tools/catalog" -> {
rejectUnknown(target.query(), Set.of());
yield tools.catalog(delegation.actor());
Map<String,Object> result=new LinkedHashMap<>(tools.catalog(delegation.actor()));
if(source!=null&&delegation.attemptId()!=null) {
result.putAll(source.catalog(releaseId,delegation.actor()));
result.put("limitations",List.of("源字段仅限当前固定发布及授权sourceFields,按列出的operators筛选;两个阶段独立筛选,不作原始记录跨表关联",
"清单仅投影明确选择的授权列并分页去重;正式指标仍由MetricsService按现有口径计算,不把原表比率列当作可平均的指标",
"字段筛选和清单能发现范围与记录,不能据此宣称生产根因;未接入的计算或操作明确停止"));
}
yield result;
}
case "POST /api/analysis/source/members" -> {
if(delegation.attemptId()==null||source==null)throw bad("字段清单需要统一任务协议");
yield source.members(boundPayload(payload,releaseId),delegation.actor());
}
case "POST /api/analysis/source/values" -> {
if(delegation.attemptId()==null||source==null)throw bad("来源值目录需要统一任务协议");
yield source.values(boundPayload(payload,releaseId),delegation.actor());
}
case "GET /api/analytics/top" -> topQuery(target.query(), releaseId, delegation);
case "POST /api/analysis/tools/details" -> {
@@ -93,6 +155,10 @@ public class NaoRuntimeQueryController {
yield tools.details(bound, delegation.actor());
}
case "POST /api/analysis/investigate" -> investigation.query(boundPayload(payload, releaseId), delegation.actor());
case "POST /api/analysis/changes" -> {
if(delegation.attemptId()==null||changes==null)throw bad("变化排名需要统一任务协议");
yield changes.query(boundPayload(payload,releaseId),delegation.actor());
}
case "POST /api/analysis/defects" -> {
Map<String, Object> bound = boundPayload(payload, releaseId);
yield defects.query(bound, delegation.actor());
@@ -115,12 +181,21 @@ public class NaoRuntimeQueryController {
NaoRuntimeAccessService.Delegation delegation) {
rejectUnknown(query, METRIC_QUERY);
requireBoundRelease(query.get("releaseId"), releaseId);
return metrics.calculate(integer(query, "year"), integer(query, "month"), query.get("quarter"),
if(query.containsKey("monthStart")||query.containsKey("monthEnd"))return metrics.calculateRange(
integer(query,"year"),integer(query,"month"),integer(query,"monthStart"),integer(query,"monthEnd"),query.get("quarter"),
query.get("department"),query.get("group"),query.get("product"),null,query.get("projectGroup"),
query.getOrDefault("view","overall"),bool(query,"includeUnassigned",false),releaseId,query.getOrDefault("trendGranularity","month"),
integer(query,"rowLimit"),integer(query,"rowOffset")==null?0:integer(query,"rowOffset"),query.get("rowOrder"),
query.get("extraFilters"),query.get("exclusions"),query.get("columnFilters"),delegation.actor());
if(!query.containsKey("columnFilters"))return metrics.calculate(integer(query, "year"), integer(query, "month"), query.get("quarter"),
query.get("department"), query.get("group"), query.get("product"), query.get("projectGroup"),
query.getOrDefault("view", "overall"), bool(query, "includeUnassigned", false), releaseId,
query.getOrDefault("trendGranularity", "month"), integer(query, "rowLimit"),
integer(query, "rowOffset") == null ? 0 : integer(query, "rowOffset"), query.get("rowOrder"), query.get("extraFilters"),
integer(query, "rowOffset") == null ? 0 : integer(query, "rowOffset"), query.get("rowOrder"), query.get("extraFilters"), query.get("exclusions"),
delegation.actor());
return metrics.calculate(integer(query,"year"),integer(query,"month"),query.get("quarter"),query.get("department"),query.get("group"),query.get("product"),null,query.get("projectGroup"),
query.getOrDefault("view","overall"),bool(query,"includeUnassigned",false),releaseId,query.getOrDefault("trendGranularity","month"),integer(query,"rowLimit"),
integer(query,"rowOffset")==null?0:integer(query,"rowOffset"),query.get("rowOrder"),query.get("extraFilters"),query.get("exclusions"),query.get("columnFilters"),delegation.actor());
}
private Object topQuery(Map<String, String> query, String releaseId,
@@ -227,6 +302,8 @@ public class NaoRuntimeQueryController {
return new ResponseStatusException(HttpStatus.BAD_REQUEST, message);
}
public record RuntimeQuery(String token, String method, String path, Map<String, Object> payload) {}
public record RuntimeQuery(String token, String method, String path, Map<String, Object> payload,String nodeId,String scopeId) {
public RuntimeQuery(String token,String method,String path,Map<String,Object> payload){this(token,method,path,payload,null,null);}
}
private record ParsedTarget(String method, String path, Map<String, String> query) {}
}
@@ -17,6 +17,7 @@ import cn.yunsheng.quality.release.ReleaseService;
import java.math.BigDecimal;
import java.math.MathContext;
import java.math.RoundingMode;
import java.util.Collections;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
@@ -66,6 +67,9 @@ public class AnalyticsController {
public Map<String, Object> top(String stage, String dimension, String category, String releaseId, Integer year, Integer month, Integer quarter, String department, String group, String product, String projectGroup, boolean includeUnassigned, int limit, AppUserPrincipal actor) {
return top(stage, dimension, category, releaseId, year, month, quarter, department, group, product, projectGroup, includeUnassigned, limit, null, actor);
}
public Map<String, Object> top(String stage, String dimension, String category, String releaseId, Integer year, Integer month, Integer quarter, String department, String group, String product, String projectGroup, boolean includeUnassigned, int limit, String extraFilters, AppUserPrincipal actor) {
return top(stage, dimension, category, releaseId, year, month, quarter, department, group, product, projectGroup, includeUnassigned, limit, extraFilters, null, actor);
}
@GetMapping("/top")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST')")
public Map<String, Object> top(@RequestParam String stage,
@@ -82,7 +86,9 @@ public class AnalyticsController {
@RequestParam(defaultValue = "false") boolean includeUnassigned,
@RequestParam(defaultValue = "20") int limit,
@RequestParam(required=false) String extraFilters,
@RequestParam(required=false) String exclusions,
@AuthenticationPrincipal AppUserPrincipal actor) {
if (exclusions != null) throw bad("top 暂不支持排除条件");
requireQueryDetail(actor);
validatePeriod(year, month, quarter);
if (!List.of("black", "bright").contains(stage)) throw bad("stage 必须是 black 或 bright");
@@ -154,7 +160,10 @@ public class AnalyticsController {
response.put("shareBasis", allComponentsVisible ? "所选筛选条件下选中原报废分类的重量总和"
: "所选筛选条件下已授权报废分类的重量总和"); response.put("limit", limit);
response.put("cumulativeThresholdApplied", false); response.put("rows", rows);
response.put("coverage", coverageSnapshot.coverage()); response.put("resultQuery", query);
var coverage = new LinkedHashMap<String, Object>(coverageSnapshot.coverage());
coverage.put("includedStageRows", Map.of("blackRows", coverageSnapshot.summary().coverage().getOrDefault("blackRows", 0L),
"brightRows", coverageSnapshot.summary().coverage().getOrDefault("brightRows", 0L)));
response.put("coverage", coverage); response.put("resultQuery", query);
String resultId = AnalyticsResultIds.top(query, actor, governance, json);
response.put("resultId", resultId);
jdbc.update("INSERT INTO analytics_result_snapshot(result_id,owner_user_id,auth_scope_hash,release_id,result_kind,query_json,result_json) VALUES(?,?,?,?, 'TOP',CAST(? AS JSON),CAST(? AS JSON)) ON DUPLICATE KEY UPDATE query_json=VALUES(query_json),result_json=VALUES(result_json),created_at=CURRENT_TIMESTAMP(6)",
@@ -714,11 +723,15 @@ public class AnalyticsController {
&& currentRule.equals(comparisonRule) && currentMapping.equals(comparisonMapping);
}
public static Map<String,String> formalCategoryFields(String stage) {
Map<String,String> fields=new LinkedHashMap<>();
if("black".equals(stage)) {fields.put("black_responsibility_scrap","责废");fields.put("black_material_scrap","料废");fields.put("black_external_scrap","外废");}
else if("bright".equals(stage)) {fields.put("bright_plating_scrap","镀废");fields.put("bright_material_scrap","材料废");fields.put("bright_processing_scrap","加工废");fields.put("bright_diffusion_scrap","扩散废");}
else throw new IllegalArgumentException("阶段无效");
return Collections.unmodifiableMap(fields);
}
private List<Component> components(String stage) {
if (stage.equals("black")) return List.of(new Component("black_responsibility_scrap", "责废"),
new Component("black_material_scrap", "料废"), new Component("black_external_scrap", "外废"));
return List.of(new Component("bright_plating_scrap", "镀废"), new Component("bright_material_scrap", "材料废"),
new Component("bright_processing_scrap", "加工废"), new Component("bright_diffusion_scrap", "扩散废"));
return formalCategoryFields(stage).entrySet().stream().map(entry->new Component(entry.getKey(),entry.getValue())).toList();
}
private String dimensionColumn(String dimension) {
@@ -1,6 +1,7 @@
package cn.yunsheng.quality.dictionary;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import cn.yunsheng.quality.metrics.MetricsService;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@@ -21,12 +22,33 @@ import tools.jackson.databind.json.JsonMapper;
public class MetricDictionaryService {
private final JdbcTemplate jdbc;
private final JsonMapper json;
private final MetricsService metrics;
private static final Set<String> CODES = Set.of("COMPOSITE_RATE", "BLACK_RATE", "BRIGHT_RATE",
"BRIGHT_RATE_WITHOUT_HEADQUARTERS_SCRAP", "COMPOSITE_RATE_WITHOUT_HEADQUARTERS_SCRAP");
public MetricDictionaryService(JdbcTemplate jdbc, JsonMapper json) { this.jdbc = jdbc; this.json = json; }
public MetricDictionaryService(JdbcTemplate jdbc, JsonMapper json, MetricsService metrics) { this.jdbc = jdbc; this.json = json; this.metrics = metrics; }
public Map<String, Object> list() { return Map.of("items", rows(false)); }
@Transactional(readOnly = true, isolation = org.springframework.transaction.annotation.Isolation.REPEATABLE_READ)
public Map<String, Object> list() {
List<Map<String, Object>> entries = rows(false);
List<String> releases = jdbc.query("SELECT release_id FROM current_release WHERE dataset_id='quality'", (rs, n) -> rs.getString(1));
if (releases.isEmpty()) return effectiveEntries(entries, Map.of(), "NO_CURRENT_DATA");
// Reuse the exact release-bound engine used by queries; active drafts are not authoritative.
return effectiveEntries(entries, metrics.rulesForRelease(releases.getFirst()).formulaDescriptions(), "CURRENT");
}
private Map<String, Object> effectiveEntries(List<Map<String, Object>> entries, Map<String, String> formulas, String status) {
Map<String, String> keys = Map.of("BLACK_RATE", "blackRate", "BRIGHT_RATE", "brightRate",
"COMPOSITE_RATE", "compositeRate", "BRIGHT_RATE_WITHOUT_HEADQUARTERS_SCRAP", "brightRateWithoutHeadquartersScrap",
"COMPOSITE_RATE_WITHOUT_HEADQUARTERS_SCRAP", "compositeRateWithoutHeadquartersScrap");
return Map.of("items", entries.stream().map(entry -> {
Map<String, Object> item = new LinkedHashMap<>(entry);
String formula = formulas.get(keys.get(String.valueOf(entry.get("code"))));
item.put("formulaStatus", formula == null && status.equals("CURRENT") ? "UNAVAILABLE" : status);
item.put("currentFormula", formula == null ? "" : formula);
return item;
}).toList());
}
private List<Map<String, Object>> rows(boolean lock) {
return jdbc.query("SELECT metric_code,display_name,aliases,description,row_version,updated_at FROM metric_dictionary ORDER BY metric_code" + (lock ? " FOR UPDATE" : ""), (rs, n) -> {
@@ -109,11 +131,11 @@ public class MetricDictionaryService {
private ResponseStatusException bad(String detail) { return new ResponseStatusException(HttpStatus.BAD_REQUEST, detail); }
private String formula(String code) {
return switch (code) {
case "COMPOSITE_RATE" -> "原黑片合格率 × 原亮片合格率;期间先汇总原分量,完整保留原调整项及空分母语义";
case "BLACK_RATE" -> "原黑片合格重量 ÷ 原黑片适用总重量(原扣除/回用/外废逻辑保持;合法结果可超过100%)";
case "BRIGHT_RATE" -> "原亮片合格重量 ÷ 原亮片适用总重量(原来料/本部/报废条件保持)";
case "BRIGHT_RATE_WITHOUT_HEADQUARTERS_SCRAP" -> "原模型亮片不含本部报废的调整度量";
case "COMPOSITE_RATE_WITHOUT_HEADQUARTERS_SCRAP" -> "原黑片合格率 × 原亮片不含本部报废度量";
case "COMPOSITE_RATE" -> "黑片合格率 × 亮片合格率;月、季、年先汇总重量再计算,不平均月度合格率";
case "BLACK_RATE" -> "黑片合格重量 ÷ 黑片适用总重量;适用总重量扣除利用料及废品库相关重量,结果允许超过100%;分母为零或空时取1";
case "BRIGHT_RATE" -> "适用亮片合格重量 ÷(适用亮片合格重量 + 适用亮片报废重量);仅统计来料标志为“是”的记录,分母为零或空时取1";
case "BRIGHT_RATE_WITHOUT_HEADQUARTERS_SCRAP" -> "(适用亮片合格重量 + 当前范围全部本部报废重量)÷ 亮片适用总重量;分母不变";
case "COMPOSITE_RATE_WITHOUT_HEADQUARTERS_SCRAP" -> "黑片合格率 × 去本部报废后的亮片合格率";
default -> throw bad("指标代码无效");
};
}
@@ -0,0 +1,819 @@
package cn.yunsheng.quality.filters;
import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.springframework.http.HttpStatus;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import tools.jackson.databind.json.JsonMapper;
/** Actor-scoped catalog and parameterized filters for published source columns. */
@Service
public class SourceColumnQueries {
private static final List<String> STAGES = List.of("black", "bright");
private static final List<String> OPERATORS = List.of("in", "not_in", "is_empty", "not_empty");
private static final Map<String, String> CANONICAL_COLUMNS = Map.of(
"product", "product", "department", "department", "group", "group_name", "projectGroup", "project_group");
private static final Map<String, String> CANONICAL_LABELS = Map.of(
"product", "产品", "department", "部门", "group", "组别", "projectGroup", "项目组");
private final JdbcTemplate jdbc;
private final GovernanceAccessService governance;
private final JsonMapper json;
public SourceColumnQueries(JdbcTemplate jdbc, GovernanceAccessService governance, JsonMapper json) {
this.jdbc = jdbc;
this.governance = governance;
this.json = json;
}
/**
* Lists visible source headers from the representative row of each release build.
* The raw header list is constrained by both actor data scope and current field authorization.
*/
public List<Map<String, Object>> catalog(String releaseId, AppUserPrincipal actor) {
requireQueryAccess(actor);
return scopedCatalog(releaseId,actor);
}
List<Map<String,Object>> scopedCatalog(String releaseId,AppUserPrincipal actor) {
if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效");
Set<String> allowed = governance.allowedFields(actor);
Map<String, Set<String>> headersByStage = new LinkedHashMap<>();
Map<String, Boolean> stagePresent = new LinkedHashMap<>();
for (String stage : STAGES) {
Set<String> headers = new LinkedHashSet<>();
StringBuilder representativeScope = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>(List.of(releaseId));
DataScope.append(representativeScope, args, actor,
"f2.department COLLATE utf8mb4_0900_as_cs", "f2.group_name COLLATE utf8mb4_0900_as_cs");
String sql = "SELECT CAST(JSON_KEYS(f.raw_values) AS CHAR) "
+ "FROM (SELECT f2.build_id,f2.source_revision_id,MIN(f2.id) AS representative_id "
+ "FROM release_member rm JOIN fact_" + stage + " f2 "
+ "ON f2.build_id=rm.transform_build_id AND f2.source_revision_id=rm.source_revision_id"
+ representativeScope + " GROUP BY f2.build_id,f2.source_revision_id) representative "
+ "JOIN fact_" + stage + " f ON f.id=representative.representative_id ORDER BY representative.build_id";
List<String> keyLists = jdbc.query(sql, (rs, row) -> rs.getString(1), args.toArray());
stagePresent.put(stage, !keyLists.isEmpty());
for (String rawKeys : keyLists) {
if (rawKeys == null) continue;
try {
Object decoded = json.readValue(rawKeys, Object.class);
if (decoded instanceof List<?> keys) {
for (Object key : keys) {
if (key instanceof String name && (allowed.contains("*") || allowed.contains(name))) headers.add(name);
}
}
} catch (Exception e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "来源列目录无法解析");
}
}
headersByStage.put(stage, headers);
}
List<ConfiguredMapping> mappings = enabledMappings();
List<ConfiguredMapping> usableMappings = usableMappings(mappings, headersByStage, allowed);
Map<String, Integer> blackUses = new HashMap<>(), brightUses = new HashMap<>();
for (ConfiguredMapping mapping : usableMappings) {
blackUses.merge(mapping.blackColumn(), 1, Integer::sum);
brightUses.merge(mapping.brightColumn(), 1, Integer::sum);
}
List<Map<String, Object>> result = new ArrayList<>();
Set<String> consumedBlack = new HashSet<>(), consumedBright = new HashSet<>();
for (ConfiguredMapping mapping : usableMappings) {
// A same-named header always keeps its name-derived key. Avoid an ambiguous cross-name override.
if (blackUses.get(mapping.blackColumn()) != 1 || brightUses.get(mapping.brightColumn()) != 1
|| headersByStage.get("bright").contains(mapping.blackColumn())
|| headersByStage.get("black").contains(mapping.brightColumn())) continue;
consumedBlack.add(mapping.blackColumn());
consumedBright.add(mapping.brightColumn());
result.add(sourceEntry(mappingKey(mapping.blackColumn(), mapping.brightColumn()), mapping.label(),
mapping.blackColumn(), mapping.brightColumn(), true, true));
}
Set<String> allHeaders = new LinkedHashSet<>();
allHeaders.addAll(headersByStage.get("black"));
allHeaders.addAll(headersByStage.get("bright"));
for (String header : allHeaders) {
boolean black = headersByStage.get("black").contains(header) && !consumedBlack.contains(header);
boolean bright = headersByStage.get("bright").contains(header) && !consumedBright.contains(header);
if (black || bright) result.add(sourceEntry(sourceKey(header), header, black ? header : null, bright ? header : null, black, bright));
}
for (String field : List.of("product", "department", "group", "projectGroup")) {
result.add(dimensionEntry(field, CANONICAL_LABELS.get(field), CANONICAL_COLUMNS.get(field),
stagePresent.get("black"), stagePresent.get("bright")));
}
return List.copyOf(result);
}
/** Parse the public columnFilters payload without coercing values to numbers or trimming them. */
public static List<Map<String, Object>> parse(Object raw) {
if (raw == null || "".equals(raw)) return List.of();
if (raw instanceof String text) {
if (text.length() > 32000) throw bad("来源列筛选条件过长");
try {
raw = JsonMapper.builder().build().readValue(text, Object.class);
} catch (Exception e) {
throw bad("来源列筛选条件格式无效");
}
}
if (!(raw instanceof List<?> rules) || rules.size() > 20) throw bad("来源列筛选条件必须是数组,最多20项");
List<Map<String, Object>> parsed = new ArrayList<>();
for (Object item : rules) {
if (!(item instanceof Map<?, ?> rule)) throw bad("来源列筛选项格式无效");
for (Object key : rule.keySet()) {
if (!(key instanceof String name) || !Set.of("field", "stages", "operator", "values").contains(name))
throw bad("来源列筛选项包含未知属性");
}
Object rawField = rule.get("field"), rawStages = rule.get("stages"), rawOperator = rule.get("operator");
if (!(rawField instanceof String field) || field.isBlank() || field.length() > 100
|| !(rawStages instanceof List<?> stages) || stages.isEmpty() || stages.size() > STAGES.size()
|| !(rawOperator instanceof String operator) || !OPERATORS.contains(operator)) {
throw bad("来源列、阶段或操作符无效");
}
List<String> normalizedStages = new ArrayList<>();
for (Object stage : stages) {
if (!(stage instanceof String value) || !STAGES.contains(value) || normalizedStages.contains(value))
throw bad("来源列筛选阶段无效或重复");
normalizedStages.add(value);
}
List<String> values = new ArrayList<>();
if (operator.equals("in") || operator.equals("not_in")) {
if (!(rule.get("values") instanceof List<?> rawValues) || rawValues.isEmpty() || rawValues.size() > 50)
throw bad("in/not_in 筛选必须提供1到50个字符串值");
for (Object value : rawValues) {
if (!(value instanceof String text) || text.isEmpty() || text.length() > 256)
throw bad("来源列筛选值必须是1到256字符的完整字符串;空值请使用空值操作符");
if (!values.contains(text)) values.add(text);
}
} else if (rule.containsKey("values")) {
throw bad("is_empty/not_empty 不接受 values");
}
Map<String, Object> normalized = new LinkedHashMap<>();
normalized.put("field", field);
normalized.put("stages", List.copyOf(normalizedStages));
normalized.put("operator", operator);
if (operator.equals("in") || operator.equals("not_in")) normalized.put("values", List.copyOf(values));
parsed.add(Collections.unmodifiableMap(normalized));
}
return List.copyOf(parsed);
}
/** Rejects keys revoked from the current actor and any stage without the requested column. */
public void validate(String releaseId, List<Map<String, Object>> filters, AppUserPrincipal actor) {
validatedCatalog(releaseId,filters,actor);
}
public List<Map<String,Object>> validatedCatalog(String releaseId,List<Map<String,Object>> filters,AppUserPrincipal actor) {
requireQueryAccess(actor);
return List.copyOf(validateAndCatalog(releaseId,filters,actor).fields().values());
}
/**
* Appends one stage's portion of a validated filter set. Rules are ANDed; each in/not_in values list
* is handled as one SQL set. The enclosing MetricsService/source-query entry already enforces live
* service authorization. Source values are trimmed like QueryFilterFields; submitted strings stay exact.
*/
public void append(StringBuilder where, List<Object> args, String stage,
List<Map<String, Object>> filters, String releaseId, AppUserPrincipal actor) {
if (!STAGES.contains(stage)) throw bad("数据阶段无效");
Validated validated = validateAndCatalog(releaseId, filters, actor);
for (Map<String, Object> rule : validated.filters()) {
@SuppressWarnings("unchecked") List<String> stages = (List<String>) rule.get("stages");
if (!stages.contains(stage)) continue;
String key = (String) rule.get("field");
Map<String, Object> field = validated.fields().get(key);
String operator = (String) rule.get("operator");
String expression;
Object column = ((Map<?, ?>) field.get("columns")).get(stage);
if ("source_value".equals(field.get("type"))) {
expression = "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL "
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
+ "COLLATE utf8mb4_0900_as_cs";
} else {
String factColumn = CANONICAL_COLUMNS.get(key);
if (factColumn == null) throw bad("规范维度无效");
expression = "NULLIF(TRIM(f." + factColumn + "), '') COLLATE utf8mb4_0900_as_cs";
}
where.append(" AND (");
if (operator.equals("in") || operator.equals("not_in")) {
@SuppressWarnings("unchecked") List<String> values = (List<String>) rule.get("values");
if (operator.equals("in")) {
where.append(expression).append(" IN (")
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append(')');
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column));
} else {
where.append('(').append(expression).append(" IS NULL OR ").append(expression).append(" NOT IN (")
.append(String.join(",", Collections.nCopies(values.size(), "?"))).append("))");
if ("source_value".equals(field.get("type"))) {
bindSourcePaths(args, String.valueOf(column));
bindSourcePaths(args, String.valueOf(column));
}
}
args.addAll(values);
} else {
where.append(expression).append(operator.equals("is_empty") ? " IS NULL" : " IS NOT NULL");
if ("source_value".equals(field.get("type"))) bindSourcePaths(args, String.valueOf(column));
}
where.append(')');
}
}
/** Returns an exact, stable page of distinct tuples from the selected source stages. */
public Map<String, Object> memberPage(String releaseId, Map<String, Object> context,
List<String> memberFields, List<String> stages,
int limit, int offset, AppUserPrincipal actor) {
requireQueryAccess(actor);
governance.requireDetail(actor);
governance.requireBoundAction("QUERY", "query.detail");
return snapshot(()->memberSnapshot(releaseId,context,memberFields,stages,limit,offset,actor));
}
private Map<String,Object> memberSnapshot(String releaseId,Map<String,Object> context,
List<String> memberFields,List<String> stages,int limit,int offset,AppUserPrincipal actor) {
if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效");
if (memberFields == null || memberFields.isEmpty() || memberFields.size() > 5
|| memberFields.stream().anyMatch(Objects::isNull)
|| new HashSet<>(memberFields).size() != memberFields.size())
throw bad("memberFields 必须选择1到5个不重复字段");
if (stages == null || stages.isEmpty() || stages.size() > STAGES.size()
|| stages.stream().anyMatch(s -> !STAGES.contains(s))
|| new HashSet<>(stages).size() != stages.size())
throw bad("selectedStages 必须是 black、bright 中的不重复项");
if (limit < 1 || limit > 200 || offset < 0 || offset > 1_000_000) throw bad("成员分页参数无效");
QueryContext queryContext = queryContext(context);
validateSelectedFilterStages(queryContext.columnFilters(), stages);
List<Map<String, Object>> availableFields = scopedCatalog(releaseId, actor);
Map<String, Map<String, Object>> byKey = new LinkedHashMap<>();
for (Map<String, Object> field : availableFields) byKey.put((String) field.get("key"), field);
for (String fieldKey : memberFields) {
Map<String, Object> field = byKey.get(fieldKey);
if (field == null) throw bad("成员字段不存在、未发布或没有权限:" + fieldKey);
@SuppressWarnings("unchecked") List<String> fieldStages = (List<String>) field.get("stages");
@SuppressWarnings("unchecked") Map<String, Object> columns = (Map<String, Object>) field.get("columns");
for (String stage : stages) {
if (!fieldStages.contains(stage) || columns.get(stage) == null)
throw bad("成员字段不支持所选阶段:" + fieldKey + " / " + stage);
}
}
List<Object> queryArgs = new ArrayList<>();
List<String> selects = new ArrayList<>();
for (String stage : stages) {
List<Object> projectionArgs = new ArrayList<>();
List<String> projections = new ArrayList<>();
for (int index = 0; index < memberFields.size(); index++) {
String fieldKey = memberFields.get(index);
Map<String, Object> field = byKey.get(fieldKey);
Object column = ((Map<?, ?>) field.get("columns")).get(stage);
projections.add(memberExpression(fieldKey, field, String.valueOf(column), projectionArgs) + " AS c" + index);
}
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> whereArgs = new ArrayList<>(List.of(releaseId));
appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor);
String fact = stage.equals("black") ? "fact_black" : "fact_bright";
selects.add("SELECT DISTINCT " + String.join(",", projections) + ", '" + stage + "' AS member_stage "
+ "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id "
+ "JOIN " + fact + " f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"
+ where);
queryArgs.addAll(projectionArgs);
queryArgs.addAll(whereArgs);
}
String unionSql = String.join(" UNION ALL ", selects);
String tupleColumns = tupleColumns(memberFields.size());
String grouped = "SELECT " + tupleColumns + " FROM (" + unionSql + ") member_rows GROUP BY " + tupleColumns;
Long total = jdbc.queryForObject("SELECT COUNT(*) FROM (" + grouped + ") member_tuples", Long.class, queryArgs.toArray());
List<Object> pageArgs = new ArrayList<>(queryArgs);
pageArgs.add(limit);
pageArgs.add(offset);
String pageSql = "SELECT " + tupleColumns + ", GROUP_CONCAT(DISTINCT member_stage ORDER BY member_stage SEPARATOR ',') AS member_stages "
+ "FROM (" + unionSql + ") member_rows GROUP BY " + tupleColumns + " ORDER BY " + tupleColumns + " LIMIT ? OFFSET ?";
List<Map<String, Object>> rows = jdbc.query(pageSql, (rs, row) -> {
Map<String, Object> values = new LinkedHashMap<>();
for (int index = 0; index < memberFields.size(); index++) values.put(memberFields.get(index), rs.getString("c" + index));
String joinedStages = rs.getString("member_stages");
List<String> rowStages = joinedStages == null || joinedStages.isEmpty() ? List.of()
: Arrays.stream(joinedStages.split(",")).filter(STAGES::contains).toList();
Map<String, Object> result = new LinkedHashMap<>();
result.put("values", Collections.unmodifiableMap(values));
result.put("stages", rowStages);
return Collections.unmodifiableMap(result);
}, pageArgs.toArray());
long totalCount = total == null ? 0 : total;
Map<String, Object> filters = queryContext.filtersResponse();
Map<String, Object> response = new LinkedHashMap<>();
response.put("rows", List.copyOf(rows));
response.put("totalCount", totalCount);
response.put("offset", offset);
response.put("limit", limit);
boolean hasMore = (long) offset + rows.size() < totalCount;
response.put("hasMore", hasMore);
response.put("complete", offset == 0 && !hasMore);
response.put("pageComplete", true);
response.put("releaseId", releaseId);
response.put("filters", Collections.unmodifiableMap(filters));
response.put("period", Collections.unmodifiableMap(queryContext.period()));
response.put("memberFields", List.copyOf(memberFields));
response.put("selectedStages", List.copyOf(stages));
return Collections.unmodifiableMap(response);
}
/** Searches authorized values before grouping, counting, ordering, and paging them in SQL. */
public Map<String, Object> values(String releaseId, Map<String, Object> context, String fieldKey,
List<String> stages, String search, int limit, int offset,
AppUserPrincipal actor) {
requireQueryAccess(actor);
return snapshot(()->valuesSnapshot(releaseId,context,fieldKey,stages,search,limit,offset,actor));
}
private Map<String,Object> valuesSnapshot(String releaseId,Map<String,Object> context,String fieldKey,
List<String> stages,String search,int limit,int offset,AppUserPrincipal actor) {
if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效");
if (fieldKey == null || fieldKey.isBlank()) throw bad("来源列标识无效");
if (stages == null || stages.isEmpty() || stages.size() > STAGES.size()
|| stages.stream().anyMatch(stage -> !STAGES.contains(stage))
|| new HashSet<>(stages).size() != stages.size())
throw bad("selectedStages 必须是 black、bright 中的不重复项");
if (search != null && search.length() > 256) throw bad("搜索文本过长");
if (limit < 1 || limit > 200 || offset < 0 || offset > 1_000_000) throw bad("候选值分页参数无效");
QueryContext queryContext = queryContext(context);
validateSelectedFilterStages(queryContext.columnFilters(), stages);
Map<String, Object> field = scopedCatalog(releaseId, actor).stream()
.filter(item -> fieldKey.equals(item.get("key"))).findFirst()
.orElseThrow(() -> bad("来源列不存在、未发布或没有权限:" + fieldKey));
@SuppressWarnings("unchecked") List<String> availableStages = (List<String>) field.get("stages");
@SuppressWarnings("unchecked") Map<String, Object> columns = (Map<String, Object>) field.get("columns");
for (String stage : stages) {
if (!availableStages.contains(stage) || columns.get(stage) == null)
throw bad("来源列在指定阶段不存在:" + fieldKey + " / " + stage);
}
List<Object> unionArgs = new ArrayList<>();
List<String> selects = new ArrayList<>();
for (String stage : stages) {
List<Object> projectionArgs = new ArrayList<>();
String expression = candidateValueExpression(fieldKey, field,
String.valueOf(columns.get(stage)), projectionArgs);
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> whereArgs = new ArrayList<>(List.of(releaseId));
appendContextFilters(where, whereArgs, stage, releaseId, queryContext, actor);
String fact = stage.equals("black") ? "fact_black" : "fact_bright";
selects.add("SELECT " + expression + " AS candidate_value "
+ "FROM release_member rm JOIN source_revision sr ON sr.id=rm.source_revision_id "
+ "JOIN " + fact + " f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id"
+ where);
unionArgs.addAll(projectionArgs);
unionArgs.addAll(whereArgs);
}
String union = String.join(" UNION ALL ", selects);
StringBuilder condition = new StringBuilder(" WHERE candidate_value IS NOT NULL");
List<Object> searchArgs = new ArrayList<>();
if (search != null && !search.isBlank()) {
condition.append(" AND LOCATE(LOWER(?),LOWER(candidate_value))>0");
searchArgs.add(search);
}
String candidates = "SELECT candidate_value FROM (" + union + ") source_values" + condition;
String grouped = "SELECT candidate_value FROM (" + candidates + ") matches GROUP BY candidate_value";
Long total = jdbc.queryForObject("SELECT COUNT(*) FROM (" + grouped + ") distinct_values", Long.class,
combine(unionArgs, searchArgs).toArray());
List<Object> pageArgs = combine(unionArgs, searchArgs);
pageArgs.add(limit);
pageArgs.add(offset);
List<Map<String, Object>> rows = jdbc.query("SELECT candidate_value FROM (" + candidates + ") matches "
+ "GROUP BY candidate_value ORDER BY candidate_value LIMIT ? OFFSET ?",
(rs, row) -> Map.of("value", rs.getString("candidate_value")),
pageArgs.toArray());
long totalCount = total == null ? 0 : total;
Map<String, Object> response = new LinkedHashMap<>();
response.put("rows", List.copyOf(rows));
response.put("totalCount", totalCount);
response.put("offset", offset);
response.put("limit", limit);
boolean hasMore = (long) offset + rows.size() < totalCount;
response.put("hasMore", hasMore);
response.put("complete", offset == 0 && !hasMore);
response.put("pageComplete", true);
response.put("releaseId", releaseId);
response.put("field", fieldKey);
response.put("selectedStages", List.copyOf(stages));
response.put("search", search == null ? "" : search);
return Collections.unmodifiableMap(response);
}
private Validated validateAndCatalog(String releaseId, List<Map<String, Object>> filters, AppUserPrincipal actor) {
List<Map<String, Object>> parsed = parse(filters);
List<Map<String, Object>> catalog = scopedCatalog(releaseId, actor);
Map<String, Map<String, Object>> fields = new LinkedHashMap<>();
for (Map<String, Object> field : catalog) fields.put((String) field.get("key"), field);
for (Map<String, Object> rule : parsed) {
String key = (String) rule.get("field");
Map<String, Object> field = fields.get(key);
if (field == null) throw bad("来源列筛选字段不存在、未发布或没有权限:" + key);
@SuppressWarnings("unchecked") List<String> stages = (List<String>) rule.get("stages");
@SuppressWarnings("unchecked") Map<String, Object> columns = (Map<String, Object>) field.get("columns");
@SuppressWarnings("unchecked") List<String> availableStages = (List<String>) field.get("stages");
for (String stage : stages) {
if (!availableStages.contains(stage) || columns.get(stage) == null)
throw bad("来源列在指定阶段不存在:" + key + " / " + stage);
}
}
return new Validated(parsed, fields);
}
private void requireQueryAccess(AppUserPrincipal actor) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "缺少用户身份");
governance.requireUserActivePolicy(actor);
governance.requireBoundAction("QUERY", "query.aggregate");
}
private <T> T snapshot(java.util.function.Supplier<T> read) {
if(jdbc.getDataSource()==null)return read.get();
// Live service authorization precedes the business snapshot. No
// global authorization row lock is retained during source scanning;
// the delegated bridge reauthorizes again before recording evidence.
var tx=new org.springframework.transaction.support.TransactionTemplate(
new org.springframework.jdbc.datasource.DataSourceTransactionManager(jdbc.getDataSource()));
tx.setPropagationBehavior(org.springframework.transaction.TransactionDefinition.PROPAGATION_REQUIRES_NEW);
tx.setIsolationLevel(org.springframework.transaction.TransactionDefinition.ISOLATION_REPEATABLE_READ);
tx.setReadOnly(true);
return tx.execute(status->read.get());
}
private QueryContext queryContext(Map<String, Object> context) {
return normalizedQueryContext(context);
}
/**
* Normalizes the shared source-query scope for read-only callers which already completed
* their endpoint-specific authorization. The returned map is also suitable for context echo.
*/
public static Map<String, Object> normalizedScope(Map<String, Object> scope) {
QueryContext context = normalizedQueryContext(scope);
Map<String, Object> normalized = new LinkedHashMap<>();
normalized.putAll(context.period());
for (String key : List.of("department", "group", "product", "projectGroup")) {
normalized.put(key, parseSelections(context.dimensions().get(key), key));
}
normalized.put("extraFilters", context.extraFilters());
normalized.put("columnFilters", context.columnFilters());
normalized.put("exclusions", context.exclusions());
normalized.put("includeUnassigned", context.includeUnassigned());
return Collections.unmodifiableMap(normalized);
}
/**
* Appends the same complete, parameterized scope used by source members and values queries.
* Callers must enforce their own endpoint authorization before invoking this method.
*/
public void appendAuthorizedScope(StringBuilder where, List<Object> args, String stage,
String releaseId, Map<String, Object> scope,
AppUserPrincipal actor) {
if (!STAGES.contains(stage)) throw bad("数据阶段无效");
if (releaseId == null || releaseId.isBlank()) throw bad("正式版标识无效");
QueryContext context = normalizedQueryContext(scope);
// A shared scope may carry rules for both facts; validate and append this stage's rules only.
List<Map<String, Object>> stageFilters = context.columnFilters().stream()
.filter(rule -> ((List<?>) rule.get("stages")).contains(stage)).toList();
QueryContext stageContext = new QueryContext(context.period(), context.dimensions(), context.extraFilters(),
stageFilters, context.exclusions(), context.includeUnassigned());
appendContextFilters(where, args, stage, releaseId, stageContext, actor);
}
private static QueryContext normalizedQueryContext(Map<String, Object> scope) {
Map<String, Object> safe = scope == null ? Map.of() : scope;
Map<String, Object> nested = objectMap(safe.get("filters"));
Map<String, Object> period = period(safe);
Map<String, Object> dimensions = new LinkedHashMap<>();
for (String key : List.of("department", "group", "product", "projectGroup")) {
Object value = safe.containsKey(key) ? safe.get(key) : nested.get(key);
if (value != null) dimensions.put(key, value);
}
Object rawExtra = safe.containsKey("extraFilters") ? safe.get("extraFilters") : nested.get("extraFilters");
Object rawColumn = safe.containsKey("columnFilters") ? safe.get("columnFilters") : nested.get("columnFilters");
Object rawExclusions = safe.containsKey("exclusions") ? safe.get("exclusions") : nested.get("exclusions");
Object rawUnassigned = safe.containsKey("includeUnassigned") ? safe.get("includeUnassigned") : nested.get("includeUnassigned");
return new QueryContext(period, dimensions, QueryFilterFields.parse(rawExtra), parse(rawColumn),
parseExclusions(rawExclusions), boolValue(rawUnassigned, false));
}
private void appendContextFilters(StringBuilder where, List<Object> args, String stage, String releaseId,
QueryContext context, AppUserPrincipal actor) {
appendPeriod(where, args, context.period());
for (String key : List.of("department", "group", "product", "projectGroup"))
appendDimensionSelection(where, args, key, context.dimensions().get(key));
appendExclusions(where, args, context.exclusions());
List<String> groups = parseSelections(context.dimensions().get("group"), "group");
if (!context.includeUnassigned() && !groups.contains("__UNASSIGNED__"))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
new QueryFilterFields(jdbc, governance).append(where, args, stage, context.extraFilters(), actor);
if (!context.columnFilters().isEmpty()) append(where, args, stage, context.columnFilters(), releaseId, actor);
DataScope.append(where, args, actor,
"f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
}
private void validateSelectedFilterStages(List<Map<String, Object>> filters, List<String> stages) {
for (Map<String, Object> rule : filters) {
@SuppressWarnings("unchecked") List<String> ruleStages = (List<String>) rule.get("stages");
if (!stages.containsAll(ruleStages)) throw bad("columnFilters 阶段必须包含在 selectedStages 中");
}
}
private String candidateValueExpression(String fieldKey, Map<String, Object> field, String column,
List<Object> args) {
if ("dimension".equals(field.get("type"))) {
String factColumn = CANONICAL_COLUMNS.get(fieldKey);
if (factColumn == null) throw bad("候选值规范维度无效");
return "NULLIF(TRIM(f." + factColumn + "), '') COLLATE utf8mb4_0900_as_cs";
}
String path = jsonPath(column);
args.add(path);
args.add(path);
args.add(path);
return "CASE WHEN JSON_EXTRACT(f.raw_values, ?) IS NULL "
+ "OR JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE NULLIF(TRIM(JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?))), '') END "
+ "COLLATE utf8mb4_0900_as_cs";
}
private static List<Object> combine(List<Object> first, List<Object> second) {
List<Object> combined = new ArrayList<>(first.size() + second.size());
combined.addAll(first);
combined.addAll(second);
return combined;
}
private String memberExpression(String fieldKey, Map<String, Object> field, String column, List<Object> args) {
if ("dimension".equals(field.get("type"))) {
String factColumn = CANONICAL_COLUMNS.get(fieldKey);
if (factColumn == null) throw bad("成员规范维度无效");
return "f." + factColumn + " COLLATE utf8mb4_0900_as_cs";
}
String path = jsonPath(column);
args.add(path);
args.add(path);
return "CASE WHEN JSON_TYPE(JSON_EXTRACT(f.raw_values, ?))='NULL' THEN NULL "
+ "ELSE JSON_UNQUOTE(JSON_EXTRACT(f.raw_values, ?)) END COLLATE utf8mb4_0900_as_cs";
}
private static String tupleColumns(int count) {
List<String> columns = new ArrayList<>();
for (int index = 0; index < count; index++) columns.add("c" + index);
return String.join(",", columns);
}
static Map<String, Object> period(Map<String, Object> context) {
Map<String, Object> nested = objectMap(context.get("period"));
Map<String, Object> result = new LinkedHashMap<>();
for (String key : List.of("year", "month", "quarter", "monthStart", "monthEnd")) {
Object raw = context.containsKey(key) ? context.get(key) : nested.get(key);
if (raw == null || String.valueOf(raw).isBlank()) continue;
int value;
try { value = Integer.parseInt(String.valueOf(raw)); }
catch (NumberFormatException e) { throw bad("期间参数无效:" + key); }
int max = key.equals("month") || key.equals("monthStart") || key.equals("monthEnd") ? 12 : key.equals("quarter") ? 4 : 9999;
int min = key.equals("year") ? 1 : 1;
if (value < min || value > max) throw bad("期间参数无效:" + key);
result.put(key, value);
}
boolean hasStart=result.containsKey("monthStart"),hasEnd=result.containsKey("monthEnd");
if(hasStart!=hasEnd || hasStart && (result.get("year") == null || result.containsKey("month") || result.containsKey("quarter")
|| (Integer)result.get("year")<2000 || (Integer)result.get("year")>2100
|| (Integer)result.get("monthStart")>(Integer)result.get("monthEnd"))
|| result.containsKey("month")&&result.containsKey("quarter")) throw bad("月份范围无效");
return result;
}
static void appendPeriod(StringBuilder where, List<Object> args, Map<String, Object> period) {
if (period.get("year") instanceof Integer year) {
where.append(" AND f.month_key LIKE ?");
args.add(String.format("%04d-%%", year));
}
if (period.get("month") instanceof Integer month) {
where.append(" AND RIGHT(f.month_key,2)=?");
args.add(String.format("%02d", month));
}
if (period.get("quarter") instanceof Integer quarter) {
int first = (quarter - 1) * 3 + 1;
where.append(" AND CAST(RIGHT(f.month_key,2) AS UNSIGNED) BETWEEN ? AND ?");
args.add(first);
args.add(first + 2);
}
if (period.get("monthStart") instanceof Integer monthStart && period.get("monthEnd") instanceof Integer monthEnd) {
int year=(Integer)period.get("year");
where.append(" AND f.month_key BETWEEN ? AND ?");
args.add(String.format("%04d-%02d",year,monthStart));
args.add(String.format("%04d-%02d",year,monthEnd));
}
}
private static Map<String, Object> objectMap(Object raw) {
if (!(raw instanceof Map<?, ?> values)) return Map.of();
Map<String, Object> result = new LinkedHashMap<>();
values.forEach((key, value) -> { if (key instanceof String name) result.put(name, value); });
return result;
}
private static boolean boolValue(Object raw, boolean defaultValue) {
if (raw == null) return defaultValue;
if (raw instanceof Boolean value) return value;
if (raw instanceof String value) {
if (value.equalsIgnoreCase("true")) return true;
if (value.equalsIgnoreCase("false")) return false;
}
throw bad("includeUnassigned 必须是布尔值");
}
private static List<String> parseSelections(Object raw, String label) {
if (raw == null) return List.of();
List<String> values;
if (raw instanceof String wire) {
try { values = SelectionValues.parse(wire); }
catch (ResponseStatusException e) { throw e; }
} else if (raw instanceof List<?> list && !list.isEmpty() && list.size() <= SelectionValues.MAX_SELECTIONS) {
values = new ArrayList<>();
for (Object value : list) {
if (!(value instanceof String text) || text.isBlank() || text.length() > SelectionValues.MAX_VALUE_LENGTH)
throw bad(label + "筛选值无效");
values.add(text);
}
} else if (raw instanceof List<?> list && list.isEmpty()) return List.of();
else throw bad(label + "筛选值格式无效");
return List.copyOf(new LinkedHashSet<>(values));
}
private static void appendDimensionSelection(StringBuilder where, List<Object> args, String key, Object raw) {
String column = switch (key) {
case "department" -> "f.department";
case "group" -> "f.group_name";
case "product" -> "f.product";
case "projectGroup" -> "f.project_group";
default -> throw bad("维度无效");
};
List<String> values = parseSelections(raw, key);
if (values.isEmpty()) return;
String exact = column + " COLLATE utf8mb4_0900_as_cs";
where.append(" AND (");
for (int index = 0; index < values.size(); index++) {
if (index > 0) where.append(" OR ");
String value = values.get(index);
if (key.equals("department") && !value.equals("__UNASSIGNED__")) value = DepartmentLabels.queryValue(value);
if (value.equals("__UNASSIGNED__")) where.append('(').append(exact).append(" IS NULL OR ").append(exact).append("='')");
else { where.append(exact).append("=?"); args.add(value); }
}
where.append(')');
}
private static Map<String, List<String>> parseExclusions(Object raw) {
if (raw == null) return Map.of();
if (!(raw instanceof Map<?, ?> map) || map.size() > 4) throw bad("exclusions 格式无效");
Set<String> supported = Set.of("department", "group", "product", "projectGroup");
Map<String, List<String>> result = new LinkedHashMap<>();
for (var entry : map.entrySet()) {
if (!(entry.getKey() instanceof String key) || !supported.contains(key)) throw bad("exclusions 维度无效");
List<String> values = parseSelections(entry.getValue(), key);
if (!values.isEmpty()) result.put(key, values);
}
return Collections.unmodifiableMap(result);
}
private static void appendExclusions(StringBuilder where, List<Object> args, Map<String, List<String>> exclusions) {
Map<String, String> columns = Map.of("department", "f.department", "group", "f.group_name",
"product", "f.product", "projectGroup", "f.project_group");
exclusions.forEach((key, values) -> {
String column = columns.get(key);
String exact = column + " COLLATE utf8mb4_0900_as_cs";
List<String> exactValues = values.stream().filter(value -> !value.equals("__UNASSIGNED__"))
.map(value -> key.equals("department") ? DepartmentLabels.queryValue(value) : value).toList();
boolean excludeUnassigned = values.contains("__UNASSIGNED__");
where.append(" AND NOT (");
boolean hasClause = false;
if (!exactValues.isEmpty()) {
where.append(exact).append(" IN (").append(String.join(",", Collections.nCopies(exactValues.size(), "?"))).append(')');
args.addAll(exactValues);
hasClause = true;
}
if (excludeUnassigned) {
if (hasClause) where.append(" OR ");
where.append('(').append(exact).append(" IS NULL OR ").append(exact).append("='')");
}
where.append(')');
});
}
private List<ConfiguredMapping> enabledMappings() {
return jdbc.query("SELECT field_key,label,black_column,bright_column FROM query_filter_field ORDER BY sort_order,field_key",
(rs, row) -> new ConfiguredMapping(rs.getString(1), rs.getString(2), rs.getString(3), rs.getString(4)));
}
private List<ConfiguredMapping> usableMappings(List<ConfiguredMapping> mappings,
Map<String, Set<String>> headersByStage, Set<String> allowed) {
return mappings.stream()
.filter(m -> m.blackColumn() != null && m.brightColumn() != null && !m.blackColumn().isBlank() && !m.brightColumn().isBlank())
.filter(m -> !Objects.equals(m.blackColumn(), m.brightColumn()))
.filter(m -> headersByStage.get("black").contains(m.blackColumn()) && headersByStage.get("bright").contains(m.brightColumn()))
.filter(m -> allowed.contains("*") || (allowed.contains(m.blackColumn()) && allowed.contains(m.brightColumn())))
.toList();
}
private Map<String, Object> sourceEntry(String key, String label, String blackColumn, String brightColumn,
boolean black, boolean bright) {
Map<String, Object> columns = new LinkedHashMap<>();
columns.put("black", black ? blackColumn : null);
columns.put("bright", bright ? brightColumn : null);
return entry(key, label, columns, black, bright, "source_value");
}
private Map<String, Object> dimensionEntry(String key, String label, String column, boolean black, boolean bright) {
Map<String, Object> columns = new LinkedHashMap<>();
columns.put("black", black ? column : null);
columns.put("bright", bright ? column : null);
return entry(key, label, columns, black, bright, "dimension");
}
private Map<String, Object> entry(String key, String label, Map<String, Object> columns,
boolean black, boolean bright, String type) {
List<String> stages = new ArrayList<>();
if (black) stages.add("black");
if (bright) stages.add("bright");
Map<String, Object> entry = new LinkedHashMap<>();
entry.put("key", key);
entry.put("label", label);
entry.put("columns", Collections.unmodifiableMap(columns));
entry.put("stages", List.copyOf(stages));
entry.put("type", type);
entry.put("operators", OPERATORS);
return Collections.unmodifiableMap(entry);
}
private String sourceKey(String header) { return "src_" + shortHash(header); }
private String mappingKey(String blackHeader, String brightHeader) { return "src_" + shortHash("mapping\u0000" + blackHeader + "\u0000" + brightHeader); }
private static String shortHash(String value) {
try {
byte[] digest = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8));
return Arrays.stream(new int[] {0, 1, 2, 3, 4, 5, 6, 7})
.mapToObj(i -> String.format("%02x", digest[i] & 0xff)).reduce("", String::concat);
} catch (Exception e) {
throw new IllegalStateException("SHA-256 unavailable", e);
}
}
private String jsonPath(String header) {
try { return "$." + json.writeValueAsString(header); }
catch (Exception e) { throw new IllegalArgumentException("来源列名无法编码", e); }
}
private void bindSourcePaths(List<Object> args, String header) {
String path = jsonPath(header);
args.add(path);
args.add(path);
args.add(path);
}
private static ResponseStatusException bad(String message) {
return new ResponseStatusException(HttpStatus.BAD_REQUEST, message);
}
private record ConfiguredMapping(String key, String label, String blackColumn, String brightColumn) { }
private record Validated(List<Map<String, Object>> filters, Map<String, Map<String, Object>> fields) { }
private record QueryContext(Map<String, Object> period, Map<String, Object> dimensions,
Map<String, List<String>> extraFilters, List<Map<String, Object>> columnFilters,
Map<String, List<String>> exclusions, boolean includeUnassigned) {
Map<String, Object> filtersResponse() {
Map<String, Object> response = new LinkedHashMap<>(dimensions);
if (!extraFilters.isEmpty()) response.put("extraFilters", extraFilters);
if (!columnFilters.isEmpty()) response.put("columnFilters", columnFilters);
if (!exclusions.isEmpty()) response.put("exclusions", exclusions);
response.put("includeUnassigned", includeUnassigned);
return Collections.unmodifiableMap(response);
}
}
}
@@ -23,6 +23,20 @@ public class GovernanceOperationsController {
long slow=r.slowRequestMillis()==null?((Number)before.get("slowRequestMillis")).longValue():r.slowRequestMillis();if(slow<100||slow>120000)throw error(HttpStatus.BAD_REQUEST,"慢请求阈值须为100到120000毫秒");
g.jdbc.update("UPDATE monitor_alert_policy SET queue_age_minutes=?,disk_free_bytes=?,backup_age_hours=?,quiet_period_minutes=?,severity_overrides=CAST(? AS JSON),silence_until=?,slow_request_millis=?,row_version=row_version+1,updated_by=? WHERE id='default'",r.queueAgeMinutes(),r.diskFreeBytes(),r.backupAgeHours(),r.quietPeriodMinutes(),g.encode(r.severityOverrides()),silence,slow,actor.getId().toString());g.audit(actor.getId().toString(),"MONITOR_ALERT_POLICY_CHANGED","monitor_alert_policy","default",Map.of("reason",r.reason(),"before",before,"after",service.alertPolicy()));return alertPolicy();}
@GetMapping("/backups")public Map<String,Object> backups(){return Map.of("items",service.backups(),"plan",g.one("SELECT * FROM backup_plan WHERE id='default'"),"liveRestoreEnabled",false);}
@GetMapping("/backups/latest")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','SYSTEM_OPERATOR','DATA_MAINTAINER','DATA_PUBLISHER')")
public Map<String,Object> latestBackup(){
Map<String,Object> latest=service.backups().stream().max(Comparator.comparing(b->String.valueOf(b.get("createdAt")))).orElse(Map.of());
Map<String,Object> safe=new LinkedHashMap<>();
for(String key:List.of("createdAt","status","restoreVerifiedAt"))if(latest.get(key)!=null)safe.put(key,latest.get(key));
if(latest.get("verification") instanceof Map<?,?> verification){
Map<String,Object> checks=new LinkedHashMap<>();
for(String key:List.of("databaseRestoreVerified","applicationFilesVerified","requiredApplicationInputsVerified","archiveFilesVerified","archiveReferencesVerified","applicationStartupVerified"))
checks.put(key,Boolean.TRUE.equals(verification.get(key)));
safe.put("verification",checks);
}
return Map.of("latest",safe,"planActive",Boolean.TRUE.equals(g.one("SELECT active FROM backup_plan WHERE id='default'").get("active")));
}
@PostMapping("/backups/jobs")public Map<String,Object> backup(@RequestBody BackupJobRequest r,@AuthenticationPrincipal AppUserPrincipal actor){return service.createBackup(r.type(),r.idempotencyKey(),r.backupId(),actor);}
@PutMapping("/backups/plan") @Transactional
public Map<String,Object> plan(@RequestBody BackupPlanRequest r,@AuthenticationPrincipal AppUserPrincipal actor){g.required(r.reason(),"理由");if(r.intervalHours()<1||r.intervalHours()>720||r.rpoHours()<1||r.rpoHours()>720||r.rtoHours()<1||r.rtoHours()>168)throw error(HttpStatus.BAD_REQUEST,"备份计划阈值无效");Map<String,Object> p=g.one("SELECT * FROM backup_plan WHERE id='default' FOR UPDATE");if(r.expectedRowVersion()==null||((Number)p.get("rowVersion")).longValue()!=r.expectedRowVersion())throw error(HttpStatus.CONFLICT,"计划版本已变化");
@@ -3,6 +3,8 @@ package cn.yunsheng.quality.governance;
import cn.yunsheng.quality.analytics.InspectionService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import cn.yunsheng.quality.identity.JdbcAppUserDetailsService;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.time.*;
import java.util.*;
@@ -16,6 +18,35 @@ import static cn.yunsheng.quality.governance.GovernanceSupport.error;
@Service @EnableScheduling
public class GovernanceOperationsService {
private static final int BACKUP_OUTPUT_LIMIT=16*1024;
private static final Map<String,String> BACKUP_FAILURE_MESSAGES=Map.ofEntries(
Map.entry("INPUTS/APPLICATION_INPUT_MISSING","备份缺少必要的应用输入文件。"),
Map.entry("INPUTS/APPLICATION_INPUT_UNSAFE","备份应用输入包含无效链接或路径。"),
Map.entry("DATABASE_EXPORT/DATABASE_DUMP_FAILED","数据库导出失败,请检查本机数据库运维状态。"),
Map.entry("DATABASE_EXPORT/DATABASE_CHANGED","备份期间正式数据发生变化,请重新执行备份。"),
Map.entry("DATABASE_EXPORT/DATABASE_READ_FAILED","数据库读取失败,请检查本机数据库运维状态。"),
Map.entry("ARCHIVE/ARCHIVE_PACK_FAILED","来源归档打包失败。"),
Map.entry("ARCHIVE/SOURCE_ARCHIVE_INVALID","来源归档缺失或与数据库指纹不一致。"),
Map.entry("APPLICATION/APPLICATION_PACK_FAILED","应用文件打包或校验失败。"),
Map.entry("MANIFEST/MANIFEST_WRITE_FAILED","备份清单写入失败。"),
Map.entry("CHECKSUM/BACKUP_CHECKSUM_MISMATCH","备份校验和不匹配,恢复已停止。"),
Map.entry("APPLICATION_RESTORE/APPLICATION_ARCHIVE_INVALID","应用归档不完整或包含不安全路径。"),
Map.entry("DATABASE_RESTORE/DATABASE_RESTORE_FAILED","隔离数据库恢复失败,正式数据库未修改。"),
Map.entry("DATABASE_RESTORE/DATABASE_CONTENT_MISMATCH","隔离数据库内容与备份清单不一致。"),
Map.entry("ARCHIVE_RESTORE/ARCHIVE_CONTENT_MISMATCH","来源归档内容与备份清单不一致。"),
Map.entry("ARCHIVE_RESTORE/ARCHIVE_ROOT_UNKNOWN","旧备份缺少来源归档根目录,无法验证迁移引用。"),
Map.entry("ARCHIVE_RESTORE/SOURCE_ARCHIVE_INVALID","来源归档缺失或与数据库指纹不一致。"),
Map.entry("ARCHIVE_REBIND/ARCHIVE_REFERENCE_INVALID","数据库中的来源归档引用无法安全重绑定。"),
Map.entry("CLEANUP/SCRATCH_CLEANUP_FAILED","隔离恢复完成后,临时数据库清理失败。"),
Map.entry("PROTOCOL/BACKUP_INTERNAL_ERROR","备份操作未完成,请检查本机运维状态。"),
Map.entry("PROCESS/BACKUP_TIMEOUT","备份操作超时,子进程已停止。"),
Map.entry("PROCESS/BACKUP_INTERRUPTED","备份操作被中断,子进程已停止。"),
Map.entry("PROTOCOL/BACKUP_OUTPUT_INVALID","备份程序未返回可识别的安全结果。"),
Map.entry("PROTOCOL/BACKUP_OUTPUT_LIMIT","备份程序输出超过安全读取上限。"),
Map.entry("INSPECTION/INSPECTION_FAILED","发布后质量检查未通过。"),
Map.entry("OPERATION/OPERATION_FAILED","后台操作未通过,请检查平台任务。")
);
private final GovernanceSupport g;private final InspectionService inspections;private final TaskRetryService retries;private final JdbcAppUserDetailsService users;
private final Path root=Path.of("").toAbsolutePath().normalize();
private final org.springframework.transaction.support.TransactionTemplate tx;
@@ -38,7 +69,13 @@ public class GovernanceOperationsService {
else{requireOperator(String.valueOf(task.get("createdBy")));result=runBackup(String.valueOf(task.get("taskType")),payload);}
result=new LinkedHashMap<>(result);result.put("elapsedSeconds",Duration.between(start,Instant.now()).toMillis()/1000.0);result.put("completedAt",Instant.now().toString());
g.jdbc.update("UPDATE task SET status='COMPLETED',stage='COMPLETED',progress=100,result=CAST(? AS JSON),error_code=NULL,error_message=NULL,lease_expires_at=NULL,heartbeat_at=CURRENT_TIMESTAMP(6) WHERE id=? AND lease_token=?",g.encode(result),id,token);
}catch(Exception e){g.jdbc.update("UPDATE task SET status='FAILED',stage='FAILED',error_code='OPERATION_FAILED',error_message='后台操作未通过,请检查平台任务和本机运维记录',retry_count=retry_count+1,next_retry_at=IF(retry_count<max_retries,TIMESTAMPADD(SECOND,30*POW(2,retry_count),CURRENT_TIMESTAMP(6)),NULL),lease_expires_at=NULL WHERE id=? AND lease_token=?",id,token);}
}catch(Exception e){
String failureStage="OPERATION",failureCode="OPERATION_FAILED",failureMessage=BACKUP_FAILURE_MESSAGES.get("OPERATION/OPERATION_FAILED");Integer exitCode=null;
if(e instanceof BackupOperationFailure failure){failureStage=failure.stage;failureCode=failure.code;failureMessage=failure.safeMessage;exitCode=failure.exitCode;}
else if(e instanceof IllegalStateException state&&"INSPECTION_FAILED".equals(state.getMessage())){failureStage="INSPECTION";failureCode="INSPECTION_FAILED";failureMessage=BACKUP_FAILURE_MESSAGES.get("INSPECTION/INSPECTION_FAILED");}
Map<String,Object> details=new LinkedHashMap<>();details.put("failureStage",failureStage);details.put("failureCode",failureCode);details.put("exitCode",exitCode);
g.jdbc.update("UPDATE task SET status='FAILED',stage='FAILED',error_code=?,error_message=?,result=CAST(? AS JSON),retry_count=retry_count+1,next_retry_at=IF(retry_count<max_retries,TIMESTAMPADD(SECOND,30*POW(2,retry_count),CURRENT_TIMESTAMP(6)),NULL),lease_expires_at=NULL WHERE id=? AND lease_token=?",failureCode,failureMessage,g.encode(details),id,token);
}
}
private void requireOperator(String user){Integer count=g.jdbc.queryForObject("SELECT COUNT(*) FROM app_user u JOIN app_user_role r ON r.user_id=u.id WHERE u.id=? AND u.active=TRUE AND r.role_code IN('SYSTEM_ADMIN','SYSTEM_OPERATOR') AND NOT EXISTS(SELECT 1 FROM user_access_policy p WHERE p.user_id=u.id AND p.valid_until<CURRENT_TIMESTAMP(6))",Integer.class,user);if(count==null||count==0)throw error(HttpStatus.FORBIDDEN,"运维授权失效");}
private Map<String,Object> runBackup(String type,Map<String,Object> payload)throws Exception{
@@ -46,14 +83,59 @@ public class GovernanceOperationsService {
List<String> command=new ArrayList<>(List.of("python3",script.toString(),"BACKUP_VERIFY".equals(type)?"verify":"create-and-verify"));Path selected=null;
if("BACKUP_VERIFY".equals(type)){selected=backupDirectory(String.valueOf(payload.get("backupId")));command.add("--directory");command.add(selected.toString());}
Set<String> before=new HashSet<>();for(Map<String,Object> backup:backups())before.add(String.valueOf(backup.get("id")));
Process process=new ProcessBuilder(command).directory(root.toFile()).redirectErrorStream(true).redirectOutput(ProcessBuilder.Redirect.DISCARD).start();
if(!process.waitFor(30,TimeUnit.MINUTES)){process.destroyForcibly();throw new IllegalStateException("Backup operation timed out");}if(process.exitValue()!=0)throw new IllegalStateException("Backup verification failed");
String expectedAction="BACKUP_VERIFY".equals(type)?"verify":"create-and-verify";
Process process;
try{process=new ProcessBuilder(command).directory(root.toFile()).redirectErrorStream(true).start();}
catch(IOException e){throw new BackupOperationFailure("PROTOCOL","BACKUP_OUTPUT_INVALID",null);}
BoundedProcessOutput output=new BoundedProcessOutput(process.getInputStream(),BACKUP_OUTPUT_LIMIT);Thread reader=new Thread(output,"quality-backup-output");reader.setDaemon(true);reader.start();
boolean completed;
try{completed=process.waitFor(30,TimeUnit.MINUTES);}catch(InterruptedException interrupted){terminateProcessTree(process);reader.join(2000);Thread.currentThread().interrupt();throw new BackupOperationFailure("PROCESS","BACKUP_INTERRUPTED",null);}
if(!completed){terminateProcessTree(process);reader.join(2000);throw new BackupOperationFailure("PROCESS","BACKUP_TIMEOUT",null);}
reader.join(5000);
if(reader.isAlive()){process.getInputStream().close();reader.join(1000);throw new BackupOperationFailure("PROTOCOL","BACKUP_OUTPUT_INVALID",process.exitValue());}
if(output.truncated)throw new BackupOperationFailure("PROTOCOL","BACKUP_OUTPUT_LIMIT",process.exitValue());
Map<?,?> event=parseBackupEvent(output.text());
if(process.exitValue()!=0){
if(event==null||!"failure".equals(event.get("event")))throw new BackupOperationFailure("PROTOCOL","BACKUP_OUTPUT_INVALID",process.exitValue());
String stage=event.get("stage") instanceof String s?s:"",code=event.get("code") instanceof String s?s:"";
String safeMessage=BACKUP_FAILURE_MESSAGES.get(stage+"/"+code);
if(safeMessage==null)throw new BackupOperationFailure("PROTOCOL","BACKUP_OUTPUT_INVALID",process.exitValue());
throw new BackupOperationFailure(stage,code,process.exitValue(),safeMessage);
}
if(event==null||!"completed".equals(event.get("event"))||!expectedAction.equals(event.get("action")))throw new BackupOperationFailure("PROTOCOL","BACKUP_OUTPUT_INVALID",process.exitValue());
if(selected!=null)return Map.of("backup",backupView(selected),"liveDatabaseChanged",false);
List<Map<String,Object>> created=backups().stream().filter(b->!before.contains(String.valueOf(b.get("id")))).toList();return Map.of("backups",created,"liveDatabaseChanged",false);
}
private Map<?,?> parseBackupEvent(String raw){
if(raw==null||raw.isBlank())return null;
String trimmed=raw.trim();
if(trimmed.indexOf('\n')>=0||trimmed.indexOf('\r')>=0)return null;
try{Object value=g.decode(trimmed);return value instanceof Map<?,?> map?map:null;}catch(RuntimeException ignored){return null;}
}
static void terminateProcessTree(Process process){
List<ProcessHandle> descendants=process.toHandle().descendants().toList();
for(int i=descendants.size()-1;i>=0;i--){ProcessHandle child=descendants.get(i);if(child.isAlive())child.destroyForcibly();}
if(process.isAlive())process.destroyForcibly();
try{process.waitFor(5,TimeUnit.SECONDS);}catch(InterruptedException interrupted){Thread.currentThread().interrupt();}
for(ProcessHandle child:descendants)if(child.isAlive())child.destroyForcibly();
}
static final class BoundedProcessOutput implements Runnable{
private final InputStream input;private final int limit;private final StringBuilder text=new StringBuilder();volatile boolean truncated;
BoundedProcessOutput(InputStream input,int limit){this.input=input;this.limit=limit;}
@Override public void run(){
try(Reader reader=new InputStreamReader(input,StandardCharsets.UTF_8)){char[] buffer=new char[2048];int count;while((count=reader.read(buffer))!=-1){int remaining=limit-text.length();if(remaining>0)text.append(buffer,0,Math.min(count,remaining));if(count>remaining)truncated=true;}}
catch(IOException ignored){/* Process termination closes the pipe; captured text remains bounded. */}
}
String text(){return text.toString();}
}
private static final class BackupOperationFailure extends Exception{
private final String stage,code,safeMessage;private final Integer exitCode;
private BackupOperationFailure(String stage,String code,Integer exitCode){this(stage,code,exitCode,BACKUP_FAILURE_MESSAGES.getOrDefault(stage+"/"+code,BACKUP_FAILURE_MESSAGES.get("PROTOCOL/BACKUP_OUTPUT_INVALID")));}
private BackupOperationFailure(String stage,String code,Integer exitCode,String safeMessage){this.stage=stage;this.code=code;this.exitCode=exitCode;this.safeMessage=safeMessage;}
}
Path backupDirectory(String id)throws java.io.IOException{if(id==null||!id.matches("[A-Za-z0-9_-]{1,80}"))throw error(HttpStatus.BAD_REQUEST,"备份编号无效");Path base=root.resolve("output/backups");Files.createDirectories(base);Path p=base.resolve(id);if(Files.isSymbolicLink(p)||!Files.isDirectory(p)||!p.toRealPath().startsWith(base.toRealPath()))throw error(HttpStatus.BAD_REQUEST,"备份不在本项目目录");return p.toRealPath();}
public List<Map<String,Object>> backups(){Path base=root.resolve("output/backups");if(!Files.isDirectory(base))return List.of();try(var paths=Files.list(base)){List<Map<String,Object>> out=new ArrayList<>();for(Path p:paths.filter(Files::isDirectory).sorted(Comparator.reverseOrder()).limit(100).toList()){if(Files.isSymbolicLink(p))continue;try{out.add(backupView(p));}catch(Exception ignored){}}return out;}catch(java.io.IOException e){return List.of();}}
@SuppressWarnings("unchecked") private Map<String,Object> backupView(Path folder)throws java.io.IOException{Object raw=g.decode(Files.readString(folder.resolve("manifest.json")));Map<String,Object> m=raw instanceof Map<?,?> map?(Map<String,Object>)map:Map.of();Map<String,Object> view=new LinkedHashMap<>();view.put("id",folder.getFileName().toString());for(String key:List.of("status","createdAt","restoreVerifiedAt","restoredTables","restoredApplicationFiles","restoreElapsedSeconds"))view.put(key,m.get(key));view.put("archiveFiles",m.get("files") instanceof Map<?,?> files?files.size():0);view.put("applicationFiles",m.get("applicationFiles") instanceof Map<?,?> files?files.size():0);return view;}
@SuppressWarnings("unchecked") private Map<String,Object> backupView(Path folder)throws java.io.IOException{Object raw=g.decode(Files.readString(folder.resolve("manifest.json")));Map<String,Object> m=raw instanceof Map<?,?> map?(Map<String,Object>)map:Map.of();Map<String,Object> view=new LinkedHashMap<>();view.put("id",folder.getFileName().toString());for(String key:List.of("status","createdAt","restoreVerifiedAt","restoredTables","restoredApplicationFiles","restoredArchiveFiles","restoreElapsedSeconds","recoveryMode","verification","archiveReferencesRebound","migrationReady","applicationRuntimePrerequisites"))view.put(key,m.get(key));view.put("archiveFiles",m.get("files") instanceof Map<?,?> files?files.size():0);view.put("applicationFiles",m.get("applicationFiles") instanceof Map<?,?> files?files.size():0);return view;}
@Transactional public Map<String,Object> createBackup(String type,String key,String backupId,AppUserPrincipal actor){if(!Set.of("BACKUP_CREATE_VERIFY","BACKUP_VERIFY").contains(type))throw error(HttpStatus.BAD_REQUEST,"备份任务类型无效");g.required(key,"幂等键");if("BACKUP_VERIFY".equals(type))try{backupDirectory(backupId);}catch(java.io.IOException e){throw error(HttpStatus.BAD_REQUEST,"备份目录不可读");}
String identity="BACKUP:"+GovernanceSupport.sha(key);List<Map<String,Object>> existing=g.rows("SELECT id,task_type,payload,created_by FROM task WHERE idempotency_key=?",identity);Map<String,Object> payload=backupId==null?Map.of():Map.of("backupId",backupId);
if(!existing.isEmpty()){Map<String,Object> prior=existing.getFirst();if(!type.equals(prior.get("taskType"))||!payload.equals(prior.get("payload"))||!actor.getId().toString().equals(prior.get("createdBy")))throw error(HttpStatus.CONFLICT,"幂等键已用于其他任务");return Map.of("task",prior);}
@@ -66,7 +148,7 @@ public class GovernanceOperationsService {
for(Map<String,Object> task:g.rows("SELECT id FROM task WHERE status='QUEUED' AND created_at<TIMESTAMPADD(MINUTE,-?,CURRENT_TIMESTAMP(6))",queueAge))alert(active,"QUEUE:"+task.get("id"),"TASK_BACKLOG","WARNING",String.valueOf(task.get("id")),"排队超过配置阈值"+queueAge+"分钟");
for(Map<String,Object> p:g.rows("SELECT id,source_slot FROM source_protocol p WHERE active=TRUE AND applicable=TRUE AND expected=TRUE AND deadline<CURRENT_TIMESTAMP(6) AND NOT EXISTS(SELECT 1 FROM source_candidate c WHERE c.source_slot=p.source_slot AND c.status='IMPORTED')"))alert(active,"SOURCE:"+p.get("id"),"SOURCE_DELAY","WARNING",String.valueOf(p.get("id")),"来源已超过协议截止时间:"+p.get("sourceSlot"));
try{if(Files.getFileStore(root).getUsableSpace()<diskFree)alert(active,"DISK","STORAGE_LOW","ERROR",null,"可用磁盘低于配置阈值");}catch(java.io.IOException ignored){}
List<Map<String,Object>> bs=backups();if(bs.stream().noneMatch(b->{try{return "ISOLATED_RESTORE_VERIFIED".equals(b.get("status"))&&Instant.parse(String.valueOf(b.get("restoreVerifiedAt"))).isAfter(Instant.now().minus(Duration.ofHours(backupAge)));}catch(Exception e){return false;}}))alert(active,"BACKUP","BACKUP_STALE","WARNING",null,"配置期间内没有通过隔离恢复验证的备份;RPO目标尚未满足");
List<Map<String,Object>> bs=backups();if(bs.stream().noneMatch(b->{try{Object raw=b.get("verification");boolean complete=raw instanceof Map<?,?> v&&Boolean.TRUE.equals(v.get("databaseRestoreVerified"))&&Boolean.TRUE.equals(v.get("applicationFilesVerified"))&&Boolean.TRUE.equals(v.get("requiredApplicationInputsVerified"))&&Boolean.TRUE.equals(v.get("archiveFilesVerified"))&&Boolean.TRUE.equals(v.get("archiveReferencesVerified"));return complete&&"ISOLATED_RESTORE_VERIFIED".equals(b.get("status"))&&Instant.parse(String.valueOf(b.get("restoreVerifiedAt"))).isAfter(Instant.now().minus(Duration.ofHours(backupAge)));}catch(Exception e){return false;}}))alert(active,"BACKUP","BACKUP_STALE","WARNING",null,"配置期间内没有通过隔离恢复验证的备份;RPO目标尚未满足");
for(Map<String,Object> old:g.rows("SELECT fingerprint FROM operation_alert WHERE status='OPEN'"))if(!active.contains(old.get("fingerprint")))g.jdbc.update("UPDATE operation_alert SET status='RECOVERED',recovered_at=CURRENT_TIMESTAMP(6) WHERE fingerprint=?",old.get("fingerprint"));
}
private void alert(Set<String> active,String fingerprint,String type,String severity,String object,String message){Map<String,Object> policy=alertPolicy();if(policy.get("severityOverrides") instanceof Map<?,?> overrides&&overrides.get(type)!=null)severity=String.valueOf(overrides.get(type));int quiet=((Number)policy.get("quietPeriodMinutes")).intValue();active.add(fingerprint);g.jdbc.update("INSERT INTO operation_alert(id,fingerprint,alert_type,severity,object_id,message) VALUES(?,?,?,?,?,?) ON DUPLICATE KEY UPDATE occurrences=IF(status='RECOVERED' OR last_seen<TIMESTAMPADD(MINUTE,-?,CURRENT_TIMESTAMP(6)),occurrences+1,occurrences),status='OPEN',severity=VALUES(severity),message=VALUES(message),last_seen=CURRENT_TIMESTAMP(6),recovered_at=NULL",UUID.randomUUID().toString(),fingerprint,type,severity,object,message,quiet);}
@@ -9,7 +9,8 @@ import org.springframework.stereotype.Service;
@Service
public class OriginalWorkbookAcceptance {
private final GovernanceSupport g;private final MetricsService metrics;private final Map<String,Object> baseline;
@SuppressWarnings("unchecked") public OriginalWorkbookAcceptance(GovernanceSupport g,MetricsService metrics){this.g=g;this.metrics=metrics;try(InputStream in=openBaseline()){if(in==null)throw new IllegalStateException("packaged migration baseline unavailable");baseline=(Map<String,Object>)g.decode(new String(in.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8));}catch(java.io.IOException e){throw new IllegalStateException("packaged migration baseline unavailable",e);}}
@org.springframework.beans.factory.annotation.Autowired @SuppressWarnings("unchecked") public OriginalWorkbookAcceptance(GovernanceSupport g,MetricsService metrics){this.g=g;this.metrics=metrics;try(InputStream in=openBaseline()){if(in==null)throw new IllegalStateException("packaged migration baseline unavailable");baseline=(Map<String,Object>)g.decode(new String(in.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8));}catch(java.io.IOException e){throw new IllegalStateException("packaged migration baseline unavailable",e);}}
OriginalWorkbookAcceptance(GovernanceSupport g,MetricsService metrics,Map<String,Object> baseline){this.g=g;this.metrics=metrics;this.baseline=Map.copyOf(baseline);}
private InputStream openBaseline(){InputStream in=getClass().getResourceAsStream("/verification/original-workbook-acceptance.json");return in!=null?in:getClass().getResourceAsStream("/verification/original-workbook-acceptance.example.json");}
public boolean isConfigured(){return !Boolean.FALSE.equals(baseline.get("configured"));}
public Map<String,Object> metadata(){Map<String,Object> result=new LinkedHashMap<>(baseline);result.remove("comparisons");result.put("configured",isConfigured());return result;}
@@ -17,16 +17,19 @@ import static cn.yunsheng.quality.governance.GovernanceSupport.error;
@Service
public class RuleRegressionService {
private final GovernanceSupport g;private final XlsxStreamingReader reader;private final MetricsService metrics;private final GovernanceAccessService access;
public RuleRegressionService(GovernanceSupport g,XlsxStreamingReader reader,MetricsService metrics,GovernanceAccessService access){this.g=g;this.reader=reader;this.metrics=metrics;this.access=access;}
private final Path configuredRoot;
@org.springframework.beans.factory.annotation.Autowired
public RuleRegressionService(GovernanceSupport g,XlsxStreamingReader reader,MetricsService metrics,GovernanceAccessService access){this(g,reader,metrics,access,configuredRoot());}
RuleRegressionService(GovernanceSupport g,XlsxStreamingReader reader,MetricsService metrics,GovernanceAccessService access,Path configuredRoot){this.g=g;this.reader=reader;this.metrics=metrics;this.access=access;this.configuredRoot=configuredRoot;}
public Map<String,Object> run(String ruleId,Map<String,Object> definition,AppUserPrincipal actor){
if(ConfigurableRuleEngine.isConfigurable(definition))return configurableRun(ruleId,definition,actor);
return originalRun(ruleId,definition,actor,true);
}
@SuppressWarnings("unchecked") private Map<String,Object> originalRun(String ruleId,Map<String,Object> definition,AppUserPrincipal actor,boolean persist){
authorize(actor);List<Map<String,Object>> cases=new ArrayList<>();List<String> errors=new ArrayList<>();String baselineHash="";
authorize(actor);List<Map<String,Object>> cases=new ArrayList<>();List<String> errors=new ArrayList<>();String baselineHash="";boolean unconfigured=false;
Object engine=g.decode(g.jdbc.queryForObject("SELECT CAST(rule_definition AS CHAR) FROM rule_version WHERE id='pq-dax-v1'",String.class));
boolean supported=Objects.equals(definition,engine);
if(!supported)errors.add("RULE_UNSUPPORTED: 仅原Power Query/DAX白名单规则包可执行,不能豁免");
if(!supported)errors.add("RULE_UNSUPPORTED: 仅支持已验证的计算规则包,不能跳过此项检查");
if(supported)try{
Path root=projectRoot(),baselinePath=root.resolve("verification/baseline.json");baselineHash=SourceScanner.sha256(baselinePath);
Map<String,Object> baseline=(Map<String,Object>)g.decode(Files.readString(baselinePath));
@@ -38,7 +41,7 @@ public class RuleRegressionService {
Map<String,Map<String,Object>> expected=new HashMap<>();for(Object o:(List<?>)baseline.get("rows")){Map<String,Object> row=(Map<String,Object>)o;expected.put(row.get("file")+":"+row.get("row"),row);}
List<BusinessRowMapper.RowValues> actual=new ArrayList<>();List<Map<String,Object>> rowFailures=new ArrayList<>();Set<String> visited=new HashSet<>();
for(Object o:(List<?>)baseline.get("source_files")){Map<String,Object> source=(Map<String,Object>)o;if(((Number)source.get("detail_rows")).longValue()==0)continue;
String relative=(String)source.get("file");Path file=root.resolve(relative).normalize();if(!file.startsWith(root)||!SourceScanner.sha256(file).equals(source.get("sha256")))throw new IllegalStateException("baseline source checksum mismatch");
String relative=(String)source.get("file");Path file=sourceFile(root,relative);if(!SourceScanner.sha256(file).equals(source.get("sha256")))throw new IllegalStateException("baseline source checksum mismatch");
Map<String,Object> exemplar=expected.entrySet().stream().filter(e->e.getKey().startsWith(relative+":")).findFirst().orElseThrow().getValue();String stage=(String)exemplar.get("kind");int year=((Number)exemplar.get("report_year")).intValue(),month=((Number)exemplar.get("month_number")).intValue();
reader.read(file,(String)source.get("sheet"),raw->{String key=relative+":"+raw.physicalRow();Map<String,Object> reference=expected.get(key);if(reference==null)return;visited.add(key);var transformed=mapper.transform(stage,raw,year,month);actual.add(transformed);
Map<String,Object> expectedComponents=(Map<String,Object>)reference.get("components");List<Map<String,Object>> differences=FormalMetricComparison.differences(expectedComponents,transformed.components(),FormalMetricComparison.WEIGHT_PRECISION,FormalMetricComparison.RATE_PRECISION);
@@ -63,19 +66,20 @@ public class RuleRegressionService {
cases.add(edgeCase("CROSS_YEAR_"+period.year()+"_"+(period.month()!=null?"M"+period.month():period.quarter()!=null?"Q"+period.quarter():"YEAR"),selected,Map.of("coverage",Map.of("blackRows",prior?1:0,"brightRows",prior?0:1),"compositeRate",prior?BigDecimal.ONE:new BigDecimal("0.8"))));
}
}catch(Exception e){errors.add("REGRESSION_EXECUTION_FAILED: 原始验收资产缺失、校验失败或回归未完成");}
}catch(AssetsUnavailable e){unconfigured=true;errors.add("REGRESSION_UNCONFIGURED: 原始工作簿验收资产尚未配置完整,请配置授权基线、对应关系及原工作簿后重试");}
catch(Exception e){errors.add("REGRESSION_EXECUTION_FAILED: 原始验收资产校验失败或回归未完成");}
List<Map<String,Object>> failed=cases.stream().filter(c->!Boolean.TRUE.equals(c.get("passed"))).toList();boolean passed=supported&&errors.isEmpty()&&failed.isEmpty()&&cases.size()>=8;
String id=UUID.randomUUID().toString();Map<String,Object> result=new LinkedHashMap<>();result.put("id",id);result.put("status",passed?"PASSED":supported?"FAILED":"UNSUPPORTED");result.put("cases",cases);result.put("failedCases",failed);result.put("errors",errors);result.put("precisionPolicy",new CandidateAcceptanceService.PrecisionPolicy(FormalMetricComparison.WEIGHT_PRECISION,FormalMetricComparison.RATE_PRECISION));result.put("baselineSha256",baselineHash);result.put("isolated",true);result.put("engineRuleVersion",BusinessRowMapper.RULE_VERSION);result.put("activatable",passed);
String id=UUID.randomUUID().toString();Map<String,Object> result=new LinkedHashMap<>();result.put("id",id);result.put("status",passed?"PASSED":unconfigured?"UNCONFIGURED":supported?"FAILED":"UNSUPPORTED");result.put("cases",cases);result.put("failedCases",failed);result.put("errors",errors);result.put("precisionPolicy",new CandidateAcceptanceService.PrecisionPolicy(FormalMetricComparison.WEIGHT_PRECISION,FormalMetricComparison.RATE_PRECISION));result.put("baselineSha256",baselineHash);result.put("isolated",true);result.put("engineRuleVersion",BusinessRowMapper.RULE_VERSION);result.put("activatable",passed);
result.put("definitionHash",GovernanceSupport.sha(g.encode(definition)));result.put("requiresDifferenceConfirmation",false);
if(persist)persist(ruleId,definition,actor,result);return result;
}
/** Semantic changes are reported for explicit review; execution and trusted-baseline failures still block activation. */
@SuppressWarnings("unchecked") private Map<String,Object> configurableRun(String ruleId,Map<String,Object> definition,AppUserPrincipal actor){
authorize(actor);List<String> errors=new ArrayList<>(ConfigurableRuleEngine.validate(definition));List<Map<String,Object>> cases=new ArrayList<>(),rowDifferences=new ArrayList<>(),differences=new ArrayList<>();
String baselineHash="",referenceVersion=BusinessRowMapper.RULE_VERSION,referenceDefinitionHash="";int totalRows=0,changedRows=0;
String baselineHash="",referenceVersion=BusinessRowMapper.RULE_VERSION,referenceDefinitionHash="";int totalRows=0,changedRows=0;boolean unconfigured=false;
Map<String,Object> originalDefinition=(Map<String,Object>)g.decode(g.jdbc.queryForObject("SELECT CAST(rule_definition AS CHAR) FROM rule_version WHERE id='pq-dax-v1'",String.class));
if(errors.isEmpty())try{
Map<String,Object> trusted=originalRun(ruleId,originalDefinition,actor,false);cases.addAll((List<Map<String,Object>>)trusted.get("cases"));if(!Boolean.TRUE.equals(trusted.get("activatable")))throw new IllegalStateException("原始规则独立基线回归失败");
Map<String,Object> trusted=originalRun(ruleId,originalDefinition,actor,false);cases.addAll((List<Map<String,Object>>)trusted.get("cases"));if("UNCONFIGURED".equals(trusted.get("status")))throw new AssetsUnavailable();if(!Boolean.TRUE.equals(trusted.get("activatable")))throw new IllegalStateException("原始规则独立基线回归失败");
baselineHash=String.valueOf(trusted.get("baselineSha256"));String active=g.jdbc.queryForObject("SELECT metadata_value FROM platform_metadata WHERE metadata_key='rules_version'",String.class);
if(active!=null&&!active.isBlank())referenceVersion=active;
Map<String,Object> referenceDefinition=referenceVersion.equals(BusinessRowMapper.RULE_VERSION)?originalDefinition:(Map<String,Object>)g.decode(g.jdbc.queryForObject("SELECT CAST(rule_definition AS CHAR) FROM rule_version WHERE id=?",String.class,referenceVersion));
@@ -86,7 +90,7 @@ public class RuleRegressionService {
BusinessRowMapper beforeMapper=new BusinessRowMapper(categories,projects,BusinessRowMapper.MAPPING_VERSION,Map.of(),Map.of(),referenceRules),afterMapper=new BusinessRowMapper(categories,projects,BusinessRowMapper.MAPPING_VERSION,Map.of(),Map.of(),candidateRules);
Map<String,Map<String,Object>> expected=new HashMap<>();for(Object o:(List<?>)baseline.get("rows")){Map<String,Object> row=(Map<String,Object>)o;expected.put(row.get("file")+":"+row.get("row"),row);}
List<BusinessRowMapper.RowValues> before=new ArrayList<>(),after=new ArrayList<>();Set<String> visited=new HashSet<>();int[] changed={0};
for(Object o:(List<?>)baseline.get("source_files")){Map<String,Object> source=(Map<String,Object>)o;if(((Number)source.get("detail_rows")).longValue()==0)continue;String relative=(String)source.get("file");Path file=root.resolve(relative).normalize();if(!file.startsWith(root)||!SourceScanner.sha256(file).equals(source.get("sha256")))throw new IllegalStateException("来源校验失败");
for(Object o:(List<?>)baseline.get("source_files")){Map<String,Object> source=(Map<String,Object>)o;if(((Number)source.get("detail_rows")).longValue()==0)continue;String relative=(String)source.get("file");Path file=sourceFile(root,relative);if(!SourceScanner.sha256(file).equals(source.get("sha256")))throw new IllegalStateException("来源校验失败");
Map<String,Object> exemplar=expected.entrySet().stream().filter(e->e.getKey().startsWith(relative+":")).findFirst().orElseThrow().getValue();String stage=(String)exemplar.get("kind");int year=((Number)exemplar.get("report_year")).intValue(),month=((Number)exemplar.get("month_number")).intValue();Set<String> headers=new HashSet<>();
reader.read(file,(String)source.get("sheet"),raw->{headers.addAll(raw.values().keySet());String key=relative+":"+raw.physicalRow();if(!expected.containsKey(key))return;if(!visited.add(key))throw new IllegalStateException("来源行重复");var b=beforeMapper.transform(stage,raw,year,month);var a=afterMapper.transform(stage,raw,year,month);if(!a.invalidNumericFields().isEmpty())throw new IllegalArgumentException("映射来源包含非数值: "+a.invalidNumericFields());before.add(b);after.add(a);List<Map<String,Object>> delta=FormalMetricComparison.differences(b.components(),a.components(),FormalMetricComparison.WEIGHT_PRECISION,FormalMetricComparison.RATE_PRECISION);if(!delta.isEmpty()){changed[0]++;if(rowDifferences.size()<100)rowDifferences.add(Map.of("sourceFile",relative,"rowNumber",raw.physicalRow(),"stage",stage,"differences",delta));}});
if(!headers.containsAll(afterMapper.requiredHeaders(stage)))throw new IllegalArgumentException("候选取数字段不存在于来源工作簿");
@@ -100,8 +104,9 @@ public class RuleRegressionService {
// Explicit ERROR policies must also execute on null/zero boundary fixtures before activation.
metrics.aggregateForGovernance(List.of(),candidateRules);for(String stage:List.of("black","bright"))metrics.aggregateForGovernance(List.of(fixture(afterMapper,stage,"0","0","0","0")),candidateRules);
cases.add(caseResult("CONFIGURABLE_SOURCE_REPLAY",true,Map.of("expectedRows",expected.size(),"actualRows",after.size(),"changedRows",changedRows)));
}catch(Exception e){errors.add("RULE_EXECUTION_FAILED: "+safeFailure(e));}
boolean executable=errors.isEmpty()&&totalRows>0;boolean changed=changedRows>0||!differences.isEmpty();Map<String,Object> result=new LinkedHashMap<>();result.put("id",UUID.randomUUID().toString());result.put("status",!executable?"FAILED":changed?"DIFFERENCES":"PASSED");result.put("activatable",executable);result.put("requiresDifferenceConfirmation",executable&&changed);result.put("totalRows",totalRows);result.put("changedRows",changedRows);result.put("rowDifferences",rowDifferences);result.put("rowDifferencesTruncated",changedRows>rowDifferences.size());result.put("differences",differences);result.put("cases",cases);result.put("failedCases",cases.stream().filter(c->!Boolean.TRUE.equals(c.get("passed"))).toList());result.put("errors",errors);result.put("isolated",true);result.put("baselineSha256",baselineHash);result.put("engineRuleVersion",ConfigurableRuleEngine.SCHEMA);result.put("definitionHash",GovernanceSupport.sha(g.encode(definition)));result.put("referenceRuleVersion",referenceVersion);result.put("referenceDefinitionHash",referenceDefinitionHash);result.put("precisionPolicy",new CandidateAcceptanceService.PrecisionPolicy(FormalMetricComparison.WEIGHT_PRECISION,FormalMetricComparison.RATE_PRECISION));persist(ruleId,definition,actor,result);return result;
}catch(AssetsUnavailable e){unconfigured=true;errors.add("REGRESSION_UNCONFIGURED: 原始工作簿验收资产尚未配置完整,请配置授权基线、对应关系及原工作簿后重试");}
catch(Exception e){errors.add("RULE_EXECUTION_FAILED: "+safeFailure(e));}
boolean executable=errors.isEmpty()&&totalRows>0;boolean changed=changedRows>0||!differences.isEmpty();Map<String,Object> result=new LinkedHashMap<>();result.put("id",UUID.randomUUID().toString());result.put("status",unconfigured?"UNCONFIGURED":!executable?"FAILED":changed?"DIFFERENCES":"PASSED");result.put("activatable",executable);result.put("requiresDifferenceConfirmation",executable&&changed);result.put("totalRows",totalRows);result.put("changedRows",changedRows);result.put("rowDifferences",rowDifferences);result.put("rowDifferencesTruncated",changedRows>rowDifferences.size());result.put("differences",differences);result.put("cases",cases);result.put("failedCases",cases.stream().filter(c->!Boolean.TRUE.equals(c.get("passed"))).toList());result.put("errors",errors);result.put("isolated",true);result.put("baselineSha256",baselineHash);result.put("engineRuleVersion",ConfigurableRuleEngine.SCHEMA);result.put("definitionHash",GovernanceSupport.sha(g.encode(definition)));result.put("referenceRuleVersion",referenceVersion);result.put("referenceDefinitionHash",referenceDefinitionHash);result.put("precisionPolicy",new CandidateAcceptanceService.PrecisionPolicy(FormalMetricComparison.WEIGHT_PRECISION,FormalMetricComparison.RATE_PRECISION));persist(ruleId,definition,actor,result);return result;
}
private void compareConfig(String name,List<BusinessRowMapper.RowValues> before,List<BusinessRowMapper.RowValues> after,ConfigurableRuleEngine beforeRules,ConfigurableRuleEngine afterRules,List<Map<String,Object>> differences){Object b=g.decode(g.encode(metrics.aggregateForGovernance(before,beforeRules))),a=g.decode(g.encode(metrics.aggregateForGovernance(after,afterRules)));var delta=FormalMetricComparison.differences(b,a,FormalMetricComparison.WEIGHT_PRECISION,FormalMetricComparison.RATE_PRECISION);if(!delta.isEmpty())differences.add(Map.of("name",name,"before",b,"after",a,"differences",delta));}
private String safeFailure(Exception failure){if(failure instanceof IllegalArgumentException&&failure.getMessage()!=null)return failure.getMessage().substring(0,Math.min(300,failure.getMessage().length()));return "原始验收资产缺失、校验失败或规则边界计算未完成";}
@@ -123,5 +128,13 @@ public class RuleRegressionService {
private BusinessRowMapper.RowValues fixture(BusinessRowMapper mapper,String stage,String good,String scrap,String extra,String reuse){return fixture(mapper,stage,good,scrap,extra,reuse,2026,1);}
private BusinessRowMapper.RowValues fixture(BusinessRowMapper mapper,String stage,String good,String scrap,String extra,String reuse,int year,int month){Map<String,XlsxStreamingReader.CellValue> v=new LinkedHashMap<>();Map<String,String> fields=new LinkedHashMap<>();fields.put("送电镀重量",good);fields.put("黑片出货重量","0");fields.put("责废总重量",scrap);fields.put("料废重量","0");fields.put("外废重量",extra);fields.put("检验备注","过账");fields.put("发料地点",reuse.equals("1")?"利用料":"普通");fields.put("是否电镀来料/二等品库来料","是");fields.put("合格重量",good);fields.put("检包入废品库重量",scrap);fields.put("本部报废重量",extra);fields.forEach((k,value)->v.put(k,new XlsxStreamingReader.CellValue(value,value,"s")));return mapper.transform(stage,new XlsxStreamingReader.RawRow("isolated",1,v,Map.of()),year,month);}
static List<BusinessRowMapper.RowValues> selectPeriod(List<BusinessRowMapper.RowValues> rows,CandidateAcceptanceService.Period period){return rows.stream().filter(row->{java.time.YearMonth ym=java.time.YearMonth.parse(row.monthKey());return ym.getYear()==period.year()&&(period.month()==null||ym.getMonthValue()==period.month())&&(period.quarter()==null||(ym.getMonthValue()-1)/3+1==period.quarter());}).toList();}
private Path projectRoot(){Path cwd=Path.of("").toAbsolutePath().normalize();for(Path p=cwd;p!=null;p=p.getParent())if(Files.isRegularFile(p.resolve("verification/baseline.json")))return p;throw new IllegalStateException("verification baseline unavailable");}
private static Path configuredRoot(){String path=System.getProperty("quality.regressionRoot",System.getenv("QUALITY_REGRESSION_ROOT"));return path==null||path.isBlank()?null:Path.of(path).toAbsolutePath().normalize();}
private Path projectRoot(){
if(configuredRoot!=null)return checkedRoot(configuredRoot);
for(Path p=Path.of("").toAbsolutePath().normalize();p!=null;p=p.getParent())if(Files.isRegularFile(p.resolve("verification/baseline.json")))return checkedRoot(p);
throw new AssetsUnavailable();
}
private Path checkedRoot(Path root){if(!Files.isRegularFile(root.resolve("verification/baseline.json"))||!Files.isRegularFile(root.resolve("verification/observed_mappings.json")))throw new AssetsUnavailable();return root;}
private Path sourceFile(Path root,String relative){Path path=root.resolve(relative).normalize();if(!path.startsWith(root))throw new IllegalArgumentException("原始验收来源路径越界");if(!Files.isRegularFile(path))throw new AssetsUnavailable();return path;}
private static class AssetsUnavailable extends IllegalStateException {}
}
@@ -6,6 +6,7 @@ import cn.yunsheng.quality.common.DepartmentLabels;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotBlank;
import java.util.Map;
import java.nio.charset.StandardCharsets;
import org.springframework.http.HttpStatus;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.core.AuthenticationException;
@@ -84,8 +85,12 @@ public class AuthController {
@AuthenticationPrincipal AppUserPrincipal actor) {
if (body.newPassword() == null || body.newPassword().length() < 6)
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "新密码至少6位");
if (body.newPassword().getBytes(StandardCharsets.UTF_8).length > 72)
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "新密码过长,请缩短后重试");
if (!passwordEncoder.matches(body.currentPassword(), actor.getPassword()))
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "当前密码错误");
if (passwordEncoder.matches(body.newPassword(), actor.getPassword()))
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "新密码不能与当前密码相同");
int updated = jdbc.update("""
UPDATE app_user SET password_hash=?,force_password_change=FALSE,row_version=row_version+1
WHERE id=? AND password_hash=? AND active=TRUE
@@ -67,10 +67,22 @@ public class CurrentUserRefreshFilter extends OncePerRequestFilter {
context.setAuthentication(refreshed);
SecurityContextHolder.setContext(context);
repository.saveContext(context, request, response);
if (authorized.isForcePasswordChange() && !passwordRecoveryRequest(request)) {
response.setStatus(HttpServletResponse.SC_FORBIDDEN);
response.setContentType("application/problem+json;charset=UTF-8");
response.getWriter().write("{\"title\":\"Forbidden\",\"status\":403,\"code\":\"PASSWORD_CHANGE_REQUIRED\",\"detail\":\"请先修改初始密码\"}");
return;
}
}
chain.doFilter(request, response);
}
private boolean passwordRecoveryRequest(HttpServletRequest request) {
String path = request.getRequestURI().substring(request.getContextPath().length());
return ("GET".equals(request.getMethod()) && java.util.Set.of("/api/auth/me", "/api/auth/csrf").contains(path))
|| ("POST".equals(request.getMethod()) && java.util.Set.of("/api/auth/login", "/api/auth/change-password", "/api/auth/logout").contains(path));
}
private void reject(HttpServletRequest request, HttpServletResponse response) throws IOException {
SecurityContextHolder.clearContext();
if (request.getSession(false) != null) request.getSession(false).invalidate();
@@ -273,7 +273,10 @@ public class ImportWorker {
}
});
if (!facts.isEmpty()) persistFacts(buildId, revisionId, facts);
if (!issues.isEmpty()) persistIssues(runId, candidate, issues);
if (!issues.isEmpty()) {
persistIssues(runId, candidate, issues);
issues.clear();
}
for (String required : parserConfig==null?rowMapper.requiredHeaders(candidate.stage()):rowMapper.configuredRequiredHeaders(candidate.stage())) {
if (!availableHeaders.contains(required)) {
issues.add(new Issue(candidate, null, "REQUIRED_HEADER_MISSING", required, "工作表缺少必需字段“" + required + "”", true));
@@ -25,7 +25,7 @@ public class MetricsController {
/** Compatibility for internal callers without source-field filters. */
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter, String department, String group, String product, String projectGroup, String view, boolean includeUnassigned, String releaseId, String trendGranularity, Integer rowLimit, int rowOffset, String rowOrder, AppUserPrincipal actor) {
return calculate(year, month, quarter, department, group, product, null, projectGroup, view, includeUnassigned, releaseId, trendGranularity, rowLimit, rowOffset, rowOrder, null, actor);
return calculate(year, month, quarter, department, group, product, null, projectGroup, view, includeUnassigned, releaseId, trendGranularity, rowLimit, rowOffset, rowOrder, null, null, actor);
}
/** Compatibility for internal callers that already pass source-field filters. */
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter, String department, String group,
@@ -33,12 +33,26 @@ public class MetricsController {
String releaseId, String trendGranularity, Integer rowLimit, int rowOffset,
String rowOrder, String extraFilters, AppUserPrincipal actor) {
return calculate(year, month, quarter, department, group, product, null, projectGroup, view, includeUnassigned,
releaseId, trendGranularity, rowLimit, rowOffset, rowOrder, extraFilters, actor);
releaseId, trendGranularity, rowLimit, rowOffset, rowOrder, extraFilters, null, actor);
}
/** Delegated callers may pass validated exclusion JSON alongside source-field filters. */
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter, String department, String group,
String product, String projectGroup, String view, boolean includeUnassigned,
String releaseId, String trendGranularity, Integer rowLimit, int rowOffset,
String rowOrder, String extraFilters, String exclusions, AppUserPrincipal actor) {
return calculate(year, month, quarter, department, group, product, null, projectGroup, view, includeUnassigned,
releaseId, trendGranularity, rowLimit, rowOffset, rowOrder, extraFilters, exclusions, actor);
}
/** Existing callers retain their exact historical parameter contract. */
public MetricsService.MetricSnapshot calculate(Integer year,Integer month,String quarter,String department,String group,String product,String productSearch,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Integer rowLimit,int rowOffset,String rowOrder,String extraFilters,String exclusions,AppUserPrincipal actor) {
return calculate(year,month,quarter,department,group,product,productSearch,projectGroup,view,includeUnassigned,releaseId,trendGranularity,rowLimit,rowOffset,rowOrder,extraFilters,exclusions,null,actor);
}
@GetMapping
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST')")
public MetricsService.MetricSnapshot calculate(@RequestParam(required = false) Integer year,
public MetricsService.MetricSnapshot calculateHttp(@RequestParam(required = false) Integer year,
@RequestParam(required = false) Integer month,
@RequestParam(required = false) Integer monthStart,
@RequestParam(required = false) Integer monthEnd,
@RequestParam(required = false) String quarter,
@RequestParam(required = false) String department,
@RequestParam(required = false) String group,
@@ -53,10 +67,43 @@ public class MetricsController {
@RequestParam(defaultValue = "0") int rowOffset,
@RequestParam(required = false) String rowOrder,
@RequestParam(required=false) String extraFilters,
@RequestParam(required=false) String exclusions,
@RequestParam(required=false) String columnFilters,
@RequestParam(required=false) String stage,
@AuthenticationPrincipal AppUserPrincipal actor) {
return calculateRangeStage(year, month, monthStart, monthEnd, quarter, department, group, product, productSearch,
projectGroup, view, includeUnassigned, releaseId, trendGranularity, rowLimit, rowOffset, rowOrder,
extraFilters, exclusions, columnFilters, stage, actor);
}
/** Existing direct callers retain the pre-range controller signature. */
public MetricsService.MetricSnapshot calculate(Integer year, Integer month, String quarter,
String department, String group, String product, String productSearch,
String projectGroup, String view, boolean includeUnassigned,
String releaseId, String trendGranularity, Integer rowLimit, int rowOffset,
String rowOrder, String extraFilters, String exclusions, String columnFilters,
AppUserPrincipal actor) {
return calculateRange(year, month, null, null, quarter, department, group, product, productSearch,
projectGroup, view, includeUnassigned, releaseId, trendGranularity, rowLimit, rowOffset, rowOrder,
extraFilters, exclusions, columnFilters, actor);
}
/** Range-aware entry point used by the delegated native runtime. */
public MetricsService.MetricSnapshot calculateRange(Integer year, Integer month, Integer monthStart, Integer monthEnd,
String quarter, String department, String group, String product,
String productSearch, String projectGroup, String view,
boolean includeUnassigned, String releaseId, String trendGranularity,
Integer rowLimit, int rowOffset, String rowOrder,
String extraFilters, String exclusions, String columnFilters,
AppUserPrincipal actor) {
return calculateRangeStage(year,month,monthStart,monthEnd,quarter,department,group,product,productSearch,projectGroup,view,includeUnassigned,releaseId,trendGranularity,rowLimit,rowOffset,rowOrder,extraFilters,exclusions,columnFilters,null,actor);
}
private MetricsService.MetricSnapshot calculateRangeStage(Integer year,Integer month,Integer monthStart,Integer monthEnd,String quarter,String department,String group,String product,String productSearch,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Integer rowLimit,int rowOffset,String rowOrder,String extraFilters,String exclusions,String columnFilters,String stage,AppUserPrincipal actor) {
Integer quarterNumber = parseQuarter(quarter);
MetricsService.MetricSnapshot visible = responseAccess.visible(metrics.calculate(new MetricsService.MetricsQuery(year, month, quarterNumber, department, group,
product, projectGroup, view, includeUnassigned, releaseId, trendGranularity).withExtraFilters(extraFilters).withProductSearch(productSearch), actor), actor);
product, projectGroup, view, includeUnassigned, releaseId, trendGranularity, parseExclusions(exclusions))
.withMonthRange(monthStart, monthEnd).withExtraFilters(extraFilters).withProductSearch(productSearch).withColumnFilters(columnFilters).withStage(stage), actor), actor);
if (rowLimit == null) return visible;
if (rowLimit < 1 || rowLimit > 500 || rowOffset < 0) throw new IllegalArgumentException("rowLimit 必须在 1 到 500 之间,rowOffset 不能为负数");
if (rowOrder != null && !rowOrder.equals("scrapDesc")) throw new IllegalArgumentException("rowOrder 无效");
@@ -72,6 +119,16 @@ public class MetricsController {
visible.trend(), visible.defects(), visible.coverage());
}
private static Map<String, List<String>> parseExclusions(String raw) {
if (raw == null) return Map.of();
if (raw.length() > 32000) throw new org.springframework.web.server.ResponseStatusException(org.springframework.http.HttpStatus.BAD_REQUEST, "排除条件过长");
Object value;
try { value = tools.jackson.databind.json.JsonMapper.builder().build().readValue(raw, Object.class); }
catch (Exception invalid) { throw new org.springframework.web.server.ResponseStatusException(org.springframework.http.HttpStatus.BAD_REQUEST, "排除条件格式无效"); }
if (!(value instanceof Map<?, ?>)) throw new org.springframework.web.server.ResponseStatusException(org.springframework.http.HttpStatus.BAD_REQUEST, "排除条件必须是对象");
return cn.yunsheng.quality.analysis.AnalysisExplorationTools.exclusions(value);
}
private BigDecimal totalScrap(MetricsService.MetricRow row) {
BigDecimal black = row.black() == null || row.black().scrapWeight() == null ? BigDecimal.ZERO : row.black().scrapWeight();
BigDecimal bright = row.bright() == null || row.bright().scrapWeight() == null ? BigDecimal.ZERO : row.bright().scrapWeight();
@@ -20,13 +20,14 @@ public class MetricsResponseAccess {
public MetricSnapshot visible(MetricSnapshot snapshot, AppUserPrincipal actor) {
governance.requireUserActivePolicy(actor);
Set<String> allowed = governance.allowedFields(actor);
MetricSnapshot filtered = allowed.contains("*") ? snapshot : restrictDefects(snapshot, allowed);
if (Boolean.TRUE.equals(governance.policy(actor.getId().toString()).get("detailAllowed"))) {
governance.requireBoundAction("QUERY", "query.detail");
Set<String> allowed = governance.allowedFields(actor);
return allowed.contains("*") ? snapshot : restrictDefects(snapshot, allowed);
return filtered;
}
Map<String, Object> defects = Map.of("black", List.of(), "bright", List.of());
return new MetricSnapshot(snapshot.release(), snapshot.context(), hideDefects(snapshot.summary()), List.of(), List.of(), defects, snapshot.coverage());
return new MetricSnapshot(filtered.release(), filtered.context(), hideDefects(filtered.summary()), List.of(), List.of(), defects, filtered.coverage());
}
private MetricRow hideDefects(MetricRow row) {
@@ -35,7 +36,7 @@ public class MetricsResponseAccess {
}
private MetricStage hideDefects(MetricStage stage) {
return new MetricStage(stage.qualifiedWeight(), stage.scrapWeight(), stage.totalWeight(), stage.rate(), List.<DefectRow>of(),
stage.headquartersScrapWeight(), stage.rateWithoutHeadquartersScrap());
stage.headquartersScrapWeight(), stage.rateWithoutHeadquartersScrap(), stage.scrapRate());
}
private MetricSnapshot restrictDefects(MetricSnapshot snapshot, Set<String> allowed) {
@@ -57,9 +58,18 @@ public class MetricsResponseAccess {
}
private MetricStage restrictDefects(MetricStage stage, String kind, Set<String> allowed) {
if (stage == null) return null;
List<DefectRow> defects = stage.defects().stream().filter(row -> allowed.contains(componentKey(kind, row.name()))).toList();
return new MetricStage(stage.qualifiedWeight(), stage.scrapWeight(), stage.totalWeight(), stage.rate(), defects,
stage.headquartersScrapWeight(), stage.rateWithoutHeadquartersScrap());
return new MetricStage(allowed.contains(kind + "_good_weight") ? stage.qualifiedWeight() : null,
allowed.contains(kind + "_scrap_weight") ? stage.scrapWeight() : null,
allowed.contains(kind + "_total_weight") ? stage.totalWeight() : null, stage.rate(), defects,
"bright".equals(kind) && allowed.contains("bright_headquarters_scrap") ? stage.headquartersScrapWeight() : null,
stage.rateWithoutHeadquartersScrap(),
allowsDerivedScrapRate(kind, allowed) ? stage.scrapRate() : null);
}
private boolean allowsDerivedScrapRate(String kind, Set<String> allowed) {
return allowed.contains("*") || allowed.contains(kind + "_scrap_weight") && allowed.contains(kind + "_total_weight");
}
private List<DefectRow> allowedDefects(Object raw, String kind, Set<String> allowed) {
@@ -2,6 +2,7 @@ package cn.yunsheng.quality.metrics;
import cn.yunsheng.quality.common.DataScope;
import cn.yunsheng.quality.filters.QueryFilterFields;
import cn.yunsheng.quality.filters.SourceColumnQueries;
import cn.yunsheng.quality.common.DepartmentLabels;
import cn.yunsheng.quality.common.SelectionValues;
import cn.yunsheng.quality.governance.GovernanceAccessService;
@@ -129,6 +130,45 @@ public class MetricsService {
return new MetricSnapshot(release.asMap(), context, summary, rows, trend, defects, coverage);
}
/** Calculates two published-period views in one repeatable-read snapshot and one grouped query per stage/period. */
public ComparisonSnapshots calculateComparisonBatch(MetricsQuery currentQuery, MetricsQuery comparisonQuery,
Map<String, String> canonicalGroupDepartments,
AppUserPrincipal actor) {
if (actor == null) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "指标查询必须绑定当前有效用户");
governance.requireUserActivePolicy(actor);
governance.requireBoundAction("QUERY", "query.aggregate");
validate(currentQuery);
validate(comparisonQuery);
if (currentQuery.releaseId() == null || currentQuery.releaseId().isBlank()
|| !Objects.equals(currentQuery.releaseId(), comparisonQuery.releaseId()))
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "两期批量比较必须绑定同一明确发布版本");
ReleaseInfo release = release(currentQuery.releaseId(), actor);
Map<String, String> owners = canonicalGroupDepartments == null ? Map.of() : canonicalGroupDepartments;
return withConsistentSnapshot(() -> {
boolean hasUnrolledBuilds = hasUnrolledBuilds(release.id());
String ruleVersion = jdbc.queryForObject("SELECT rule_version FROM data_release WHERE id=?", String.class, release.id());
ConfigurableRuleEngine rules = rulesForVersion(ruleVersion);
MetricSnapshot current = calculateComparisonSide(currentQuery, actor, release, hasUnrolledBuilds, owners, rules, ruleVersion);
MetricSnapshot comparison = calculateComparisonSide(comparisonQuery, actor, release, hasUnrolledBuilds, owners, rules, ruleVersion);
return new ComparisonSnapshots(current, comparison);
});
}
private MetricSnapshot calculateComparisonSide(MetricsQuery query, AppUserPrincipal actor, ReleaseInfo release,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments,
ConfigurableRuleEngine rules, String ruleVersion) {
AggregateSet aggregates = queryAggregates(release.id(), query, actor, hasUnrolledBuilds, canonicalGroupDepartments);
MetricRow summary = rules.applyMetrics(aggregates.summary().toMetricRow("overall", "总体", null, null, null));
List<MetricRow> rows = groupedRows(query.view(), aggregates).stream().map(rules::applyMetrics).toList();
Map<String, Object> coverage = queryCoverage(release.id(), query, actor, hasUnrolledBuilds);
Map<String, Object> context = context(query, query.includeUnassigned());
context.put("ruleVersionId", ruleVersion == null ? BusinessRowMapper.RULE_VERSION : ruleVersion);
context.put("calculationRules", rules.calculationRules());
context.put("metricFormulaDescriptions", rules.formulaDescriptions());
Map<String, Object> defects = Map.of("black", summary.black().defects(), "bright", summary.bright().defects());
return new MetricSnapshot(release.asMap(), context, summary, rows, List.of(), defects, coverage);
}
/** The immutable release binding decides formulas, never the current active rule metadata. */
@SuppressWarnings("unchecked")
public ConfigurableRuleEngine rulesForRelease(String releaseId) {
@@ -174,18 +214,25 @@ public class MetricsService {
}
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor, boolean hasUnrolledBuilds) {
return queryAggregates(releaseId, query, actor, hasUnrolledBuilds, null);
}
private AggregateSet queryAggregates(String releaseId, MetricsQuery query, AppUserPrincipal actor,
boolean hasUnrolledBuilds, Map<String, String> canonicalGroupDepartments) {
AggregateSet result = new AggregateSet();
if (query.extraFilters().isEmpty()) aggregateStage(result, "black", query.view(), releaseId, query, actor, false);
if (query.extraFilters().isEmpty()) aggregateStage(result, "bright", query.view(), releaseId, query, actor, false);
if (hasUnrolledBuilds || !query.extraFilters().isEmpty()) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, true);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true);
if ((query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) aggregateStage(result, "black", query.view(), releaseId, query, actor, false, canonicalGroupDepartments);
if ((query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) aggregateStage(result, "bright", query.view(), releaseId, query, actor, false, canonicalGroupDepartments);
if (hasUnrolledBuilds || !(query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) {
aggregateStage(result, "black", query.view(), releaseId, query, actor, true, canonicalGroupDepartments);
aggregateStage(result, "bright", query.view(), releaseId, query, actor, true, canonicalGroupDepartments);
}
return result;
}
private void aggregateStage(AggregateSet result, String stage, String view, String releaseId,
MetricsQuery query, AppUserPrincipal actor, boolean factFallback) {
MetricsQuery query, AppUserPrincipal actor, boolean factFallback,
Map<String, String> canonicalGroupDepartments) {
if(query.stage()!=null&&!query.stage().equals(stage))return;
String table = factFallback ? (stage.equals("black") ? "fact_black" : "fact_bright") : "metric_rollup";
String dimensionColumn = view.equals("overall") ? null : switch (view) {
case "department" -> "f.department";
@@ -196,21 +243,27 @@ public class MetricsService {
};
String bucket = dimensionColumn == null ? "'overall'"
: "COALESCE(" + dimensionColumn + " COLLATE utf8mb4_0900_as_cs, '__UNASSIGNED__') COLLATE utf8mb4_0900_as_cs";
boolean canonicalGroupRows = "group".equals(view) && canonicalGroupDepartments != null;
String ownerDepartment = canonicalGroupRows ? ", f.department COLLATE utf8mb4_0900_as_cs AS owner_department" : "";
String aggregateKeys = canonicalGroupRows ? "trend_period, bucket_key, owner_department" : "trend_period, bucket_key";
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
if (factFallback && query.extraFilters().isEmpty()) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback)new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
if (!effectiveIncludeUnassigned(query))
where.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
String trendBucket = trendBucketExpression(query.trendGranularity());
String sql = """
SELECT %s AS trend_period, %s AS bucket_key,
SELECT %s AS trend_period, %s AS bucket_key%s,
SUM(f.qualified_weight) AS qualified_weight,
SUM(f.scrap_weight) AS scrap_weight,
SUM(f.total_weight) AS total_weight,
@@ -225,14 +278,16 @@ public class MetricsService {
%s AS row_count
FROM release_member rm JOIN %s f ON f.build_id=rm.transform_build_id
%s
GROUP BY trend_period, bucket_key
ORDER BY trend_period, bucket_key
GROUP BY %s
ORDER BY %s
""".formatted(trendBucket, bucket,
ownerDepartment,
componentSum("black_responsibility_scrap"), componentSum("black_material_scrap"),
componentSum("black_external_scrap"), componentSum("bright_headquarters_scrap"),
componentSum("bright_plating_scrap"), componentSum("bright_material_scrap"),
componentSum("bright_processing_scrap"), componentSum("bright_diffusion_scrap"),
factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, where);
factFallback ? "COUNT(*)" : "SUM(f.source_row_count)", table, where,
aggregateKeys, aggregateKeys);
jdbc.query(sql, (RowCallbackHandler) rs -> {
String period = rs.getString("trend_period");
String key = rs.getString("bucket_key");
@@ -240,7 +295,8 @@ public class MetricsService {
result.summary().merge(stage, stats);
result.periods().computeIfAbsent(period, ignored -> new AggregateStats()).merge(stage, stats);
if (dimensionColumn != null) {
result.dimensions().computeIfAbsent(key, ignored -> new AggregateStats()).merge(stage, stats);
if (!canonicalGroupRows || canonicalGroupOwnerMatches(canonicalGroupDepartments, key, rs.getString("owner_department")))
result.dimensions().computeIfAbsent(key, ignored -> new AggregateStats()).merge(stage, stats);
}
}, args.toArray());
}
@@ -252,7 +308,7 @@ public class MetricsService {
+ "ELSE CAST(JSON_UNQUOTE(" + extracted + ") AS DECIMAL(38,8)) END)";
}
private void appendTime(StringBuilder sql, List<Object> args, MetricsQuery query) {
void appendTime(StringBuilder sql, List<Object> args, MetricsQuery query) {
if (query.year() != null) { sql.append(" AND f.month_key LIKE ?"); args.add(String.format("%04d-%%", query.year())); }
if (query.month() != null) { sql.append(" AND RIGHT(f.month_key,2)=?"); args.add(String.format("%02d", query.month())); }
if (query.quarter() != null) {
@@ -260,6 +316,11 @@ public class MetricsService {
sql.append(" AND CAST(RIGHT(f.month_key,2) AS UNSIGNED) BETWEEN ? AND ?");
args.add(first); args.add(first + 2);
}
if (query.monthStart() != null && query.monthEnd() != null) {
sql.append(" AND f.month_key BETWEEN ? AND ?");
args.add(String.format("%04d-%02d", query.year(), query.monthStart()));
args.add(String.format("%04d-%02d", query.year(), query.monthEnd()));
}
}
private void appendDimensions(StringBuilder sql, List<Object> args, MetricsQuery query) {
@@ -345,11 +406,11 @@ public class MetricsService {
return trend;
}
private Map<String, Object> queryCoverage(String releaseId, MetricsQuery query, AppUserPrincipal actor, boolean hasUnrolledBuilds) {
Map<String, Object> queryCoverage(String releaseId, MetricsQuery query, AppUserPrincipal actor, boolean hasUnrolledBuilds) {
TreeMap<String, CoverageMonth> months = new TreeMap<>();
if (query.extraFilters().isEmpty()) coverageStage(months, "black", releaseId, query, actor, false);
if (query.extraFilters().isEmpty()) coverageStage(months, "bright", releaseId, query, actor, false);
if (hasUnrolledBuilds || !query.extraFilters().isEmpty()) {
if ((query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) coverageStage(months, "black", releaseId, query, actor, false);
if ((query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) coverageStage(months, "bright", releaseId, query, actor, false);
if (hasUnrolledBuilds || !(query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) {
coverageStage(months, "black", releaseId, query, actor, true);
coverageStage(months, "bright", releaseId, query, actor, true);
}
@@ -371,12 +432,14 @@ public class MetricsService {
if (query.year() != null) years.add(query.year());
else months.keySet().forEach(month -> years.add(Integer.parseInt(month.substring(0, 4))));
for (int year : years) {
int firstMonth = query.quarter() == null ? (query.month() == null ? 1 : query.month()) : (query.quarter() - 1) * 3 + 1;
int lastMonth = query.quarter() == null ? (query.month() == null ? 12 : query.month()) : firstMonth + 2;
int firstMonth = query.monthStart() != null ? query.monthStart()
: query.quarter() == null ? (query.month() == null ? 1 : query.month()) : (query.quarter() - 1) * 3 + 1;
int lastMonth = query.monthEnd() != null ? query.monthEnd()
: query.quarter() == null ? (query.month() == null ? 12 : query.month()) : firstMonth + 2;
for (int month = firstMonth; month <= lastMonth; month++) {
CoverageMonth counts = months.getOrDefault(String.format("%04d-%02d", year, month), new CoverageMonth());
if (counts.blackRows == 0) missingStages.add(Map.of("year", year, "month", month, "stage", "black"));
if (counts.brightRows == 0) missingStages.add(Map.of("year", year, "month", month, "stage", "bright"));
if ((query.stage()==null||query.stage().equals("black"))&&counts.blackRows == 0) missingStages.add(Map.of("year", year, "month", month, "stage", "black"));
if ((query.stage()==null||query.stage().equals("bright"))&&counts.brightRows == 0) missingStages.add(Map.of("year", year, "month", month, "stage", "bright"));
}
}
return Map.of("rawRows", rawRows, "includedRows", includedRows(rawRows, excludedBlank, query),
@@ -387,16 +450,20 @@ public class MetricsService {
private void coverageStage(Map<String, CoverageMonth> months, String stage, String releaseId,
MetricsQuery query, AppUserPrincipal actor, boolean factFallback) {
if(query.stage()!=null&&!query.stage().equals(stage))return;
String table = factFallback ? (stage.equals("black") ? "fact_black" : "fact_bright") : "metric_rollup";
StringBuilder where = new StringBuilder(" WHERE rm.release_id=?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
if (factFallback && query.extraFilters().isEmpty()) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
if (factFallback && (query.extraFilters().isEmpty() && query.columnFilters().isEmpty())) where.append(" AND NOT EXISTS (SELECT 1 FROM metric_rollup_build rb WHERE rb.build_id=f.build_id)");
else if (!factFallback) { where.append(" AND f.stage=?"); args.add(stage); }
appendTime(where, args, query);
appendDimensions(where, args, query);
appendExclusions(where, args, query);
if(factFallback)new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
if(factFallback) {
new QueryFilterFields(jdbc,governance).append(where,args,stage,query.extraFilters(),actor);
if(!query.columnFilters().isEmpty())new SourceColumnQueries(jdbc,governance,JsonMapper.builder().build()).append(where,args,stage,query.columnFilters(),releaseId,actor);
}
DataScope.append(where, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
String countExpression = factFallback ? "COUNT(*)" : "SUM(f.source_row_count)";
String excludedExpression = factFallback
@@ -425,6 +492,13 @@ public class MetricsService {
}, args.toArray());
}
static boolean canonicalGroupOwnerMatches(Map<String, String> canonicalGroupDepartments, String group, String department) {
if (canonicalGroupDepartments == null || group == null || department == null) return false;
String owner = canonicalGroupDepartments.get(group);
return owner != null && !owner.isBlank()
&& DepartmentLabels.queryValue(owner).equals(department);
}
private BigDecimal add(BigDecimal left, BigDecimal right) {
if (left == null && right == null) return null;
return (left == null ? BigDecimal.ZERO : left).add(right == null ? BigDecimal.ZERO : right);
@@ -435,6 +509,11 @@ public class MetricsService {
return numerator.divide(denominator, RATE_CONTEXT);
}
private static BigDecimal calculateScrapRate(BigDecimal scrapWeight, BigDecimal totalWeight) {
if (scrapWeight == null || totalWeight == null || totalWeight.signum() == 0) return null;
return scrapWeight.divide(totalWeight, RATE_CONTEXT);
}
private BigDecimal multiply(BigDecimal left, BigDecimal right) {
if (left == null || right == null) return null;
return left.multiply(right, RATE_CONTEXT);
@@ -561,9 +640,11 @@ public class MetricsService {
long totalRows() { return blackRows + brightRows; }
}
private Map<String, Object> context(MetricsQuery query, boolean includeUnassigned) {
Map<String, Object> context(MetricsQuery query, boolean includeUnassigned) {
Map<String, Object> result = new LinkedHashMap<>();
result.put("year", query.year()); result.put("month", query.month()); result.put("quarter", query.quarter());
if(query.monthStart()!=null)result.put("monthStart",query.monthStart());
if(query.monthEnd()!=null)result.put("monthEnd",query.monthEnd());
result.put("department", query.department()); result.put("group", query.group()); result.put("product", query.product());
if (query.productSearch() != null) result.put("productSearch", query.productSearch());
result.put("projectGroup", query.projectGroup()); result.put("view", query.view());
@@ -571,6 +652,8 @@ public class MetricsService {
result.put("trendGranularity", query.trendGranularity());
result.put("exclusions", query.exclusions());
result.put("extraFilters", query.extraFilters());
if(query.stage()!=null)result.put("stage",query.stage());
if(!query.columnFilters().isEmpty())result.put("columnFilters",query.columnFilters());
return result;
}
@@ -603,10 +686,16 @@ public class MetricsService {
Map.of("black", empty.black().defects(), "bright", empty.bright().defects()), coverage);
}
private void validate(MetricsQuery query) {
void validate(MetricsQuery query) {
if (query.year() != null && (query.year() < 2000 || query.year() > 2100)) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "年份无效");
if (query.month() != null && (query.month() < 1 || query.month() > 12)) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "月份无效");
if (query.quarter() != null && (query.quarter() < 1 || query.quarter() > 4)) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "季度无效");
boolean hasMonthStart = query.monthStart() != null, hasMonthEnd = query.monthEnd() != null;
if (hasMonthStart != hasMonthEnd) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "monthStart 和 monthEnd 必须成对提供");
if (hasMonthStart && (query.year() == null || query.month() != null || query.quarter() != null
|| query.monthStart() < 1 || query.monthStart() > 12 || query.monthEnd() < 1 || query.monthEnd() > 12
|| query.monthStart() > query.monthEnd()))
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "月份范围无效,且不能与 month 或 quarter 并用");
if (!List.of("overall", "department", "group", "product", "projectGroup").contains(query.view())) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "view 无效");
if (!List.of("month", "quarter", "year").contains(query.trendGranularity())) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "trendGranularity 必须是 month、quarter 或 year");
SelectionValues.parse(query.department());
@@ -631,8 +720,11 @@ public class MetricsService {
public record MetricsQuery(Integer year, Integer month, Integer quarter, String department, String group,
String product, String projectGroup, String view, boolean includeUnassigned,
String releaseId, String trendGranularity, Map<String, List<String>> exclusions, Map<String,List<String>> extraFilters,
String productSearch) {
String productSearch, List<Map<String,Object>> columnFilters, Integer monthStart, Integer monthEnd, String stage) {
public MetricsQuery {
columnFilters=SourceColumnQueries.parse(columnFilters);
if(stage!=null&&!List.of("black","bright").contains(stage))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"stage 必须是 black 或 bright");
if(stage!=null)for(var filter:columnFilters)if(!List.of(stage).containsAll((List<?>)filter.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段");
extraFilters=QueryFilterFields.parse(extraFilters);
productSearch = productSearch == null || productSearch.isBlank() ? null : productSearch.trim();
if (exclusions == null) exclusions = Map.of();
@@ -643,6 +735,25 @@ public class MetricsService {
exclusions = java.util.Collections.unmodifiableMap(copy);
}
}
public MetricsQuery(Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Map<String,List<String>> exclusions,Map<String,List<String>> extraFilters,String productSearch) {
this(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,extraFilters,productSearch,List.of(),null,null);
}
/** Preserve the former canonical constructor while adding an explicit range period. */
public MetricsQuery(Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Map<String,List<String>> exclusions,Map<String,List<String>> extraFilters,String productSearch,List<Map<String,Object>> columnFilters) {
this(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,extraFilters,productSearch,columnFilters,null,null);
}
public MetricsQuery(Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,String view,boolean includeUnassigned,String releaseId,String trendGranularity,Map<String,List<String>> exclusions,Map<String,List<String>> extraFilters,String productSearch,List<Map<String,Object>> columnFilters,Integer monthStart,Integer monthEnd) {
this(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,extraFilters,productSearch,columnFilters,monthStart,monthEnd,null);
}
public MetricsQuery withStage(String value) {
return new MetricsQuery(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,extraFilters,productSearch,columnFilters,monthStart,monthEnd,value);
}
public MetricsQuery withColumnFilters(Object filters) {
return new MetricsQuery(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,extraFilters,productSearch,SourceColumnQueries.parse(filters),monthStart,monthEnd,stage);
}
public MetricsQuery withMonthRange(Integer start, Integer end) {
return new MetricsQuery(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,extraFilters,productSearch,columnFilters,start,end,stage);
}
public MetricsQuery(Integer year, Integer month, Integer quarter, String department, String group,
String product, String projectGroup, String view, boolean includeUnassigned,
String releaseId, String trendGranularity, Map<String, List<String>> exclusions,
@@ -654,11 +765,11 @@ public class MetricsService {
this(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,Map.of(),null);
}
public MetricsQuery withExtraFilters(Object filters) {
return new MetricsQuery(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,QueryFilterFields.parse(filters),productSearch);
return new MetricsQuery(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,trendGranularity,exclusions,QueryFilterFields.parse(filters),productSearch,columnFilters,monthStart,monthEnd,stage);
}
public MetricsQuery withProductSearch(String value) {
return new MetricsQuery(year,month,quarter,department,group,product,projectGroup,view,includeUnassigned,releaseId,
trendGranularity,exclusions,extraFilters,value);
trendGranularity,exclusions,extraFilters,value,columnFilters,monthStart,monthEnd,stage);
}
/** Preserve the historic 10-argument constructor used by service callers. */
public MetricsQuery(Integer year, Integer month, Integer quarter, String department, String group,
@@ -689,7 +800,15 @@ public class MetricsService {
public record MetricStage(BigDecimal qualifiedWeight, BigDecimal scrapWeight, BigDecimal totalWeight,
BigDecimal rate, List<DefectRow> defects, BigDecimal headquartersScrapWeight,
BigDecimal rateWithoutHeadquartersScrap) {}
BigDecimal rateWithoutHeadquartersScrap, BigDecimal scrapRate) {
/** Preserve the existing constructor contract while exposing the stage's formal scrap ratio. */
public MetricStage(BigDecimal qualifiedWeight, BigDecimal scrapWeight, BigDecimal totalWeight,
BigDecimal rate, List<DefectRow> defects, BigDecimal headquartersScrapWeight,
BigDecimal rateWithoutHeadquartersScrap) {
this(qualifiedWeight, scrapWeight, totalWeight, rate, defects, headquartersScrapWeight,
rateWithoutHeadquartersScrap, MetricsService.calculateScrapRate(scrapWeight, totalWeight));
}
}
public record DefectRow(String name, BigDecimal weight, BigDecimal rate) {}
public record MetricRow(String key, String label, String department, String group, String product, String projectGroup,
BigDecimal compositeRate, MetricStage black, MetricStage bright,
@@ -710,6 +829,7 @@ public class MetricsService {
public record MetricSnapshot(Map<String, Object> release, Map<String, Object> context, MetricRow summary,
List<MetricRow> rows, List<TrendMetricRow> trend, Map<String, Object> defects,
Map<String, Object> coverage) {}
public record ComparisonSnapshots(MetricSnapshot current, MetricSnapshot comparison) {}
private record Fact(String stage, String month, String department, String group, String projectGroup, String product,
BigDecimal qualified, BigDecimal scrap, BigDecimal total, Map<String, Object> raw,
@@ -109,6 +109,13 @@ public class OptionsController {
}).distinct().toList();
Map<String, Object> response = new LinkedHashMap<>();
response.put("years", years);
List<String> periods = values.stream().map(row -> (String) row.get("month_key"))
.filter(value -> value != null && value.matches("[0-9]{4}-(0[1-9]|1[0-2])"))
.distinct().sorted().toList();
response.put("periods", periods);
response.put("latestDataPeriod", periods.isEmpty() ? Map.of() : Map.of(
"year", Integer.parseInt(periods.getLast().substring(0, 4)),
"month", Integer.parseInt(periods.getLast().substring(5, 7))));
response.put("departments", departments);
response.put("groups", groups);
response.put("groupDepartments", BusinessGroupOwnership.resolve(mappingDefinition, groups, departments));
@@ -81,9 +81,12 @@ public class RecordController {
return export(year, month, quarter, department, group, product, null, projectGroup, includeUnassigned,
releaseId, extraFilters, actor);
}
public Map<String,Object> records(String stage,Integer year,Integer month,Integer quarter,String department,String group,String product,String productSearch,String projectGroup,boolean includeUnassigned,String releaseId,int page,int pageSize,String field,String search,String sort,String direction,List<String> columns,String extraFilters,AppUserPrincipal actor) {
return recordsFiltered(stage,year,month,quarter,department,group,product,productSearch,projectGroup,includeUnassigned,releaseId,page,pageSize,field,search,sort,direction,columns,extraFilters,null,actor);
}
@GetMapping
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST')")
public Map<String, Object> records(@RequestParam String stage,
public Map<String, Object> recordsFiltered(@RequestParam String stage,
@RequestParam(required = false) Integer year,
@RequestParam(required = false) Integer month,
@RequestParam(required = false) Integer quarter,
@@ -102,6 +105,7 @@ public class RecordController {
@RequestParam(defaultValue = "asc") String direction,
@RequestParam(required = false) List<String> columns,
@RequestParam(required=false) String extraFilters,
@RequestParam(required=false) String columnFilters,
@AuthenticationPrincipal AppUserPrincipal actor) {
governance.requireDetail(actor);
governance.requireBoundAction("QUERY", "query.detail");
@@ -140,6 +144,9 @@ public class RecordController {
args.add(jsonPath(selectedField));
}
new QueryFilterFields(jdbc,governance).append(predicate,args,stage,QueryFilterFields.parse(extraFilters),actor);
var sourceFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters);
for(var rule:sourceFilters)if(!List.of(stage).containsAll((List<?>)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段");
if(!sourceFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,stage,sourceFilters,selectedRelease,actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs", "f.group_name COLLATE utf8mb4_0900_as_cs");
Long total = jdbc.queryForObject("""
@@ -167,9 +174,12 @@ public class RecordController {
"pageSize", pageSize, "releaseId", selectedRelease);
}
public ResponseEntity<StreamingResponseBody> export(Integer year,Integer month,Integer quarter,String department,String group,String product,String productSearch,String projectGroup,boolean includeUnassigned,String releaseId,String extraFilters,AppUserPrincipal actor) {
return exportFiltered(year,month,quarter,department,group,product,productSearch,projectGroup,includeUnassigned,releaseId,extraFilters,null,null,actor);
}
@GetMapping("/export")
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','QUALITY_VIEWER','BUSINESS_ANALYST')")
public ResponseEntity<StreamingResponseBody> export(@RequestParam(required = false) Integer year,
public ResponseEntity<StreamingResponseBody> exportFiltered(@RequestParam(required = false) Integer year,
@RequestParam(required = false) Integer month,
@RequestParam(required = false) Integer quarter,
@RequestParam(required = false) String department,
@@ -180,12 +190,14 @@ public class RecordController {
@RequestParam(defaultValue = "false") boolean includeUnassigned,
@RequestParam(required = false) String releaseId,
@RequestParam(required=false) String extraFilters,
@RequestParam(required=false) String stage,
@RequestParam(required=false) String columnFilters,
@AuthenticationPrincipal AppUserPrincipal actor) {
governance.requireDetail(actor);
governance.requireBoundAction("QUERY", "query.detail");
if (productSearch != null && productSearch.length() > SelectionValues.MAX_VALUE_LENGTH) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "产品搜索文本过长");
Path file = workbookExport.export(year, month, quarter, department, group, product, projectGroup,
includeUnassigned, releaseId, actor, QueryFilterFields.parse(extraFilters), productSearch);
Path file = stage==null&&columnFilters==null ? workbookExport.export(year,month,quarter,department,group,product,projectGroup,includeUnassigned,releaseId,actor,QueryFilterFields.parse(extraFilters),productSearch) : workbookExport.exportFiltered(year, month, quarter, department, group, product, projectGroup,
includeUnassigned, releaseId, actor, QueryFilterFields.parse(extraFilters), productSearch,stage,cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters));
StreamingResponseBody body = output -> {
try (InputStream input = Files.newInputStream(file)) {
input.transferTo(output);
@@ -69,6 +69,12 @@ public class RecordsWorkbookExportService {
return export(year, month, quarter, department, group, product, projectGroup, includeUnassigned, releaseId, actor, extraFilters, null);
}
public Path export(Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,boolean includeUnassigned,String releaseId,AppUserPrincipal actor,Map<String,List<String>> extraFilters,String productSearch) {
return exportFiltered(year,month,quarter,department,group,product,projectGroup,includeUnassigned,releaseId,actor,extraFilters,productSearch,null,List.of());
}
public Path exportFiltered(Integer year,Integer month,Integer quarter,String department,String group,String product,String projectGroup,boolean includeUnassigned,String releaseId,AppUserPrincipal actor,Map<String,List<String>> extraFilters,String productSearch,String selectedStage,List<Map<String,Object>> columnFilters) {
if(selectedStage!=null&&!List.of("black","bright").contains(selectedStage))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"stage 必须是 black 或 bright");
columnFilters=cn.yunsheng.quality.filters.SourceColumnQueries.parse(columnFilters);
if(selectedStage!=null)for(var rule:columnFilters)if(!List.of(selectedStage).containsAll((List<?>)rule.get("stages")))throw new ResponseStatusException(HttpStatus.BAD_REQUEST,"来源字段筛选必须属于当前阶段");
RecordController.validatePeriodFilters(year, month, quarter);
if (productSearch != null && productSearch.length() > SelectionValues.MAX_VALUE_LENGTH)
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "产品搜索文本过长");
@@ -81,10 +87,10 @@ public class RecordsWorkbookExportService {
try (workbook) {
workbook.setCompressTempFiles(true);
CellStyle headerStyle = createHeaderStyle(workbook);
createStageSheet(workbook, headerStyle, "黑片", "fact_black", selectedRelease, year, month, quarter,
department, group, product, projectGroup, includeUnassigned, allowedFields, actor, extraFilters, productSearch);
createStageSheet(workbook, headerStyle, "亮片", "fact_bright", selectedRelease, year, month, quarter,
department, group, product, projectGroup, includeUnassigned, allowedFields, actor, extraFilters, productSearch);
if(selectedStage==null||selectedStage.equals("black"))createStageSheet(workbook, headerStyle, "黑片", "fact_black", selectedRelease, year, month, quarter,
department, group, product, projectGroup, includeUnassigned, allowedFields, actor, extraFilters, productSearch,columnFilters);
if(selectedStage==null||selectedStage.equals("bright"))createStageSheet(workbook, headerStyle, "亮片", "fact_bright", selectedRelease, year, month, quarter,
department, group, product, projectGroup, includeUnassigned, allowedFields, actor, extraFilters, productSearch,columnFilters);
try (var stream = Files.newOutputStream(output)) {
workbook.write(stream);
}
@@ -105,12 +111,12 @@ public class RecordsWorkbookExportService {
String releaseId, Integer year, Integer month, Integer quarter, String department,
String group, String product, String projectGroup, boolean includeUnassigned,
Set<String> allowedFields, AppUserPrincipal actor, Map<String,List<String>> extraFilters,
String productSearch) throws IOException {
String productSearch,List<Map<String,Object>> columnFilters) throws IOException {
Sheet sheet = workbook.createSheet(sheetName);
LinkedHashSet<String> rawFields = new LinkedHashSet<>();
if (releaseId != null) {
QueryScope rawScope = queryScope(fact, releaseId, year, month, quarter, department, group, product,
projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch);
projectGroup, includeUnassigned, "f.raw_values", actor, extraFilters, productSearch,columnFilters);
scan(rawScope, rs -> rawFields.addAll(visibleRaw(readJson(rs.getString(1)), allowedFields).keySet()));
}
if (rawFields.size() + HEADER_COUNT > MAX_COLUMNS)
@@ -132,7 +138,7 @@ public class RecordsWorkbookExportService {
QueryScope rowScope = queryScope(fact, releaseId, year, month, quarter, department, group, product,
projectGroup, includeUnassigned,
"sr.source_file, f.sheet_name, f.physical_row_no, sr.id AS source_revision_id, f.month_key, " +
"f.department, f.group_name, f.project_group, f.product, f.raw_values", actor, extraFilters, productSearch);
"f.department, f.group_name, f.project_group, f.product, f.raw_values", actor, extraFilters, productSearch,columnFilters);
int[] rowIndex = {1};
scan(rowScope, rs -> writeRow(sheet, rowIndex, rs, rawFields, allowedFields));
}
@@ -140,7 +146,7 @@ public class RecordsWorkbookExportService {
private QueryScope queryScope(String fact, String releaseId, Integer year, Integer month, Integer quarter,
String department, String group, String product, String projectGroup,
boolean includeUnassigned, String select, AppUserPrincipal actor, Map<String,List<String>> extraFilters,
String productSearch) {
String productSearch,List<Map<String,Object>> columnFilters) {
StringBuilder predicate = new StringBuilder(" WHERE rm.release_id = ?");
List<Object> args = new ArrayList<>();
args.add(releaseId);
@@ -156,6 +162,7 @@ public class RecordsWorkbookExportService {
if (!includeUnassigned && !SelectionValues.parse(group).contains("__UNASSIGNED__"))
predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name <> '')");
new QueryFilterFields(jdbc,governance).append(predicate,args,fact.equals("fact_black")?"black":"bright",extraFilters,actor);
if(!columnFilters.isEmpty())new cn.yunsheng.quality.filters.SourceColumnQueries(jdbc,governance,mapper).append(predicate,args,fact.equals("fact_black")?"black":"bright",columnFilters,releaseId,actor);
DataScope.append(predicate, args, actor, "f.department COLLATE utf8mb4_0900_as_cs",
"f.group_name COLLATE utf8mb4_0900_as_cs");
String columns = select == null ? "f.raw_values" : select;
@@ -13,6 +13,7 @@ import cn.yunsheng.quality.ingestion.ConfigurableRuleEngine;
import cn.yunsheng.quality.ingestion.ImportRunService;
import cn.yunsheng.quality.ingestion.ImportWorker;
import cn.yunsheng.quality.ingestion.MappingValueCodec;
import cn.yunsheng.quality.metrics.MetricsService;
import cn.yunsheng.quality.release.ReleaseService;
import cn.yunsheng.quality.source.SourceCatalogService;
import cn.yunsheng.quality.source.SourceDeletionService;
@@ -30,6 +31,7 @@ import java.util.UUID;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicReference;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.TaskExecutor;
import org.springframework.http.HttpStatus;
@@ -62,6 +64,7 @@ public class DataWorkflowService {
private final JdbcAppUserDetailsService users;
private final TaskExecutor executor;
private final TransactionTemplate transaction;
private final MetricsService metrics;
private final AtomicReference<String> runningOperation = new AtomicReference<>();
private final AtomicReference<String> runningLease = new AtomicReference<>();
@@ -76,6 +79,19 @@ public class DataWorkflowService {
JdbcAppUserDetailsService users,
@Qualifier("dataWorkflowExecutor") TaskExecutor executor,
PlatformTransactionManager manager) {
this(jdbc,json,imports,importWorker,releases,catalog,deletion,sourceGovernance,dependencies,
sourceAuthorization,access,users,executor,manager,new MetricsService(jdbc,access));
}
@Autowired
public DataWorkflowService(JdbcTemplate jdbc, JsonMapper json, ImportRunService imports,
ImportWorker importWorker, ReleaseService releases, SourceCatalogService catalog,
SourceDeletionService deletion, SourceGovernanceService sourceGovernance,
GovernanceDependencyService dependencies,
SourceAuthorizationService sourceAuthorization, GovernanceAccessService access,
JdbcAppUserDetailsService users,
@Qualifier("dataWorkflowExecutor") TaskExecutor executor,
PlatformTransactionManager manager, MetricsService metrics) {
this.jdbc = jdbc;
this.json = json;
this.imports = imports;
@@ -90,6 +106,7 @@ public class DataWorkflowService {
this.users = users;
this.executor = executor;
this.transaction = new TransactionTemplate(manager);
this.metrics=metrics;
}
public Map<String, Object> status(AppUserPrincipal actor) {
@@ -144,13 +161,17 @@ public class DataWorkflowService {
return actor.getRoleCodes().stream().anyMatch(set::contains);
}
static String configurationName(java.time.Instant savedAt) {
return java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(java.time.ZoneId.of("Asia/Shanghai")).format(savedAt);
}
public Map<String, Object> apply(DataWorkflowController.ApplyRequest request, AppUserPrincipal actor) {
requireEnabled();
requirePublisher(actor);
if (request == null || !Set.of("mapping", "rule").contains(request.kind()))
throw bad("kind 须为 mapping 或 rule");
String name = required(request.name(), "名称不能为空");
if (name.length() > 190) throw bad("名称不能超过190字");
String name = configurationName(java.time.Instant.now());
Map<String, Object> definition = asMap(request.definition());
if (definition == null) throw bad("definition 须为 JSON 对象");
String current = currentReleaseId();
@@ -326,7 +347,13 @@ public class DataWorkflowService {
Map<String, Object> definition = asMap(payload.get("definition"));
if (definition == null) throw failure("INVALID_DEFINITION", "definition 须为 JSON 对象");
if ("mapping".equals(kind)) {
completeMappingTargets(definition);
Map<String,Object> previous=Map.of();
String base=boundVersion("mapping_version","mapping_version",operation.expectedCurrentReleaseId());
if(base!=null) {
Map<String,Object> version=versionView("mapping_version","definition",base,false);
if(version!=null&&asMap(version.get("definition"))!=null)previous=asMap(version.get("definition"));
}
completeMappingTargets(definition,previous);
List<String> errors = validateMapping(definition);
if (!errors.isEmpty()) throw failure("MAPPING_INVALID", String.join(";", errors));
} else {
@@ -498,6 +525,7 @@ public class DataWorkflowService {
transaction.executeWithoutResult(status -> {
if (!Objects.equals(currentReleaseId(), operation.expectedCurrentReleaseId()))
throw conflict("当前正式版本已变化;候选保留为未发布状态");
validateCandidateMetrics(releaseId,actor);
if (activateDefinitions) activateDefinitions(mappingId, ruleId, actor);
String reason = "TEST_WORKFLOW:" + operationId;
releases.transitionInTestWorkflow(releaseId, action,
@@ -507,6 +535,26 @@ public class DataWorkflowService {
complete(operationId, token, currentReleaseId(), releaseId, Map.of("published", true));
}
/** Execute the candidate's own formulas before changing any active definition or pointer. */
private void validateCandidateMetrics(String releaseId,AppUserPrincipal actor) {
Set<CandidatePeriod> periods=new LinkedHashSet<>();
for(Map<String,Object> row:jdbc.queryForList("SELECT DISTINCT d.source_year,d.source_month FROM release_member m JOIN source_document d ON d.id=m.source_document_id WHERE m.release_id=? ORDER BY d.source_year,d.source_month",releaseId)) {
int year=((Number)row.get("source_year")).intValue(),month=((Number)row.get("source_month")).intValue();
periods.add(new CandidatePeriod(year,month,null));
periods.add(new CandidatePeriod(year,null,(month-1)/3+1));
periods.add(new CandidatePeriod(year,null,null));
}
if(periods.isEmpty())periods.add(new CandidatePeriod(java.time.LocalDate.now(java.time.ZoneId.of("Asia/Shanghai")).getYear(),null,null));
try {
for(CandidatePeriod period:periods)for(String dimension:List.of("overall","department","group","product","projectGroup"))
metrics.calculateCandidateForGovernance(new MetricsService.MetricsQuery(period.year(),period.month(),period.quarter(),
null,null,null,null,dimension,true,releaseId,"month",Map.of(),Map.of(),null),actor);
} catch(RuntimeException error) {
String reason=error instanceof ConfigurableRuleEngine.RuleEvaluationException?error.getMessage():"计算未完成,请检查规则、来源或查询权限";
throw conflict("候选指标计算未通过,当前规则和报表保持不变:"+safeMessage(reason,"指标不可执行"));
}
}
private void activateDefinitions(String mappingId, String ruleId, AppUserPrincipal actor) {
if (mappingId != null) {
dependencies.freeze("mapping", mappingId, actor.getId().toString());
@@ -713,44 +761,57 @@ public class DataWorkflowService {
}
private void completeMappingTargets(Map<String, Object> definition) {
completeMappingTargets(definition,Map.of());
}
private void completeMappingTargets(Map<String,Object> definition,Map<String,Object> previous) {
Map<String,Object> priorMaster=asMap(previous.get("masterData"));
if(priorMaster==null)priorMaster=Map.of();
Map<String, Object> master = asMap(definition.get("masterData"));
if (master == null) master = new LinkedHashMap<>();
if (master == null) master = new LinkedHashMap<>(priorMaster);
definition.put("masterData", master);
Map<String, String> lookupDimension = Map.of("departmentLookup", "departments", "categoryLookup", "groups",
"productLookup", "products", "projectGroupLookup", "projectGroups");
for (Map.Entry<String, String> entry : lookupDimension.entrySet()) {
List<Map<String, Object>> rows = rows(master.get(entry.getValue()));
List<Map<String, Object>> rows = rows(master.containsKey(entry.getValue())?master.get(entry.getValue()):priorMaster.get(entry.getValue()));
Set<String> names = new LinkedHashSet<>();
rows.forEach(row -> { if (row.get("name") != null) names.add(String.valueOf(row.get("name"))); });
Set<String> disabled = disabledKeys(definition, entry.getKey());
Map<String, Object> lookup = asMap(definition.get(entry.getKey()));
if (lookup != null) for (Map.Entry<String, Object> mapping : lookup.entrySet()) {
addTarget(rows, names, entry.getValue(), mapping.getValue(), disabled.contains(mapping.getKey()));
addTarget(rows, names, entry.getValue(), mapping.getValue(), disabled.contains(mapping.getKey()),previousNames(priorMaster,entry.getValue()));
}
master.put(entry.getValue(), rows);
}
Object rawRules = definition.get("matchRules");
Object rawRules = effectiveMatchRules(definition);
if (rawRules instanceof List<?> rules) for (Object raw : rules) if (raw instanceof Map<?, ?> rule && Boolean.TRUE.equals(rule.get("enabled"))) {
String target = String.valueOf(rule.get("target"));
String dimension = switch (target) { case "department" -> "departments"; case "group" -> "groups"; case "product" -> "products"; case "projectGroup" -> "projectGroups"; default -> null; };
if (dimension != null) {
List<Map<String, Object>> rows = rows(master.get(dimension));
Set<String> names = new LinkedHashSet<>(); rows.forEach(row -> { if (row.get("name") != null) names.add(String.valueOf(row.get("name"))); });
addTarget(rows, names, dimension, rule.get("value"), false);
addTarget(rows, names, dimension, rule.get("value"), false,previousNames(priorMaster,dimension));
master.put(dimension, rows);
}
}
if (master.isEmpty()) definition.remove("masterData");
}
private void addTarget(List<Map<String, Object>> rows, Set<String> names, String dimension, Object raw, boolean disabled) {
private Set<String> previousNames(Map<String,Object> master,String dimension) {
Set<String> names=new LinkedHashSet<>();
rows(master.get(dimension)).forEach(row->{if(row.get("name") instanceof String name)names.add(name);});
return names;
}
private Object effectiveMatchRules(Map<String,Object> definition) {
return definition.containsKey("matchRules")?definition.get("matchRules"):cn.yunsheng.quality.ingestion.ConfigurableMatchRules.defaultRules();
}
private void addTarget(List<Map<String, Object>> rows, Set<String> names, String dimension, Object raw, boolean disabled,Set<String> previousNames) {
if (raw == null || disabled) return;
if (!(raw instanceof String name) || name.isBlank()) return;
Map<String, Object> existing = rows.stream().filter(row -> name.equals(row.get("name"))).findFirst().orElse(null);
if (existing != null) {
existing.put("enabled", true);
return;
}
if (existing != null || previousNames.contains(name)) return;
if (names.add(name)) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("code", "AUTO_" + digest(dimension + "\n" + name).substring(0, 24));
@@ -769,6 +830,16 @@ public class DataWorkflowService {
errors.addAll(MappingGovernanceValidator.validate(definition));
errors.addAll(MasterDataSupport.validate(definition));
errors.addAll(cn.yunsheng.quality.ingestion.ConfigurableMatchRules.validate(definition));
Map<String,Object> master=asMap(definition.get("masterData"));
if(master!=null&&effectiveMatchRules(definition) instanceof List<?> rules)for(Object raw:rules)
if(raw instanceof Map<?,?> rule&&Boolean.TRUE.equals(rule.get("enabled"))) {
String dimension=switch(String.valueOf(rule.get("target"))) {
case "department"->"departments";case "group"->"groups";case "product"->"products";case "projectGroup"->"projectGroups";default->null;
};
if(dimension!=null&&master.containsKey(dimension)&&rows(master.get(dimension)).stream().noneMatch(row->
Boolean.TRUE.equals(row.get("enabled"))&&Objects.equals(row.get("name"),rule.get("value"))))
errors.add("条件匹配目标未登记或已停用:"+rule.get("value"));
}
return errors.stream().distinct().toList();
}
@@ -1014,6 +1085,7 @@ public class DataWorkflowService {
String expectedCurrentReleaseId, String mappingVersionId, String ruleVersionId,
String importRunId, String releaseId, String sourceChangeRequestId,
String createdBy, String createdAt, String updatedAt) {}
private record CandidatePeriod(int year,Integer month,Integer quarter) {}
private static final class WorkflowFailure extends RuntimeException {
private final String code;
@@ -0,0 +1,80 @@
package db.migration;
import java.sql.Connection;
import java.sql.SQLException;
import org.flywaydb.core.api.migration.BaseJavaMigration;
import org.flywaydb.core.api.migration.Context;
/** Correct only columns formerly written as UTC wall time into a non-UTC MySQL session.
* Includes the report lease written with SQL UTC_TIMESTAMP; SQL CURRENT_TIMESTAMP fields stay intact. */
public class V25__Normalize_legacy_instant_columns extends BaseJavaMigration {
private static final String MARKER = "legacy-utc-instants-v25";
private static final String[][] COLUMNS = {
{"source_protocol", "deadline", ""},
{"user_access_policy", "valid_until", ",updated_at=updated_at"},
{"monitor_alert_policy", "silence_until", ",updated_at=updated_at"},
{"analysis_report_plan", "next_run_at,last_run_at", ",updated_at=updated_at"},
{"analysis_report", "lease_until", ",updated_at=updated_at"}
};
@Override public void migrate(Context context) throws Exception {
Connection connection = context.getConnection();
// This marker commits with the data shift. A lost commit acknowledgement must never shift twice.
try (var statement = connection.createStatement()) {
statement.execute("CREATE TABLE IF NOT EXISTS quality_data_migration (migration_key VARCHAR(100) PRIMARY KEY,"
+ "offset_seconds INT NOT NULL,affected_rows BIGINT NOT NULL,completed_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)) ENGINE=InnoDB");
try (var result = statement.executeQuery("SELECT migration_key FROM quality_data_migration WHERE migration_key='" + MARKER + "'")) {
if (result.next()) return;
}
}
long rows = 0;
for (String[] entry : COLUMNS) {
String where = String.join(" IS NOT NULL OR ", entry[1].split(",")) + " IS NOT NULL";
try (var statement = connection.createStatement();
var result = statement.executeQuery("SELECT COUNT(*) FROM " + entry[0] + " WHERE " + where)) {
result.next(); rows += result.getLong(1);
}
}
String raw = context.getConfiguration().getPlaceholders().get("legacyTimestampOffsetSeconds");
if (rows == 0) return;
int offset = verifiedOffset(raw);
if (offset == 0) return;
// Flyway may run a Java migration without a transaction on MySQL. Keep all shifts atomic.
boolean ownTransaction = connection.getAutoCommit();
if (ownTransaction) connection.setAutoCommit(false);
try {
for (String[] entry : COLUMNS) {
String[] columns = entry[1].split(",");
String assignments = java.util.Arrays.stream(columns)
.map(column -> column + "=TIMESTAMPADD(SECOND,?," + column + ")")
.collect(java.util.stream.Collectors.joining(","));
String where = String.join(" IS NOT NULL OR ", columns) + " IS NOT NULL";
try (var update = connection.prepareStatement("UPDATE " + entry[0] + " SET " + assignments + entry[2] + " WHERE " + where)) {
for (int i = 0; i < columns.length; i++) update.setInt(i + 1, offset);
update.executeUpdate();
}
}
try (var marker = connection.prepareStatement("INSERT INTO quality_data_migration(migration_key,offset_seconds,affected_rows) VALUES(?,?,?)")) {
marker.setString(1, MARKER); marker.setInt(2, offset); marker.setLong(3, rows); marker.executeUpdate();
}
if (ownTransaction) connection.commit();
} catch (Exception failure) {
if (ownTransaction) connection.rollback();
throw failure;
} finally {
if (ownTransaction) connection.setAutoCommit(true);
}
}
static int verifiedOffset(String raw) throws SQLException {
try {
int value = Integer.parseInt(raw);
if (Math.abs((long) value) > 14 * 3600 || value % 60 != 0) throw new NumberFormatException();
return value;
} catch (RuntimeException failure) {
throw new SQLException("Existing UTC-written timestamps require a verified legacy session offset. "
+ "Back up first, then set MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS (0 for a previously UTC session). "
+ "No historical timestamps were changed.");
}
}
}
@@ -0,0 +1,7 @@
{
"version": 1,
"maxOutputTokens": 32768,
"turnTimeoutSeconds": 3000,
"requestTimeoutSeconds": 2400,
"finalizationReserveSeconds": 300
}
+5 -1
View File
@@ -13,7 +13,7 @@ spring:
application:
name: quality-platform-backend
datasource:
url: ${MYSQL_URL:jdbc:mysql://127.0.0.1:3306/quality_platform?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true}
url: ${MYSQL_URL:jdbc:mysql://127.0.0.1:3306/quality_platform?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true}
username: ${MYSQL_USER:quality_app}
password: ${MYSQL_PASSWORD:}
hikari:
@@ -25,6 +25,8 @@ spring:
enabled: true
locations: classpath:db/migration
validate-on-migrate: true
placeholders:
legacyTimestampOffsetSeconds: ${MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS:UNCONFIRMED}
servlet:
multipart:
max-file-size: 256MB
@@ -44,6 +46,8 @@ management:
quality:
analysis:
visuals-enabled: ${QUALITY_ANALYSIS_VISUALS_ENABLED:true}
autonomous-enabled: ${QUALITY_ANALYSIS_AUTONOMOUS_ENABLED:false}
protocol-v1-enabled: ${QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED:false}
archive-root: ${QUALITY_ARCHIVE_ROOT:./runtimedata/archive}
allowed-roots: ${QUALITY_ALLOWED_ROOTS:}
max-upload-bytes: ${QUALITY_MAX_UPLOAD_BYTES:268435456}
@@ -0,0 +1,35 @@
-- Independent protocol tables. Existing analysis sessions and context are not backfilled.
CREATE TABLE analysis_protocol_session (
session_id CHAR(36) NOT NULL PRIMARY KEY,
version BIGINT NOT NULL DEFAULT 0,
current_revision BIGINT,
confirmed_revision BIGINT,
current_execution_id CHAR(36),
status VARCHAR(24) NOT NULL DEFAULT 'EMPTY',
CONSTRAINT fk_protocol_session FOREIGN KEY (session_id) REFERENCES analysis_session(id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE analysis_protocol_revision (
session_id CHAR(36) NOT NULL,
revision BIGINT NOT NULL,
disposition VARCHAR(24) NOT NULL,
task_json JSON NOT NULL,
created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
PRIMARY KEY (session_id, revision),
CONSTRAINT fk_protocol_revision_session FOREIGN KEY (session_id) REFERENCES analysis_protocol_session(session_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE analysis_protocol_execution (
execution_id CHAR(36) NOT NULL PRIMARY KEY,
session_id CHAR(36) NOT NULL,
revision BIGINT NOT NULL,
request_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin NOT NULL,
request_digest CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
status VARCHAR(24) NOT NULL,
result_json JSON,
created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
CONSTRAINT fk_protocol_execution_revision FOREIGN KEY (session_id, revision) REFERENCES analysis_protocol_revision(session_id, revision),
UNIQUE KEY uq_protocol_request (session_id, request_key),
KEY ix_protocol_execution_revision (session_id, revision)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
@@ -0,0 +1,8 @@
-- The legacy protocol remains unchanged; bindings are populated only by the new transaction.
CREATE TABLE analysis_protocol_execution_turn (
execution_id CHAR(36) NOT NULL PRIMARY KEY,
turn_id CHAR(36) NOT NULL,
CONSTRAINT fk_protocol_binding_execution FOREIGN KEY (execution_id) REFERENCES analysis_protocol_execution(execution_id),
CONSTRAINT fk_protocol_binding_turn FOREIGN KEY (turn_id) REFERENCES analysis_turn(id),
KEY ix_protocol_binding_turn (turn_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
@@ -0,0 +1,35 @@
ALTER TABLE analysis_protocol_session
ADD COLUMN current_attempt_id CHAR(36),
ADD COLUMN base_context_json JSON;
CREATE TABLE analysis_protocol_attempt (
attempt_id CHAR(36) NOT NULL PRIMARY KEY,
session_id CHAR(36) NOT NULL,
turn_id CHAR(36) NOT NULL,
status VARCHAR(32) NOT NULL,
revision BIGINT,
execution_id CHAR(36),
input_json JSON NOT NULL,
created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
CONSTRAINT fk_protocol_attempt_session FOREIGN KEY (session_id) REFERENCES analysis_protocol_session(session_id),
CONSTRAINT fk_protocol_attempt_turn FOREIGN KEY (turn_id) REFERENCES analysis_turn(id),
KEY ix_protocol_attempt_turn (turn_id, created_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE analysis_protocol_query (
query_id CHAR(36) NOT NULL PRIMARY KEY,
attempt_id CHAR(36) NOT NULL,
execution_id CHAR(36) NOT NULL,
revision BIGINT NOT NULL,
auth_scope_hash CHAR(64) NOT NULL,
parameters_json JSON NOT NULL,
response_hash CHAR(64) NOT NULL,
created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
CONSTRAINT fk_protocol_query_attempt FOREIGN KEY (attempt_id) REFERENCES analysis_protocol_attempt(attempt_id),
CONSTRAINT fk_protocol_query_execution FOREIGN KEY (execution_id) REFERENCES analysis_protocol_execution(execution_id),
KEY ix_protocol_query_attempt (attempt_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
ALTER TABLE analysis_turn_event
ADD COLUMN protocol_attempt_id CHAR(36),
ADD COLUMN protocol_execution_id CHAR(36);
@@ -0,0 +1,2 @@
ALTER TABLE analysis_protocol_query ADD COLUMN query_target VARCHAR(128) NULL;
ALTER TABLE analysis_protocol_query ADD COLUMN coverage_json JSON NULL;
@@ -0,0 +1 @@
ALTER TABLE analysis_protocol_query ADD COLUMN response_json JSON NULL;
@@ -0,0 +1,8 @@
-- Optional exact execution-node bindings and server-produced ranking selector snapshots.
-- Existing queries and legacy execution remain readable without these fields.
ALTER TABLE analysis_protocol_query
ADD COLUMN protocol_node_id VARCHAR(32) NULL,
ADD COLUMN protocol_scope_id CHAR(64) NULL,
ADD COLUMN rank_artifact_json JSON NULL;
CREATE INDEX idx_analysis_protocol_query_node
ON analysis_protocol_query (execution_id, protocol_node_id, protocol_scope_id);
@@ -0,0 +1,6 @@
-- Exact response bytes for authorized historical evidence, separate from MySQL JSON normalization.
-- Existing queries remain valid; NULL means there is no reusable lossless snapshot.
ALTER TABLE analysis_protocol_query
ADD COLUMN response_snapshot MEDIUMBLOB NULL,
ADD COLUMN response_snapshot_bytes INT NULL,
ADD COLUMN response_snapshot_encoding VARCHAR(24) NULL;
@@ -0,0 +1,19 @@
-- A history import belongs to its current report execution; old business query proofs are not rebound.
CREATE TABLE analysis_protocol_history_import (
import_id CHAR(36) NOT NULL PRIMARY KEY,
attempt_id CHAR(36) NOT NULL,
execution_id CHAR(36) NOT NULL,
revision BIGINT NOT NULL,
auth_scope_hash CHAR(64) NOT NULL,
release_id CHAR(36) NOT NULL,
task_hash CHAR(64) NOT NULL,
selection_hash CHAR(64) NOT NULL,
snapshot_hash CHAR(64) NOT NULL,
snapshot_payload MEDIUMBLOB NOT NULL,
snapshot_bytes INT NOT NULL,
snapshot_encoding VARCHAR(24) NOT NULL,
created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
UNIQUE KEY uq_protocol_history_attempt (attempt_id),
CONSTRAINT fk_protocol_history_attempt FOREIGN KEY (attempt_id) REFERENCES analysis_protocol_attempt(attempt_id),
CONSTRAINT fk_protocol_history_execution FOREIGN KEY (execution_id) REFERENCES analysis_protocol_execution(execution_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
@@ -0,0 +1,23 @@
-- Presentation recovery is separate from immutable formal responses and original executions.
CREATE TABLE analysis_protocol_visual_recovery (
recovery_id CHAR(36) NOT NULL PRIMARY KEY,
turn_id CHAR(36) NOT NULL,
attempt_id CHAR(36) NOT NULL,
execution_id CHAR(36) NOT NULL,
revision BIGINT NOT NULL,
release_id CHAR(36) NOT NULL,
auth_scope_hash CHAR(64) NOT NULL,
source_hash CHAR(64) NOT NULL,
claim_token CHAR(36) NOT NULL,
status VARCHAR(24) NOT NULL,
result_payload MEDIUMBLOB,
result_hash CHAR(64),
result_bytes INT,
result_encoding VARCHAR(24),
created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
UNIQUE KEY uq_visual_recovery_source(turn_id, attempt_id, source_hash),
CONSTRAINT fk_visual_recovery_turn FOREIGN KEY (turn_id) REFERENCES analysis_turn(id),
CONSTRAINT fk_visual_recovery_attempt FOREIGN KEY (attempt_id) REFERENCES analysis_protocol_attempt(attempt_id),
CONSTRAINT fk_visual_recovery_execution FOREIGN KEY (execution_id) REFERENCES analysis_protocol_execution(execution_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
@@ -0,0 +1,18 @@
-- Only newly initialized sessions read the deployment switch. Existing sessions stay static.
ALTER TABLE analysis_protocol_session ADD autonomous_enabled BOOLEAN NOT NULL DEFAULT FALSE;
CREATE TABLE analysis_protocol_exploration_step (
step_id CHAR(36) NOT NULL PRIMARY KEY,
attempt_id CHAR(36) NOT NULL,
execution_id CHAR(36) NOT NULL,
request_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
request_digest CHAR(64) NOT NULL,
node_id VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
node_json JSON NOT NULL,
scope_json JSON NOT NULL,
parent_query_ids_json JSON NOT NULL,
created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
UNIQUE KEY uq_exploration_request(execution_id,request_key),
UNIQUE KEY uq_exploration_node(execution_id,node_id),
CONSTRAINT fk_exploration_attempt FOREIGN KEY(attempt_id) REFERENCES analysis_protocol_attempt(attempt_id),
CONSTRAINT fk_exploration_execution FOREIGN KEY(execution_id) REFERENCES analysis_protocol_execution(execution_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;