修复:查询局部刷新与智能分析会话交互,完善分析收尾和配置切换
This commit is contained in:
@@ -20,6 +20,8 @@ import java.util.concurrent.Flow;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -65,7 +67,7 @@ public class AnalysisApiSettingsService {
|
||||
|
||||
public synchronized Map<String, Object> metadata() {
|
||||
Config c = load();
|
||||
return view(c);
|
||||
return view(c, profiles(c));
|
||||
}
|
||||
|
||||
public synchronized Map<String, Object> save(Settings request) {
|
||||
@@ -73,8 +75,13 @@ public class AnalysisApiSettingsService {
|
||||
if (request == null || request.revision() == null) throw bad("请先读取当前 API 配置");
|
||||
if (!current.revision().equals(request.revision()))
|
||||
throw new ResponseStatusException(HttpStatus.CONFLICT, "API 配置已被修改,请重新打开后再保存");
|
||||
Config next = resolve(request, current);
|
||||
List<Profile> profiles = profiles(current);
|
||||
Profile selected = selectedProfile(request.profileId(), current, profiles);
|
||||
Config next = resolve(request, selected.config());
|
||||
next = new Config(next.baseUrl(), next.apiKey(), next.model(), UUID.randomUUID().toString());
|
||||
List<Profile> updated = new ArrayList<>();
|
||||
for (Profile profile : profiles) updated.add(profile.id().equals(selected.id())
|
||||
? new Profile(profile.id(), profile.name(), next) : profile);
|
||||
Path target = settingsPath();
|
||||
Path temp = null;
|
||||
try {
|
||||
@@ -82,10 +89,13 @@ public class AnalysisApiSettingsService {
|
||||
if (Files.isSymbolicLink(target.getParent()) || Files.isSymbolicLink(target)) throw new IOException();
|
||||
temp = Files.createTempFile(target.getParent(), ".nao-model-", ".tmp");
|
||||
Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------"));
|
||||
Files.writeString(temp, json.writeValueAsString(Map.of("schemaVersion", 1, "baseUrl", next.baseUrl(),
|
||||
"apiKey", next.apiKey(), "model", next.model(), "revision", next.revision())), StandardCharsets.UTF_8);
|
||||
List<Map<String,Object>> storedProfiles = updated.stream().map(p -> Map.<String,Object>of(
|
||||
"id",p.id(),"name",p.name(),"baseUrl",p.config().baseUrl(),"apiKey",p.config().apiKey(),"model",p.config().model())).toList();
|
||||
Files.writeString(temp, json.writeValueAsString(Map.of("schemaVersion", 2, "baseUrl", next.baseUrl(),
|
||||
"apiKey", next.apiKey(), "model", next.model(), "revision", next.revision(),
|
||||
"activeProfileId", selected.id(), "profiles", storedProfiles)), StandardCharsets.UTF_8);
|
||||
Files.move(temp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
|
||||
return view(next);
|
||||
return view(next, updated);
|
||||
} catch (IOException failure) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "API 配置保存失败,请检查本机配置目录");
|
||||
} finally {
|
||||
@@ -95,7 +105,10 @@ public class AnalysisApiSettingsService {
|
||||
|
||||
public Map<String, Object> test(Settings request) {
|
||||
Config c;
|
||||
synchronized (this) { c = resolve(request, load()); }
|
||||
synchronized (this) {
|
||||
Config current = load();
|
||||
c = resolve(request, selectedProfile(request == null ? null : request.profileId(), current, profiles(current)).config());
|
||||
}
|
||||
long started = System.nanoTime();
|
||||
try {
|
||||
String body = json.writeValueAsString(Map.of("model", c.model(), "stream", false, "max_tokens", 32,
|
||||
@@ -214,13 +227,73 @@ public class AnalysisApiSettingsService {
|
||||
}
|
||||
|
||||
private Path settingsPath() { return root.resolve(".runtime/nao-model-config.json"); }
|
||||
private Map<String, Object> view(Config c) {
|
||||
return Map.of("baseUrl", c.baseUrl(), "model", c.model(), "hasApiKey", !c.apiKey().isBlank(), "revision", c.revision());
|
||||
private List<Profile> profiles(Config current) {
|
||||
try {
|
||||
List<Profile> result = new ArrayList<>();
|
||||
if (Files.exists(settingsPath())) {
|
||||
Map<?,?> saved = json.readValue(Files.readString(settingsPath()), Map.class);
|
||||
if (saved.get("profiles") instanceof List<?> stored) {
|
||||
if (stored.isEmpty() || stored.size() > 8) throw new IOException();
|
||||
for (Object raw : stored) {
|
||||
if (!(raw instanceof Map<?,?> p) || !(p.get("id") instanceof String id)
|
||||
|| !id.matches("[A-Za-z0-9_-]{1,64}") || !(p.get("name") instanceof String name)
|
||||
|| name.isBlank() || name.length() > 80 || result.stream().anyMatch(v->v.id().equals(id))) throw new IOException();
|
||||
Config config = resolve(new Settings((String)p.get("baseUrl"),(String)p.get("apiKey"),
|
||||
(String)p.get("model"),current.revision()),new Config("","","",current.revision()));
|
||||
result.add(new Profile(id,name,config));
|
||||
}
|
||||
return List.copyOf(result);
|
||||
}
|
||||
}
|
||||
// Import the existing two private configurations once, without sending keys to the browser.
|
||||
if (Files.isRegularFile(legacy) && Files.size(legacy) <= 16384) {
|
||||
byte[] bytes = Files.readAllBytes(legacy);
|
||||
String text = bytes.length >= 2 && ((bytes[0] == (byte)0xff && bytes[1] == (byte)0xfe)
|
||||
|| (bytes[0] == (byte)0xfe && bytes[1] == (byte)0xff))
|
||||
? new String(bytes, StandardCharsets.UTF_16) : new String(bytes, StandardCharsets.UTF_8).replace("\ufeff", "");
|
||||
List<String> lines = text.lines().map(String::strip).filter(v->!v.isBlank()).toList();
|
||||
if (lines.size()%3 != 0 || lines.size()/3 > 8) throw new IOException();
|
||||
for (int i=0;i<lines.size();i+=3) {
|
||||
String base = lines.get(i).replaceAll("/+$", "");
|
||||
if (base.endsWith("/chat/completions")) base=base.substring(0,base.length()-17);
|
||||
if (!base.endsWith("/v1")) base+="/v1";
|
||||
Config config=resolve(new Settings(base,lines.get(i+1),lines.get(i+2),current.revision()),current);
|
||||
String model=config.model().toLowerCase(java.util.Locale.ROOT);
|
||||
result.add(new Profile("profile-"+(i/3+1),model.contains("mimo")?"MiMo":model.contains("qwen")?"Qwen":"配置 "+(i/3+1),config));
|
||||
}
|
||||
}
|
||||
int matching = -1;
|
||||
for (int i=0;i<result.size();i++) if (result.get(i).config().baseUrl().equals(current.baseUrl())) matching=i;
|
||||
if (matching>=0) {
|
||||
Profile profile=result.get(matching);result.set(matching,new Profile(profile.id(),profile.name(),current));
|
||||
} else result.add(new Profile("current","当前配置",current));
|
||||
return List.copyOf(result);
|
||||
} catch (Exception invalid) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,"已保存的 API 配置无法读取,请检查本机配置文件");
|
||||
}
|
||||
}
|
||||
private Profile selectedProfile(String id, Config current, List<Profile> profiles) {
|
||||
String selected = id == null || id.isBlank() ? activeProfileId(current,profiles) : id;
|
||||
return profiles.stream().filter(p->p.id().equals(selected)).findFirst().orElseThrow(()->bad("所选 API 配置不存在,请重新读取"));
|
||||
}
|
||||
private String activeProfileId(Config current,List<Profile> profiles) {
|
||||
return profiles.stream().filter(p->p.config().baseUrl().equals(current.baseUrl()) && p.config().model().equals(current.model()))
|
||||
.map(Profile::id).findFirst().orElse(profiles.getFirst().id());
|
||||
}
|
||||
private Map<String, Object> view(Config c,List<Profile> profiles) {
|
||||
var result = new LinkedHashMap<String,Object>();
|
||||
result.putAll(Map.of("baseUrl", c.baseUrl(), "model", c.model(), "hasApiKey", !c.apiKey().isBlank(), "revision", c.revision()));
|
||||
result.put("activeProfileId",activeProfileId(c,profiles));
|
||||
result.put("profiles",profiles.stream().map(p->Map.of("id",p.id(),"name",p.name(),"baseUrl",p.config().baseUrl(),
|
||||
"model",p.config().model(),"hasApiKey",!p.config().apiKey().isBlank())).toList());
|
||||
return result;
|
||||
}
|
||||
private static ResponseStatusException bad(String message) { return new ResponseStatusException(HttpStatus.BAD_REQUEST, message); }
|
||||
public record Settings(String baseUrl, String apiKey, String model, String revision) {
|
||||
public record Settings(String baseUrl, String apiKey, String model, String revision, String profileId) {
|
||||
public Settings(String baseUrl,String apiKey,String model,String revision) { this(baseUrl,apiKey,model,revision,null); }
|
||||
@Override public String toString() { return "Settings[credentials redacted]"; }
|
||||
}
|
||||
private record Profile(String id,String name,Config config) { }
|
||||
private record Config(String baseUrl, String apiKey, String model, String revision) {
|
||||
@Override public String toString() { return "Config[credentials redacted]"; }
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import org.springframework.transaction.support.TransactionTemplate;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PatchMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
@@ -68,6 +69,21 @@ public class AnalysisController {
|
||||
actor.getId().toString(), scopeHash(actor)));
|
||||
}
|
||||
|
||||
@PatchMapping("/sessions/{id}")
|
||||
public Map<String, Object> renameSession(@PathVariable String id, @RequestBody RenameSessionRequest request,
|
||||
@AuthenticationPrincipal AppUserPrincipal actor) {
|
||||
if (id == null || !id.matches("[a-fA-F0-9-]{36}")) throw new IllegalArgumentException("会话编号无效");
|
||||
String title = request == null || request.title() == null ? "" : request.title().strip();
|
||||
if (title.isEmpty() || title.length() > 160 || title.chars().anyMatch(Character::isISOControl))
|
||||
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "会话名称须为1至160个字符,不包含控制字符");
|
||||
int changed = jdbc.update("UPDATE analysis_session SET title=?,updated_at=CURRENT_TIMESTAMP(6) WHERE id=? AND user_id=? AND auth_scope_hash=?",
|
||||
title, id, actor.getId().toString(), scopeHash(actor));
|
||||
if (changed == 0) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "会话不存在或不可访问");
|
||||
return Map.of("sessionId", id, "title", title);
|
||||
}
|
||||
|
||||
public record RenameSessionRequest(String title) {}
|
||||
|
||||
@DeleteMapping("/sessions/{id}")
|
||||
public Map<String, Object> deleteSession(@PathVariable String id, @AuthenticationPrincipal AppUserPrincipal actor) {
|
||||
if (id == null || !id.matches("[a-fA-F0-9-]{36}")) throw new IllegalArgumentException("会话编号无效");
|
||||
|
||||
@@ -115,19 +115,19 @@ public class AnalysisFineDefectsService {
|
||||
|
||||
Aggregator aggregate = new Aggregator(request, visibleFields, categories, componentCodes, json);
|
||||
String sql = """
|
||||
SELECT %s AS raw_fields, %s AS formal_components%s
|
||||
SELECT %s AS raw_fields, %s AS formal_components, f.group_name AS business_group%s
|
||||
FROM release_member rm
|
||||
JOIN transform_build b ON b.id=rm.transform_build_id AND b.source_revision_id=rm.source_revision_id
|
||||
JOIN source_revision sr ON sr.id=rm.source_revision_id
|
||||
JOIN %s ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id
|
||||
%s
|
||||
""".formatted(jsonObject("f.raw_values", visibleFields.stream().map(DefectField::key).toList()),
|
||||
""".formatted(jsonObject("f.raw_values", java.util.stream.Stream.concat(visibleFields.stream().map(DefectField::key),categories.stream().map(c->ExpectedSourceDetailPolicy.totalField(c.componentCode())).filter(java.util.Objects::nonNull).filter(k->isAllowed(k,allowed))).distinct().toList()),
|
||||
jsonObject("f.components", new ArrayList<>(componentCodes)),
|
||||
request.dimension().equals("product") ? ", f.product AS product_key" : "", factJoin, predicate);
|
||||
jdbc.query(sql, (RowCallbackHandler) rs -> {
|
||||
try {
|
||||
aggregate.add(readObject(rs.getString("raw_fields")), readObject(rs.getString("formal_components")),
|
||||
request.dimension().equals("product") ? rs.getString("product_key") : null);
|
||||
request.dimension().equals("product") ? rs.getString("product_key") : null,rs.getString("business_group"));
|
||||
} catch (SQLException error) {
|
||||
throw new IllegalStateException(error);
|
||||
}
|
||||
@@ -372,6 +372,8 @@ public class AnalysisFineDefectsService {
|
||||
private final Map<String, ProductStats> products = new LinkedHashMap<>();
|
||||
private final JsonMapper json;
|
||||
private long sourceRows;
|
||||
private final Map<String,Long> expectedRows=new LinkedHashMap<>();
|
||||
private final Map<String,BigDecimal> expectedWeights=new LinkedHashMap<>();
|
||||
|
||||
Aggregator(Request request, List<DefectField> fields, List<Category> categories, Set<String> componentCodes, JsonMapper json) {
|
||||
this.request = request; this.fields = fields; this.categories = categories; this.componentCodes = componentCodes; this.json = json;
|
||||
@@ -379,8 +381,12 @@ public class AnalysisFineDefectsService {
|
||||
componentCodes.forEach(code -> componentStats.put(code, new FieldStats()));
|
||||
}
|
||||
|
||||
void add(Map<String, Object> raw, Map<String, Object> components, String product) {
|
||||
void add(Map<String, Object> raw, Map<String, Object> components, String product) { add(raw,components,product,null); }
|
||||
void add(Map<String, Object> raw, Map<String, Object> components, String product,String group) {
|
||||
sourceRows++;
|
||||
for(Category category:categories){List<String> selected=fields.stream().filter(f->f.category().equals(category.name())).map(DefectField::key).toList();long full=fields(request.stage()).stream().filter(f->f.category().equals(category.name())).count();String total=ExpectedSourceDetailPolicy.totalField(category.componentCode());
|
||||
if(total!=null&&selected.size()==full&&ExpectedSourceDetailPolicy.applies(request.stage(),group,selected,raw,components.get(category.componentCode()),raw.get(total))){expectedRows.merge(category.name(),1L,Long::sum);expectedWeights.merge(category.name(),AnalysisInvestigationService.decimal(components.get(category.componentCode())),BigDecimal::add);}}
|
||||
|
||||
for (DefectField field : fields) {
|
||||
FieldStats stats = fieldStats.get(field.key());
|
||||
stats.add(raw.get(field.key()));
|
||||
@@ -479,7 +485,8 @@ public class AnalysisFineDefectsService {
|
||||
}
|
||||
if (!category.available()) reasons.add(category.unavailableReason());
|
||||
if (sourceRows == 0) reasons.add("筛选范围没有来源行,当前不可用");
|
||||
String status = sourceRows == 0 || !componentAllowed ? "UNAVAILABLE"
|
||||
boolean totalsOnly=sourceRows>0&&expectedRows.getOrDefault(category.name(),0L)==sourceRows;
|
||||
String status = totalsOnly?"EXPECTED_TOTALS_ONLY":sourceRows == 0 || !componentAllowed ? "UNAVAILABLE"
|
||||
: reconciled ? "RECONCILED" : !rawComplete || !formalComplete ? "INCOMPLETE" : "DIFF";
|
||||
result.add(mapOf("category", category.name(), "sourceRows", sourceRows, "components", items,
|
||||
"subitemWeightSum", subitemSum, "validSubitemCount", validFields,
|
||||
@@ -492,7 +499,7 @@ public class AnalysisFineDefectsService {
|
||||
"negativeRows", formal.negativeRows, "sourceRows", sourceRows),
|
||||
"diff", diff, "isReconciled", reconciled,
|
||||
"reconciliationStatus", status,
|
||||
"reconciliationReasons", reasons));
|
||||
"reconciliationReasons", reasons,"expectedTotalsOnly",mapOf("policyId",ExpectedSourceDetailPolicy.ID,"canonicalGroup",ExpectedSourceDetailPolicy.GROUP,"rows",expectedRows.getOrDefault(category.name(),0L),"formalWeightRetained",expectedWeights.getOrDefault(category.name(),BigDecimal.ZERO),"fineDetailStatus","未提供/不适用;不是零缺陷"),"requiresFurtherInvestigation",!reconciled && !(expectedRows.getOrDefault(category.name(),0L)==sourceRows&&sourceRows>0)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package cn.yunsheng.quality.analysis;
|
||||
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.governance.GovernanceAccessService;
|
||||
import java.util.Map;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
import org.springframework.security.core.annotation.AuthenticationPrincipal;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
/** Read-only, release-bound investigation; never accepts SQL or filesystem paths. */
|
||||
@RestController
|
||||
@RequestMapping("/api/analysis/investigate")
|
||||
public class AnalysisInvestigationController {
|
||||
private final AnalysisInvestigationService service;
|
||||
private final GovernanceAccessService access;
|
||||
public AnalysisInvestigationController(AnalysisInvestigationService service, GovernanceAccessService access) {
|
||||
this.service = service;
|
||||
this.access = access;
|
||||
}
|
||||
@PostMapping
|
||||
@PreAuthorize("hasAnyRole('SYSTEM_ADMIN','BUSINESS_ANALYST','QUALITY_VIEWER')")
|
||||
public Map<String,Object> query(@RequestBody Map<String,Object> request, @AuthenticationPrincipal AppUserPrincipal actor) {
|
||||
// Bound service checks can lock their identity row; keep them outside the read-only scan transaction.
|
||||
access.requireBoundAction("QUERY", "query.detail");
|
||||
return service.query(request, actor);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package cn.yunsheng.quality.analysis;
|
||||
|
||||
import cn.yunsheng.quality.common.DataScope;
|
||||
import cn.yunsheng.quality.common.DepartmentLabels;
|
||||
import cn.yunsheng.quality.common.SelectionValues;
|
||||
import cn.yunsheng.quality.filters.QueryFilterFields;
|
||||
import cn.yunsheng.quality.governance.GovernanceAccessService;
|
||||
import cn.yunsheng.quality.identity.AppUserPrincipal;
|
||||
import cn.yunsheng.quality.ingestion.ConfigurableRuleEngine;
|
||||
import cn.yunsheng.quality.ingestion.XlsxStreamingReader;
|
||||
import cn.yunsheng.quality.release.ReleaseService;
|
||||
import cn.yunsheng.quality.source.SourceParserConfig;
|
||||
import cn.yunsheng.quality.source.SourceScanner;
|
||||
import java.math.BigDecimal;
|
||||
import java.nio.file.*;
|
||||
import java.sql.SQLException;
|
||||
import java.util.*;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.RowCallbackHandler;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Isolation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/** Full-scope statistics, explicit record pages and verified archive comparisons. No business writes. */
|
||||
@Service
|
||||
public class AnalysisInvestigationService {
|
||||
static final BigDecimal TOLERANCE = new BigDecimal("0.000001");
|
||||
static final int MAX_SOURCE_ROWS = 100_000;
|
||||
private static final Set<String> EXTRA_KEYS = Set.of("action","offset","recordFilter","sourceRevisionId","fields");
|
||||
private static final Map<String,String> RAW_TOTALS = Map.of("black_responsibility_scrap","责废总重量","black_material_scrap","料废重量","black_external_scrap","外废重量","bright_plating_scrap","镀废重量合计","bright_material_scrap","材料废重量合计","bright_processing_scrap","加工废重量合计","bright_diffusion_scrap","晶界扩散报废合计重量");
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ReleaseService releases;
|
||||
private final GovernanceAccessService access;
|
||||
private final JsonMapper json;
|
||||
private final SourceScanner scanner;
|
||||
private final XlsxStreamingReader reader;
|
||||
|
||||
public AnalysisInvestigationService(JdbcTemplate jdbc, ReleaseService releases, GovernanceAccessService access, JsonMapper json, SourceScanner scanner, XlsxStreamingReader reader) {
|
||||
this.jdbc=jdbc;this.releases=releases;this.access=access;this.json=json;this.scanner=scanner;this.reader=reader;
|
||||
}
|
||||
|
||||
@Transactional(readOnly=true, isolation=Isolation.REPEATABLE_READ)
|
||||
public Map<String,Object> query(Map<String,Object> input, AppUserPrincipal actor) {
|
||||
access.requireEvidence(actor);
|
||||
if(input==null)throw bad("调查参数不能为空");
|
||||
if(input.keySet().stream().anyMatch(Set.of("mode","dimension","defect")::contains))throw bad("调查接口不接受mode/dimension/defect;请使用category和fields");
|
||||
String action=text(input.getOrDefault("action","overview"));
|
||||
if(!Set.of("overview","rows","fields","source").contains(action))throw bad("action 必须是 overview/rows/fields/source");
|
||||
if(!action.equals("rows")&&(input.containsKey("offset")||input.containsKey("recordFilter")))throw bad("offset/recordFilter只用于rows分页");
|
||||
String recordFilter=text(input.getOrDefault("recordFilter","all"));
|
||||
if(!Set.of("all","differences","unexpectedDifferences").contains(recordFilter))throw bad("recordFilter 必须是 all/differences/unexpectedDifferences");
|
||||
int offset=integer(input.getOrDefault("offset",0),0,10_000_000,"offset");
|
||||
String revision=text(input.get("sourceRevisionId"));
|
||||
if(revision!=null&&!revision.matches("[a-zA-Z0-9-]{1,64}"))throw bad("sourceRevisionId 无效");
|
||||
Set<String> allowed=access.allowedFields(actor);
|
||||
Map<String,Object> base=new LinkedHashMap<>(input);EXTRA_KEYS.forEach(base::remove);base.put("mode","top");base.put("dimension","defect");
|
||||
var request=AnalysisFineDefectsService.parse(base,allowed);
|
||||
String release=releases.resolvePublishedRelease(request.releaseId());
|
||||
|
||||
List<AnalysisFineDefectsService.Category> categories=AnalysisFineDefectsService.categories(request.stage()).stream().filter(c->request.category()==null||request.category().equals(c.name())).toList();
|
||||
for(var c:categories)if(!permitted(c.componentCode(),allowed))throw forbidden("当前身份未授权调查正式分项:"+c.componentCode());
|
||||
List<String> selectedFields=fields(input.get("fields"));
|
||||
List<String> needed=new ArrayList<>();
|
||||
for(var c:categories){AnalysisFineDefectsService.fields(request.stage()).stream().filter(f->f.category().equals(c.name())).map(AnalysisFineDefectsService.DefectField::key).forEach(needed::add);needed.add(RAW_TOTALS.get(c.componentCode()));}
|
||||
// Missing field permission is a blocker, never silently reinterpreted as an empty column.
|
||||
for(String field:needed)if(!permitted(field,allowed))throw forbidden("当前身份缺少调查所需源字段:"+field);
|
||||
for(String field:selectedFields)if(!permitted(field,allowed)||sensitive(field))throw forbidden("字段未授权或包含敏感身份信息");
|
||||
if(selectedFields.isEmpty())selectedFields=List.copyOf(new LinkedHashSet<>(needed));
|
||||
if(selectedFields.size()>32)throw bad("每页最多32个字段,请按字段字典分批选择");
|
||||
State state=new State(request,categories,selectedFields,allowed,action,offset,recordFilter);
|
||||
StringBuilder predicate=new StringBuilder(" WHERE rm.release_id=?");List<Object> args=new ArrayList<>(List.of(release));
|
||||
AnalysisFineDefectsService.appendPeriod(predicate,args,request);
|
||||
SelectionValues.appendPredicate(predicate,args,"f.department",request.department(),DepartmentLabels::queryValue);
|
||||
SelectionValues.appendPredicate(predicate,args,"f.group_name",request.group(),x->x);
|
||||
SelectionValues.appendPredicate(predicate,args,"f.product",request.product(),x->x);
|
||||
if(request.productSearch()!=null){predicate.append(" AND LOCATE(?, f.product)>0");args.add(request.productSearch());}
|
||||
SelectionValues.appendPredicate(predicate,args,"f.project_group",request.projectGroup(),x->x);
|
||||
if(!request.includeUnassigned()&&!SelectionValues.parse(request.group()).contains("__UNASSIGNED__"))predicate.append(" AND (f.group_name IS NOT NULL AND f.group_name<>'')");
|
||||
if(revision!=null){predicate.append(" AND f.source_revision_id=?");args.add(revision);}
|
||||
new QueryFilterFields(jdbc,access).append(predicate,args,request.stage(),request.extraFilters(),actor);
|
||||
DataScope.append(predicate,args,actor,"f.department COLLATE utf8mb4_0900_as_cs","f.group_name COLLATE utf8mb4_0900_as_cs");
|
||||
String sql="SELECT f.id,f.source_revision_id,f.build_id,f.sheet_name,f.physical_row_no,f.group_name,f.product,f.raw_values,f.components,sr.source_file,sr.sha256,sr.file_size,sr.archive_path,b.parser_version,b.parser_config,b.parser_config_hash,b.rule_version,b.mapping_version,rv.rule_definition FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id AND b.source_revision_id=rm.source_revision_id JOIN source_revision sr ON sr.id=rm.source_revision_id JOIN fact_"+request.stage()+" f ON f.build_id=rm.transform_build_id AND f.source_revision_id=rm.source_revision_id JOIN rule_version rv ON rv.id=b.rule_version"+predicate+" ORDER BY f.id";
|
||||
jdbc.query(sql,(RowCallbackHandler)rs->{try{
|
||||
Map<String,Object> raw=decode(rs.getString("raw_values")),components=decode(rs.getString("components"));
|
||||
Map<String,Object> meta=map("sourceRevisionId",rs.getString("source_revision_id"),"buildId",rs.getString("build_id"),"sourceFile",rs.getString("source_file"),"sha256",rs.getString("sha256"),"fileSize",rs.getLong("file_size"),"parserVersion",rs.getString("parser_version"),"parserConfigHash",rs.getString("parser_config_hash"),"ruleVersion",rs.getString("rule_version"),"mappingVersion",rs.getString("mapping_version"));
|
||||
state.add(rs.getLong("id"),rs.getString("sheet_name"),rs.getInt("physical_row_no"),rs.getString("product"),rs.getString("group_name"),raw,components,meta,decode(rs.getString("parser_config")),decode(rs.getString("rule_definition")),rs.getString("archive_path"));
|
||||
}catch(SQLException e){throw new IllegalStateException(e);}},args.toArray());
|
||||
Map<String,Object> result=state.response(release,input);
|
||||
if(action.equals("source")){
|
||||
Map<String,Object> proof=verifySources(state);result.put("sourceVerification",proof);
|
||||
if(!Boolean.TRUE.equals(proof.get("complete"))||!Boolean.TRUE.equals(proof.get("allSelectedValuesMatchStored"))){
|
||||
// Totals-only is never an exemption for a broken or unverified import/archive.
|
||||
for(Object item:(List<?>)result.get("categories"))if(item instanceof Map<?,?>){
|
||||
@SuppressWarnings("unchecked") Map<String,Object> category=(Map<String,Object>)item;
|
||||
@SuppressWarnings("unchecked") Map<String,Object> expected=(Map<String,Object>)category.get("expectedTotalsOnly");
|
||||
expected.put("candidateRows",expected.get("rows"));expected.put("rows",0L);expected.put("applicationStatus","NOT_APPLIED_SOURCE_UNVERIFIED");
|
||||
category.put("unexpectedDifferenceRows",category.get("differenceRows"));category.put("unexpectedDifferenceFormalMinusSubitems",category.get("differenceFormalMinusSubitems"));category.put("requiresFurtherInvestigation",true);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private Map<String,Object> verifySources(State state){
|
||||
if(state.totalRows==0)return map("complete",false,"available",false,"verifiedRows",0,"expectedRows",0,"missingRows",0,"mismatchRows",0,"blocker","当前期间及筛选没有可核验的相关行,不能称原表核验通过");
|
||||
if(state.totalRows>MAX_SOURCE_ROWS)return map("complete",false,"verifiedRows",0,"expectedRows",state.totalRows,"blocker","单次原件核验超过100000行,请按sourceRevisionId或月份拆分;不能把未核验部分当成通过");
|
||||
long verified=0,mismatches=0,missing=0;List<Map<String,Object>> files=new ArrayList<>();
|
||||
for(Source source:state.sources.values()){
|
||||
Map<String,Object> proof=map("sourceRevisionId",source.meta.get("sourceRevisionId"),"sourceFile",source.meta.get("sourceFile"),"sha256",source.meta.get("sha256"),"expectedRows",source.rows.size(),"valueBasis","归档XLSX保存的单元格值;未执行Excel刷新或公式重算");
|
||||
try{
|
||||
Path path=Path.of(source.path);if(Files.isSymbolicLink(path)||!Files.isRegularFile(path,LinkOption.NOFOLLOW_LINKS)||!path.toRealPath().startsWith(scanner.archiveRoot())||Files.size(path)!=((Number)source.meta.get("fileSize")).longValue()||!SourceScanner.sha256(path).equals(source.meta.get("sha256")))throw new java.io.IOException("归档路径、大小或指纹校验失败");
|
||||
Set<String> seen=new HashSet<>();Map<String,Count> fieldCounts=new LinkedHashMap<>();source.fields.forEach(f->fieldCounts.put(f,new Count()));long[] difference={0};List<Map<String,Object>> examples=new ArrayList<>();
|
||||
for(String sheet:source.sheets){
|
||||
SourceParserConfig config=source.parser.isEmpty()?null:new SourceParserConfig(state.request.stage(),sheet,((Number)source.parser.getOrDefault("headerRow",1)).intValue(),strings(source.parser.get("fieldMap")),null);
|
||||
reader.read(path,sheet,config,row->{String key=sheet+":"+row.physicalRow();Map<String,Object> stored=source.rows.get(key);if(stored==null)return;seen.add(key);boolean same=true;
|
||||
for(String field:source.fields){Object actual=row.rawView().get(field);fieldCounts.get(field).add(actual,row.rawView().containsKey(field));if(row.rawView().containsKey(field)!=stored.containsKey(field)||!sameValue(actual,stored.get(field))){same=false;if(examples.size()<10)examples.add(map("sheet",sheet,"rowNumber",row.physicalRow(),"field",field,"sourceValue",safe(actual),"storedValue",safe(stored.get(field))));}}
|
||||
if(!same)difference[0]++;
|
||||
});
|
||||
}
|
||||
verified+=seen.size();mismatches+=difference[0];long absent=source.rows.size()-seen.size();missing+=absent;
|
||||
proof.putAll(map("archiveFingerprintMatches",true,"verifiedRows",seen.size(),"missingRows",absent,"mismatchRows",difference[0],"fields",fieldCounts.entrySet().stream().map(e->map("field",e.getKey(),"sourceCoverage",e.getValue().view())).toList(),"mismatchExamples",examples,"complete",absent==0));
|
||||
}catch(Exception failure){proof.putAll(map("complete",false,"blocker","归档无法完成核验;请检查文件可用性、指纹、冻结工作表和表头配置"));missing+=source.rows.size();}
|
||||
files.add(proof);
|
||||
}
|
||||
return map("complete",verified==state.totalRows&&missing==0,"expectedRows",state.totalRows,"verifiedRows",verified,"mismatchRows",mismatches,"missingRows",missing,"allSelectedValuesMatchStored",verified==state.totalRows&&missing==0&&mismatches==0,"files",files,"boundary","只核对当前发布、期间、筛选及组织授权命中的行和所选字段;未返回其他行,不推断现场生产原因");
|
||||
}
|
||||
|
||||
private final class State {
|
||||
final AnalysisFineDefectsService.Request request;final List<String> fields;final Set<String> allowed;final String action,recordFilter;final int offset;
|
||||
final Map<String,CategoryCount> categories=new LinkedHashMap<>();final Map<String,Count> dictionary=new TreeMap<>();final Map<String,Source> sources=new LinkedHashMap<>();final List<Map<String,Object>> page=new ArrayList<>();
|
||||
final PriorityQueue<Map<String,Object>> examples=new PriorityQueue<>(Comparator.comparing((Map<String,Object> r)->((BigDecimal)r.get("differenceMagnitude"))));
|
||||
long totalRows,matchingRows;
|
||||
State(AnalysisFineDefectsService.Request request,List<AnalysisFineDefectsService.Category> selected,List<String> fields,Set<String> allowed,String action,int offset,String recordFilter){this.request=request;this.fields=fields;this.allowed=allowed;this.action=action;this.offset=offset;this.recordFilter=recordFilter;for(var category:selected)categories.put(category.name(),new CategoryCount(category));}
|
||||
void add(long id,String sheet,int row,String product,String group,Map<String,Object> raw,Map<String,Object> components,Map<String,Object> meta,Map<String,Object> parser,Map<String,Object> rule,String path){
|
||||
totalRows++;raw.forEach((k,v)->{if(permitted(k,allowed)&&!sensitive(k))dictionary.computeIfAbsent(k,x->new Count()).add(v,true);});
|
||||
Source source=sources.computeIfAbsent(String.valueOf(meta.get("sourceRevisionId")),x->new Source(meta,parser,rule,path));if(!Objects.equals(source.meta.get("buildId"),meta.get("buildId")))throw new ResponseStatusException(HttpStatus.CONFLICT,"同一来源存在多个构建,请按明确发布范围核验");source.count++;source.sheets.add(sheet);
|
||||
boolean different=false,unexpected=false;BigDecimal magnitude=BigDecimal.ZERO;List<Map<String,Object>> differences=new ArrayList<>();
|
||||
for(var c:categories.values()){
|
||||
String canonical=RAW_TOTALS.get(c.category.componentCode());String sourceColumn=sourceColumn(canonical,request.stage(),parser,rule);
|
||||
if(!permitted(sourceColumn,allowed))throw forbidden("冻结映射所需源列未授权:"+sourceColumn);
|
||||
c.rawTotal.add(raw.get(sourceColumn),raw.containsKey(sourceColumn));c.formal.add(components.get(c.category.componentCode()),components.containsKey(c.category.componentCode()));
|
||||
BigDecimal sum=BigDecimal.ZERO;
|
||||
for(String field:c.fields){Object value=raw.get(field);c.counts.computeIfAbsent(field,x->new Count()).add(value,raw.containsKey(field));sum=sum.add(nonnegative(decimal(value)));}
|
||||
c.subitems=c.subitems.add(sum);BigDecimal formalValue=decimal(components.get(c.category.componentCode()));boolean comparable=formalValue!=null&&formalValue.signum()>=0&&!c.fields.isEmpty();BigDecimal delta=comparable?formalValue.subtract(sum):null;if(comparable){c.delta=c.delta.add(delta);c.comparableRows++;}
|
||||
boolean gap=comparable&&delta.abs().compareTo(TOLERANCE)>0;if(gap)c.differenceRows++;
|
||||
boolean expected=ExpectedSourceDetailPolicy.applies(request.stage(),group,c.fields,raw,components.get(c.category.componentCode()),raw.get(sourceColumn));
|
||||
if(expected){c.expectedRows++;c.expectedWeight=c.expectedWeight.add(formalValue);if(gap)c.expectedDifferenceRows++;}
|
||||
else if(gap){unexpected=true;c.unexpectedRows++;c.unexpectedDifference=c.unexpectedDifference.add(delta);}
|
||||
Map<String,Object> grouped=c.groups.computeIfAbsent(Objects.toString(group,"__UNASSIGNED__"),g->map("group",g,"rows",0L,"formalWeight",BigDecimal.ZERO,"subitemWeightSum",BigDecimal.ZERO,"differenceRows",0L,"differenceFormalMinusSubitems",BigDecimal.ZERO,"expectedTotalsOnlyRows",0L));
|
||||
grouped.put("rows",(long)grouped.get("rows")+1);grouped.put("formalWeight",((BigDecimal)grouped.get("formalWeight")).add(zero(formalValue)));grouped.put("subitemWeightSum",((BigDecimal)grouped.get("subitemWeightSum")).add(sum));if(gap){grouped.put("differenceRows",(long)grouped.get("differenceRows")+1);grouped.put("differenceFormalMinusSubitems",((BigDecimal)grouped.get("differenceFormalMinusSubitems")).add(delta));}if(expected)grouped.put("expectedTotalsOnlyRows",(long)grouped.get("expectedTotalsOnlyRows")+1);
|
||||
c.rawFormalMismatchRows+=sameValue(raw.get(sourceColumn),components.get(c.category.componentCode()))?0:1;
|
||||
if(gap){different=true;magnitude=magnitude.add(delta.abs());differences.add(map("category",c.category.name(),"subitemWeightSum",sum,"formalWeight",decimal(components.get(c.category.componentCode())),"differenceFormalMinusSubitems",delta,"rawTotalField",sourceColumn,"rawTotalValue",safe(raw.get(sourceColumn))));}
|
||||
}
|
||||
Map<String,Object> record=map("recordId",id,"sourceRevisionId",meta.get("sourceRevisionId"),"buildId",meta.get("buildId"),"sourceFile",meta.get("sourceFile"),"sheet",sheet,"rowNumber",row,"product",product,"group",group,"values",selected(raw,fields),"differences",differences,"differenceMagnitude",magnitude);
|
||||
if(recordFilter.equals("all")||recordFilter.equals("differences")&&different||recordFilter.equals("unexpectedDifferences")&&unexpected){long position=matchingRows++;if(position>=offset&&page.size()<request.limit())page.add(record);}
|
||||
if(different){examples.add(record);if(examples.size()>5)examples.poll();}
|
||||
if(action.equals("source")&&totalRows<=MAX_SOURCE_ROWS){List<String> verifyFields=new ArrayList<>(fields);for(var c:categories.values()){String col=sourceColumn(RAW_TOTALS.get(c.category.componentCode()),request.stage(),parser,rule);if(!verifyFields.contains(col))verifyFields.add(col);}source.fields.addAll(verifyFields);source.rows.put(sheet+":"+row,unclipped(raw,verifyFields));}
|
||||
}
|
||||
Map<String,Object> response(String release,Map<String,Object> input){
|
||||
Map<String,Object> context=map("releaseId",release,"stage",request.stage(),"year",request.year(),"month",request.month(),"quarter",request.quarter(),"department",request.department(),"group",request.group(),"product",request.product(),"projectGroup",request.projectGroup(),"includeUnassigned",request.includeUnassigned(),"extraFilters",request.extraFilters(),"category",request.category(),"productSearch",request.productSearch(),"sourceRevisionId",input.get("sourceRevisionId"));
|
||||
List<Map<String,Object>> lineage=sources.values().stream().map(s->{Map<String,Object> m=new LinkedHashMap<>(s.meta);m.put("matchedRows",s.count);m.put("sheets",s.sheets);Map<String,Object> mappings=new LinkedHashMap<>();strings(s.parser.get("fieldMap")).forEach((k,v)->{if((action.equals("fields")||fields.contains(k))&&permitted(k,allowed)&&permitted(v,allowed)&&!sensitive(k)&&!sensitive(v))mappings.put(k,v);});m.put("frozenFieldMap",mappings);m.put("componentRules",rulesFor(s.rule,categories.values().stream().map(c->c.category.componentCode()).toList()));m.put("formulaSourceAliases",safeAliases(s.rule,request.stage(),allowed));return m;}).toList();
|
||||
for(String field:fields)if(input.containsKey("fields")&&totalRows>0&&!dictionary.containsKey(field))throw new ResponseStatusException(HttpStatus.CONFLICT,"所选源字段不在命中行字段字典中:"+field);
|
||||
Map<String,Object> result=map("kind","quality_investigation","action",action,"context",context,"totalRows",totalRows,"aggregateComplete",true,"aggregateRowLimit",null,"categories",categories.values().stream().map(CategoryCount::view).toList(),"lineage",lineage,"selectedFields",fields,"availableFieldCount",dictionary.size(),"fieldDictionary",dictionary.entrySet().stream().filter(e->action.equals("fields")||fields.contains(e.getKey())).map(e->map("field",e.getKey(),"coverage",e.getValue().view(totalRows))).toList(),"largestDifferenceExamples",examples.stream().sorted(Comparator.comparing((Map<String,Object> r)->((BigDecimal)r.get("differenceMagnitude"))).reversed()).toList(),"exampleLimit",5,"examplesAreNotFullRows",true,"basis","所有统计扫描全部命中事实行;空/非法/负数细分不作有效重量;差额是正式合计减已知非负细分和,缺失覆盖单列,不能把差额认作某缺失字段的值。不是新的正式KPI,也不推断设备、工艺或人员原因。","nextSteps",action.equals("source")?List.of("结合原件核验与冻结规则解释表内差额;未核验部分明确列为卡点"):List.of("存在差额或字段缺失时调用action=source核验全部相关原表行;多来源/超限按sourceRevisionId拆分","action=rows配合recordFilter=differences查具体行;用nextOffset翻页,不能把一页当全量"));
|
||||
if(action.equals("rows"))result.putAll(map("rows",page,"offset",offset,"limit",request.limit(),"returnedRows",page.size(),"matchingRows",matchingRows,"hasMore",offset+page.size()<matchingRows,"nextOffset",offset+page.size()<matchingRows?offset+page.size():null,"pageComplete",offset+page.size()>=matchingRows,"recordFilter",recordFilter));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
static final class Count {
|
||||
long rows,absent,nulls,invalid,negative,present;BigDecimal sum=BigDecimal.ZERO;
|
||||
void add(Object v,boolean exists){rows++;if(!exists)absent++;if(v==null||v instanceof String s&&s.isBlank()){nulls++;return;}BigDecimal n=decimal(v);if(n==null){invalid++;return;}present++;if(n.signum()<0)negative++;sum=sum.add(n);}
|
||||
Map<String,Object> view(long expected){Map<String,Object> result=view();result.put("rows",expected);result.put("keyAbsentRows",absent+expected-rows);result.put("nullOrBlankRows",nulls+expected-rows);return result;}
|
||||
Map<String,Object> view(){return map("rows",rows,"keyAbsentRows",absent,"nullOrBlankRows",nulls,"numericRows",present,"invalidNumericRows",invalid,"negativeRows",negative,"numericSum",present==0?null:sum);}
|
||||
}
|
||||
static final class CategoryCount {
|
||||
final AnalysisFineDefectsService.Category category;final List<String> fields;final Map<String,Count> counts=new LinkedHashMap<>();final Count formal=new Count(),rawTotal=new Count();
|
||||
BigDecimal subitems=BigDecimal.ZERO,delta=BigDecimal.ZERO,expectedWeight=BigDecimal.ZERO,unexpectedDifference=BigDecimal.ZERO;long differenceRows,rawFormalMismatchRows,comparableRows,expectedRows,expectedDifferenceRows,unexpectedRows;final Map<String,Map<String,Object>> groups=new TreeMap<>();
|
||||
CategoryCount(AnalysisFineDefectsService.Category category){this.category=category;this.fields=AnalysisFineDefectsService.fields(category.componentCode().startsWith("black")?"black":"bright").stream().filter(f->f.category().equals(category.name())).map(AnalysisFineDefectsService.DefectField::key).toList();}
|
||||
Map<String,Object> view(){return map("category",category.name(),"formalComponent",category.componentCode(),"formalWeight",formal.present==0?null:formal.sum,"rawTotalWeight",rawTotal.present==0?null:rawTotal.sum,"subitemWeightSum",fields.isEmpty()?null:subitems,"differenceFormalMinusSubitems",fields.isEmpty()||comparableRows!=formal.rows?null:delta,"knownFormalRowsDifference",fields.isEmpty()?null:delta,"comparableRows",comparableRows,"missingOrInvalidFormalRows",fields.isEmpty()?null:formal.rows-comparableRows,"differenceRows",differenceRows,"rawFormalMismatchRows",rawFormalMismatchRows,"formalCoverage",formal.view(),"rawTotalCoverage",rawTotal.view(),"fields",counts.entrySet().stream().map(e->map("field",e.getKey(),"coverage",e.getValue().view())).toList(),"noFineFields",fields.isEmpty(),"byGroup",new ArrayList<>(groups.values()),"expectedTotalsOnly",map("policyId",ExpectedSourceDetailPolicy.ID,"canonicalGroup",ExpectedSourceDetailPolicy.GROUP,"rows",expectedRows,"differenceRows",expectedDifferenceRows,"formalWeightRetained",expectedWeight,"fineDetailStatus","未提供/不适用;不是零缺陷,正式总量保留"),"unexpectedDifferenceRows",unexpectedRows,"unexpectedDifferenceFormalMinusSubitems",unexpectedDifference,"requiresFurtherInvestigation",unexpectedRows>0||rawFormalMismatchRows>0||(!fields.isEmpty()&&comparableRows<formal.rows));}
|
||||
}
|
||||
private static final class Source {
|
||||
final Map<String,Object> meta,parser,rule;final String path;final Set<String> sheets=new LinkedHashSet<>(),fields=new LinkedHashSet<>();final Map<String,Map<String,Object>> rows=new LinkedHashMap<>();long count;
|
||||
Source(Map<String,Object> meta,Map<String,Object> parser,Map<String,Object> rule,String path){this.meta=meta;this.parser=parser;this.rule=rule;this.path=path;}
|
||||
}
|
||||
static String sourceColumn(String canonical,String stage,Map<String,Object> parser,Map<String,Object> rule){Map<String,String> frozen=strings(parser.get("fieldMap"));if(frozen.containsKey(canonical))return frozen.get(canonical);for(var item:ConfigurableRuleEngine.SOURCE_FIELDS.get(stage).entrySet())if(item.getValue().equals(canonical)){Map<String,Object> stages=record(rule.get("sourceFields"));return String.valueOf(record(stages.get(stage)).getOrDefault(item.getKey(),canonical));}return canonical;}
|
||||
private static List<Object> rulesFor(Map<String,Object> rule,List<String> components){return rule.get("rowRules") instanceof List<?> rows?rows.stream().filter(x->x instanceof Map<?,?> m&&components.contains(m.get("target"))).map(x->(Object)x).toList():List.of();}
|
||||
private static Map<String,Object> safeAliases(Map<String,Object> rule,String stage,Set<String> allowed){Map<String,Object> result=new LinkedHashMap<>();record(record(rule.get("sourceFields")).get(stage)).forEach((k,v)->{if(permitted(String.valueOf(v),allowed)&&!sensitive(String.valueOf(v)))result.put(k,v);});return result;}
|
||||
private Map<String,Object> decode(String value){if(value==null||value.isBlank())return Map.of();try{return record(json.readValue(value,Map.class));}catch(Exception e){throw new ResponseStatusException(HttpStatus.CONFLICT,"冻结来源或规则JSON无法读取");}}
|
||||
private static Map<String,Object> record(Object v){Map<String,Object> result=new LinkedHashMap<>();if(v instanceof Map<?,?> m)m.forEach((k,x)->result.put(String.valueOf(k),x));return result;}
|
||||
private static Map<String,String> strings(Object v){Map<String,String> r=new LinkedHashMap<>();record(v).forEach((k,x)->r.put(k,String.valueOf(x)));return r;}
|
||||
private static List<String> fields(Object raw){if(raw==null)return List.of();if(!(raw instanceof List<?> values)||values.isEmpty()||values.size()>32)throw bad("fields 须为1至32个源字段");List<String> result=new ArrayList<>();for(Object v:values){if(!(v instanceof String s)||s.isBlank()||s.length()>200)throw bad("fields 包含无效字段");if(!result.contains(s))result.add(s);}return result;}
|
||||
private static Map<String,Object> selected(Map<String,Object> raw,List<String> fields){Map<String,Object> m=new LinkedHashMap<>();fields.forEach(f->m.put(f,safe(raw.get(f))));return m;}
|
||||
private static Object safe(Object v){
|
||||
if(v instanceof Map<?,?>||v instanceof Collection<?>)return "(非标量内容未展示)";
|
||||
if(!(v instanceof String))return v;
|
||||
String s=((String)v).replaceAll("(?i)(bearer\\s+)[A-Za-z0-9._~+/-]+", "$1[REDACTED]")
|
||||
.replaceAll("(?i)(api[_-]?key|password|token|secret|authorization)(\\s*[=::]\\s*)[^\\s,,;;]+", "$1$2[REDACTED]");
|
||||
return s.length()>500?s.substring(0,500)+"(内容截断)":s;
|
||||
}
|
||||
static BigDecimal decimal(Object v){if(v==null||v instanceof Boolean)return null;try{return new BigDecimal(v.toString().trim());}catch(Exception e){return null;}}
|
||||
static boolean sameValue(Object a,Object b){BigDecimal x=decimal(a),y=decimal(b);if(x!=null&&y!=null)return x.subtract(y).abs().compareTo(TOLERANCE)<=0;return Objects.equals(a==null?null:a.toString(),b==null?null:b.toString());}
|
||||
private static Map<String,Object> unclipped(Map<String,Object> raw,List<String> fields){Map<String,Object> r=new LinkedHashMap<>();fields.forEach(f->{if(raw.containsKey(f))r.put(f,raw.get(f));});return r;}
|
||||
private static BigDecimal nonnegative(BigDecimal v){return v==null||v.signum()<0?BigDecimal.ZERO:v;}
|
||||
private static BigDecimal zero(BigDecimal v){return v==null?BigDecimal.ZERO:v;}
|
||||
private static boolean permitted(String field,Set<String> allowed){return allowed.contains("*")||allowed.contains(field);}
|
||||
private static boolean sensitive(String field){return field.matches("(?i).*(密码|密钥|令牌|身份证|手机号|联系电话|银行账号|银行卡|邮箱|姓名|联系人|电话|手机|销售员|业务组长|产品工程师|交付工程师|生产主管|操作人|操作者|业务员|email|passport|password|secret|token|api.?key|authorization).*");}
|
||||
private static String text(Object v){return v==null?null:String.valueOf(v);}
|
||||
private static int integer(Object v,int min,int max,String key){try{String s=String.valueOf(v);if(!s.matches("[0-9]{1,8}"))throw new NumberFormatException();int n=Integer.parseInt(s);if(n<min||n>max)throw new NumberFormatException();return n;}catch(Exception e){throw bad(key+" 超出允许范围");}}
|
||||
static Map<String,Object> map(Object... pairs){Map<String,Object> r=new LinkedHashMap<>();for(int i=0;i<pairs.length;i+=2)r.put(String.valueOf(pairs[i]),pairs[i+1]);return r;}
|
||||
private static ResponseStatusException bad(String m){return new ResponseStatusException(HttpStatus.BAD_REQUEST,m);}
|
||||
private static ResponseStatusException forbidden(String m){return new ResponseStatusException(HttpStatus.FORBIDDEN,m);}
|
||||
}
|
||||
@@ -289,6 +289,8 @@ public class AnalysisTurnService {
|
||||
"reportMarkdown", "reportHtml", "nativeChatId", "toolCallCount"))
|
||||
if (answer.containsKey(field)) result.put(field, answer.get(field));
|
||||
result.put("stopReason", turn.stopReason()); result.put("budget", budget);
|
||||
result.putAll(jdbc.queryForObject("SELECT UNIX_TIMESTAMP(created_at)*1000,UNIX_TIMESTAMP(started_at)*1000,UNIX_TIMESTAMP(finished_at)*1000 FROM analysis_turn WHERE id=? AND user_id=?",
|
||||
(rs, n) -> turnTimes(rs.getBigDecimal(1), rs.getBigDecimal(2), rs.getBigDecimal(3)), turnId, actor.getId().toString()));
|
||||
result.put("feedback", feedback(turnId));
|
||||
return result;
|
||||
}
|
||||
@@ -497,12 +499,21 @@ public class AnalysisTurnService {
|
||||
|
||||
public Map<String, Object> listTurns(String sessionId, AppUserPrincipal actor) {
|
||||
ownedSession(sessionId, actor);
|
||||
List<Map<String, Object>> items = jdbc.query("SELECT id,question,status,created_at,finished_at FROM analysis_turn WHERE session_id=? AND user_id=? ORDER BY created_at DESC",
|
||||
List<Map<String, Object>> items = jdbc.query("SELECT id,question,status,UNIX_TIMESTAMP(created_at)*1000,UNIX_TIMESTAMP(started_at)*1000,UNIX_TIMESTAMP(finished_at)*1000 FROM analysis_turn WHERE session_id=? AND user_id=? ORDER BY created_at DESC",
|
||||
(rs, n) -> { Map<String, Object> m = new LinkedHashMap<>(); m.put("id", rs.getString(1)); m.put("question", rs.getString(2)); m.put("status", rs.getString(3));
|
||||
m.put("createdAt", rs.getTimestamp(4).toInstant().toString()); if (rs.getTimestamp(5) != null) m.put("finishedAt", rs.getTimestamp(5).toInstant().toString()); return m; }, sessionId, actor.getId().toString());
|
||||
m.putAll(turnTimes(rs.getBigDecimal(4),rs.getBigDecimal(5),rs.getBigDecimal(6))); return m; }, sessionId, actor.getId().toString());
|
||||
return Map.of("items", items);
|
||||
}
|
||||
|
||||
static Map<String,Object> turnTimes(BigDecimal created, BigDecimal started, BigDecimal finished) {
|
||||
Map<String,Object> times = new LinkedHashMap<>();
|
||||
if (created != null) times.put("createdAt", Instant.ofEpochMilli(created.longValue()).toString());
|
||||
if (started != null) times.put("startedAt", Instant.ofEpochMilli(started.longValue()).toString());
|
||||
if (finished != null) times.put("finishedAt", Instant.ofEpochMilli(finished.longValue()).toString());
|
||||
if (started != null && finished != null) times.put("durationMs", Math.max(0L,finished.longValue()-started.longValue()));
|
||||
return times;
|
||||
}
|
||||
|
||||
private void submitWorker(String turnId) {
|
||||
executor.execute(() -> run(turnId));
|
||||
}
|
||||
@@ -792,14 +803,18 @@ public class AnalysisTurnService {
|
||||
progress -> {
|
||||
String message = Objects.toString(progress.getOrDefault("message", "正在分析质量数据…"), "正在分析质量数据…");
|
||||
if (progress.get("steps") instanceof List<?> steps) plan.put("steps", steps);
|
||||
if (progress.get("timeline") instanceof List<?> timeline) plan.put("timeline", timeline);
|
||||
plan.put("toolCount", progress.getOrDefault("toolCount", 0));
|
||||
plan.put("progressMessage", message);
|
||||
plan.put("updatedAt", Instant.now().toString());
|
||||
List<Map<String, Object>> steps = mapList(plan.get("steps"));
|
||||
persistState(turnId, "RUNNING", plan, steps, List.of(), null);
|
||||
appendEvent(turnId, "progress", Map.of("turnId", turnId, "status", "RUNNING",
|
||||
"message", message, "toolCount", plan.get("toolCount"), "steps", steps,
|
||||
"startedAt", plan.get("startedAt"), "updatedAt", plan.get("updatedAt")));
|
||||
Map<String, Object> event = new LinkedHashMap<>();
|
||||
event.put("turnId", turnId); event.put("status", "RUNNING");
|
||||
event.put("message", message); event.put("toolCount", plan.get("toolCount")); event.put("steps", steps);
|
||||
event.put("startedAt", plan.get("startedAt")); event.put("updatedAt", plan.get("updatedAt"));
|
||||
if (plan.containsKey("timeline")) event.put("timeline", plan.get("timeline"));
|
||||
appendEvent(turnId, "progress", event);
|
||||
},
|
||||
answer -> persistNativeAnswer(turnId, UUID.fromString(turn.userId()), turn.authScopeHash(),
|
||||
session.releaseId(), context, answer),
|
||||
@@ -817,6 +832,7 @@ public class AnalysisTurnService {
|
||||
plan.put("steps", steps);
|
||||
plan.put("toolCount", steps.size());
|
||||
plan.put("stopReason", stopReason);
|
||||
if (result.details().get("timeline") instanceof List<?>) plan.put("timeline", NaoAnalysisRunner.safeTimeline(result.details().get("timeline")));
|
||||
Map<String, Object> answer = nativeAnswer(result.details(), answerText, context, session.releaseId(), status, stopReason);
|
||||
persistContext(session.id(), finalActor, context);
|
||||
assertScope(currentUser(UUID.fromString(turn.userId())), turn.authScopeHash());
|
||||
@@ -875,6 +891,7 @@ public class AnalysisTurnService {
|
||||
answer.put("stopReason", Objects.requireNonNullElse(stopReason, ""));
|
||||
for (String key : List.of("reportMarkdown", "reportHtml", "nativeChatId"))
|
||||
if (details.get(key) instanceof String value && value.length() <= 2_000_000) answer.put(key, value);
|
||||
if (details.get("timeline") instanceof List<?>) answer.put("timeline", NaoAnalysisRunner.safeTimeline(details.get("timeline")));
|
||||
Object count = details.get("toolCount");
|
||||
if (count instanceof Number) answer.put("toolCallCount", count);
|
||||
return answer;
|
||||
@@ -897,11 +914,11 @@ public class AnalysisTurnService {
|
||||
Object state = call.get("status");
|
||||
step.put("status", state == null ? "SUCCEEDED" : cap(state.toString(), 40));
|
||||
if (Boolean.TRUE.equals(call.get("recovered"))) step.put("recovered", true);
|
||||
for (String field : List.of("label", "outputSummary", "startedAt", "finishedAt"))
|
||||
for (String field : List.of("toolCallId", "label", "outputSummary", "startedAt", "finishedAt"))
|
||||
if (call.get(field) instanceof String text) step.put(field, cap(text, 500));
|
||||
if (call.get("input") instanceof Map<?, ?> input) {
|
||||
Map<String, Object> safeInput = new LinkedHashMap<>();
|
||||
for (String field : List.of("mode", "scope", "period", "comparisonPeriod", "filters", "view", "stage", "dimension", "defect", "category", "limit", "sort")) {
|
||||
for (String field : List.of("mode", "scope", "period", "comparisonPeriod", "filters", "view", "stage", "dimension", "defect", "category", "limit", "sort", "action", "offset", "recordFilter")) {
|
||||
Object item = input.get(field);
|
||||
if (item instanceof Map<?, ?> nested) {
|
||||
Map<String, Object> safeNested = new LinkedHashMap<>();
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package cn.yunsheng.quality.analysis;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/** Recorded business exception: canonical VCM bright group supplies category totals only.
|
||||
* Never changes facts, formal totals, or the KPI formula. A nonzero fine value, invalid value,
|
||||
* unknown group, missing total, or raw/formal disagreement prevents the exception. */
|
||||
final class ExpectedSourceDetailPolicy {
|
||||
static final String ID="VCM_BRIGHT_TOTALS_ONLY";
|
||||
static final String GROUP="工业电机-VCM";
|
||||
static boolean applies(String stage,String group,List<String> fields,Map<String,Object> raw,Object formal,Object rawTotal) {
|
||||
if(!"bright".equals(stage)||!GROUP.equals(group)||fields.isEmpty())return false;
|
||||
BigDecimal f=AnalysisInvestigationService.decimal(formal),t=AnalysisInvestigationService.decimal(rawTotal);
|
||||
if(f==null||t==null||f.signum()<0||t.signum()<0||!AnalysisInvestigationService.sameValue(f,t))return false;
|
||||
for(String field:fields){Object value=raw.get(field);if(value==null||value instanceof String s&&s.isBlank())continue;
|
||||
BigDecimal n=AnalysisInvestigationService.decimal(value);if(n==null||n.signum()!=0)return false;}
|
||||
return true;
|
||||
}
|
||||
static String totalField(String code){return switch(code){case "bright_plating_scrap"->"镀废重量合计";case "bright_material_scrap"->"材料废重量合计";case "bright_processing_scrap"->"加工废重量合计";case "bright_diffusion_scrap"->"晶界扩散报废合计重量";default->null;};}
|
||||
}
|
||||
@@ -176,9 +176,42 @@ public class NaoAnalysisRunner {
|
||||
safe.put("toolCount", number.intValue());
|
||||
if (message.get("toolCalls") instanceof List<?>)
|
||||
safe.put("steps", AnalysisTurnService.nativeSteps(message.get("toolCalls")));
|
||||
if (message.get("timeline") instanceof List<?>) safe.put("timeline", safeTimeline(message.get("timeline")));
|
||||
return safe;
|
||||
}
|
||||
|
||||
static List<Map<String, Object>> safeTimeline(Object value) {
|
||||
if (!(value instanceof List<?> entries)) return List.of();
|
||||
var safe = new java.util.ArrayList<Map<String, Object>>();
|
||||
long lastSequence = 0;
|
||||
for (Object raw : entries.stream().limit(3000).toList()) {
|
||||
if (!(raw instanceof Map<?, ?> entry) || !(entry.get("sequence") instanceof Number number)) continue;
|
||||
long sequence = number.longValue();
|
||||
String kind = Objects.toString(entry.get("kind"), "");
|
||||
if (sequence <= lastSequence || !List.of("assistant", "tool_start", "tool_result").contains(kind)) continue;
|
||||
var row = new LinkedHashMap<String, Object>();
|
||||
row.put("sequence", sequence); row.put("kind", kind);
|
||||
for (String key : List.of("at", "toolCallId", "tool", "label", "status", "text"))
|
||||
if (entry.get(key) instanceof String text) row.put(key, publicText(cap(text, key.equals("text") ? 8000 : 160)));
|
||||
if (entry.get("input") instanceof Map<?, ?> input) {
|
||||
var steps = AnalysisTurnService.nativeSteps(List.of(Map.of("input", input)));
|
||||
row.put("input", steps.getFirst().get("input"));
|
||||
}
|
||||
if(entry.get("evidenceChecks") instanceof Map<?,?> checks){
|
||||
Map<String,String> verified=new LinkedHashMap<>();
|
||||
checks.entrySet().stream().limit(64).forEach(check->{String key=String.valueOf(check.getKey()), state=String.valueOf(check.getValue());if(key.matches("[a-f0-9]{16}")&&List.of("unresolved","checked","expected").contains(state))verified.put(key,state);});
|
||||
if(!verified.isEmpty())row.put("evidenceChecks",verified);
|
||||
}
|
||||
safe.add(row); lastSequence = sequence;
|
||||
}
|
||||
return List.copyOf(safe);
|
||||
}
|
||||
|
||||
private static String publicText(String value) {
|
||||
return value.replaceAll("(?i)Bearer\\s+[A-Za-z0-9._~+/-]+", "Bearer [REDACTED]")
|
||||
.replaceAll("(?i)(api[_-]?key|password|token|secret|authorization)(\\s*[=::]\\s*)[^\\s,,;;]+", "$1$2[REDACTED]");
|
||||
}
|
||||
|
||||
private void writeCancel(BufferedWriter stdin) throws IOException {
|
||||
stdin.write("{\"type\":\"cancel\"}");
|
||||
stdin.newLine();
|
||||
@@ -282,7 +315,11 @@ public class NaoAnalysisRunner {
|
||||
if (cancellationRequested || "CANCELLED".equals(status)) {
|
||||
status = "CANCELLED";
|
||||
reason = "用户请求停止";
|
||||
} else if ("COMPLETED".equals(status) && !"stop".equalsIgnoreCase(reason)) {
|
||||
} else if ("COMPLETED".equals(status) && !"stop".equalsIgnoreCase(reason)
|
||||
&& !("evidence_finalized".equals(reason)
|
||||
&& "bounded_evidence_finalization".equals(message.get("completionMethod"))
|
||||
&& "stop".equals(message.get("finishReason"))
|
||||
&& !Boolean.FALSE.equals(message.get("nativeStopSucceeded")))) {
|
||||
status = "PARTIAL";
|
||||
if (reason.isBlank()) reason = "原装 NAO 未以 stop 原因正常结束";
|
||||
} else if (!List.of("COMPLETED", "PARTIAL", "FAILED").contains(status)) {
|
||||
|
||||
@@ -42,11 +42,12 @@ public class NaoRuntimeQueryController {
|
||||
private final AnalysisToolsController tools;
|
||||
private final AnalyticsController analytics;
|
||||
private final AnalysisFineDefectsController defects;
|
||||
private final AnalysisInvestigationController investigation;
|
||||
|
||||
public NaoRuntimeQueryController(NaoRuntimeAccessService access, ReleaseController releases,
|
||||
OptionsController options, MetricsController metrics,
|
||||
AnalysisToolsController tools, AnalyticsController analytics,
|
||||
AnalysisFineDefectsController defects) {
|
||||
AnalysisFineDefectsController defects, AnalysisInvestigationController investigation) {
|
||||
this.access = access;
|
||||
this.releases = releases;
|
||||
this.options = options;
|
||||
@@ -54,6 +55,7 @@ public class NaoRuntimeQueryController {
|
||||
this.tools = tools;
|
||||
this.analytics = analytics;
|
||||
this.defects = defects;
|
||||
this.investigation = investigation;
|
||||
}
|
||||
|
||||
@PostMapping("/query")
|
||||
@@ -90,6 +92,7 @@ public class NaoRuntimeQueryController {
|
||||
Map<String, Object> bound = boundPayload(payload, releaseId);
|
||||
yield tools.details(bound, delegation.actor());
|
||||
}
|
||||
case "POST /api/analysis/investigate" -> investigation.query(boundPayload(payload, releaseId), delegation.actor());
|
||||
case "POST /api/analysis/defects" -> {
|
||||
Map<String, Object> bound = boundPayload(payload, releaseId);
|
||||
yield defects.query(bound, delegation.actor());
|
||||
|
||||
Reference in New Issue
Block a user