完善:同月多文件导入与当前用户离线迁移更新

This commit is contained in:
2026-11-04 11:35:24 +08:00
parent 147b56d0fd
commit f04d4e96e3
89 changed files with 6634 additions and 201 deletions
@@ -88,7 +88,9 @@ public class AnalysisApiSettingsService {
Files.createDirectories(target.getParent());
if (Files.isSymbolicLink(target.getParent()) || Files.isSymbolicLink(target)) throw new IOException();
temp = Files.createTempFile(target.getParent(), ".nao-model-", ".tmp");
Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------"));
if (Files.getFileStore(temp).supportsFileAttributeView("posix")) {
Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------"));
}
List<Map<String,Object>> storedProfiles = updated.stream().map(p -> Map.<String,Object>of(
"id",p.id(),"name",p.name(),"baseUrl",p.config().baseUrl(),"apiKey",p.config().apiKey(),"model",p.config().model(),"thinkingEnabled",p.config().thinkingEnabled(),"thinkingEffort",p.config().thinkingEffort())).toList();
Files.writeString(temp, json.writeValueAsString(Map.of("schemaVersion", 3, "baseUrl", next.baseUrl(),
@@ -19,8 +19,18 @@ import org.springframework.stereotype.Service;
}
static boolean overlaps(Coverage a,Coverage b) {
if(!Objects.equals(a.stage(),b.stage())||a.year()!=b.year()||a.month()!=b.month()||!SourceIdentity.intersects(a.scope(),b.scope()))return false;
return Objects.equals(SourceIdentity.canonicalSlot(a.slot()),SourceIdentity.canonicalSlot(b.slot()))
||!"SHARD_SNAPSHOT".equals(a.mode())||!"SHARD_SNAPSHOT".equals(b.mode())||Objects.equals(a.sha(),b.sha());
if(Objects.equals(SourceIdentity.canonicalSlot(a.slot()),SourceIdentity.canonicalSlot(b.slot()))||Objects.equals(a.sha(),b.sha()))return true;
if("SHARD_SNAPSHOT".equals(a.mode())&&"SHARD_SNAPSHOT".equals(b.mode()))return false;
// APPEND_SNAPSHOT is created only by the explicit upload workflow. Its scope must match
// exactly before it can augment a complete monthly file; intersecting declarations still conflict.
if(isUploadAppend(a.mode())||isUploadAppend(b.mode())){
boolean supportedPair=(isUploadAppend(a.mode())&&isUploadAppend(b.mode()))
||("MONTHLY_SNAPSHOT".equals(a.mode())&&isUploadAppend(b.mode()))
||("MONTHLY_SNAPSHOT".equals(b.mode())&&isUploadAppend(a.mode()));
return !supportedPair||!Objects.equals(SourceIdentity.scope(a.scope()),SourceIdentity.scope(b.scope()));
}
return true;
}
private static boolean isUploadAppend(String mode){return "APPEND_SNAPSHOT".equals(mode);}
record Coverage(String stage,int year,int month,String scope,String mode,String sha,String slot) {}
}
@@ -255,8 +255,7 @@ public class ImportWorker {
issues.add(new Issue(candidate, row, "INVALID_NUMBER", field, "数值字段不能转换为重量数值,原值已保留在raw中", true));
}
if (value.unmappedCategory()) {
blocking[0]++;
issues.add(new Issue(candidate, row, "CATEGORY_MAPPING_MISSING", "PDT代码/销售片区", "非空来源键未命中已验证的产业部-地区映射", true));
issues.add(new Issue(candidate, row, "CATEGORY_MAPPING_MISSING", "PDT代码/销售片区", "未命中业务分组对应关系,已保留为未分组,可在对应关系中补充归类", false));
} else if ("source_key_blank".equals(value.components().get("categoryMappingStatus"))) {
blankCategory[0]++;
issues.add(new Issue(candidate, row, "SOURCE_CATEGORY_KEY_BLANK", "PDT代码/销售片区", "原始匹配键为空,按源模型趋势上下文默认排除;原始行保留", false));
@@ -19,6 +19,8 @@ final class TransformBuildKey {
Integer.toString(source.year()), Integer.toString(source.month()), parserVersion,
rulesVersion, mappingVersion);
if(parserConfigHash!=null&&!"legacy".equals(parserConfigHash))identity+="\n"+parserConfigHash;
// Validation policy is part of cache identity; old mapping rejections must not survive a retry.
identity += "\nunassigned-mapping-nonblocking-v1";
try {
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(identity.getBytes(StandardCharsets.UTF_8)));
} catch (NoSuchAlgorithmException e) {
@@ -32,8 +32,8 @@ public class SourceUploadService {
}
public Map<String,Object> options(AppUserPrincipal original){
access.requireUserActivePolicy(original);AppUserPrincipal actor=access.resolvePrincipal(original);String current=current(false);List<Map<String,Object>> targets=new ArrayList<>();
if(current!=null)for(Map<String,Object> row:g.rows("SELECT d.id AS document_id,r.id AS revision_id,d.source_slot,d.stage,d.source_year AS year,d.source_month AS month,d.scope_code,r.source_file,r.sha256,r.parser_config_hash FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id=? ORDER BY d.source_year,d.source_month,d.stage",current)){
try{sourceAccess.requireSource((String)row.get("documentId"),(String)row.get("revisionId"),actor);targets.add(row);}catch(ResponseStatusException denied){if(denied.getStatusCode()!=HttpStatus.FORBIDDEN)throw denied;}
if(current!=null)for(Map<String,Object> row:g.rows("SELECT d.id AS document_id,r.id AS revision_id,d.source_slot,d.source_mode,d.stage,d.source_year AS year,d.source_month AS month,d.scope_code,r.source_file,r.sha256,r.parser_config_hash FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id=? ORDER BY d.source_year,d.source_month,d.stage,d.source_slot",current)){
try{sourceAccess.requireSource((String)row.get("documentId"),(String)row.get("revisionId"),actor);row.put("fileId",fileId(row.get("sourceSlot")));targets.add(row);}catch(ResponseStatusException denied){if(denied.getStatusCode()!=HttpStatus.FORBIDDEN)throw denied;}
}
List<String> currentSourceIds=new ArrayList<>();
for(var row:targets){var ids=jdbc.query("SELECT id FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND sha256=? AND parser_config_hash=? AND deleted_at IS NULL AND status IN('DISCOVERED','IMPORTED') AND NOT "+SourceDeletionService.pendingDeletionSql("source_candidate")+" ORDER BY discovered_at DESC,id DESC LIMIT 1",(rs,n)->rs.getString(1),SourceIdentity.canonicalSlot((String)row.get("sourceSlot")),row.get("sha256"),row.get("parserConfigHash"));if(!ids.isEmpty())currentSourceIds.add(ids.getFirst());}
@@ -41,39 +41,90 @@ public class SourceUploadService {
}
@Transactional public Map<String,Object> configure(ConfigureRequest r,AppUserPrincipal original)throws IOException {
requireActor(original);AppUserPrincipal actor=access.resolvePrincipal(original);
if(r.year()<1900||r.year()>2100||r.month()<1||r.month()>12||!Set.of("ADD","REPLACE").contains(r.mode()==null?"":r.mode()))throw new IllegalArgumentException("请选择有效年月和新增/替换模式");
if(r.year()<1900||r.year()>2100||r.month()<1||r.month()>12||!Set.of("ADD","REPLACE","APPEND").contains(r.mode()==null?"":r.mode()))throw new IllegalArgumentException("请选择有效年月和新增、追加或替换模式");
boolean append="APPEND".equals(r.mode()),appendReplacement="REPLACE".equals(r.mode())&&r.fileId()!=null&&!r.fileId().isBlank();
if((append||appendReplacement)&&!validFileId(r.fileId()))throw new IllegalArgumentException("追加或替换追加文件时必须提供有效fileId");
if(!append&&!appendReplacement&&r.fileId()!=null&&!r.fileId().isBlank())throw new IllegalArgumentException("仅追加文件替换可提供fileId");
if(r.templateId()!=null&&!r.templateId().isBlank())throw new IllegalArgumentException("表头模板功能已停用,请直接提交工作表、表头行和字段映射");
String reason=g.required(r.reason(),"操作理由");String scope=SourceIdentity.scope(r.scopeCode()==null?"ALL":r.scopeCode());
if(scope==null||scope.isBlank()||scope.length()>160||scope.contains("|")||!SourceIdentity.fullOrganizationScope(actor)&&(!actor.getDepartments().stream().map(SourceIdentity::scope).toList().contains(scope)||!actor.getGroups().isEmpty()&&!actor.getGroups().contains("*")))throw new ResponseStatusException(HttpStatus.FORBIDDEN,"来源范围须完整属于当前组织及分组授权");
Map<String,Object> draft=draft(r.uploadId(),actor,true);if(draft.get("candidateId")!=null)throw new ResponseStatusException(HttpStatus.CONFLICT,"此上传已确认,请刷新来源列表或重新上传");
Map<String,Object> draft=draft(r.uploadId(),actor,true);String existingCandidateId=(String)draft.get("candidateId");Map<String,Object> priorCandidate=existingCandidateId==null?null:reconfigurableCandidate(existingCandidateId,actor,draft);
Path file=verified(draft);SourceParserConfig config=new SourceParserConfig(r.stage(),r.sheetName(),r.headerRow(),r.fieldMap(),null);
Map<String,Object> actual=reader.preview(file,r.sheetName(),r.headerRow());Map<?,?> preview=(Map<?,?>)actual.get("preview");config.validate((List<String>)preview.get("headers"));
String current=current(true);if(!Objects.equals(current,r.expectedCurrentReleaseId()))throw new ResponseStatusException(HttpStatus.CONFLICT,"当前正式版本已变化,请刷新后重新确认");
String slot=SourceIdentity.slot(r.stage(),r.year(),r.month(),scope,"MAIN");
String shard=(append||appendReplacement)?"UPLOAD_"+r.fileId():"MAIN";
String slot=SourceIdentity.slot(r.stage(),r.year(),r.month(),scope,shard);
if(slot.length()>190)throw new IllegalArgumentException("来源身份过长");
var documents=g.rows("SELECT id FROM source_document WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? FOR UPDATE",slot);
String replacementRevision=null,replacementDocument=null;
if("REPLACE".equals(r.mode())){
if(r.targetDocumentId()==null||r.targetRevisionId()==null||current==null)throw new IllegalArgumentException("请选择当前正式版中的替换来源");
var old=g.rows("SELECT d.id,r.source_revision_id FROM release_member r JOIN source_document d ON d.id=r.source_document_id WHERE r.release_id=? AND d.id=? AND r.source_revision_id=? AND "+SourceIdentity.canonicalSlotSql("d.source_slot")+"=? FOR UPDATE",current,r.targetDocumentId(),r.targetRevisionId(),slot);
if(old.size()!=1)throw new ResponseStatusException(HttpStatus.CONFLICT,"替换目标的月份、阶段、范围或旧修订已变化");sourceAccess.requireSource(r.targetDocumentId(),r.targetRevisionId(),actor);replacementDocument=r.targetDocumentId();replacementRevision=r.targetRevisionId();
}else if(!documents.isEmpty())throw new ResponseStatusException(HttpStatus.CONFLICT,"该月份、阶段及范围已有来源,请选择替换旧数据");
var old=g.rows("SELECT d.id,r.source_revision_id,d.source_mode FROM release_member r JOIN source_document d ON d.id=r.source_document_id WHERE r.release_id=? AND d.id=? AND r.source_revision_id=? AND "+SourceIdentity.canonicalSlotSql("d.source_slot")+"=? FOR UPDATE",current,r.targetDocumentId(),r.targetRevisionId(),slot);
if(old.size()!=1)throw new ResponseStatusException(HttpStatus.CONFLICT,"替换目标的月份、阶段、范围、追加文件ID或旧修订已变化");
if(appendReplacement&&!("APPEND_SNAPSHOT".equals(old.getFirst().get("sourceMode"))))throw new ResponseStatusException(HttpStatus.CONFLICT,"替换目标不是由追加上传创建的文件批次");
sourceAccess.requireSource(r.targetDocumentId(),r.targetRevisionId(),actor);replacementDocument=r.targetDocumentId();replacementRevision=r.targetRevisionId();
}else if(!documents.isEmpty())throw new ResponseStatusException(HttpStatus.CONFLICT,append?"该fileId已有来源,请选择这份追加文件并替换其当前修订":"该月份、阶段及范围已有来源,请选择替换旧数据");
if(append){
var prior=g.rows("SELECT id FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND deleted_at IS NULL AND status IN('DISCOVERED','QUEUED','RUNNING','IMPORTED') FOR UPDATE",slot);
if(prior.stream().anyMatch(row->!Objects.equals(existingCandidateId,row.get("id"))))throw new ResponseStatusException(HttpStatus.CONFLICT,"该fileId已登记来源,请选择已有来源替换,或为新的独立文件批次生成fileId");
}
Integer pending=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND status IN('QUEUED','RUNNING')",Integer.class,slot);if(pending!=null&&pending>0)throw new ResponseStatusException(HttpStatus.CONFLICT,"该来源正在构建,请稍后重试");
String protocol=protocol(slot,r,scope,actor,reason);String key=GovernanceSupport.sha(slot+"\n"+draft.get("sha256")+"\n"+config.hash()+"\n"+r.uploadId());String id=UUID.randomUUID().toString();
Map<String,Object> frozenConfiguration=new LinkedHashMap<>(config.snapshot());frozenConfiguration.put("sourceSlot",slot);
jdbc.update("INSERT INTO source_candidate(id,candidate_key,source_slot,stage,source_year,source_month,scope_code,source_file,source_path,sha256,file_size,status,protocol_id,resolution,resolution_reason,parser_config,parser_config_hash,expected_current_release_id,replacement_document_id,replacement_revision_id,configured_by) VALUES(?,?,?,?,?,?,?,?,?,?,?,'DISCOVERED',?,?,?,CAST(? AS JSON),?,?,?,?,?)",id,key,slot,r.stage(),r.year(),r.month(),scope,draft.get("sourceFile"),file.toString(),draft.get("sha256"),draft.get("fileSize"),protocol,"REPLACE".equals(r.mode())?"REPLACE":"CONFIRMED",reason,json.writeValueAsString(frozenConfiguration),config.hash(),current,replacementDocument,replacementRevision,actor.getId().toString());
jdbc.update("UPDATE source_upload_draft SET candidate_id=? WHERE id=?",id,r.uploadId());g.audit(actor.getId().toString(),"SOURCE_UPLOAD_CONFIGURED","source_candidate",id,Map.of("mode",r.mode(),"slot",slot,"reason",reason,"configuration",frozenConfiguration));
Map<String,Object> result=new LinkedHashMap<>();result.put("source",new SourceCatalogService.SourceView(id,(String)draft.get("sourceFile"),r.stage(),r.year(),r.month(),((Number)draft.get("fileSize")).longValue(),(String)draft.get("sha256"),"DISCOVERED"));result.put("configuration",frozenConfiguration);result.put("currentReleaseId",current);return result;
ensureUniqueContent((String)draft.get("sha256"),r.stage(),r.year(),r.month(),scope,current,replacementDocument,existingCandidateId);
String protocol=protocol(slot,r,scope,actor,reason,shard);String key=GovernanceSupport.sha(slot+"\n"+draft.get("sha256")+"\n"+config.hash()+"\n"+r.uploadId());String id=existingCandidateId==null?UUID.randomUUID().toString():existingCandidateId;
Map<String,Object> frozenConfiguration=new LinkedHashMap<>(config.snapshot());frozenConfiguration.put("sourceSlot",slot);if(append||appendReplacement)frozenConfiguration.put("fileId",r.fileId());
String resolution=append?"APPEND_CONFIRMED":"REPLACE".equals(r.mode())?"REPLACE":"CONFIRMED";
if(priorCandidate==null){
jdbc.update("INSERT INTO source_candidate(id,candidate_key,source_slot,stage,source_year,source_month,scope_code,source_file,source_path,sha256,file_size,status,protocol_id,resolution,resolution_reason,parser_config,parser_config_hash,expected_current_release_id,replacement_document_id,replacement_revision_id,configured_by) VALUES(?,?,?,?,?,?,?,?,?,?,?,'DISCOVERED',?,?,?,CAST(? AS JSON),?,?,?,?,?)",id,key,slot,r.stage(),r.year(),r.month(),scope,draft.get("sourceFile"),file.toString(),draft.get("sha256"),draft.get("fileSize"),protocol,resolution,reason,json.writeValueAsString(frozenConfiguration),config.hash(),current,replacementDocument,replacementRevision,actor.getId().toString());
}else{
String oldSlot=SourceIdentity.canonicalSlot((String)priorCandidate.get("sourceSlot"));String oldProtocol=Objects.toString(priorCandidate.get("protocolId"),null);
if(!Objects.equals(oldSlot,SourceIdentity.canonicalSlot(slot))&&oldProtocol!=null)deactivateUnusedUploadProtocol(oldProtocol,id);
int changed=jdbc.update("UPDATE source_candidate SET candidate_key=?,source_slot=?,stage=?,source_year=?,source_month=?,scope_code=?,source_file=?,source_path=?,sha256=?,file_size=?,protocol_id=?,resolution=?,resolution_reason=?,parser_config=CAST(? AS JSON),parser_config_hash=?,expected_current_release_id=?,replacement_document_id=?,replacement_revision_id=?,configured_by=? WHERE id=? AND status='DISCOVERED' AND deleted_at IS NULL",key,slot,r.stage(),r.year(),r.month(),scope,draft.get("sourceFile"),file.toString(),draft.get("sha256"),draft.get("fileSize"),protocol,resolution,reason,json.writeValueAsString(frozenConfiguration),config.hash(),current,replacementDocument,replacementRevision,actor.getId().toString(),id);
if(changed!=1)throw new ResponseStatusException(HttpStatus.CONFLICT,"来源已排队或状态变化,请刷新来源列表后重试");
}
jdbc.update("UPDATE source_upload_draft SET candidate_id=? WHERE id=?",id,r.uploadId());g.audit(actor.getId().toString(),priorCandidate==null?"SOURCE_UPLOAD_CONFIGURED":"SOURCE_UPLOAD_RECONFIGURED","source_candidate",id,Map.of("mode",r.mode(),"slot",slot,"reason",reason,"configuration",frozenConfiguration));
Map<String,Object> result=new LinkedHashMap<>();result.put("source",new SourceCatalogService.SourceView(id,(String)draft.get("sourceFile"),r.stage(),r.year(),r.month(),((Number)draft.get("fileSize")).longValue(),(String)draft.get("sha256"),"DISCOVERED"));result.put("configuration",frozenConfiguration);result.put("currentReleaseId",current);result.put("reconfigured",priorCandidate!=null);return result;
}
private String protocol(String slot,ConfigureRequest r,String scope,AppUserPrincipal actor,String reason){
private String protocol(String slot,ConfigureRequest r,String scope,AppUserPrincipal actor,String reason,String shard){
String mode="MAIN".equals(shard)?"MONTHLY_SNAPSHOT":"APPEND_SNAPSHOT";
var existing=g.rows("SELECT id,source_mode,applicable FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND active=TRUE FOR UPDATE",slot);
if(existing.size()>1)throw new IllegalArgumentException("来源协议存在重复,请先完成来源治理");
if(!existing.isEmpty()){if(!"MONTHLY_SNAPSHOT".equals(existing.getFirst().get("sourceMode"))||!Boolean.TRUE.equals(existing.getFirst().get("applicable")))throw new IllegalArgumentException("此向导仅接收适用阶段的完整月度快照");return (String)existing.getFirst().get("id");}
String id=UUID.randomUUID().toString();Integer version=jdbc.queryForObject("SELECT COALESCE(MAX(version_no),0)+1 FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=?",Integer.class,slot);jdbc.update("INSERT INTO source_protocol(id,source_slot,version_no,stage,source_year,source_month,scope_code,source_mode,required_sheet,applicable,expected,reason,created_by) VALUES(?,?,?,?,?,?,?,'MONTHLY_SNAPSHOT',?,TRUE,TRUE,?,?)",id,slot,version,r.stage(),r.year(),r.month(),scope,r.sheetName(),reason,actor.getId().toString());return id;
if(!existing.isEmpty()){if(!mode.equals(existing.getFirst().get("sourceMode"))||!Boolean.TRUE.equals(existing.getFirst().get("applicable")))throw new IllegalArgumentException("来源协议与完整快照/追加批次身份不匹配,须先完成来源治理");return (String)existing.getFirst().get("id");}
var versions=g.rows("SELECT version_no FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? ORDER BY version_no DESC LIMIT 1 FOR UPDATE",slot);int version=versions.isEmpty()?1:((Number)versions.getFirst().get("versionNo")).intValue()+1;
String id=UUID.randomUUID().toString();jdbc.update("INSERT INTO source_protocol(id,source_slot,version_no,stage,source_year,source_month,scope_code,shard_code,source_mode,required_sheet,applicable,expected,reason,created_by) VALUES(?,?,?,?,?,?,?,?,?,?,TRUE,TRUE,?,?)",id,slot,version,r.stage(),r.year(),r.month(),scope,shard,mode,r.sheetName(),reason,actor.getId().toString());return id;
}
private Map<String,Object> reconfigurableCandidate(String id,AppUserPrincipal actor,Map<String,Object> draft){
var rows=g.rows("SELECT * FROM source_candidate WHERE id=? AND configured_by=? FOR UPDATE",id,actor.getId().toString());
if(rows.isEmpty())throw new ResponseStatusException(HttpStatus.CONFLICT,"已确认来源不属于当前上传人,请重新上传");Map<String,Object> candidate=rows.getFirst();
if(!"DISCOVERED".equals(candidate.get("status"))||!Objects.equals(candidate.get("sha256"),draft.get("sha256"))||!Objects.equals(candidate.get("sourcePath"),draft.get("sourcePath")))throw new ResponseStatusException(HttpStatus.CONFLICT,"此来源已排队、处理或内容变化,不能重新确认");
Integer runs=jdbc.queryForObject("SELECT COUNT(*) FROM import_run WHERE JSON_CONTAINS(requested_sources,JSON_QUOTE(?))",Integer.class,id);
String sameSlot=SourceIdentity.canonicalSlotSql("d.source_slot")+"="+SourceIdentity.canonicalSlotSql("c.source_slot");
Integer documents=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate c JOIN source_document d ON "+sameSlot+" JOIN source_revision r ON r.source_document_id=d.id AND r.sha256=c.sha256 AND r.parser_config_hash=c.parser_config_hash JOIN release_member m ON m.source_document_id=d.id AND m.source_revision_id=r.id WHERE c.id=?",Integer.class,id);
if((runs!=null&&runs>0)||(documents!=null&&documents>0))throw new ResponseStatusException(HttpStatus.CONFLICT,"来源已被导入任务或版本引用,不能重新确认");
return candidate;
}
private void deactivateUnusedUploadProtocol(String protocolId,String candidateId){
var protocol=g.rows("SELECT source_slot,active FROM source_protocol WHERE id=? AND active=TRUE FOR UPDATE",protocolId);if(protocol.isEmpty())return;
Integer others=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate WHERE protocol_id=? AND id<>? AND deleted_at IS NULL AND status NOT IN('RETURNED','SUPERSEDED')",Integer.class,protocolId,candidateId);
Integer documents=jdbc.queryForObject("SELECT COUNT(*) FROM source_document WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=?",Integer.class,SourceIdentity.canonicalSlot((String)protocol.getFirst().get("sourceSlot")));
if((others==null||others==0)&&(documents==null||documents==0))jdbc.update("UPDATE source_protocol SET active=FALSE WHERE id=? AND active=TRUE",protocolId);
}
private void ensureUniqueContent(String sha,String stage,int year,int month,String scope,String releaseId,String excludedDocument,String excludedCandidate){
if(releaseId!=null){
List<Map<String,Object>> currentRows=g.rows("SELECT d.id,d.scope_code FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id=? AND r.sha256=? AND d.stage=? AND d.source_year=? AND d.source_month=? FOR UPDATE",releaseId,sha,stage,year,month);
if(currentRows.stream().anyMatch(row->!Objects.equals(excludedDocument,row.get("id"))&&SourceIdentity.intersects(scope,(String)row.get("scopeCode"))))throw duplicateContent();
}
List<Map<String,Object>> pending=g.rows("SELECT id,source_slot,scope_code FROM source_candidate WHERE sha256=? AND stage=? AND source_year=? AND source_month=? AND deleted_at IS NULL AND status IN('DISCOVERED','QUEUED','RUNNING') AND NOT "+SourceDeletionService.pendingDeletionSql("source_candidate")+" FOR UPDATE",sha,stage,year,month);
if(pending.stream().anyMatch(row->!Objects.equals(excludedCandidate,row.get("id"))&&SourceIdentity.intersects(scope,(String)row.get("scopeCode"))))throw duplicateContent();
}
private ResponseStatusException duplicateContent(){return new ResponseStatusException(HttpStatus.CONFLICT,"该工作簿内容已用于同一年月、阶段及重叠范围的另一份来源,请勿重复计入;更正版请精确选择原来源替换");}
private static boolean validFileId(String value){return value!=null&&value.length()<=80&&value.matches("[A-Za-z0-9_-]{1,80}");}
private static String fileId(Object slotValue){if(!(slotValue instanceof String slot))return null;String[] parts=SourceIdentity.canonicalSlot(slot).split("\\|",-1);if(parts.length!=5||!parts[4].startsWith("UPLOAD_"))return null;String value=parts[4].substring("UPLOAD_".length());return validFileId(value)?value:null;}
private Map<String,Object> draft(String id,AppUserPrincipal actor,boolean lock){var rows=g.rows("SELECT * FROM source_upload_draft WHERE id=? AND created_by=? AND expires_at>CURRENT_TIMESTAMP(6)"+(lock?" FOR UPDATE":""),id,actor.getId().toString());if(rows.isEmpty())throw new ResponseStatusException(HttpStatus.NOT_FOUND,"上传预览已过期或不属于当前用户,请重新上传");return rows.getFirst();}
private Path verified(Map<String,Object> draft)throws IOException{Path path=Path.of((String)draft.get("sourcePath"));if(Files.isSymbolicLink(path)||!Files.isRegularFile(path)||!path.toRealPath().startsWith(scanner.archiveRoot())||Files.size(path)!=((Number)draft.get("fileSize")).longValue()||!SourceScanner.sha256(path).equals(draft.get("sha256")))throw new IllegalArgumentException("上传文件已变化,请重新上传");return path.toRealPath();}
private String current(boolean lock){return jdbc.queryForObject("SELECT release_id FROM current_release WHERE dataset_id='quality'"+(lock?" FOR UPDATE":""),String.class);}
private void requireActor(AppUserPrincipal actor){access.requireUserActivePolicy(actor);access.requireBoundAction("RECEIVER","source.receive");}
/** templateId remains an explicitly disabled compatibility field for older clients. */
public record ConfigureRequest(String uploadId,String stage,int year,int month,String sheetName,int headerRow,Map<String,String> fieldMap,String templateId,String mode,String expectedCurrentReleaseId,String targetDocumentId,String targetRevisionId,String reason,String scopeCode){}
public record ConfigureRequest(String uploadId,String stage,int year,int month,String sheetName,int headerRow,Map<String,String> fieldMap,String templateId,String mode,String expectedCurrentReleaseId,String targetDocumentId,String targetRevisionId,String reason,String scopeCode,String fileId){
public ConfigureRequest(String uploadId,String stage,int year,int month,String sheetName,int headerRow,Map<String,String> fieldMap,String templateId,String mode,String expectedCurrentReleaseId,String targetDocumentId,String targetRevisionId,String reason,String scopeCode){this(uploadId,stage,year,month,sheetName,headerRow,fieldMap,templateId,mode,expectedCurrentReleaseId,targetDocumentId,targetRevisionId,reason,scopeCode,null);}
}
}
@@ -0,0 +1,27 @@
package cn.yunsheng.quality.update;
import java.util.Map;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/system-update")
@PreAuthorize("hasRole('SYSTEM_ADMIN')")
public class SystemUpdateController {
private final SystemUpdateService updates;
public SystemUpdateController(SystemUpdateService updates) { this.updates = updates; }
@GetMapping
public Map<String, Object> status() { return updates.status(); }
// Both mutations use the application's existing session + CSRF filter.
@PostMapping("/check")
public Map<String, Object> check() { return updates.check(); }
@PostMapping("/apply")
public Map<String, Object> apply() { return updates.apply(); }
}
@@ -0,0 +1,39 @@
package cn.yunsheng.quality.update;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Set;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.OncePerRequestFilter;
/** Keeps the startup-to-update-commit interval free of business writes. */
@Component
@Order(Ordered.HIGHEST_PRECEDENCE + 20)
public class SystemUpdateMaintenanceFilter extends OncePerRequestFilter {
private static final Set<String> READ_METHODS = Set.of("GET", "HEAD", "OPTIONS", "TRACE");
private static final Set<String> AUTH_ROUTES = Set.of("/api/auth/login", "/api/auth/logout", "/api/auth/csrf", "/api/auth/me");
private final SystemUpdateService updates;
public SystemUpdateMaintenanceFilter(SystemUpdateService updates) { this.updates = updates; }
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain)
throws ServletException, IOException {
String path = request.getRequestURI().substring(request.getContextPath().length());
if (!READ_METHODS.contains(request.getMethod()) && path.startsWith("/api/")
&& !path.equals("/api/system-update") && !path.startsWith("/api/system-update/")
&& !AUTH_ROUTES.contains(path) && updates.maintenanceActive()) {
response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
response.setContentType("application/problem+json;charset=UTF-8");
response.setHeader("Retry-After", "10");
response.getWriter().write("{\"title\":\"Service Unavailable\",\"status\":503,\"detail\":\"系统正在更新,请稍后重试。\"}");
return;
}
chain.doFilter(request, response);
}
}
@@ -0,0 +1,138 @@
package cn.yunsheng.quality.update;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
import tools.jackson.databind.json.JsonMapper;
/** Restricted bridge to the packaged updater; neither URLs nor paths are input. */
@Service
public class SystemUpdateService {
private static final Set<String> RESPONSE_FIELDS = Set.of("enabled", "currentVersion", "state", "message",
"checkedAt", "version", "available", "startedAt", "finishedAt");
private final JsonMapper json;
private final Path root;
private final boolean enabled;
public SystemUpdateService(JsonMapper json,
@Value("${QUALITY_PROJECT_ROOT:}") String root,
@Value("${QUALITY_UPDATE_ENABLED:false}") boolean requested) {
this.json = json;
this.root = root.isBlank() ? null : Path.of(root).toAbsolutePath().normalize();
this.enabled = requested && System.getProperty("os.name", "").startsWith("Windows")
&& this.root != null && Files.isRegularFile(this.root.resolve("manifest.json"))
&& Files.isRegularFile(this.root.resolve("runtimes/python/python.exe"))
&& Files.isRegularFile(this.root.resolve("deploy/offline/update_manager.py"))
&& Files.isRegularFile(this.root.resolve("deploy/offline/update-public-key.pem"));
}
public synchronized Map<String, Object> status() {
if (!enabled) return Map.of("enabled", false, "state", "disabled", "message", "此部署不支持软件内更新。");
return invoke("status");
}
public synchronized Map<String, Object> check() {
requireEnabled();
return invoke("check");
}
public synchronized Map<String, Object> apply() {
requireEnabled();
return invoke("launch");
}
/** Fail closed during installation/recovery, including a process crash. */
public boolean maintenanceActive() {
if (!enabled) return false;
try {
Map<?, ?> state = readMetadata("status.json");
if (state.get("state") instanceof String phase
&& Set.of("backingUp", "installing", "restarting", "rollingBack").contains(phase)) return true;
Map<?, ?> journal = readMetadata("journal.json");
return journal.get("phase") instanceof String phase && Set.of("stopping", "installing").contains(phase);
} catch (Exception failure) {
return true;
}
}
private Map<?, ?> readMetadata(String name) throws IOException {
Path path = root.resolve(".runtime/update/" + name);
if (!Files.exists(path)) return Map.of();
try (var stream = Files.newInputStream(path)) {
byte[] raw = stream.readNBytes(8193);
if (raw.length > 8192) throw new IOException("Oversized update state");
Object decoded = json.readValue(raw, Object.class);
if (!(decoded instanceof Map<?, ?> value)) throw new IOException("Invalid update state");
return value;
}
}
private void requireEnabled() {
if (!enabled) throw new ResponseStatusException(HttpStatus.CONFLICT, "此部署不支持软件内更新。");
}
private Map<String, Object> invoke(String action) {
Process process = null;
try {
ProcessBuilder builder = new ProcessBuilder(root.resolve("runtimes/python/python.exe").toString(),
root.resolve("deploy/offline/update_manager.py").toString(), action);
builder.directory(root.toFile());
builder.redirectError(ProcessBuilder.Redirect.DISCARD);
builder.environment().put("PYTHONUTF8", "1");
builder.environment().put("PYTHONIOENCODING", "utf-8");
builder.environment().put("PYTHONDONTWRITEBYTECODE", "1");
process = builder.start();
process.getOutputStream().close();
// The CLI prints one small, allowlisted JSON receipt. Waiting first
// also bounds hangs caused by a failed DNS/TLS/network operation.
if (!process.waitFor(45, TimeUnit.SECONDS)) {
process.destroyForcibly();
throw new ResponseStatusException(HttpStatus.GATEWAY_TIMEOUT, "更新服务器暂未响应,请稍后重试。");
}
byte[] raw = process.getInputStream().readNBytes(8193);
if (raw.length > 8192) throw new IOException("Oversized updater receipt");
Object decoded = json.readValue(new String(raw, StandardCharsets.UTF_8), Object.class);
if (!(decoded instanceof Map<?, ?> value)) throw new IOException("Invalid updater receipt");
if (process.exitValue() != 0) {
Object detail = value.get("error");
String message = detail instanceof String s && s.length() <= 300 ? s : "更新操作未完成,请稍后重试。";
throw new ResponseStatusException(HttpStatus.CONFLICT, message);
}
return safeResponse(value);
} catch (ResponseStatusException failure) {
throw failure;
} catch (InterruptedException failure) {
Thread.currentThread().interrupt();
throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "更新操作已中断。");
} catch (Exception failure) {
throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "更新服务暂不可用,请稍后重试。");
} finally {
if (process != null && process.isAlive()) process.destroyForcibly();
}
}
static Map<String, Object> safeResponse(Map<?, ?> value) throws IOException {
Map<String, Object> result = new LinkedHashMap<>();
for (String field : RESPONSE_FIELDS) {
Object item = value.get(field);
if (item == null) continue;
if ((field.equals("enabled") || field.equals("available")) && !(item instanceof Boolean))
throw new IOException("Invalid updater boolean");
if (!(item instanceof Boolean) && !(item instanceof String s && s.length() <= 300))
throw new IOException("Invalid updater field");
result.put(field, item);
}
if (!(result.get("state") instanceof String) || !(result.get("enabled") instanceof Boolean))
throw new IOException("Incomplete updater receipt");
return result;
}
}
@@ -0,0 +1,97 @@
package cn.yunsheng.quality.update;
import jakarta.annotation.PreDestroy;
import java.lang.reflect.Method;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.aopalliance.intercept.MethodInterceptor;
import org.springframework.aop.framework.AopProxyUtils;
import org.springframework.aop.framework.ProxyFactory;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.config.BeanPostProcessor;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.AnnotatedElementUtils;
import org.springframework.core.env.Environment;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.util.ReflectionUtils;
/** Defers business recovery and suppresses schedules until the update commits. */
@Component
public class SystemUpdateStartupBarrier implements BeanPostProcessor, Ordered {
private static final Set<String> RECOVERY_METHODS = Set.of(
"cn.yunsheng.quality.ingestion.ImportRecoveryRunner#run",
"cn.yunsheng.quality.analysis.AnalysisTurnService#recoverQueuedTurns",
"cn.yunsheng.quality.analysis.AnalysisReportService#recoverPendingReports",
"cn.yunsheng.quality.analytics.InspectionService#recoverPendingDrilldowns");
private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(SystemUpdateStartupBarrier.class);
private final ObjectProvider<SystemUpdateService> updates;
private final Environment environment;
private final ConcurrentLinkedQueue<Runnable> pending = new ConcurrentLinkedQueue<>();
private final ScheduledExecutorService release = Executors.newSingleThreadScheduledExecutor(task -> {
Thread thread = new Thread(task, "quality-update-recovery");
thread.setDaemon(true);
return thread;
});
public SystemUpdateStartupBarrier(ObjectProvider<SystemUpdateService> updates, Environment environment) {
this.updates = updates;
this.environment = environment;
release.scheduleWithFixedDelay(this::drain, 500, 500, TimeUnit.MILLISECONDS);
}
@Override public int getOrder() { return Ordered.HIGHEST_PRECEDENCE + 50; }
@Override
public Object postProcessAfterInitialization(Object bean, String name) {
if (!System.getProperty("os.name", "").startsWith("Windows")
|| !environment.getProperty("QUALITY_UPDATE_ENABLED", Boolean.class, false)) return bean;
Class<?> type = AopProxyUtils.ultimateTargetClass(bean);
if (!type.getName().startsWith("cn.yunsheng.quality.") || type.getName().contains(".update.")) return bean;
boolean candidate = false;
for (Method method : ReflectionUtils.getAllDeclaredMethods(type))
if (isRecovery(type, method) || AnnotatedElementUtils.hasAnnotation(method, Scheduled.class)) candidate = true;
return candidate ? wrap(bean, type) : bean;
}
Object wrap(Object bean, Class<?> type) {
ProxyFactory proxy = new ProxyFactory(bean);
proxy.setProxyTargetClass(true);
proxy.addAdvice((MethodInterceptor) invocation -> {
Method method = ReflectionUtils.findMethod(type, invocation.getMethod().getName(), invocation.getMethod().getParameterTypes());
if (method == null || !updates.getObject().maintenanceActive()) return invocation.proceed();
if (isRecovery(type, method)) {
pending.add(() -> {
try { invocation.proceed(); }
catch (Throwable failure) { LOG.error("Deferred application recovery failed", failure); }
});
return null;
}
if (AnnotatedElementUtils.hasAnnotation(method, Scheduled.class)) return null;
return invocation.proceed();
});
return proxy.getProxy(type.getClassLoader());
}
private static boolean isRecovery(Class<?> type, Method method) {
return RECOVERY_METHODS.contains(type.getName() + "#" + method.getName());
}
void drain() {
if (pending.isEmpty()) return;
try {
while (!updates.getObject().maintenanceActive()) {
Runnable task = pending.poll();
if (task == null) return;
task.run();
}
} catch (Exception failure) {
LOG.warn("Waiting for application update to complete");
}
}
@PreDestroy public void close() { release.shutdownNow(); pending.clear(); }
}
@@ -506,10 +506,40 @@ public class DataWorkflowService {
throw failure("CANDIDATE_VERSION_MISMATCH", "候选版本未使用本次工作流冻结的映射和规则,候选不会发布");
Integer invalidBuilds = jdbc.queryForObject("""
SELECT COUNT(*) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id
WHERE rm.release_id=? AND (b.mapping_version<>? OR b.rule_version<>? OR b.status<>'COMPLETED' OR b.rejected_count>0)
WHERE rm.release_id=? AND (b.mapping_version<>? OR b.rule_version<>?)
""", Integer.class, releaseId, mappingId, ruleId);
if (invalidBuilds != null && invalidBuilds > 0)
throw failure("CANDIDATE_BUILD_VERSION_MISMATCH", "候选中存在未使用本次冻结版本的转换产物,候选不会发布");
Integer unfinishedBuilds = jdbc.queryForObject("""
SELECT COUNT(*) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id
WHERE rm.release_id=? AND b.status<>'COMPLETED'
""", Integer.class, releaseId);
if (unfinishedBuilds != null && unfinishedBuilds > 0)
throw failure("CANDIDATE_BUILD_INCOMPLETE", "有 " + unfinishedBuilds + " 个来源尚未完成转换,候选不会发布");
Integer rejected = jdbc.queryForObject("""
SELECT COALESCE(SUM(b.rejected_count),0) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id
WHERE rm.release_id=?
""", Integer.class, releaseId);
if (rejected != null && rejected > 0) {
List<Map<String,Object>> issues = jdbc.queryForList("""
SELECT issue_code AS issueCode,field_name AS fieldName,COUNT(*) AS issueCount FROM import_issue
WHERE import_run_id=(SELECT id FROM import_run WHERE result_release_id=? ORDER BY created_at DESC LIMIT 1)
AND JSON_EXTRACT(details,'$.blocking')=TRUE
GROUP BY issue_code,field_name ORDER BY issueCount DESC,issue_code,field_name LIMIT 3
""", releaseId);
String detail = issues.stream().map(issue -> {
String reason = switch (Objects.toString(issue.get("issueCode"), "")) {
case "INVALID_NUMBER" -> "数值格式无效";
case "REQUIRED_HEADER_MISSING" -> "缺少必需列";
case "NO_DETAIL_ROWS" -> "没有可导入明细";
default -> "来源校验未通过";
};
String field = Objects.toString(issue.get("fieldName"), "");
return (field.isBlank() ? "" : field + ":") + reason + "(" + issue.get("issueCount") + "项)";
}).collect(java.util.stream.Collectors.joining(";"));
throw failure("CANDIDATE_SOURCE_VALIDATION_FAILED", "来源数据有 " + rejected + " 项阻断问题,候选不会发布。"
+ (detail.isBlank() ? "请查看导入记录中的问题明细" : detail));
}
}
private void publishExisting(WorkflowOperation operation, AppUserPrincipal actor, String token) {