完善:同月多文件导入与当前用户离线迁移更新

This commit is contained in:
2026-11-04 11:35:24 +08:00
parent 147b56d0fd
commit f04d4e96e3
89 changed files with 6634 additions and 201 deletions
@@ -88,7 +88,9 @@ public class AnalysisApiSettingsService {
Files.createDirectories(target.getParent()); Files.createDirectories(target.getParent());
if (Files.isSymbolicLink(target.getParent()) || Files.isSymbolicLink(target)) throw new IOException(); if (Files.isSymbolicLink(target.getParent()) || Files.isSymbolicLink(target)) throw new IOException();
temp = Files.createTempFile(target.getParent(), ".nao-model-", ".tmp"); temp = Files.createTempFile(target.getParent(), ".nao-model-", ".tmp");
Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------")); if (Files.getFileStore(temp).supportsFileAttributeView("posix")) {
Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------"));
}
List<Map<String,Object>> storedProfiles = updated.stream().map(p -> Map.<String,Object>of( List<Map<String,Object>> storedProfiles = updated.stream().map(p -> Map.<String,Object>of(
"id",p.id(),"name",p.name(),"baseUrl",p.config().baseUrl(),"apiKey",p.config().apiKey(),"model",p.config().model(),"thinkingEnabled",p.config().thinkingEnabled(),"thinkingEffort",p.config().thinkingEffort())).toList(); "id",p.id(),"name",p.name(),"baseUrl",p.config().baseUrl(),"apiKey",p.config().apiKey(),"model",p.config().model(),"thinkingEnabled",p.config().thinkingEnabled(),"thinkingEffort",p.config().thinkingEffort())).toList();
Files.writeString(temp, json.writeValueAsString(Map.of("schemaVersion", 3, "baseUrl", next.baseUrl(), Files.writeString(temp, json.writeValueAsString(Map.of("schemaVersion", 3, "baseUrl", next.baseUrl(),
@@ -19,8 +19,18 @@ import org.springframework.stereotype.Service;
} }
static boolean overlaps(Coverage a,Coverage b) { static boolean overlaps(Coverage a,Coverage b) {
if(!Objects.equals(a.stage(),b.stage())||a.year()!=b.year()||a.month()!=b.month()||!SourceIdentity.intersects(a.scope(),b.scope()))return false; if(!Objects.equals(a.stage(),b.stage())||a.year()!=b.year()||a.month()!=b.month()||!SourceIdentity.intersects(a.scope(),b.scope()))return false;
return Objects.equals(SourceIdentity.canonicalSlot(a.slot()),SourceIdentity.canonicalSlot(b.slot())) if(Objects.equals(SourceIdentity.canonicalSlot(a.slot()),SourceIdentity.canonicalSlot(b.slot()))||Objects.equals(a.sha(),b.sha()))return true;
||!"SHARD_SNAPSHOT".equals(a.mode())||!"SHARD_SNAPSHOT".equals(b.mode())||Objects.equals(a.sha(),b.sha()); if("SHARD_SNAPSHOT".equals(a.mode())&&"SHARD_SNAPSHOT".equals(b.mode()))return false;
// APPEND_SNAPSHOT is created only by the explicit upload workflow. Its scope must match
// exactly before it can augment a complete monthly file; intersecting declarations still conflict.
if(isUploadAppend(a.mode())||isUploadAppend(b.mode())){
boolean supportedPair=(isUploadAppend(a.mode())&&isUploadAppend(b.mode()))
||("MONTHLY_SNAPSHOT".equals(a.mode())&&isUploadAppend(b.mode()))
||("MONTHLY_SNAPSHOT".equals(b.mode())&&isUploadAppend(a.mode()));
return !supportedPair||!Objects.equals(SourceIdentity.scope(a.scope()),SourceIdentity.scope(b.scope()));
}
return true;
} }
private static boolean isUploadAppend(String mode){return "APPEND_SNAPSHOT".equals(mode);}
record Coverage(String stage,int year,int month,String scope,String mode,String sha,String slot) {} record Coverage(String stage,int year,int month,String scope,String mode,String sha,String slot) {}
} }
@@ -255,8 +255,7 @@ public class ImportWorker {
issues.add(new Issue(candidate, row, "INVALID_NUMBER", field, "数值字段不能转换为重量数值,原值已保留在raw中", true)); issues.add(new Issue(candidate, row, "INVALID_NUMBER", field, "数值字段不能转换为重量数值,原值已保留在raw中", true));
} }
if (value.unmappedCategory()) { if (value.unmappedCategory()) {
blocking[0]++; issues.add(new Issue(candidate, row, "CATEGORY_MAPPING_MISSING", "PDT代码/销售片区", "未命中业务分组对应关系,已保留为未分组,可在对应关系中补充归类", false));
issues.add(new Issue(candidate, row, "CATEGORY_MAPPING_MISSING", "PDT代码/销售片区", "非空来源键未命中已验证的产业部-地区映射", true));
} else if ("source_key_blank".equals(value.components().get("categoryMappingStatus"))) { } else if ("source_key_blank".equals(value.components().get("categoryMappingStatus"))) {
blankCategory[0]++; blankCategory[0]++;
issues.add(new Issue(candidate, row, "SOURCE_CATEGORY_KEY_BLANK", "PDT代码/销售片区", "原始匹配键为空,按源模型趋势上下文默认排除;原始行保留", false)); issues.add(new Issue(candidate, row, "SOURCE_CATEGORY_KEY_BLANK", "PDT代码/销售片区", "原始匹配键为空,按源模型趋势上下文默认排除;原始行保留", false));
@@ -19,6 +19,8 @@ final class TransformBuildKey {
Integer.toString(source.year()), Integer.toString(source.month()), parserVersion, Integer.toString(source.year()), Integer.toString(source.month()), parserVersion,
rulesVersion, mappingVersion); rulesVersion, mappingVersion);
if(parserConfigHash!=null&&!"legacy".equals(parserConfigHash))identity+="\n"+parserConfigHash; if(parserConfigHash!=null&&!"legacy".equals(parserConfigHash))identity+="\n"+parserConfigHash;
// Validation policy is part of cache identity; old mapping rejections must not survive a retry.
identity += "\nunassigned-mapping-nonblocking-v1";
try { try {
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(identity.getBytes(StandardCharsets.UTF_8))); return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(identity.getBytes(StandardCharsets.UTF_8)));
} catch (NoSuchAlgorithmException e) { } catch (NoSuchAlgorithmException e) {
@@ -32,8 +32,8 @@ public class SourceUploadService {
} }
public Map<String,Object> options(AppUserPrincipal original){ public Map<String,Object> options(AppUserPrincipal original){
access.requireUserActivePolicy(original);AppUserPrincipal actor=access.resolvePrincipal(original);String current=current(false);List<Map<String,Object>> targets=new ArrayList<>(); access.requireUserActivePolicy(original);AppUserPrincipal actor=access.resolvePrincipal(original);String current=current(false);List<Map<String,Object>> targets=new ArrayList<>();
if(current!=null)for(Map<String,Object> row:g.rows("SELECT d.id AS document_id,r.id AS revision_id,d.source_slot,d.stage,d.source_year AS year,d.source_month AS month,d.scope_code,r.source_file,r.sha256,r.parser_config_hash FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id=? ORDER BY d.source_year,d.source_month,d.stage",current)){ if(current!=null)for(Map<String,Object> row:g.rows("SELECT d.id AS document_id,r.id AS revision_id,d.source_slot,d.source_mode,d.stage,d.source_year AS year,d.source_month AS month,d.scope_code,r.source_file,r.sha256,r.parser_config_hash FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id=? ORDER BY d.source_year,d.source_month,d.stage,d.source_slot",current)){
try{sourceAccess.requireSource((String)row.get("documentId"),(String)row.get("revisionId"),actor);targets.add(row);}catch(ResponseStatusException denied){if(denied.getStatusCode()!=HttpStatus.FORBIDDEN)throw denied;} try{sourceAccess.requireSource((String)row.get("documentId"),(String)row.get("revisionId"),actor);row.put("fileId",fileId(row.get("sourceSlot")));targets.add(row);}catch(ResponseStatusException denied){if(denied.getStatusCode()!=HttpStatus.FORBIDDEN)throw denied;}
} }
List<String> currentSourceIds=new ArrayList<>(); List<String> currentSourceIds=new ArrayList<>();
for(var row:targets){var ids=jdbc.query("SELECT id FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND sha256=? AND parser_config_hash=? AND deleted_at IS NULL AND status IN('DISCOVERED','IMPORTED') AND NOT "+SourceDeletionService.pendingDeletionSql("source_candidate")+" ORDER BY discovered_at DESC,id DESC LIMIT 1",(rs,n)->rs.getString(1),SourceIdentity.canonicalSlot((String)row.get("sourceSlot")),row.get("sha256"),row.get("parserConfigHash"));if(!ids.isEmpty())currentSourceIds.add(ids.getFirst());} for(var row:targets){var ids=jdbc.query("SELECT id FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND sha256=? AND parser_config_hash=? AND deleted_at IS NULL AND status IN('DISCOVERED','IMPORTED') AND NOT "+SourceDeletionService.pendingDeletionSql("source_candidate")+" ORDER BY discovered_at DESC,id DESC LIMIT 1",(rs,n)->rs.getString(1),SourceIdentity.canonicalSlot((String)row.get("sourceSlot")),row.get("sha256"),row.get("parserConfigHash"));if(!ids.isEmpty())currentSourceIds.add(ids.getFirst());}
@@ -41,39 +41,90 @@ public class SourceUploadService {
} }
@Transactional public Map<String,Object> configure(ConfigureRequest r,AppUserPrincipal original)throws IOException { @Transactional public Map<String,Object> configure(ConfigureRequest r,AppUserPrincipal original)throws IOException {
requireActor(original);AppUserPrincipal actor=access.resolvePrincipal(original); requireActor(original);AppUserPrincipal actor=access.resolvePrincipal(original);
if(r.year()<1900||r.year()>2100||r.month()<1||r.month()>12||!Set.of("ADD","REPLACE").contains(r.mode()==null?"":r.mode()))throw new IllegalArgumentException("请选择有效年月和新增/替换模式"); if(r.year()<1900||r.year()>2100||r.month()<1||r.month()>12||!Set.of("ADD","REPLACE","APPEND").contains(r.mode()==null?"":r.mode()))throw new IllegalArgumentException("请选择有效年月和新增、追加或替换模式");
boolean append="APPEND".equals(r.mode()),appendReplacement="REPLACE".equals(r.mode())&&r.fileId()!=null&&!r.fileId().isBlank();
if((append||appendReplacement)&&!validFileId(r.fileId()))throw new IllegalArgumentException("追加或替换追加文件时必须提供有效fileId");
if(!append&&!appendReplacement&&r.fileId()!=null&&!r.fileId().isBlank())throw new IllegalArgumentException("仅追加文件替换可提供fileId");
if(r.templateId()!=null&&!r.templateId().isBlank())throw new IllegalArgumentException("表头模板功能已停用,请直接提交工作表、表头行和字段映射"); if(r.templateId()!=null&&!r.templateId().isBlank())throw new IllegalArgumentException("表头模板功能已停用,请直接提交工作表、表头行和字段映射");
String reason=g.required(r.reason(),"操作理由");String scope=SourceIdentity.scope(r.scopeCode()==null?"ALL":r.scopeCode()); String reason=g.required(r.reason(),"操作理由");String scope=SourceIdentity.scope(r.scopeCode()==null?"ALL":r.scopeCode());
if(scope==null||scope.isBlank()||scope.length()>160||scope.contains("|")||!SourceIdentity.fullOrganizationScope(actor)&&(!actor.getDepartments().stream().map(SourceIdentity::scope).toList().contains(scope)||!actor.getGroups().isEmpty()&&!actor.getGroups().contains("*")))throw new ResponseStatusException(HttpStatus.FORBIDDEN,"来源范围须完整属于当前组织及分组授权"); if(scope==null||scope.isBlank()||scope.length()>160||scope.contains("|")||!SourceIdentity.fullOrganizationScope(actor)&&(!actor.getDepartments().stream().map(SourceIdentity::scope).toList().contains(scope)||!actor.getGroups().isEmpty()&&!actor.getGroups().contains("*")))throw new ResponseStatusException(HttpStatus.FORBIDDEN,"来源范围须完整属于当前组织及分组授权");
Map<String,Object> draft=draft(r.uploadId(),actor,true);if(draft.get("candidateId")!=null)throw new ResponseStatusException(HttpStatus.CONFLICT,"此上传已确认,请刷新来源列表或重新上传"); Map<String,Object> draft=draft(r.uploadId(),actor,true);String existingCandidateId=(String)draft.get("candidateId");Map<String,Object> priorCandidate=existingCandidateId==null?null:reconfigurableCandidate(existingCandidateId,actor,draft);
Path file=verified(draft);SourceParserConfig config=new SourceParserConfig(r.stage(),r.sheetName(),r.headerRow(),r.fieldMap(),null); Path file=verified(draft);SourceParserConfig config=new SourceParserConfig(r.stage(),r.sheetName(),r.headerRow(),r.fieldMap(),null);
Map<String,Object> actual=reader.preview(file,r.sheetName(),r.headerRow());Map<?,?> preview=(Map<?,?>)actual.get("preview");config.validate((List<String>)preview.get("headers")); Map<String,Object> actual=reader.preview(file,r.sheetName(),r.headerRow());Map<?,?> preview=(Map<?,?>)actual.get("preview");config.validate((List<String>)preview.get("headers"));
String current=current(true);if(!Objects.equals(current,r.expectedCurrentReleaseId()))throw new ResponseStatusException(HttpStatus.CONFLICT,"当前正式版本已变化,请刷新后重新确认"); String current=current(true);if(!Objects.equals(current,r.expectedCurrentReleaseId()))throw new ResponseStatusException(HttpStatus.CONFLICT,"当前正式版本已变化,请刷新后重新确认");
String slot=SourceIdentity.slot(r.stage(),r.year(),r.month(),scope,"MAIN"); String shard=(append||appendReplacement)?"UPLOAD_"+r.fileId():"MAIN";
String slot=SourceIdentity.slot(r.stage(),r.year(),r.month(),scope,shard);
if(slot.length()>190)throw new IllegalArgumentException("来源身份过长");
var documents=g.rows("SELECT id FROM source_document WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? FOR UPDATE",slot); var documents=g.rows("SELECT id FROM source_document WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? FOR UPDATE",slot);
String replacementRevision=null,replacementDocument=null; String replacementRevision=null,replacementDocument=null;
if("REPLACE".equals(r.mode())){ if("REPLACE".equals(r.mode())){
if(r.targetDocumentId()==null||r.targetRevisionId()==null||current==null)throw new IllegalArgumentException("请选择当前正式版中的替换来源"); if(r.targetDocumentId()==null||r.targetRevisionId()==null||current==null)throw new IllegalArgumentException("请选择当前正式版中的替换来源");
var old=g.rows("SELECT d.id,r.source_revision_id FROM release_member r JOIN source_document d ON d.id=r.source_document_id WHERE r.release_id=? AND d.id=? AND r.source_revision_id=? AND "+SourceIdentity.canonicalSlotSql("d.source_slot")+"=? FOR UPDATE",current,r.targetDocumentId(),r.targetRevisionId(),slot); var old=g.rows("SELECT d.id,r.source_revision_id,d.source_mode FROM release_member r JOIN source_document d ON d.id=r.source_document_id WHERE r.release_id=? AND d.id=? AND r.source_revision_id=? AND "+SourceIdentity.canonicalSlotSql("d.source_slot")+"=? FOR UPDATE",current,r.targetDocumentId(),r.targetRevisionId(),slot);
if(old.size()!=1)throw new ResponseStatusException(HttpStatus.CONFLICT,"替换目标的月份、阶段、范围或旧修订已变化");sourceAccess.requireSource(r.targetDocumentId(),r.targetRevisionId(),actor);replacementDocument=r.targetDocumentId();replacementRevision=r.targetRevisionId(); if(old.size()!=1)throw new ResponseStatusException(HttpStatus.CONFLICT,"替换目标的月份、阶段、范围、追加文件ID或旧修订已变化");
}else if(!documents.isEmpty())throw new ResponseStatusException(HttpStatus.CONFLICT,"该月份、阶段及范围已有来源,请选择替换旧数据"); if(appendReplacement&&!("APPEND_SNAPSHOT".equals(old.getFirst().get("sourceMode"))))throw new ResponseStatusException(HttpStatus.CONFLICT,"替换目标不是由追加上传创建的文件批次");
sourceAccess.requireSource(r.targetDocumentId(),r.targetRevisionId(),actor);replacementDocument=r.targetDocumentId();replacementRevision=r.targetRevisionId();
}else if(!documents.isEmpty())throw new ResponseStatusException(HttpStatus.CONFLICT,append?"该fileId已有来源,请选择这份追加文件并替换其当前修订":"该月份、阶段及范围已有来源,请选择替换旧数据");
if(append){
var prior=g.rows("SELECT id FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND deleted_at IS NULL AND status IN('DISCOVERED','QUEUED','RUNNING','IMPORTED') FOR UPDATE",slot);
if(prior.stream().anyMatch(row->!Objects.equals(existingCandidateId,row.get("id"))))throw new ResponseStatusException(HttpStatus.CONFLICT,"该fileId已登记来源,请选择已有来源替换,或为新的独立文件批次生成fileId");
}
Integer pending=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND status IN('QUEUED','RUNNING')",Integer.class,slot);if(pending!=null&&pending>0)throw new ResponseStatusException(HttpStatus.CONFLICT,"该来源正在构建,请稍后重试"); Integer pending=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND status IN('QUEUED','RUNNING')",Integer.class,slot);if(pending!=null&&pending>0)throw new ResponseStatusException(HttpStatus.CONFLICT,"该来源正在构建,请稍后重试");
String protocol=protocol(slot,r,scope,actor,reason);String key=GovernanceSupport.sha(slot+"\n"+draft.get("sha256")+"\n"+config.hash()+"\n"+r.uploadId());String id=UUID.randomUUID().toString(); ensureUniqueContent((String)draft.get("sha256"),r.stage(),r.year(),r.month(),scope,current,replacementDocument,existingCandidateId);
Map<String,Object> frozenConfiguration=new LinkedHashMap<>(config.snapshot());frozenConfiguration.put("sourceSlot",slot); String protocol=protocol(slot,r,scope,actor,reason,shard);String key=GovernanceSupport.sha(slot+"\n"+draft.get("sha256")+"\n"+config.hash()+"\n"+r.uploadId());String id=existingCandidateId==null?UUID.randomUUID().toString():existingCandidateId;
jdbc.update("INSERT INTO source_candidate(id,candidate_key,source_slot,stage,source_year,source_month,scope_code,source_file,source_path,sha256,file_size,status,protocol_id,resolution,resolution_reason,parser_config,parser_config_hash,expected_current_release_id,replacement_document_id,replacement_revision_id,configured_by) VALUES(?,?,?,?,?,?,?,?,?,?,?,'DISCOVERED',?,?,?,CAST(? AS JSON),?,?,?,?,?)",id,key,slot,r.stage(),r.year(),r.month(),scope,draft.get("sourceFile"),file.toString(),draft.get("sha256"),draft.get("fileSize"),protocol,"REPLACE".equals(r.mode())?"REPLACE":"CONFIRMED",reason,json.writeValueAsString(frozenConfiguration),config.hash(),current,replacementDocument,replacementRevision,actor.getId().toString()); Map<String,Object> frozenConfiguration=new LinkedHashMap<>(config.snapshot());frozenConfiguration.put("sourceSlot",slot);if(append||appendReplacement)frozenConfiguration.put("fileId",r.fileId());
jdbc.update("UPDATE source_upload_draft SET candidate_id=? WHERE id=?",id,r.uploadId());g.audit(actor.getId().toString(),"SOURCE_UPLOAD_CONFIGURED","source_candidate",id,Map.of("mode",r.mode(),"slot",slot,"reason",reason,"configuration",frozenConfiguration)); String resolution=append?"APPEND_CONFIRMED":"REPLACE".equals(r.mode())?"REPLACE":"CONFIRMED";
Map<String,Object> result=new LinkedHashMap<>();result.put("source",new SourceCatalogService.SourceView(id,(String)draft.get("sourceFile"),r.stage(),r.year(),r.month(),((Number)draft.get("fileSize")).longValue(),(String)draft.get("sha256"),"DISCOVERED"));result.put("configuration",frozenConfiguration);result.put("currentReleaseId",current);return result; if(priorCandidate==null){
jdbc.update("INSERT INTO source_candidate(id,candidate_key,source_slot,stage,source_year,source_month,scope_code,source_file,source_path,sha256,file_size,status,protocol_id,resolution,resolution_reason,parser_config,parser_config_hash,expected_current_release_id,replacement_document_id,replacement_revision_id,configured_by) VALUES(?,?,?,?,?,?,?,?,?,?,?,'DISCOVERED',?,?,?,CAST(? AS JSON),?,?,?,?,?)",id,key,slot,r.stage(),r.year(),r.month(),scope,draft.get("sourceFile"),file.toString(),draft.get("sha256"),draft.get("fileSize"),protocol,resolution,reason,json.writeValueAsString(frozenConfiguration),config.hash(),current,replacementDocument,replacementRevision,actor.getId().toString());
}else{
String oldSlot=SourceIdentity.canonicalSlot((String)priorCandidate.get("sourceSlot"));String oldProtocol=Objects.toString(priorCandidate.get("protocolId"),null);
if(!Objects.equals(oldSlot,SourceIdentity.canonicalSlot(slot))&&oldProtocol!=null)deactivateUnusedUploadProtocol(oldProtocol,id);
int changed=jdbc.update("UPDATE source_candidate SET candidate_key=?,source_slot=?,stage=?,source_year=?,source_month=?,scope_code=?,source_file=?,source_path=?,sha256=?,file_size=?,protocol_id=?,resolution=?,resolution_reason=?,parser_config=CAST(? AS JSON),parser_config_hash=?,expected_current_release_id=?,replacement_document_id=?,replacement_revision_id=?,configured_by=? WHERE id=? AND status='DISCOVERED' AND deleted_at IS NULL",key,slot,r.stage(),r.year(),r.month(),scope,draft.get("sourceFile"),file.toString(),draft.get("sha256"),draft.get("fileSize"),protocol,resolution,reason,json.writeValueAsString(frozenConfiguration),config.hash(),current,replacementDocument,replacementRevision,actor.getId().toString(),id);
if(changed!=1)throw new ResponseStatusException(HttpStatus.CONFLICT,"来源已排队或状态变化,请刷新来源列表后重试");
}
jdbc.update("UPDATE source_upload_draft SET candidate_id=? WHERE id=?",id,r.uploadId());g.audit(actor.getId().toString(),priorCandidate==null?"SOURCE_UPLOAD_CONFIGURED":"SOURCE_UPLOAD_RECONFIGURED","source_candidate",id,Map.of("mode",r.mode(),"slot",slot,"reason",reason,"configuration",frozenConfiguration));
Map<String,Object> result=new LinkedHashMap<>();result.put("source",new SourceCatalogService.SourceView(id,(String)draft.get("sourceFile"),r.stage(),r.year(),r.month(),((Number)draft.get("fileSize")).longValue(),(String)draft.get("sha256"),"DISCOVERED"));result.put("configuration",frozenConfiguration);result.put("currentReleaseId",current);result.put("reconfigured",priorCandidate!=null);return result;
} }
private String protocol(String slot,ConfigureRequest r,String scope,AppUserPrincipal actor,String reason){ private String protocol(String slot,ConfigureRequest r,String scope,AppUserPrincipal actor,String reason,String shard){
String mode="MAIN".equals(shard)?"MONTHLY_SNAPSHOT":"APPEND_SNAPSHOT";
var existing=g.rows("SELECT id,source_mode,applicable FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND active=TRUE FOR UPDATE",slot); var existing=g.rows("SELECT id,source_mode,applicable FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND active=TRUE FOR UPDATE",slot);
if(existing.size()>1)throw new IllegalArgumentException("来源协议存在重复,请先完成来源治理"); if(existing.size()>1)throw new IllegalArgumentException("来源协议存在重复,请先完成来源治理");
if(!existing.isEmpty()){if(!"MONTHLY_SNAPSHOT".equals(existing.getFirst().get("sourceMode"))||!Boolean.TRUE.equals(existing.getFirst().get("applicable")))throw new IllegalArgumentException("此向导仅接收适用阶段的完整月度快照");return (String)existing.getFirst().get("id");} if(!existing.isEmpty()){if(!mode.equals(existing.getFirst().get("sourceMode"))||!Boolean.TRUE.equals(existing.getFirst().get("applicable")))throw new IllegalArgumentException("来源协议与完整快照/追加批次身份不匹配,须先完成来源治理");return (String)existing.getFirst().get("id");}
String id=UUID.randomUUID().toString();Integer version=jdbc.queryForObject("SELECT COALESCE(MAX(version_no),0)+1 FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=?",Integer.class,slot);jdbc.update("INSERT INTO source_protocol(id,source_slot,version_no,stage,source_year,source_month,scope_code,source_mode,required_sheet,applicable,expected,reason,created_by) VALUES(?,?,?,?,?,?,?,'MONTHLY_SNAPSHOT',?,TRUE,TRUE,?,?)",id,slot,version,r.stage(),r.year(),r.month(),scope,r.sheetName(),reason,actor.getId().toString());return id; var versions=g.rows("SELECT version_no FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? ORDER BY version_no DESC LIMIT 1 FOR UPDATE",slot);int version=versions.isEmpty()?1:((Number)versions.getFirst().get("versionNo")).intValue()+1;
String id=UUID.randomUUID().toString();jdbc.update("INSERT INTO source_protocol(id,source_slot,version_no,stage,source_year,source_month,scope_code,shard_code,source_mode,required_sheet,applicable,expected,reason,created_by) VALUES(?,?,?,?,?,?,?,?,?,?,TRUE,TRUE,?,?)",id,slot,version,r.stage(),r.year(),r.month(),scope,shard,mode,r.sheetName(),reason,actor.getId().toString());return id;
} }
private Map<String,Object> reconfigurableCandidate(String id,AppUserPrincipal actor,Map<String,Object> draft){
var rows=g.rows("SELECT * FROM source_candidate WHERE id=? AND configured_by=? FOR UPDATE",id,actor.getId().toString());
if(rows.isEmpty())throw new ResponseStatusException(HttpStatus.CONFLICT,"已确认来源不属于当前上传人,请重新上传");Map<String,Object> candidate=rows.getFirst();
if(!"DISCOVERED".equals(candidate.get("status"))||!Objects.equals(candidate.get("sha256"),draft.get("sha256"))||!Objects.equals(candidate.get("sourcePath"),draft.get("sourcePath")))throw new ResponseStatusException(HttpStatus.CONFLICT,"此来源已排队、处理或内容变化,不能重新确认");
Integer runs=jdbc.queryForObject("SELECT COUNT(*) FROM import_run WHERE JSON_CONTAINS(requested_sources,JSON_QUOTE(?))",Integer.class,id);
String sameSlot=SourceIdentity.canonicalSlotSql("d.source_slot")+"="+SourceIdentity.canonicalSlotSql("c.source_slot");
Integer documents=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate c JOIN source_document d ON "+sameSlot+" JOIN source_revision r ON r.source_document_id=d.id AND r.sha256=c.sha256 AND r.parser_config_hash=c.parser_config_hash JOIN release_member m ON m.source_document_id=d.id AND m.source_revision_id=r.id WHERE c.id=?",Integer.class,id);
if((runs!=null&&runs>0)||(documents!=null&&documents>0))throw new ResponseStatusException(HttpStatus.CONFLICT,"来源已被导入任务或版本引用,不能重新确认");
return candidate;
}
private void deactivateUnusedUploadProtocol(String protocolId,String candidateId){
var protocol=g.rows("SELECT source_slot,active FROM source_protocol WHERE id=? AND active=TRUE FOR UPDATE",protocolId);if(protocol.isEmpty())return;
Integer others=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate WHERE protocol_id=? AND id<>? AND deleted_at IS NULL AND status NOT IN('RETURNED','SUPERSEDED')",Integer.class,protocolId,candidateId);
Integer documents=jdbc.queryForObject("SELECT COUNT(*) FROM source_document WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=?",Integer.class,SourceIdentity.canonicalSlot((String)protocol.getFirst().get("sourceSlot")));
if((others==null||others==0)&&(documents==null||documents==0))jdbc.update("UPDATE source_protocol SET active=FALSE WHERE id=? AND active=TRUE",protocolId);
}
private void ensureUniqueContent(String sha,String stage,int year,int month,String scope,String releaseId,String excludedDocument,String excludedCandidate){
if(releaseId!=null){
List<Map<String,Object>> currentRows=g.rows("SELECT d.id,d.scope_code FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id=? AND r.sha256=? AND d.stage=? AND d.source_year=? AND d.source_month=? FOR UPDATE",releaseId,sha,stage,year,month);
if(currentRows.stream().anyMatch(row->!Objects.equals(excludedDocument,row.get("id"))&&SourceIdentity.intersects(scope,(String)row.get("scopeCode"))))throw duplicateContent();
}
List<Map<String,Object>> pending=g.rows("SELECT id,source_slot,scope_code FROM source_candidate WHERE sha256=? AND stage=? AND source_year=? AND source_month=? AND deleted_at IS NULL AND status IN('DISCOVERED','QUEUED','RUNNING') AND NOT "+SourceDeletionService.pendingDeletionSql("source_candidate")+" FOR UPDATE",sha,stage,year,month);
if(pending.stream().anyMatch(row->!Objects.equals(excludedCandidate,row.get("id"))&&SourceIdentity.intersects(scope,(String)row.get("scopeCode"))))throw duplicateContent();
}
private ResponseStatusException duplicateContent(){return new ResponseStatusException(HttpStatus.CONFLICT,"该工作簿内容已用于同一年月、阶段及重叠范围的另一份来源,请勿重复计入;更正版请精确选择原来源替换");}
private static boolean validFileId(String value){return value!=null&&value.length()<=80&&value.matches("[A-Za-z0-9_-]{1,80}");}
private static String fileId(Object slotValue){if(!(slotValue instanceof String slot))return null;String[] parts=SourceIdentity.canonicalSlot(slot).split("\\|",-1);if(parts.length!=5||!parts[4].startsWith("UPLOAD_"))return null;String value=parts[4].substring("UPLOAD_".length());return validFileId(value)?value:null;}
private Map<String,Object> draft(String id,AppUserPrincipal actor,boolean lock){var rows=g.rows("SELECT * FROM source_upload_draft WHERE id=? AND created_by=? AND expires_at>CURRENT_TIMESTAMP(6)"+(lock?" FOR UPDATE":""),id,actor.getId().toString());if(rows.isEmpty())throw new ResponseStatusException(HttpStatus.NOT_FOUND,"上传预览已过期或不属于当前用户,请重新上传");return rows.getFirst();} private Map<String,Object> draft(String id,AppUserPrincipal actor,boolean lock){var rows=g.rows("SELECT * FROM source_upload_draft WHERE id=? AND created_by=? AND expires_at>CURRENT_TIMESTAMP(6)"+(lock?" FOR UPDATE":""),id,actor.getId().toString());if(rows.isEmpty())throw new ResponseStatusException(HttpStatus.NOT_FOUND,"上传预览已过期或不属于当前用户,请重新上传");return rows.getFirst();}
private Path verified(Map<String,Object> draft)throws IOException{Path path=Path.of((String)draft.get("sourcePath"));if(Files.isSymbolicLink(path)||!Files.isRegularFile(path)||!path.toRealPath().startsWith(scanner.archiveRoot())||Files.size(path)!=((Number)draft.get("fileSize")).longValue()||!SourceScanner.sha256(path).equals(draft.get("sha256")))throw new IllegalArgumentException("上传文件已变化,请重新上传");return path.toRealPath();} private Path verified(Map<String,Object> draft)throws IOException{Path path=Path.of((String)draft.get("sourcePath"));if(Files.isSymbolicLink(path)||!Files.isRegularFile(path)||!path.toRealPath().startsWith(scanner.archiveRoot())||Files.size(path)!=((Number)draft.get("fileSize")).longValue()||!SourceScanner.sha256(path).equals(draft.get("sha256")))throw new IllegalArgumentException("上传文件已变化,请重新上传");return path.toRealPath();}
private String current(boolean lock){return jdbc.queryForObject("SELECT release_id FROM current_release WHERE dataset_id='quality'"+(lock?" FOR UPDATE":""),String.class);} private String current(boolean lock){return jdbc.queryForObject("SELECT release_id FROM current_release WHERE dataset_id='quality'"+(lock?" FOR UPDATE":""),String.class);}
private void requireActor(AppUserPrincipal actor){access.requireUserActivePolicy(actor);access.requireBoundAction("RECEIVER","source.receive");} private void requireActor(AppUserPrincipal actor){access.requireUserActivePolicy(actor);access.requireBoundAction("RECEIVER","source.receive");}
/** templateId remains an explicitly disabled compatibility field for older clients. */ /** templateId remains an explicitly disabled compatibility field for older clients. */
public record ConfigureRequest(String uploadId,String stage,int year,int month,String sheetName,int headerRow,Map<String,String> fieldMap,String templateId,String mode,String expectedCurrentReleaseId,String targetDocumentId,String targetRevisionId,String reason,String scopeCode){} public record ConfigureRequest(String uploadId,String stage,int year,int month,String sheetName,int headerRow,Map<String,String> fieldMap,String templateId,String mode,String expectedCurrentReleaseId,String targetDocumentId,String targetRevisionId,String reason,String scopeCode,String fileId){
public ConfigureRequest(String uploadId,String stage,int year,int month,String sheetName,int headerRow,Map<String,String> fieldMap,String templateId,String mode,String expectedCurrentReleaseId,String targetDocumentId,String targetRevisionId,String reason,String scopeCode){this(uploadId,stage,year,month,sheetName,headerRow,fieldMap,templateId,mode,expectedCurrentReleaseId,targetDocumentId,targetRevisionId,reason,scopeCode,null);}
}
} }
@@ -0,0 +1,27 @@
package cn.yunsheng.quality.update;
import java.util.Map;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/system-update")
@PreAuthorize("hasRole('SYSTEM_ADMIN')")
public class SystemUpdateController {
private final SystemUpdateService updates;
public SystemUpdateController(SystemUpdateService updates) { this.updates = updates; }
@GetMapping
public Map<String, Object> status() { return updates.status(); }
// Both mutations use the application's existing session + CSRF filter.
@PostMapping("/check")
public Map<String, Object> check() { return updates.check(); }
@PostMapping("/apply")
public Map<String, Object> apply() { return updates.apply(); }
}
@@ -0,0 +1,39 @@
package cn.yunsheng.quality.update;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Set;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.OncePerRequestFilter;
/** Keeps the startup-to-update-commit interval free of business writes. */
@Component
@Order(Ordered.HIGHEST_PRECEDENCE + 20)
public class SystemUpdateMaintenanceFilter extends OncePerRequestFilter {
private static final Set<String> READ_METHODS = Set.of("GET", "HEAD", "OPTIONS", "TRACE");
private static final Set<String> AUTH_ROUTES = Set.of("/api/auth/login", "/api/auth/logout", "/api/auth/csrf", "/api/auth/me");
private final SystemUpdateService updates;
public SystemUpdateMaintenanceFilter(SystemUpdateService updates) { this.updates = updates; }
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain)
throws ServletException, IOException {
String path = request.getRequestURI().substring(request.getContextPath().length());
if (!READ_METHODS.contains(request.getMethod()) && path.startsWith("/api/")
&& !path.equals("/api/system-update") && !path.startsWith("/api/system-update/")
&& !AUTH_ROUTES.contains(path) && updates.maintenanceActive()) {
response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
response.setContentType("application/problem+json;charset=UTF-8");
response.setHeader("Retry-After", "10");
response.getWriter().write("{\"title\":\"Service Unavailable\",\"status\":503,\"detail\":\"系统正在更新,请稍后重试。\"}");
return;
}
chain.doFilter(request, response);
}
}
@@ -0,0 +1,138 @@
package cn.yunsheng.quality.update;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
import tools.jackson.databind.json.JsonMapper;
/** Restricted bridge to the packaged updater; neither URLs nor paths are input. */
@Service
public class SystemUpdateService {
private static final Set<String> RESPONSE_FIELDS = Set.of("enabled", "currentVersion", "state", "message",
"checkedAt", "version", "available", "startedAt", "finishedAt");
private final JsonMapper json;
private final Path root;
private final boolean enabled;
public SystemUpdateService(JsonMapper json,
@Value("${QUALITY_PROJECT_ROOT:}") String root,
@Value("${QUALITY_UPDATE_ENABLED:false}") boolean requested) {
this.json = json;
this.root = root.isBlank() ? null : Path.of(root).toAbsolutePath().normalize();
this.enabled = requested && System.getProperty("os.name", "").startsWith("Windows")
&& this.root != null && Files.isRegularFile(this.root.resolve("manifest.json"))
&& Files.isRegularFile(this.root.resolve("runtimes/python/python.exe"))
&& Files.isRegularFile(this.root.resolve("deploy/offline/update_manager.py"))
&& Files.isRegularFile(this.root.resolve("deploy/offline/update-public-key.pem"));
}
public synchronized Map<String, Object> status() {
if (!enabled) return Map.of("enabled", false, "state", "disabled", "message", "此部署不支持软件内更新。");
return invoke("status");
}
public synchronized Map<String, Object> check() {
requireEnabled();
return invoke("check");
}
public synchronized Map<String, Object> apply() {
requireEnabled();
return invoke("launch");
}
/** Fail closed during installation/recovery, including a process crash. */
public boolean maintenanceActive() {
if (!enabled) return false;
try {
Map<?, ?> state = readMetadata("status.json");
if (state.get("state") instanceof String phase
&& Set.of("backingUp", "installing", "restarting", "rollingBack").contains(phase)) return true;
Map<?, ?> journal = readMetadata("journal.json");
return journal.get("phase") instanceof String phase && Set.of("stopping", "installing").contains(phase);
} catch (Exception failure) {
return true;
}
}
private Map<?, ?> readMetadata(String name) throws IOException {
Path path = root.resolve(".runtime/update/" + name);
if (!Files.exists(path)) return Map.of();
try (var stream = Files.newInputStream(path)) {
byte[] raw = stream.readNBytes(8193);
if (raw.length > 8192) throw new IOException("Oversized update state");
Object decoded = json.readValue(raw, Object.class);
if (!(decoded instanceof Map<?, ?> value)) throw new IOException("Invalid update state");
return value;
}
}
private void requireEnabled() {
if (!enabled) throw new ResponseStatusException(HttpStatus.CONFLICT, "此部署不支持软件内更新。");
}
private Map<String, Object> invoke(String action) {
Process process = null;
try {
ProcessBuilder builder = new ProcessBuilder(root.resolve("runtimes/python/python.exe").toString(),
root.resolve("deploy/offline/update_manager.py").toString(), action);
builder.directory(root.toFile());
builder.redirectError(ProcessBuilder.Redirect.DISCARD);
builder.environment().put("PYTHONUTF8", "1");
builder.environment().put("PYTHONIOENCODING", "utf-8");
builder.environment().put("PYTHONDONTWRITEBYTECODE", "1");
process = builder.start();
process.getOutputStream().close();
// The CLI prints one small, allowlisted JSON receipt. Waiting first
// also bounds hangs caused by a failed DNS/TLS/network operation.
if (!process.waitFor(45, TimeUnit.SECONDS)) {
process.destroyForcibly();
throw new ResponseStatusException(HttpStatus.GATEWAY_TIMEOUT, "更新服务器暂未响应,请稍后重试。");
}
byte[] raw = process.getInputStream().readNBytes(8193);
if (raw.length > 8192) throw new IOException("Oversized updater receipt");
Object decoded = json.readValue(new String(raw, StandardCharsets.UTF_8), Object.class);
if (!(decoded instanceof Map<?, ?> value)) throw new IOException("Invalid updater receipt");
if (process.exitValue() != 0) {
Object detail = value.get("error");
String message = detail instanceof String s && s.length() <= 300 ? s : "更新操作未完成,请稍后重试。";
throw new ResponseStatusException(HttpStatus.CONFLICT, message);
}
return safeResponse(value);
} catch (ResponseStatusException failure) {
throw failure;
} catch (InterruptedException failure) {
Thread.currentThread().interrupt();
throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "更新操作已中断。");
} catch (Exception failure) {
throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "更新服务暂不可用,请稍后重试。");
} finally {
if (process != null && process.isAlive()) process.destroyForcibly();
}
}
static Map<String, Object> safeResponse(Map<?, ?> value) throws IOException {
Map<String, Object> result = new LinkedHashMap<>();
for (String field : RESPONSE_FIELDS) {
Object item = value.get(field);
if (item == null) continue;
if ((field.equals("enabled") || field.equals("available")) && !(item instanceof Boolean))
throw new IOException("Invalid updater boolean");
if (!(item instanceof Boolean) && !(item instanceof String s && s.length() <= 300))
throw new IOException("Invalid updater field");
result.put(field, item);
}
if (!(result.get("state") instanceof String) || !(result.get("enabled") instanceof Boolean))
throw new IOException("Incomplete updater receipt");
return result;
}
}
@@ -0,0 +1,97 @@
package cn.yunsheng.quality.update;
import jakarta.annotation.PreDestroy;
import java.lang.reflect.Method;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.aopalliance.intercept.MethodInterceptor;
import org.springframework.aop.framework.AopProxyUtils;
import org.springframework.aop.framework.ProxyFactory;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.config.BeanPostProcessor;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.AnnotatedElementUtils;
import org.springframework.core.env.Environment;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.util.ReflectionUtils;
/** Defers business recovery and suppresses schedules until the update commits. */
@Component
public class SystemUpdateStartupBarrier implements BeanPostProcessor, Ordered {
private static final Set<String> RECOVERY_METHODS = Set.of(
"cn.yunsheng.quality.ingestion.ImportRecoveryRunner#run",
"cn.yunsheng.quality.analysis.AnalysisTurnService#recoverQueuedTurns",
"cn.yunsheng.quality.analysis.AnalysisReportService#recoverPendingReports",
"cn.yunsheng.quality.analytics.InspectionService#recoverPendingDrilldowns");
private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(SystemUpdateStartupBarrier.class);
private final ObjectProvider<SystemUpdateService> updates;
private final Environment environment;
private final ConcurrentLinkedQueue<Runnable> pending = new ConcurrentLinkedQueue<>();
private final ScheduledExecutorService release = Executors.newSingleThreadScheduledExecutor(task -> {
Thread thread = new Thread(task, "quality-update-recovery");
thread.setDaemon(true);
return thread;
});
public SystemUpdateStartupBarrier(ObjectProvider<SystemUpdateService> updates, Environment environment) {
this.updates = updates;
this.environment = environment;
release.scheduleWithFixedDelay(this::drain, 500, 500, TimeUnit.MILLISECONDS);
}
@Override public int getOrder() { return Ordered.HIGHEST_PRECEDENCE + 50; }
@Override
public Object postProcessAfterInitialization(Object bean, String name) {
if (!System.getProperty("os.name", "").startsWith("Windows")
|| !environment.getProperty("QUALITY_UPDATE_ENABLED", Boolean.class, false)) return bean;
Class<?> type = AopProxyUtils.ultimateTargetClass(bean);
if (!type.getName().startsWith("cn.yunsheng.quality.") || type.getName().contains(".update.")) return bean;
boolean candidate = false;
for (Method method : ReflectionUtils.getAllDeclaredMethods(type))
if (isRecovery(type, method) || AnnotatedElementUtils.hasAnnotation(method, Scheduled.class)) candidate = true;
return candidate ? wrap(bean, type) : bean;
}
Object wrap(Object bean, Class<?> type) {
ProxyFactory proxy = new ProxyFactory(bean);
proxy.setProxyTargetClass(true);
proxy.addAdvice((MethodInterceptor) invocation -> {
Method method = ReflectionUtils.findMethod(type, invocation.getMethod().getName(), invocation.getMethod().getParameterTypes());
if (method == null || !updates.getObject().maintenanceActive()) return invocation.proceed();
if (isRecovery(type, method)) {
pending.add(() -> {
try { invocation.proceed(); }
catch (Throwable failure) { LOG.error("Deferred application recovery failed", failure); }
});
return null;
}
if (AnnotatedElementUtils.hasAnnotation(method, Scheduled.class)) return null;
return invocation.proceed();
});
return proxy.getProxy(type.getClassLoader());
}
private static boolean isRecovery(Class<?> type, Method method) {
return RECOVERY_METHODS.contains(type.getName() + "#" + method.getName());
}
void drain() {
if (pending.isEmpty()) return;
try {
while (!updates.getObject().maintenanceActive()) {
Runnable task = pending.poll();
if (task == null) return;
task.run();
}
} catch (Exception failure) {
LOG.warn("Waiting for application update to complete");
}
}
@PreDestroy public void close() { release.shutdownNow(); pending.clear(); }
}
@@ -506,10 +506,40 @@ public class DataWorkflowService {
throw failure("CANDIDATE_VERSION_MISMATCH", "候选版本未使用本次工作流冻结的映射和规则,候选不会发布"); throw failure("CANDIDATE_VERSION_MISMATCH", "候选版本未使用本次工作流冻结的映射和规则,候选不会发布");
Integer invalidBuilds = jdbc.queryForObject(""" Integer invalidBuilds = jdbc.queryForObject("""
SELECT COUNT(*) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id SELECT COUNT(*) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id
WHERE rm.release_id=? AND (b.mapping_version<>? OR b.rule_version<>? OR b.status<>'COMPLETED' OR b.rejected_count>0) WHERE rm.release_id=? AND (b.mapping_version<>? OR b.rule_version<>?)
""", Integer.class, releaseId, mappingId, ruleId); """, Integer.class, releaseId, mappingId, ruleId);
if (invalidBuilds != null && invalidBuilds > 0) if (invalidBuilds != null && invalidBuilds > 0)
throw failure("CANDIDATE_BUILD_VERSION_MISMATCH", "候选中存在未使用本次冻结版本的转换产物,候选不会发布"); throw failure("CANDIDATE_BUILD_VERSION_MISMATCH", "候选中存在未使用本次冻结版本的转换产物,候选不会发布");
Integer unfinishedBuilds = jdbc.queryForObject("""
SELECT COUNT(*) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id
WHERE rm.release_id=? AND b.status<>'COMPLETED'
""", Integer.class, releaseId);
if (unfinishedBuilds != null && unfinishedBuilds > 0)
throw failure("CANDIDATE_BUILD_INCOMPLETE", "有 " + unfinishedBuilds + " 个来源尚未完成转换,候选不会发布");
Integer rejected = jdbc.queryForObject("""
SELECT COALESCE(SUM(b.rejected_count),0) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id
WHERE rm.release_id=?
""", Integer.class, releaseId);
if (rejected != null && rejected > 0) {
List<Map<String,Object>> issues = jdbc.queryForList("""
SELECT issue_code AS issueCode,field_name AS fieldName,COUNT(*) AS issueCount FROM import_issue
WHERE import_run_id=(SELECT id FROM import_run WHERE result_release_id=? ORDER BY created_at DESC LIMIT 1)
AND JSON_EXTRACT(details,'$.blocking')=TRUE
GROUP BY issue_code,field_name ORDER BY issueCount DESC,issue_code,field_name LIMIT 3
""", releaseId);
String detail = issues.stream().map(issue -> {
String reason = switch (Objects.toString(issue.get("issueCode"), "")) {
case "INVALID_NUMBER" -> "数值格式无效";
case "REQUIRED_HEADER_MISSING" -> "缺少必需列";
case "NO_DETAIL_ROWS" -> "没有可导入明细";
default -> "来源校验未通过";
};
String field = Objects.toString(issue.get("fieldName"), "");
return (field.isBlank() ? "" : field + ":") + reason + "(" + issue.get("issueCount") + "项)";
}).collect(java.util.stream.Collectors.joining(";"));
throw failure("CANDIDATE_SOURCE_VALIDATION_FAILED", "来源数据有 " + rejected + " 项阻断问题,候选不会发布。"
+ (detail.isBlank() ? "请查看导入记录中的问题明细" : detail));
}
} }
private void publishExisting(WorkflowOperation operation, AppUserPrincipal actor, String token) { private void publishExisting(WorkflowOperation operation, AppUserPrincipal actor, String token) {
@@ -132,7 +132,7 @@ class AnalysisAgentLoopTest {
private static final class Fixture { private static final class Fixture {
final AppUserPrincipal actor = new AppUserPrincipal(UUID.fromString("29000000-0000-0000-0000-000000000001"), final AppUserPrincipal actor = new AppUserPrincipal(UUID.fromString("29000000-0000-0000-0000-000000000001"),
"agent-test", "unused", "Agent Test", true, false, List.of("BUSINESS_ANALYST"), "agent-test", "unused", "Agent Test", true, false, List.of("USER"),
List.of("质量中心"), List.of("锻造"), List.of()); List.of("质量中心"), List.of("锻造"), List.of());
final JdbcTemplate jdbc = mock(JdbcTemplate.class); final JdbcTemplate jdbc = mock(JdbcTemplate.class);
final JsonMapper json = JsonMapper.builder().build(); final JsonMapper json = JsonMapper.builder().build();
@@ -184,7 +184,7 @@ class AnalysisApiSettingsTest {
context.registerBean(AnalysisApiSettingsService.class, this::service); context.registerBean(AnalysisApiSettingsService.class, this::service);
context.register(AnalysisApiSettingsController.class); context.refresh(); context.register(AnalysisApiSettingsController.class); context.refresh();
var c = context.getBean(AnalysisApiSettingsController.class); var c = context.getBean(AnalysisApiSettingsController.class);
SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken("viewer", "", List.of(new SimpleGrantedAuthority("ROLE_QUALITY_VIEWER")))); SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken("viewer", "", List.of(new SimpleGrantedAuthority("ROLE_USER"))));
assertThrows(AccessDeniedException.class, c::get); assertThrows(AccessDeniedException.class, c::get);
assertThrows(AccessDeniedException.class, () -> c.test(null)); assertThrows(AccessDeniedException.class, () -> c.test(null));
assertThrows(AccessDeniedException.class, () -> c.save(null)); assertThrows(AccessDeniedException.class, () -> c.save(null));
@@ -27,7 +27,7 @@ class AnalysisChangesServiceTest {
private final GovernanceAccessService governance = mock(GovernanceAccessService.class); private final GovernanceAccessService governance = mock(GovernanceAccessService.class);
private final OptionsController options = mock(OptionsController.class); private final OptionsController options = mock(OptionsController.class);
private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "analyst", "unused", "Analyst", true, private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "analyst", "unused", "Analyst", true,
false, List.of("BUSINESS_ANALYST"), List.of("消费电子产业部"), List.of("消费电子-3C"), List.of()); false, List.of("USER"), List.of("消费电子产业部"), List.of("消费电子-3C"), List.of());
private final AnalysisChangesService service = new AnalysisChangesService(metrics, responseAccess, governance, options); private final AnalysisChangesService service = new AnalysisChangesService(metrics, responseAccess, governance, options);
@Test @Test
@@ -56,7 +56,7 @@ class AnalysisControllerHistoryAccessTest {
} }
private static final class Fixture { private static final class Fixture {
final String session=UUID.randomUUID().toString(),turn=UUID.randomUUID().toString(),release=UUID.randomUUID().toString(); final String session=UUID.randomUUID().toString(),turn=UUID.randomUUID().toString(),release=UUID.randomUUID().toString();
final AppUserPrincipal actor=new AppUserPrincipal(UUID.randomUUID(),"history-viewer","unused","测试",true,false,List.of("QUALITY_VIEWER"),List.of(),List.of(),List.of()); final AppUserPrincipal actor=new AppUserPrincipal(UUID.randomUUID(),"history-viewer","unused","测试",true,false,List.of("USER"),List.of(),List.of(),List.of());
final AnalysisProtocolService protocol=mock(AnalysisProtocolService.class); final AnalysisProtocolService protocol=mock(AnalysisProtocolService.class);
final MetricsService metrics=mock(MetricsService.class); final MetricsService metrics=mock(MetricsService.class);
final AnalysisController controller; final AnalysisController controller;
@@ -43,7 +43,7 @@ class AnalysisExplorationToolsTest {
private final JdbcTemplate jdbc = mock(JdbcTemplate.class); private final JdbcTemplate jdbc = mock(JdbcTemplate.class);
private final AnalysisExplorationTools tools = new AnalysisExplorationTools(metrics, dictionary, governance, responseAccess, jdbc); private final AnalysisExplorationTools tools = new AnalysisExplorationTools(metrics, dictionary, governance, responseAccess, jdbc);
private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "analyst", "unused", "Analyst", true, private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "analyst", "unused", "Analyst", true,
false, List.of("BUSINESS_ANALYST"), List.of("质量中心"), List.of("锻造"), List.of()); false, List.of("USER"), List.of("质量中心"), List.of("锻造"), List.of());
@Test @Test
void detailsReturnsOnlyAuthorizedFieldsFromFilteredPublishedReleaseSample() throws Exception { void detailsReturnsOnlyAuthorizedFieldsFromFilteredPublishedReleaseSample() throws Exception {
@@ -157,7 +157,7 @@ class AnalysisNativeAgentContextTest {
JdbcTemplate jdbc = mock(JdbcTemplate.class); JdbcTemplate jdbc = mock(JdbcTemplate.class);
GovernanceAccessService governance = mock(GovernanceAccessService.class); GovernanceAccessService governance = mock(GovernanceAccessService.class);
AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "agent-context-test", "unused", "测试", true, false, AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "agent-context-test", "unused", "测试", true, false,
List.of("QUALITY_VIEWER"), List.of(), List.of(), List.of()); List.of("USER"), List.of(), List.of(), List.of());
when(governance.allowedFields(actor)).thenReturn(Set.of("black_good_weight", "black_total_weight")); when(governance.allowedFields(actor)).thenReturn(Set.of("black_good_weight", "black_total_weight"));
doReturn(List.of()).when(jdbc).query(org.mockito.ArgumentMatchers.anyString(), doReturn(List.of()).when(jdbc).query(org.mockito.ArgumentMatchers.anyString(),
org.mockito.ArgumentMatchers.any(RowMapper.class)); org.mockito.ArgumentMatchers.any(RowMapper.class));
@@ -112,7 +112,7 @@ class AnalysisNativeRuntimeRecoveryTest {
final GovernanceAccessService governance = mock(GovernanceAccessService.class); final GovernanceAccessService governance = mock(GovernanceAccessService.class);
final NaoAnalysisRunner runner = mock(NaoAnalysisRunner.class); final NaoAnalysisRunner runner = mock(NaoAnalysisRunner.class);
final AppUserPrincipal actor = new AppUserPrincipal(UUID.fromString("32cd8848-cfa1-4b10-8d2f-91c7bf433103"), final AppUserPrincipal actor = new AppUserPrincipal(UUID.fromString("32cd8848-cfa1-4b10-8d2f-91c7bf433103"),
"runtime-review", "unused", "隔离测试", true, false, List.of("QUALITY_VIEWER"), List.of("D"), List.of("G"), List.of()); "runtime-review", "unused", "隔离测试", true, false, List.of("USER"), List.of("D"), List.of("G"), List.of());
final JsonMapper json = JsonMapper.builder().build(); final JsonMapper json = JsonMapper.builder().build();
final AnalysisTurnService service; final AnalysisTurnService service;
final Map<String, Long> firstEvents = new LinkedHashMap<>(); final Map<String, Long> firstEvents = new LinkedHashMap<>();
@@ -25,7 +25,7 @@ class AnalysisProtocolServiceMySqlTest {
private String database; private String database;
private boolean created; private boolean created;
private final String session=UUID.randomUUID().toString(),release=UUID.randomUUID().toString(); private final String session=UUID.randomUUID().toString(),release=UUID.randomUUID().toString();
private final AppUserPrincipal actor=new AppUserPrincipal(UUID.randomUUID(),"protocol-test","unused","测试",true,false,List.of("QUALITY_VIEWER"),List.of(),List.of(),List.of()); private final AppUserPrincipal actor=new AppUserPrincipal(UUID.randomUUID(),"protocol-test","unused","测试",true,false,List.of("USER"),List.of(),List.of(),List.of());
private final GovernanceAccessService governance=mock(GovernanceAccessService.class); private final GovernanceAccessService governance=mock(GovernanceAccessService.class);
private final OptionsController options=mock(OptionsController.class); private final OptionsController options=mock(OptionsController.class);
private final AnalysisToolsController tools=mock(AnalysisToolsController.class); private final AnalysisToolsController tools=mock(AnalysisToolsController.class);
@@ -67,7 +67,7 @@ class AnalysisSemanticsTest {
@Test void modelIntentResolvesAutomotiveAliasToAllAuthorizedGroupsAndDefaultsGenericRateToComposite() throws Exception { @Test void modelIntentResolvesAutomotiveAliasToAllAuthorizedGroupsAndDefaultsGenericRateToComposite() throws Exception {
AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "semantic-test", "unused", "Semantic Test", true, AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "semantic-test", "unused", "Semantic Test", true,
false, List.of("BUSINESS_ANALYST"), List.of("工业电机产业部", "新能源汽车磁钢", "欧洲磁钢", "消费电子产业部", "美洲产业部"), false, List.of("USER"), List.of("工业电机产业部", "新能源汽车磁钢", "欧洲磁钢", "消费电子产业部", "美洲产业部"),
List.of("汽车交通-国内", "汽车交通-海外"), List.of()); List.of("汽车交通-国内", "汽车交通-海外"), List.of());
List<Map<String,Object>> catalog = List.of(Map.of("code", "COMPOSITE_RATE", "name", "综合合格率", "enabled", true), List<Map<String,Object>> catalog = List.of(Map.of("code", "COMPOSITE_RATE", "name", "综合合格率", "enabled", true),
Map.of("code", "BLACK_RATE", "name", "黑片合格率", "enabled", true), Map.of("code", "BLACK_RATE", "name", "黑片合格率", "enabled", true),
@@ -110,7 +110,7 @@ class AnalysisSemanticsTest {
@Test void modelCannotSilentlyDropAnUnmappedDimension() throws Exception { @Test void modelCannotSilentlyDropAnUnmappedDimension() throws Exception {
AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "semantic-test", "unused", "Semantic Test", true, AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "semantic-test", "unused", "Semantic Test", true,
false, List.of("BUSINESS_ANALYST"), List.of("工业电机产业部"), List.of("汽车交通-国内"), List.of()); false, List.of("USER"), List.of("工业电机产业部"), List.of("汽车交通-国内"), List.of());
List<Map<String,Object>> catalog = List.of(Map.of("code", "COMPOSITE_RATE", "name", "综合合格率", "enabled", true)); List<Map<String,Object>> catalog = List.of(Map.of("code", "COMPOSITE_RATE", "name", "综合合格率", "enabled", true));
JsonNode parsed = mapper.readTree("{\"intent\":\"metric\",\"metricCode\":\"COMPOSITE_RATE\",\"group\":\"其他业务组\"}"); JsonNode parsed = mapper.readTree("{\"intent\":\"metric\",\"metricCode\":\"COMPOSITE_RATE\",\"group\":\"其他业务组\"}");
assertThrows(IllegalArgumentException.class, () -> AnalysisModelGateway.decodeIntent(parsed, assertThrows(IllegalArgumentException.class, () -> AnalysisModelGateway.decodeIntent(parsed,
@@ -12,7 +12,7 @@ import org.springframework.web.server.ResponseStatusException;
class AnalysisSourceQueryControllerTest { class AnalysisSourceQueryControllerTest {
@Test void aPolicyChangeWhileScanningPreventsReturningTheProjectedData() { @Test void aPolicyChangeWhileScanningPreventsReturningTheProjectedData() {
var user=new AppUserPrincipal(UUID.randomUUID(),"source-reader","unused","测试",true,false,List.of("QUALITY_VIEWER"),List.of(),List.of(),List.of()); var user=new AppUserPrincipal(UUID.randomUUID(),"source-reader","unused","测试",true,false,List.of("USER"),List.of(),List.of(),List.of());
var governance=mock(cn.yunsheng.quality.governance.GovernanceAccessService.class); var governance=mock(cn.yunsheng.quality.governance.GovernanceAccessService.class);
var users=mock(cn.yunsheng.quality.identity.JdbcAppUserDetailsService.class); var users=mock(cn.yunsheng.quality.identity.JdbcAppUserDetailsService.class);
when(governance.allowedFields(user)).thenReturn(Set.of("*")); when(governance.allowedFields(user)).thenReturn(Set.of("*"));
@@ -222,7 +222,7 @@ class AnalysisTurnSnapshotTest {
private final MetricsService metrics = mock(MetricsService.class); private final MetricsService metrics = mock(MetricsService.class);
private final JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); private final JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class);
private final GovernanceAccessService governance = mock(GovernanceAccessService.class); private final GovernanceAccessService governance = mock(GovernanceAccessService.class);
private final AppUserPrincipal submittedActor = actor("submitted-user", "QUALITY_VIEWER"); private final AppUserPrincipal submittedActor = actor("submitted-user", "USER");
private final AppUserPrincipal freshActor = actor("fresh-user", "SYSTEM_ADMIN"); private final AppUserPrincipal freshActor = actor("fresh-user", "SYSTEM_ADMIN");
private final String fingerprint; private final String fingerprint;
private final AnalysisTurnService service; private final AnalysisTurnService service;
@@ -643,7 +643,7 @@ class AnalysisVisualSnapshotAccessTest {
JdbcTemplate jdbc = mock(JdbcTemplate.class); JdbcTemplate jdbc = mock(JdbcTemplate.class);
GovernanceAccessService governance = mock(GovernanceAccessService.class); GovernanceAccessService governance = mock(GovernanceAccessService.class);
AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "visual-test", "unused", "测试", true, false, AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "visual-test", "unused", "测试", true, false,
List.of("QUALITY_VIEWER"), List.of(), List.of(), List.of()); List.of("USER"), List.of(), List.of(), List.of());
when(governance.allowedFields(actor)).thenReturn(RATE_FIELDS); when(governance.allowedFields(actor)).thenReturn(RATE_FIELDS);
doReturn(List.of()).when(jdbc).query(anyString(), any(RowMapper.class)); doReturn(List.of()).when(jdbc).query(anyString(), any(RowMapper.class));
AnalysisTurnService service = new AnalysisTurnService(jdbc, JsonMapper.builder().build(), null, null, AnalysisTurnService service = new AnalysisTurnService(jdbc, JsonMapper.builder().build(), null, null,
@@ -43,7 +43,7 @@ class FineDefectRollupsMySqlTest {
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))"); jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))");
jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))"); jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))");
for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))"); for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))");
for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql")) { for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql")) {
try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) { try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) {
for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql); for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql);
} }
@@ -108,7 +108,7 @@ class FineDefectRollupsMySqlTest {
@Test void fieldAndOrganizationPermissionsRemainEffective() { @Test void fieldAndOrganizationPermissionsRemainEffective() {
var viewer = new AppUserPrincipal(UUID.randomUUID(), "viewer", "", "Viewer", true, false, var viewer = new AppUserPrincipal(UUID.randomUUID(), "viewer", "", "Viewer", true, false,
List.of("QUALITY_VIEWER"), List.of("A"), List.of("G"), List.of()); List.of("USER"), List.of("A"), List.of("G"), List.of());
when(governance.allowedFields(viewer)).thenReturn(Set.of("责废尺寸重量")); when(governance.allowedFields(viewer)).thenReturn(Set.of("责废尺寸重量"));
equivalent(request("black", Map.of()), viewer); equivalent(request("black", Map.of()), viewer);
when(governance.allowedFields(actor)).thenReturn(Set.of("镀废缺角重量", "bright_plating_scrap")); when(governance.allowedFields(actor)).thenReturn(Set.of("镀废缺角重量", "bright_plating_scrap"));
@@ -139,6 +139,27 @@ class FineDefectRollupsMySqlTest {
equivalent(request("bright", Map.of()), actor); equivalent(request("bright", Map.of()), actor);
} }
@Test void excludingASmallTypeSubtractsExactCountsAndWeightsAndLargeExclusionsKeepSourceFallback() {
String version=UUID.randomUUID().toString();
jdbc.update("INSERT INTO sales_order_type_version(id,source_file,source_sha256,import_mode,row_count,created_by) VALUES(?,'fixture',?,'REPLACE',2,?)",version,"0".repeat(64),actor.getId().toString());
jdbc.update("INSERT INTO sales_order_type_entry VALUES(?,'sample','样品销售订单'),(?,'normal','标准销售订单')",version,version);
jdbc.update("UPDATE sales_order_type_current SET version_id=?",version);
jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64),label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)");
for(String stage:List.of("black","bright")) jdbc.update("UPDATE fact_"+stage+" SET raw_values=JSON_SET(raw_values,?,CASE WHEN id=1 THEN 'sample' WHEN id=2 THEN NULL ELSE 'normal' END)","$.\""+cn.yunsheng.quality.filters.SalesOrderTypes.column(stage)+"\"");
var typed=new cn.yunsheng.quality.metrics.SalesOrderMetricRollups(jdbc);typed.prepareBuild(version,build);
for(String stage:List.of("black","bright")) for(var selection:List.of(List.of("样品销售订单"),List.of("标准销售订单"),List.of("__EMPTY__"),List.of("样品销售订单","__EMPTY__"))) {
for(Map<String,Object> scope:List.<Map<String,Object>>of(Map.of(),Map.of("includeUnassigned",false),Map.of("department","A"),Map.of("productSearch","P0"),Map.of("month",1))) {
var input=new LinkedHashMap<>(scope);input.put("extraFilters",Map.of("salesOrderTypeNotIn",selection));
var query=request(stage,input);
assertEquals(canonical(source.query(query,actor)),canonical(fast.query(query,actor)),stage+selection+scope);
}
}
// An unavailable fine-defect snapshot must use the original complete query.
jdbc.update("DELETE FROM fine_defect_rollup_build");
var query=request("black",Map.of("extraFilters",Map.of("salesOrderTypeNotIn",List.of("样品销售订单"))));
assertEquals(canonical(source.query(query,actor)),canonical(fast.query(query,actor)));
}
@AfterEach void cleanup() { @AfterEach void cleanup() {
if (admin != null && database != null && database.matches("quality_fine_rollup_[a-f0-9]{32}")) admin.execute("DROP DATABASE " + database); if (admin != null && database != null && database.matches("quality_fine_rollup_[a-f0-9]{32}")) admin.execute("DROP DATABASE " + database);
} }
@@ -40,7 +40,7 @@ class NaoAnalysisRunnerRevocationTest {
return null; return null;
}).when(access).authorizeModel("offline-delegation"); }).when(access).authorizeModel("offline-delegation");
AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "offline", "unused", "测试", true, false, AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "offline", "unused", "测试", true, false,
List.of("QUALITY_VIEWER"), List.of("D"), List.of(), List.of()); List.of("USER"), List.of("D"), List.of(), List.of());
NaoAnalysisRunner runner = new NaoAnalysisRunner(JsonMapper.builder().build(), access); NaoAnalysisRunner runner = new NaoAnalysisRunner(JsonMapper.builder().build(), access);
try { try {
System.setProperty("quality.project.root", root.toString()); System.setProperty("quality.project.root", root.toString());
@@ -23,7 +23,7 @@ class NaoProtocolRunnerTest {
print(json.dumps({'type':'result','status':'COMPLETED','answer':'verified','stopReason':'stop'}),flush=True) print(json.dumps({'type':'result','status':'COMPLETED','answer':'verified','stopReason':'stop'}),flush=True)
"""); """);
var access=mock(NaoRuntimeAccessService.class);when(access.issueProtocol(any(),anyString(),anyString(),anyString())).thenReturn("synthetic-token"); var access=mock(NaoRuntimeAccessService.class);when(access.issueProtocol(any(),anyString(),anyString(),anyString())).thenReturn("synthetic-token");
var actor=new AppUserPrincipal(UUID.randomUUID(),"test","unused","测试",true,false,List.of("QUALITY_VIEWER"),List.of(),List.of(),List.of()); var actor=new AppUserPrincipal(UUID.randomUUID(),"test","unused","测试",true,false,List.of("USER"),List.of(),List.of(),List.of());
var runner=new NaoAnalysisRunner(JsonMapper.builder().build(),access); var runner=new NaoAnalysisRunner(JsonMapper.builder().build(),access);
String prior=System.getProperty("quality.project.root"),command=System.getProperty("quality.analysis.nao.command"); String prior=System.getProperty("quality.project.root"),command=System.getProperty("quality.analysis.nao.command");
try { try {
@@ -100,8 +100,8 @@ class NaoRuntimeAccessServiceTest {
private final AtomicInteger policyVersion = new AtomicInteger(1); private final AtomicInteger policyVersion = new AtomicInteger(1);
private final MutableClock clock = new MutableClock(Instant.parse("2026-10-22T04:00:00Z")); private final MutableClock clock = new MutableClock(Instant.parse("2026-10-22T04:00:00Z"));
private final AppUserPrincipal actor = new AppUserPrincipal(USER_ID, "quality-user", "hash", "质量用户", true, private final AppUserPrincipal actor = new AppUserPrincipal(USER_ID, "quality-user", "hash", "质量用户", true,
false, List.of("QUALITY_VIEWER"), List.of("D-1"), List.of("G-1"), false, List.of("USER"), List.of("D-1"), List.of("G-1"),
List.of(new SimpleGrantedAuthority("ROLE_QUALITY_VIEWER"))); List.of(new SimpleGrantedAuthority("ROLE_USER")));
private final JdbcTemplate jdbc = mock(JdbcTemplate.class); private final JdbcTemplate jdbc = mock(JdbcTemplate.class);
private final JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); private final JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class);
private final GovernanceAccessService governance = mock(GovernanceAccessService.class); private final GovernanceAccessService governance = mock(GovernanceAccessService.class);
@@ -55,8 +55,8 @@ class NaoRuntimeQueryControllerTest {
org.mockito.Mockito.verify(protocol,org.mockito.Mockito.never()).recordQuery(org.mockito.ArgumentMatchers.anyString(),org.mockito.ArgumentMatchers.anyString(),org.mockito.ArgumentMatchers.anyString(),org.mockito.ArgumentMatchers.anyMap(),org.mockito.ArgumentMatchers.any(),org.mockito.ArgumentMatchers.any(),org.mockito.ArgumentMatchers.any(),org.mockito.ArgumentMatchers.any()); org.mockito.Mockito.verify(protocol,org.mockito.Mockito.never()).recordQuery(org.mockito.ArgumentMatchers.anyString(),org.mockito.ArgumentMatchers.anyString(),org.mockito.ArgumentMatchers.anyString(),org.mockito.ArgumentMatchers.anyMap(),org.mockito.ArgumentMatchers.any(),org.mockito.ArgumentMatchers.any(),org.mockito.ArgumentMatchers.any(),org.mockito.ArgumentMatchers.any());
} }
private static final AppUserPrincipal ACTOR = new AppUserPrincipal(UUID.randomUUID(), "viewer", "hash", "查看者", private static final AppUserPrincipal ACTOR = new AppUserPrincipal(UUID.randomUUID(), "viewer", "hash", "查看者",
true, false, List.of("QUALITY_VIEWER"), List.of(), List.of(), true, false, List.of("USER"), List.of(), List.of(),
List.of(new SimpleGrantedAuthority("ROLE_QUALITY_VIEWER"))); List.of(new SimpleGrantedAuthority("ROLE_USER")));
@Test void delegatedModelAuthorizationRechecksTheBoundModelService() { @Test void delegatedModelAuthorizationRechecksTheBoundModelService() {
Fixture fixture = new Fixture(); Fixture fixture = new Fixture();
@@ -9,8 +9,10 @@ import org.junit.jupiter.api.Test;
class NaoSemanticCompletionTest { class NaoSemanticCompletionTest {
@Test void historyCompletionNeedsCurrentImportAndCompleteCoverage() { @Test void historyCompletionNeedsCurrentImportAndCompleteCoverage() {
var message=new LinkedHashMap<String,Object>(Map.of("type","result","status","COMPLETED","answer","","stopReason","history_report_complete","finishReason","stop","completionMethod","confirmed_history_report","agentTask",Map.of("kind","report"),"historyEvidence",Map.of("importId","current"),"historyCoverage",Map.of("complete",true))); var message=new LinkedHashMap<String,Object>(Map.of("type","result","status","COMPLETED","answer","历史报告证据已完整核对。","stopReason","history_report_complete","finishReason","stop","completionMethod","confirmed_history_report","agentTask",Map.of("kind","report"),"historyEvidence",Map.of("importId","current"),"historyCoverage",Map.of("complete",true)));
assertEquals("COMPLETED",NaoAnalysisRunner.Result.from(message,"",false).status()); assertEquals("COMPLETED",NaoAnalysisRunner.Result.from(message,"",false).status());
var blank = new LinkedHashMap<>(message); blank.put("answer", "");
assertEquals("PARTIAL", NaoAnalysisRunner.Result.from(blank,"",false).status());
message.put("historyCoverage",Map.of("complete",false));assertEquals("PARTIAL",NaoAnalysisRunner.Result.from(message,"",false).status()); message.put("historyCoverage",Map.of("complete",false));assertEquals("PARTIAL",NaoAnalysisRunner.Result.from(message,"",false).status());
message.put("historyCoverage",Map.of("complete",true));message.remove("historyEvidence");assertEquals("PARTIAL",NaoAnalysisRunner.Result.from(message,"",false).status()); message.put("historyCoverage",Map.of("complete",true));message.remove("historyEvidence");assertEquals("PARTIAL",NaoAnalysisRunner.Result.from(message,"",false).status());
} }
@@ -0,0 +1,45 @@
package cn.yunsheng.quality.analysis;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import tools.jackson.databind.json.JsonMapper;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import static org.mockito.ArgumentMatchers.*;
class RoleMigrationFingerprintTest {
@Test @SuppressWarnings("unchecked") void roleRenamePreservesHistoryWhileRealGrantChangesStillInvalidateIt() {
JdbcTemplate jdbc = mock(JdbcTemplate.class);
List<String> oldRoles = List.of("SYSTEM_ADMIN","QUALITY_VIEWER","BUSINESS_ANALYST","DATA_MAINTAINER","DATA_PUBLISHER","RULE_MAINTAINER","SYSTEM_OPERATOR","AUDIT_VIEWER");
when(jdbc.query(contains("user_role_migration_anchor"),any(RowMapper.class),any(Object[].class)))
.thenReturn(List.of(Map.of("role","SYSTEM_ADMIN","previous",oldRoles)));
when(jdbc.query(contains("FROM user_access_policy"),any(RowMapper.class),any(Object[].class))).thenReturn(List.of());
GovernanceAccessService access = new GovernanceAccessService(jdbc,JsonMapper.builder().build());
UUID id=UUID.randomUUID();
String original=AuthorizationFingerprint.of(user(id,oldRoles,List.of("A"),List.of("G")),access);
assertEquals(original,AuthorizationFingerprint.of(user(id,List.of("SYSTEM_ADMIN"),List.of("A"),List.of("G")),access));
assertNotEquals(original,AuthorizationFingerprint.of(user(id,List.of("USER"),List.of("A"),List.of("G")),access));
assertNotEquals(original,AuthorizationFingerprint.of(user(id,List.of("SYSTEM_ADMIN"),List.of("B"),List.of("G")),access));
assertNotEquals(original,AuthorizationFingerprint.of(user(id,List.of("SYSTEM_ADMIN"),List.of("A"),List.of("H")),access));
when(jdbc.query(contains("FROM user_access_policy"),any(RowMapper.class),any(Object[].class)))
.thenReturn(List.of(Map.of("detailAllowed",false,"evidenceAllowed",false,"originalDownloadAllowed",false,"allowedFields",List.of("product"),"rowVersion",1L)));
assertNotEquals(original,AuthorizationFingerprint.of(user(id,List.of("SYSTEM_ADMIN"),List.of("A"),List.of("G")),access));
}
@Test @SuppressWarnings("unchecked") void inconsistentAnchorCannotKeepAnOldAdministratorFingerprintForAUser() {
JdbcTemplate jdbc=mock(JdbcTemplate.class);
when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenReturn(List.of(Map.of("role","USER","previous",List.of("SYSTEM_ADMIN"))));
GovernanceAccessService access=new GovernanceAccessService(jdbc,JsonMapper.builder().build());
assertEquals(List.of("USER"),access.fingerprintRoles(user(UUID.randomUUID(),List.of("USER"),List.of("A"),List.of())));
}
private AppUserPrincipal user(UUID id,List<String> roles,List<String> departments,List<String> groups) {
return new AppUserPrincipal(id,"test","unused","test",true,false,roles,departments,groups,List.of());
}
}
@@ -16,15 +16,15 @@ class ComparisonViewSharingServiceTest {
void legacyDepartmentSpellingUsesTheSameSharingScope() { void legacyDepartmentSpellingUsesTheSameSharingScope() {
var policy = Map.<String,Object>of("detailAllowed", true, "evidenceAllowed", false, var policy = Map.<String,Object>of("detailAllowed", true, "evidenceAllowed", false,
"originalDownloadAllowed", false, "inheritDepartmentChildren", false); "originalDownloadAllowed", false, "inheritDepartmentChildren", false);
var oldName = principal(List.of("QUALITY_VIEWER"), List.of("质量管理部"), List.of()); var oldName = principal(List.of("USER"), List.of("质量管理部"), List.of());
var currentName = principal(List.of("QUALITY_VIEWER"), List.of("质量中心"), List.of()); var currentName = principal(List.of("USER"), List.of("质量中心"), List.of());
assertEquals(ComparisonViewSharingService.semanticFingerprint(oldName, Set.of("*"), policy), assertEquals(ComparisonViewSharingService.semanticFingerprint(oldName, Set.of("*"), policy),
ComparisonViewSharingService.semanticFingerprint(currentName, Set.of("*"), policy)); ComparisonViewSharingService.semanticFingerprint(currentName, Set.of("*"), policy));
} }
@Test @Test
void equivalentAuthorizationSemanticsIgnorePolicyEditCountAndCollectionOrder() { void equivalentAuthorizationSemanticsIgnorePolicyEditCountAndCollectionOrder() {
AppUserPrincipal first = principal(List.of("QUALITY_VIEWER", "BUSINESS_ANALYST"), List.of("部二", "部一"), List.of("组B", "组A")); AppUserPrincipal first = principal(List.of("USER"), List.of("部二", "部一"), List.of("组B", "组A"));
AppUserPrincipal second = principal(List.of("BUSINESS_ANALYST", "QUALITY_VIEWER"), List.of("部一", "部二"), List.of("组A", "组B")); AppUserPrincipal second = principal(List.of("USER"), List.of("部一", "部二"), List.of("组A", "组B"));
String firstFingerprint = ComparisonViewSharingService.semanticFingerprint(first, Set.of("序号", "批次"), String firstFingerprint = ComparisonViewSharingService.semanticFingerprint(first, Set.of("序号", "批次"),
Map.of("detailAllowed", true, "evidenceAllowed", false, "originalDownloadAllowed", false, Map.of("detailAllowed", true, "evidenceAllowed", false, "originalDownloadAllowed", false,
@@ -38,8 +38,8 @@ class ComparisonViewSharingServiceTest {
@Test @Test
void changedOrganizationFieldOrPolicySemanticsCannotShare() { void changedOrganizationFieldOrPolicySemanticsCannotShare() {
AppUserPrincipal owner = principal(List.of("QUALITY_VIEWER"), List.of("部一"), List.of("组A")); AppUserPrincipal owner = principal(List.of("USER"), List.of("部一"), List.of("组A"));
AppUserPrincipal narrowed = principal(List.of("QUALITY_VIEWER"), List.of("部二"), List.of("组A")); AppUserPrincipal narrowed = principal(List.of("USER"), List.of("部二"), List.of("组A"));
Map<String, Object> policy = Map.of("detailAllowed", true, "evidenceAllowed", false, Map<String, Object> policy = Map.of("detailAllowed", true, "evidenceAllowed", false,
"originalDownloadAllowed", false, "inheritDepartmentChildren", false, "validUntil", "2026-12-31T00:00:00Z"); "originalDownloadAllowed", false, "inheritDepartmentChildren", false, "validUntil", "2026-12-31T00:00:00Z");
String original = ComparisonViewSharingService.semanticFingerprint(owner, Set.of("序号"), policy); String original = ComparisonViewSharingService.semanticFingerprint(owner, Set.of("序号"), policy);
@@ -14,7 +14,7 @@ class WorkspaceStateControllerTest {
final WorkspaceStateController controller = new WorkspaceStateController(jdbc, access); final WorkspaceStateController controller = new WorkspaceStateController(jdbc, access);
final UUID id = UUID.randomUUID(); final UUID id = UUID.randomUUID();
AppUserPrincipal actor(List<String> departments) { AppUserPrincipal actor(List<String> departments) {
return new AppUserPrincipal(id,"test","","test",true,false,List.of("QUALITY_VIEWER"),departments,List.of("*"),List.of()); return new AppUserPrincipal(id,"test","","test",true,false,List.of("USER"),departments,List.of("*"),List.of());
} }
@Test void stableDigestIgnoresScopeOrderAndDoesNotExposePolicy() { @Test void stableDigestIgnoresScopeOrderAndDoesNotExposePolicy() {
when(jdbc.queryForList(contains("user_access_policy"), any(Object[].class))).thenReturn(List.of(Map.of("allowed_fields","private-field"))); when(jdbc.queryForList(contains("user_access_policy"), any(Object[].class))).thenReturn(List.of(Map.of("allowed_fields","private-field")));
@@ -33,7 +33,7 @@ class SourceColumnQueriesMySqlTest {
private boolean created; private boolean created;
private final GovernanceAccessService governance = mock(GovernanceAccessService.class); private final GovernanceAccessService governance = mock(GovernanceAccessService.class);
private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "source-column-test", "", "测试", private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "source-column-test", "", "测试",
true, false, List.of("QUALITY_VIEWER"), List.of("部门A"), List.of("组A"), List.of()); true, false, List.of("USER"), List.of("部门A"), List.of("组A"), List.of());
private SourceColumnQueries queries; private SourceColumnQueries queries;
@BeforeEach void setup() { @BeforeEach void setup() {
@@ -19,7 +19,7 @@ class SourceColumnQueriesTest {
private static final String SOURCE = "src_0123456789abcdef"; private static final String SOURCE = "src_0123456789abcdef";
private final GovernanceAccessService governance = mock(GovernanceAccessService.class); private final GovernanceAccessService governance = mock(GovernanceAccessService.class);
private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "source-test", "", "test", private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "source-test", "", "test",
true, false, List.of("QUALITY_VIEWER"), List.of(), List.of(), List.of()); true, false, List.of("USER"), List.of(), List.of(), List.of());
@Test void parsesJsonStringAndKeepsLeadingZeroAndInjectionTextAsExactStrings() { @Test void parsesJsonStringAndKeepsLeadingZeroAndInjectionTextAsExactStrings() {
List<Map<String, Object>> parsed = SourceColumnQueries.parse( List<Map<String, Object>> parsed = SourceColumnQueries.parse(
@@ -37,7 +37,7 @@ class GovernanceAccessServiceTest {
when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenAnswer(call->call.getArgument(0,String.class).startsWith("SELECT detail_allowed")?List.of(Map.of("inheritDepartmentChildren",true)):List.of(definition)); when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenAnswer(call->call.getArgument(0,String.class).startsWith("SELECT detail_allowed")?List.of(Map.of("inheritDepartmentChildren",true)):List.of(definition));
when(jdbc.query(anyString(),any(RowMapper.class))).thenReturn(List.of(definition)); when(jdbc.query(anyString(),any(RowMapper.class))).thenReturn(List.of(definition));
GovernanceAccessService service=new GovernanceAccessService(jdbc,JsonMapper.builder().build()); GovernanceAccessService service=new GovernanceAccessService(jdbc,JsonMapper.builder().build());
AppUserPrincipal original=new AppUserPrincipal(UUID.randomUUID(),"reader","unchanged","Reader",true,false,List.of("QUALITY_VIEWER"),List.of("parent"),List.of("G1"),List.of()); AppUserPrincipal original=new AppUserPrincipal(UUID.randomUUID(),"reader","unchanged","Reader",true,false,List.of("USER"),List.of("parent"),List.of("G1"),List.of());
AppUserPrincipal expanded=service.resolvePrincipal(original); AppUserPrincipal expanded=service.resolvePrincipal(original);
assertEquals(List.of("parent"),original.getDepartments());assertEquals(Set.of("parent","child"),new HashSet<>(expanded.getDepartments())); assertEquals(List.of("parent"),original.getDepartments());assertEquals(Set.of("parent","child"),new HashSet<>(expanded.getDepartments()));
assertEquals(original.getId(),expanded.getId());assertEquals(original.getRoleCodes(),expanded.getRoleCodes());assertEquals(original.getGroups(),expanded.getGroups()); assertEquals(original.getId(),expanded.getId());assertEquals(original.getRoleCodes(),expanded.getRoleCodes());assertEquals(original.getGroups(),expanded.getGroups());
@@ -47,7 +47,7 @@ class GovernanceAccessServiceTest {
when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenReturn(List.of(Map.of("inheritDepartmentChildren",true))); when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenReturn(List.of(Map.of("inheritDepartmentChildren",true)));
when(jdbc.query(anyString(),any(RowMapper.class))).thenReturn(List.of(definition)); when(jdbc.query(anyString(),any(RowMapper.class))).thenReturn(List.of(definition));
GovernanceAccessService service=new GovernanceAccessService(jdbc,JsonMapper.builder().build()); GovernanceAccessService service=new GovernanceAccessService(jdbc,JsonMapper.builder().build());
AppUserPrincipal reader=new AppUserPrincipal(UUID.randomUUID(),"reader","unused","Reader",true,false,List.of("QUALITY_VIEWER"),List.of("p"),List.of(),List.of()); AppUserPrincipal reader=new AppUserPrincipal(UUID.randomUUID(),"reader","unused","Reader",true,false,List.of("USER"),List.of("p"),List.of(),List.of());
assertThrows(ResponseStatusException.class,()->service.resolvePrincipal(reader)); assertThrows(ResponseStatusException.class,()->service.resolvePrincipal(reader));
} }
} }
@@ -18,7 +18,7 @@ import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*; import static org.mockito.Mockito.*;
class SourceChangeAuthorizationBoundaryTest { class SourceChangeAuthorizationBoundaryTest {
private AppUserPrincipal actor(){return new AppUserPrincipal(UUID.randomUUID(),"a","unused","A",true,false,List.of("DATA_MAINTAINER","DATA_PUBLISHER"),List.of("A"),List.of(),List.of());} private AppUserPrincipal actor(){return new AppUserPrincipal(UUID.randomUUID(),"a","unused","A",true,false,List.of("DATA_ADMIN"),List.of("A"),List.of(),List.of());}
private SourceAuthorizationService denyingAccess(){SourceAuthorizationService access=mock(SourceAuthorizationService.class);doThrow(new ResponseStatusException(HttpStatus.FORBIDDEN,"B is outside scope")).when(access).requireSource(eq("doc-b"),eq("rev-b"),any(AppUserPrincipal.class));return access;} private SourceAuthorizationService denyingAccess(){SourceAuthorizationService access=mock(SourceAuthorizationService.class);doThrow(new ResponseStatusException(HttpStatus.FORBIDDEN,"B is outside scope")).when(access).requireSource(eq("doc-b"),eq("rev-b"),any(AppUserPrincipal.class));return access;}
private GovernanceSupport support(JdbcTemplate jdbc){return new GovernanceSupport(jdbc,JsonMapper.builder().build());} private GovernanceSupport support(JdbcTemplate jdbc){return new GovernanceSupport(jdbc,JsonMapper.builder().build());}
@@ -13,7 +13,7 @@ import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*; import static org.mockito.Mockito.*;
class SourceSecurityRegressionTest { class SourceSecurityRegressionTest {
private AppUserPrincipal reader(String department){return new AppUserPrincipal(UUID.randomUUID(),"reader","unused","Reader",true,false,List.of("DATA_MAINTAINER","DATA_PUBLISHER"),List.of(department),List.of(),List.of());} private AppUserPrincipal reader(String department){return new AppUserPrincipal(UUID.randomUUID(),"reader","unused","Reader",true,false,List.of("DATA_ADMIN"),List.of(department),List.of(),List.of());}
@Test @SuppressWarnings("unchecked") void parsedAuthorizedRowsCannotGrantWholeWorkbookDownload(){ @Test @SuppressWarnings("unchecked") void parsedAuthorizedRowsCannotGrantWholeWorkbookDownload(){
JdbcTemplate jdbc=mock(JdbcTemplate.class); JdbcTemplate jdbc=mock(JdbcTemplate.class);
@@ -35,6 +35,17 @@ class SourceSecurityRegressionTest {
assertFalse(SourceCoverageGuard.overlaps(old,nextMonth)); assertFalse(SourceCoverageGuard.overlaps(old,nextMonth));
} }
@Test void uploadAppendOnlyCoexistsWithExactScopeMonthlySnapshotAndOtherAppendBatches(){
var monthly=new SourceCoverageGuard.Coverage("black",2026,2,"质量管理部","MONTHLY_SNAPSHOT","sha-month","black|2026|02|质量管理部");
var appendA=new SourceCoverageGuard.Coverage("black",2026,2,"质量中心","APPEND_SNAPSHOT","sha-a","black|2026|02|质量管理部|UPLOAD_A");
var appendB=new SourceCoverageGuard.Coverage("black",2026,2,"质量管理部","APPEND_SNAPSHOT","sha-b","black|2026|02|质量管理部|UPLOAD_B");
assertFalse(SourceCoverageGuard.overlaps(monthly,appendA));
assertFalse(SourceCoverageGuard.overlaps(appendA,appendB));
assertTrue(SourceCoverageGuard.overlaps(monthly,new SourceCoverageGuard.Coverage("black",2026,2,"ALL","APPEND_SNAPSHOT","sha-c","black|2026|02|ALL|UPLOAD_C")));
assertTrue(SourceCoverageGuard.overlaps(monthly,new SourceCoverageGuard.Coverage("black",2026,2,"质量管理部","APPEND_SNAPSHOT","sha-month","black|2026|02|质量管理部|UPLOAD_D")));
assertTrue(SourceCoverageGuard.overlaps(monthly,new SourceCoverageGuard.Coverage("black",2026,2,"质量管理部","SHARD_SNAPSHOT","sha-shard","black|2026|02|质量管理部|P1")));
}
@Test @SuppressWarnings("unchecked") void localMaintainerCannotRemoveAnotherDepartmentsSource(){ @Test @SuppressWarnings("unchecked") void localMaintainerCannotRemoveAnotherDepartmentsSource(){
JdbcTemplate jdbc=mock(JdbcTemplate.class); JdbcTemplate jdbc=mock(JdbcTemplate.class);
when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenReturn(List.of("B")); when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenReturn(List.of("B"));
@@ -0,0 +1,34 @@
package cn.yunsheng.quality.identity;
import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.springframework.web.server.ResponseStatusException;
import static org.junit.jupiter.api.Assertions.*;
class AppRolesTest {
@Test void exactlyOneOfThreeRolesCanBeAssigned() {
for (String role : AppRoles.CODES) assertEquals(List.of(role), AppRoles.validate(List.of(role)));
for (List<String> roles : List.of(List.<String>of(), List.of("USER", "DATA_ADMIN"),
List.of("USER", "USER"), List.of("QUALITY_VIEWER"), Arrays.asList((String) null)))
assertEquals(400, assertThrows(ResponseStatusException.class, () -> AppRoles.validate(roles)).getStatusCode().value());
assertThrows(ResponseStatusException.class, () -> AppRoles.validate(null));
}
@Test void operationAuthoritiesAreInheritedWhileAssignedRoleStaysSingle() {
assertEquals(List.of("ROLE_USER"), codes("USER"));
assertEquals(List.of("ROLE_DATA_ADMIN", "ROLE_USER"), codes("DATA_ADMIN"));
assertEquals(List.of("ROLE_SYSTEM_ADMIN", "ROLE_DATA_ADMIN", "ROLE_USER"), codes("SYSTEM_ADMIN"));
assertEquals(List.of("DATA_ADMIN"), AppRoles.validate(List.of("DATA_ADMIN")));
}
@Test void unknownOrMultipleAssignmentsCannotGainAuthorities() {
assertTrue(AppRoles.authorities(List.of("QUALITY_VIEWER")).isEmpty());
assertTrue(AppRoles.authorities(List.of("USER", "SYSTEM_ADMIN")).isEmpty());
assertTrue(AppRoles.authorities(Arrays.asList((String) null)).isEmpty());
}
private List<String> codes(String role) {
return AppRoles.authorities(List.of(role)).stream().map(a -> a.getAuthority()).toList();
}
}
@@ -45,7 +45,7 @@ class CurrentUserRefreshFilterTest {
@Test void refreshesRolesAndScopesFromTheCurrentUserRecord() throws Exception { @Test void refreshesRolesAndScopesFromTheCurrentUserRecord() throws Exception {
UUID id = UUID.randomUUID(); UUID id = UUID.randomUUID();
AppUserPrincipal old = principal(id, List.of("SYSTEM_ADMIN"), List.of("*"), List.of("*"), true); AppUserPrincipal old = principal(id, List.of("SYSTEM_ADMIN"), List.of("*"), List.of("*"), true);
AppUserPrincipal current = principal(id, List.of("QUALITY_VIEWER"), List.of("质量管理部"), List.of("VCM"), true); AppUserPrincipal current = principal(id, List.of("USER"), List.of("质量管理部"), List.of("VCM"), true);
JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class);
when(users.loadUserByUsername("alice")).thenReturn(current); when(users.loadUserByUsername("alice")).thenReturn(current);
SecurityContextRepository repository = mock(SecurityContextRepository.class); SecurityContextRepository repository = mock(SecurityContextRepository.class);
@@ -60,14 +60,14 @@ class CurrentUserRefreshFilterTest {
assertTrue(continued.get()); assertTrue(continued.get());
AppUserPrincipal refreshed = (AppUserPrincipal) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); AppUserPrincipal refreshed = (AppUserPrincipal) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
assertEquals(List.of("QUALITY_VIEWER"), refreshed.getRoleCodes()); assertEquals(List.of("USER"), refreshed.getRoleCodes());
assertEquals(List.of("质量管理部"), refreshed.getDepartments()); assertEquals(List.of("质量管理部"), refreshed.getDepartments());
assertEquals(List.of("VCM"), refreshed.getGroups()); assertEquals(List.of("VCM"), refreshed.getGroups());
verify(repository).saveContext(any(), eq(request), eq(response)); verify(repository).saveContext(any(), eq(request), eq(response));
} }
@Test void rejectsAndInvalidatesARevokedSessionBeforeTheRequestRuns() throws Exception { @Test void rejectsAndInvalidatesARevokedSessionBeforeTheRequestRuns() throws Exception {
AppUserPrincipal old = principal(UUID.randomUUID(), List.of("QUALITY_VIEWER"), List.of("*"), List.of("*"), true); AppUserPrincipal old = principal(UUID.randomUUID(), List.of("USER"), List.of("*"), List.of("*"), true);
JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class);
when(users.loadUserByUsername("alice")).thenThrow(new UsernameNotFoundException("inactive")); when(users.loadUserByUsername("alice")).thenThrow(new UsernameNotFoundException("inactive"));
SecurityContextHolder.getContext().setAuthentication(UsernamePasswordAuthenticationToken.authenticated( SecurityContextHolder.getContext().setAuthentication(UsernamePasswordAuthenticationToken.authenticated(
@@ -87,7 +87,7 @@ class CurrentUserRefreshFilterTest {
} }
@Test void expiresAnExistingSessionBeforeAnyBusinessRequestRuns() throws Exception { @Test void expiresAnExistingSessionBeforeAnyBusinessRequestRuns() throws Exception {
AppUserPrincipal current = principal(UUID.randomUUID(), List.of("QUALITY_VIEWER"), List.of("*"), List.of("*"), true); AppUserPrincipal current = principal(UUID.randomUUID(), List.of("USER"), List.of("*"), List.of("*"), true);
JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class);
when(users.loadUserByUsername("alice")).thenReturn(current); when(users.loadUserByUsername("alice")).thenReturn(current);
GovernanceAccessService access = mock(GovernanceAccessService.class); GovernanceAccessService access = mock(GovernanceAccessService.class);
@@ -111,8 +111,8 @@ class CurrentUserRefreshFilterTest {
@Test void usesTheServerResolvedHierarchyForTheRefreshedIdentity() throws Exception { @Test void usesTheServerResolvedHierarchyForTheRefreshedIdentity() throws Exception {
UUID id = UUID.randomUUID(); UUID id = UUID.randomUUID();
AppUserPrincipal current = principal(id, List.of("QUALITY_VIEWER"), List.of("parent"), List.of(), true); AppUserPrincipal current = principal(id, List.of("USER"), List.of("parent"), List.of(), true);
AppUserPrincipal expanded = principal(id, List.of("QUALITY_VIEWER"), List.of("parent", "child"), List.of(), true); AppUserPrincipal expanded = principal(id, List.of("USER"), List.of("parent", "child"), List.of(), true);
JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class);
when(users.loadUserByUsername("alice")).thenReturn(current); when(users.loadUserByUsername("alice")).thenReturn(current);
GovernanceAccessService access = mock(GovernanceAccessService.class); GovernanceAccessService access = mock(GovernanceAccessService.class);
@@ -26,12 +26,12 @@ class ImportBeforeImageAuthorizationTest {
} }
private GovernanceAccessService access(){GovernanceAccessService access=mock(GovernanceAccessService.class);when(access.resolvePrincipal(any())).thenAnswer(call->call.getArgument(0));return access;} private GovernanceAccessService access(){GovernanceAccessService access=mock(GovernanceAccessService.class);when(access.resolvePrincipal(any())).thenAnswer(call->call.getArgument(0));return access;}
@Test void historicalCandidateCannotReplaceCurrentBeforeImageOutsideMaintainerScope(){ @Test void historicalCandidateCannotReplaceCurrentBeforeImageOutsideMaintainerScope(){
JdbcTemplate jdbc=jdbc("B","DATA_MAINTAINER"); JdbcTemplate jdbc=jdbc("B","DATA_ADMIN");
var denied=assertThrows(ResponseStatusException.class,()->ImportRunService.requireBeforeImages(jdbc,access(),List.of("historical-candidate-with-only-A-rows"),actor,"current-release")); var denied=assertThrows(ResponseStatusException.class,()->ImportRunService.requireBeforeImages(jdbc,access(),List.of("historical-candidate-with-only-A-rows"),actor,"current-release"));
assertEquals(HttpStatus.FORBIDDEN,denied.getStatusCode());verify(jdbc,never()).update(anyString(),any(Object[].class)); assertEquals(HttpStatus.FORBIDDEN,denied.getStatusCode());verify(jdbc,never()).update(anyString(),any(Object[].class));
} }
@Test void currentRevisionIsCheckedInsteadOfHistoricalCandidateRevision(){ @Test void currentRevisionIsCheckedInsteadOfHistoricalCandidateRevision(){
JdbcTemplate jdbc=jdbc("A","DATA_MAINTAINER");SourceAuthorizationService sourceAccess=mock(SourceAuthorizationService.class); JdbcTemplate jdbc=jdbc("A","DATA_ADMIN");SourceAuthorizationService sourceAccess=mock(SourceAuthorizationService.class);
ImportRunService.requireBeforeImages(jdbc,access(),sourceAccess,List.of("historical-candidate"),actor,"current-release"); ImportRunService.requireBeforeImages(jdbc,access(),sourceAccess,List.of("historical-candidate"),actor,"current-release");
verify(sourceAccess).requireSource(eq("current-doc"),eq("current-revision"),argThat(user->user.getDepartments().equals(List.of("A")))); verify(sourceAccess).requireSource(eq("current-doc"),eq("current-revision"),argThat(user->user.getDepartments().equals(List.of("A"))));
} }
@@ -0,0 +1,126 @@
package cn.yunsheng.quality.ingestion;
import cn.yunsheng.quality.analytics.TargetVersionService;
import cn.yunsheng.quality.governance.*;
import cn.yunsheng.quality.metrics.MetricsRollupService;
import cn.yunsheng.quality.source.*;
import java.nio.file.Path;
import java.util.*;
import java.util.function.Consumer;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.*;
import org.springframework.test.util.ReflectionTestUtils;
import tools.jackson.databind.json.JsonMapper;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import static org.mockito.ArgumentMatchers.*;
class ImportUnassignedRowsTest {
@Test
@org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable(named="QUALITY_UNASSIGNED_REPLAY_INPUT", matches=".+")
@SuppressWarnings("unchecked")
void failedDecemberWorkbookKeepsAllRowsAndItsTwelveUnassignedMappings() throws Exception {
JsonMapper json = JsonMapper.builder().build();
Map<String,Object> input = json.readValue(java.nio.file.Files.readString(Path.of(System.getenv("QUALITY_UNASSIGNED_REPLAY_INPUT"))), Map.class);
Map<String,Object> mapping = (Map<String,Object>) input.get("mapping");
Map<String,Object> config = (Map<String,Object>) input.get("parserConfig");
JdbcTemplate jdbc = mock(JdbcTemplate.class);
SourceScanner scanner = mock(SourceScanner.class);
Path file = Path.of((String) input.get("archivePath"));
when(scanner.archiveRevision(any())).thenReturn(file);
when(jdbc.queryForObject(anyString(), eq(String.class), any(Object[].class))).thenAnswer(call -> {
String sql = call.getArgument(0);
return sql.contains("CAST(parser_config") ? json.writeValueAsString(config) : sql.contains("parser_config_hash") ? input.get("parserConfigHash") : sql.contains("scope_code") ? "ALL" : "actor";
});
when(jdbc.query(anyString(), any(RowMapper.class), any(Object[].class))).thenAnswer(call -> {
String sql = call.getArgument(0);
return sql.startsWith("SELECT id FROM source_document") ? List.of("doc") : sql.startsWith("SELECT id FROM source_revision") ? List.of("revision") : List.of();
});
int[] counters = {0, 0, -1};
Set<Integer> unmappedRows = new TreeSet<>();
doAnswer(call -> {
String sql = call.getArgument(0);
if (sql.contains("INSERT INTO fact_bright")) {
counters[0]++;
if (call.getArgument(7) == null) counters[1]++;
Map<String,Object> components = json.readValue(call.getArgument(14, String.class), Map.class);
if ("unmapped".equals(components.get("categoryMappingStatus"))) unmappedRows.add(call.getArgument(4, Integer.class));
}
if (sql.contains("UPDATE transform_build SET status='COMPLETED'")) counters[2] = call.getArgument(2, Integer.class);
return 1;
}).when(jdbc).update(anyString(), any(Object[].class));
var rules = ConfigurableRuleEngine.compile((Map<String,Object>) input.get("rules"));
var mapper = new BusinessRowMapper(lookup(mapping,"categoryLookup"), lookup(mapping,"projectGroupLookup"), (String)input.get("mappingVersion"), lookup(mapping,"departmentLookup"), lookup(mapping,"productLookup"), rules, mapping);
var worker = new ImportWorker(jdbc, mock(SourceCatalogService.class), scanner, new XlsxStreamingReader(), json, mock(GovernanceAccessService.class), mock(TargetVersionService.class), mock(SourceCoverageGuard.class), mock(TaskProgressService.class), mock(CandidateAcceptanceService.class), mock(MappingIssueService.class), mock(MetricsRollupService.class));
var candidate = new SourceFileCandidate(UUID.randomUUID(), "replay", "2025:bright:12", "bright", 2025, 12, "12月亮片.xlsx", file, (String)input.get("sha256"), java.nio.file.Files.size(file), "DISCOVERED");
ReflectionTestUtils.invokeMethod(worker, "build", candidate, mapper, input.get("ruleVersion"), "replay-run");
assertArrayEquals(new int[]{18627,13,0}, counters, "all original rows survive, including unassigned rows; no blocking data errors");
assertEquals(Set.of(9104,11838,12576,12682,13205,13682,13773,13774,13945,14347,15419,17290), unmappedRows);
}
@SuppressWarnings("unchecked")
private Map<String,String> lookup(Map<String,Object> definition, String name) {
Map<String,String> result = new HashMap<>();
((Map<String,Object>)definition.getOrDefault(name, Map.of())).forEach((key,value) -> result.put(key, value == null ? null : MappingValueCodec.fromStorage(String.valueOf(value))));
return result;
}
@Test void missingBusinessGroupIsStoredWithoutBlockingEitherStage() throws Exception {
for (String stage : List.of("black", "bright")) checkBuild(stage, false, false);
}
@Test void unassignedRowsStillRejectInvalidNumbersAndMissingHeaders() throws Exception {
checkBuild("black", true, false);
checkBuild("bright", false, true);
}
@SuppressWarnings("unchecked")
private void checkBuild(String stage, boolean invalidNumber, boolean missingHeader) throws Exception {
JdbcTemplate jdbc = mock(JdbcTemplate.class);
SourceScanner scanner = mock(SourceScanner.class);
XlsxStreamingReader reader = mock(XlsxStreamingReader.class);
when(jdbc.queryForObject(anyString(), eq(String.class), any(Object[].class))).thenAnswer(call -> {
String sql = call.getArgument(0);
return sql.contains("CAST(parser_config") ? "{}" : sql.contains("parser_config_hash") ? "legacy" : sql.contains("scope_code") ? "ALL" : "actor";
});
when(jdbc.query(anyString(), any(RowMapper.class), any(Object[].class))).thenAnswer(call -> {
String sql = call.getArgument(0);
return sql.startsWith("SELECT id FROM source_document") ? List.of("doc") : sql.startsWith("SELECT id FROM source_revision") ? List.of("revision") : List.of();
});
when(scanner.archiveRevision(any())).thenReturn(Path.of("/unused"));
BusinessRowMapper mapper = new BusinessRowMapper(Map.of(), Map.of(), "mapping");
Map<String, XlsxStreamingReader.CellValue> cells = new LinkedHashMap<>();
for (String header : mapper.requiredHeaders(stage)) cells.put(header, cell("0"));
cells.put("PDT代码", cell("P001")); cells.put("销售片区", cell("欧洲事业部"));
cells.put("事业部", cell("新能源汽车磁钢")); cells.put("成品规格", cell("32.05X18X3.7+T"));
if (invalidNumber) cells.put("送电镀重量", cell("bad"));
if (missingHeader) cells.remove("晶界扩散报废合计重量");
String sheet = stage.equals("black") ? "报废明细" : "明细";
doAnswer(call -> {
Consumer<XlsxStreamingReader.RawRow> consumer = call.getArgument(3);
consumer.accept(new XlsxStreamingReader.RawRow(sheet, 9104, cells, Map.of("PDT代码", "P001", "销售片区", "欧洲事业部")));
return new XlsxStreamingReader.ReadSummary(sheet, 1);
}).when(reader).read(any(Path.class), anyString(), isNull(), any(Consumer.class));
List<List<Object>> facts = new ArrayList<>(), issues = new ArrayList<>();
int[] rejected = {-1};
doAnswer(call -> {
String sql = call.getArgument(0);
List<Object> args = Arrays.asList(call.getArguments()).subList(1, call.getArguments().length);
if (sql.contains("INSERT INTO fact_")) facts.add(new ArrayList<>(args));
if (sql.contains("INSERT INTO import_issue")) issues.add(new ArrayList<>(args));
if (sql.contains("UPDATE transform_build SET status='COMPLETED'")) rejected[0] = ((Number) args.get(1)).intValue();
return 1;
}).when(jdbc).update(anyString(), any(Object[].class));
ImportWorker worker = new ImportWorker(jdbc, mock(SourceCatalogService.class), scanner, reader, JsonMapper.builder().build(), mock(GovernanceAccessService.class), mock(TargetVersionService.class), mock(SourceCoverageGuard.class), mock(TaskProgressService.class), mock(CandidateAcceptanceService.class), mock(MappingIssueService.class), mock(MetricsRollupService.class));
var candidate = new SourceFileCandidate(UUID.randomUUID(), "key", "2025:" + stage + ":12", stage, 2025, 12, "synthetic.xlsx", Path.of("/unused"), "a".repeat(64), 1, "DISCOVERED");
ReflectionTestUtils.invokeMethod(worker, "build", candidate, mapper, "pq-dax-v1", "run");
assertEquals(1, facts.size());
assertNull(facts.getFirst().get(6), "unmatched business group must remain unassigned");
assertTrue(String.valueOf(facts.getFirst().get(12)).contains("欧洲事业部"), "original values are retained");
var unassigned = issues.stream().filter(issue -> "CATEGORY_MAPPING_MISSING".equals(issue.get(5))).findFirst().orElseThrow();
assertTrue(String.valueOf(unassigned.get(8)).contains("\"blocking\":false"));
assertEquals(invalidNumber || missingHeader ? 1 : 0, rejected[0]);
}
private XlsxStreamingReader.CellValue cell(String value) { return new XlsxStreamingReader.CellValue(value, value, "s"); }
}
@@ -11,6 +11,14 @@ import org.junit.jupiter.api.Test;
class TransformBuildKeyTest { class TransformBuildKeyTest {
private static final String SAME_FILE_HASH = "a".repeat(64); private static final String SAME_FILE_HASH = "a".repeat(64);
@Test
void previousBlockingMappingCacheIsNotReusedAfterUnassignedPolicyChange() throws Exception {
var source = source("2026:black:01", "black", 1);
String oldIdentity = String.join("\n", source.sha256(), source.sourceSlot(), "revision", source.stage(), "2026", "1", "parser", "rule", "mapping");
String oldKey = java.util.HexFormat.of().formatHex(java.security.MessageDigest.getInstance("SHA-256").digest(oldIdentity.getBytes(java.nio.charset.StandardCharsets.UTF_8)));
assertNotEquals(oldKey, TransformBuildKey.create(source, "revision", "parser", "rule", "mapping"));
}
@Test @Test
void identicalWorkbookBytesAreSeparatedBySourcePeriodAndRevision() { void identicalWorkbookBytesAreSeparatedBySourcePeriodAndRevision() {
SourceFileCandidate january = source("2026:black:01", "black", 1); SourceFileCandidate january = source("2026:black:01", "black", 1);
@@ -30,6 +30,9 @@ class UnclassifiedMappingRebuildMySqlTest {
jdbc.execute("CREATE TABLE defect_component(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),stage VARCHAR(16),sheet_name VARCHAR(100),physical_row_no INT,defect_name VARCHAR(100),weight DECIMAL(24,8),source_field VARCHAR(100))"); jdbc.execute("CREATE TABLE defect_component(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),stage VARCHAR(16),sheet_name VARCHAR(100),physical_row_no INT,defect_name VARCHAR(100),weight DECIMAL(24,8),source_field VARCHAR(100))");
try(var stream=getClass().getResourceAsStream("/db/migration/V16__metric_rollups.sql")){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);} try(var stream=getClass().getResourceAsStream("/db/migration/V16__metric_rollups.sql")){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);}
try(var stream=getClass().getResourceAsStream("/db/migration/V36__fine_defect_rollups.sql")){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);} try(var stream=getClass().getResourceAsStream("/db/migration/V36__fine_defect_rollups.sql")){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);}
for(String migration:List.of("V39__sales_order_types.sql","V40__sales_order_fact_indexes.sql","V41__sales_order_metric_rollups.sql")) {
try(var stream=getClass().getResourceAsStream("/db/migration/"+migration)){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);}
}
jdbc.execute("CREATE TABLE import_run(id CHAR(36) PRIMARY KEY,idempotency_key VARCHAR(190),status VARCHAR(32),stage VARCHAR(80),requested_sources JSON,expected_current_release_id CHAR(36),created_by CHAR(36),processed BIGINT,total BIGINT,result_release_id CHAR(36),expected_mapping_version VARCHAR(100),expected_rule_version VARCHAR(100),workflow_operation_id CHAR(36),validation JSON,started_at TIMESTAMP(6),completed_at TIMESTAMP(6))"); jdbc.execute("CREATE TABLE import_run(id CHAR(36) PRIMARY KEY,idempotency_key VARCHAR(190),status VARCHAR(32),stage VARCHAR(80),requested_sources JSON,expected_current_release_id CHAR(36),created_by CHAR(36),processed BIGINT,total BIGINT,result_release_id CHAR(36),expected_mapping_version VARCHAR(100),expected_rule_version VARCHAR(100),workflow_operation_id CHAR(36),validation JSON,started_at TIMESTAMP(6),completed_at TIMESTAMP(6))");
jdbc.execute("CREATE TABLE release_protocol_dependency(release_id CHAR(36),protocol_id VARCHAR(100),PRIMARY KEY(release_id,protocol_id))");jdbc.execute("CREATE TABLE release_source_exclusion(release_id CHAR(36),source_document_id VARCHAR(100),request_id VARCHAR(100))"); jdbc.execute("CREATE TABLE release_protocol_dependency(release_id CHAR(36),protocol_id VARCHAR(100),PRIMARY KEY(release_id,protocol_id))");jdbc.execute("CREATE TABLE release_source_exclusion(release_id CHAR(36),source_document_id VARCHAR(100),request_id VARCHAR(100))");
jdbc.update("INSERT INTO data_release VALUES(?,'parent','PUBLISHED',NULL,NULL,'rule','old-map',NULL,TRUE,JSON_OBJECT(),?)",parent,UUID.randomUUID().toString()); jdbc.update("INSERT INTO data_release VALUES(?,'parent','PUBLISHED',NULL,NULL,'rule','old-map',NULL,TRUE,JSON_OBJECT(),?)",parent,UUID.randomUUID().toString());
@@ -15,7 +15,7 @@ import org.junit.jupiter.api.Test;
class MetricsResponseAccessTest { class MetricsResponseAccessTest {
final GovernanceAccessService governance = mock(GovernanceAccessService.class); final GovernanceAccessService governance = mock(GovernanceAccessService.class);
final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "scope", "unused", "测试", true, false, final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "scope", "unused", "测试", true, false,
List.of("QUALITY_VIEWER"), List.of("D"), List.of(), List.of()); List.of("USER"), List.of("D"), List.of(), List.of());
final MetricStage stage = new MetricStage(new BigDecimal("90"), new BigDecimal("10"), new BigDecimal("100"), final MetricStage stage = new MetricStage(new BigDecimal("90"), new BigDecimal("10"), new BigDecimal("100"),
new BigDecimal("0.9"), List.of(new DefectRow("责废", BigDecimal.TEN, BigDecimal.ONE)), BigDecimal.ONE, new BigDecimal("0.91")); new BigDecimal("0.9"), List.of(new DefectRow("责废", BigDecimal.TEN, BigDecimal.ONE)), BigDecimal.ONE, new BigDecimal("0.91"));
final MetricRow row = new MetricRow("x", "x", "D", "G", "P", null, new BigDecimal("0.81"), stage, stage, Map.of(), new BigDecimal("0.82")); final MetricRow row = new MetricRow("x", "x", "D", "G", "P", null, new BigDecimal("0.81"), stage, stage, Map.of(), new BigDecimal("0.82"));
@@ -19,7 +19,7 @@ class MetricsTrendControllerTest {
MetricsService metrics = mock(MetricsService.class); MetricsService metrics = mock(MetricsService.class);
MetricsResponseAccess access = mock(MetricsResponseAccess.class); MetricsResponseAccess access = mock(MetricsResponseAccess.class);
AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "trend", "unused", "趋势", true, false, AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "trend", "unused", "趋势", true, false,
List.of("QUALITY_VIEWER"), List.of("D"), List.of("G"), List.of()); List.of("USER"), List.of("D"), List.of("G"), List.of());
MetricsService.MetricTrendSnapshot response = new MetricsService.MetricTrendSnapshot( MetricsService.MetricTrendSnapshot response = new MetricsService.MetricTrendSnapshot(
Map.of(), Map.of(), List.of(), Map.of()); Map.of(), Map.of(), List.of(), Map.of());
when(metrics.calculateTrend(any(), eq(actor))).thenReturn(response); when(metrics.calculateTrend(any(), eq(actor))).thenReturn(response);
@@ -62,7 +62,7 @@ class SourceDeletionServiceTest {
verify(jdbc,never()).update(anyString(),any(Object[].class));verify(governance,never()).request(any(),any()); verify(jdbc,never()).update(anyString(),any(Object[].class));verify(governance,never()).request(any(),any());
} }
@Test void restrictedMaintainerCannotHideARevisionOutsideTheirOrganization(){ @Test void restrictedMaintainerCannotHideARevisionOutsideTheirOrganization(){
currentUsed=true;var service=service();var restricted=new AppUserPrincipal(actor.getId(),"A","","",true,false,List.of("DATA_MAINTAINER"),List.of("A"),List.of(),List.of()); currentUsed=true;var service=service();var restricted=new AppUserPrincipal(actor.getId(),"A","","",true,false,List.of("DATA_ADMIN"),List.of("A"),List.of(),List.of());
doThrow(new ResponseStatusException(HttpStatus.FORBIDDEN,"B outside scope")).when(sourceAccess).requireSource(eq("document"),eq("current-revision"),any()); doThrow(new ResponseStatusException(HttpStatus.FORBIDDEN,"B outside scope")).when(sourceAccess).requireSource(eq("document"),eq("current-revision"),any());
assertEquals(HttpStatus.FORBIDDEN,assertThrows(ResponseStatusException.class,()->service.delete("candidate",new SourceDeletionService.DeleteRequest(pointer,0L,"current-revision","理由","key"),restricted)).getStatusCode()); assertEquals(HttpStatus.FORBIDDEN,assertThrows(ResponseStatusException.class,()->service.delete("candidate",new SourceDeletionService.DeleteRequest(pointer,0L,"current-revision","理由","key"),restricted)).getStatusCode());
verify(jdbc,never()).update(anyString(),any(Object[].class)); verify(jdbc,never()).update(anyString(),any(Object[].class));
@@ -0,0 +1,284 @@
package cn.yunsheng.quality.source;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.mock;
import cn.yunsheng.quality.analytics.TargetVersionService;
import cn.yunsheng.quality.config.QualityProperties;
import cn.yunsheng.quality.governance.*;
import cn.yunsheng.quality.identity.AppRoles;
import cn.yunsheng.quality.identity.AppUserPrincipal;
import cn.yunsheng.quality.ingestion.*;
import cn.yunsheng.quality.metrics.MetricsRollupService;
import cn.yunsheng.quality.metrics.MetricsService;
import cn.yunsheng.quality.release.ReleaseService;
import java.io.ByteArrayOutputStream;
import java.math.BigDecimal;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.flywaydb.core.Flyway;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.http.HttpStatus;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.web.server.ResponseStatusException;
import tools.jackson.databind.json.JsonMapper;
/** Exercises the upload, import, release, and metric paths against a real isolated MySQL schema. */
@EnabledIfEnvironmentVariable(named="QUALITY_MULTIFILE_TEST_CREATE_DATABASE",matches="true")
class SourceUploadAppendMySqlTest {
@TempDir Path tempDir;
private JdbcTemplate admin,jdbc;
private DriverManagerDataSource dataSource;
private TransactionTemplate tx;
private String database;
private Path archiveRoot;
private UUID adminId;
private AppUserPrincipal adminActor;
private JsonMapper json;
private GovernanceAccessService access;
private SourceAuthorizationService sourceAccess;
private SourceScanner scanner;
private XlsxStreamingReader reader;
private SourceUploadService uploads;
private SourceCatalogService sources;
private ImportRunService imports;
private ImportWorker worker;
private SourceCoverageGuard coverage;
private ReleaseService releases;
private SourceGovernanceService governance;
private MetricsService metrics;
@BeforeEach void setup() throws Exception {
String serverUrl=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_SERVER_URL");
assertNotNull(serverUrl,"set the isolated local MySQL server URL");
assertTrue(serverUrl.startsWith("jdbc:mysql://127.0.0.1:")&&serverUrl.contains("/?"));
java.util.regex.Matcher port=java.util.regex.Pattern.compile("jdbc:mysql://127\\.0\\.0\\.1:(\\d+)/\\?").matcher(serverUrl);
assertTrue(port.find());assertNotEquals("53306",port.group(1),"tests must never connect to the formal MySQL port");
String user=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_USER"),password=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_PASSWORD");
admin=new JdbcTemplate(new DriverManagerDataSource(serverUrl,user,password));
database="quality_verify_multifile_"+UUID.randomUUID().toString().replace("-","");
admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
dataSource=new DriverManagerDataSource(serverUrl.replaceFirst("/\\?","/"+database+"?"),user,password);
Flyway.configure().dataSource(dataSource).locations("classpath:db/migration")
.placeholders(Map.of("legacyTimestampOffsetSeconds","0")).load().migrate();
jdbc=new JdbcTemplate(dataSource);tx=new TransactionTemplate(new DataSourceTransactionManager(dataSource));
archiveRoot=tempDir.resolve("archive");Files.createDirectories(archiveRoot);
adminId=UUID.randomUUID();
jdbc.update("INSERT INTO app_user(id,username,display_name,password_hash,active) VALUES(?,?,?,?,TRUE)",adminId.toString(),"multifile-test-admin","Multifile test admin","fixture-hash");
jdbc.update("INSERT INTO app_user_role(user_id,role_code) VALUES(?,'SYSTEM_ADMIN')",adminId.toString());
adminActor=new AppUserPrincipal(adminId,"multifile-test-admin","fixture-hash","Multifile test admin",true,false,List.of("SYSTEM_ADMIN"),List.of("*"),List.of("*"),AppRoles.authorities(List.of("SYSTEM_ADMIN")));
QualityProperties properties=new QualityProperties();properties.setArchiveRoot(archiveRoot);scanner=new SourceScanner(properties);reader=new XlsxStreamingReader();
json=JsonMapper.builder().build();access=new GovernanceAccessService(jdbc,json);sourceAccess=new SourceAuthorizationService(jdbc);
GovernanceSupport support=new GovernanceSupport(jdbc,json);sources=new SourceCatalogService(jdbc,scanner);coverage=new SourceCoverageGuard(jdbc);
uploads=new SourceUploadService(jdbc,scanner,reader,sourceAccess,access,json,support);
imports=new ImportRunService(jdbc,sources,json,access);
MetricsRollupService rollups=new MetricsRollupService(jdbc);
worker=new ImportWorker(jdbc,sources,scanner,reader,json,access,new TargetVersionService(jdbc),coverage,new TaskProgressService(jdbc),mock(CandidateAcceptanceService.class),mock(MappingIssueService.class),rollups);
ReflectionTestUtils.setField(worker,"testWorkflowMode",true);
releases=new ReleaseService(jdbc,json,mock(ApplicationEventPublisher.class),scanner,coverage,sourceAccess,mock(CandidateAcceptanceService.class));
ReflectionTestUtils.setField(releases,"testWorkflowMode",true);
governance=new SourceGovernanceService(support,imports,worker,coverage,sourceAccess,mock(CandidateAcceptanceService.class));
ReflectionTestUtils.setField(governance,"testWorkflowMode",true);
metrics=new MetricsService(jdbc,access);
}
@AfterEach void cleanup(){
if(admin!=null&&database!=null&&database.matches("quality_verify_multifile_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);
}
@Test void mainAndTwoAppendFilesImportPublishReplaceWithdrawAndKeepHistory() throws Exception {
ConfiguredSource mainBlack=configure("main-black.xlsx","black","ADD",null,11,"main black",null);
ConfiguredSource mainBright=configure("main-bright.xlsx","bright","ADD",null,5,"main bright",null);
String initialCandidate=importSources(List.of(mainBlack.sourceId(),mainBright.sourceId()),null);
publish(initialCandidate,null,"publish-main");
String mainRelease=currentRelease();
Map<String,Member> originalMembers=members(mainRelease);
assertEquals(2,originalMembers.size());
assertWeight(mainRelease,11);
ConfiguredSource appendA=configure("append-a.xlsx","black","APPEND","batch-A",4,"append batch A",mainRelease);
Map<String,Object> retry=configureAgain(appendA,"black","APPEND","batch-A",4,"retry same file identity",mainRelease);
assertEquals(true,retry.get("reconfigured"));
assertEquals(appendA.sourceId(),sourceId(retry));
ConfiguredSource appendB=configure("append-b.xlsx","black","APPEND","batch-B",6,"append batch B",mainRelease);
String appendCandidate=importSources(List.of(appendA.sourceId(),appendB.sourceId()),mainRelease);
assertTrue(Boolean.TRUE.equals(jdbc.queryForObject("SELECT complete FROM data_release WHERE id=?",Boolean.class,appendCandidate)));
assertEquals(0,coverage.overlapCount(appendCandidate));
publish(appendCandidate,mainRelease,"publish-appends");
String appendRelease=currentRelease();
Map<String,Member> appendedMembers=members(appendRelease);
assertEquals(4,appendedMembers.size());
assertEquals(originalMembers.get("black|2026|02|ALL"),appendedMembers.get("black|2026|02|ALL"));
assertEquals(originalMembers.get("bright|2026|02|ALL"),appendedMembers.get("bright|2026|02|ALL"));
assertEquals("APPEND_SNAPSHOT",sourceMode("black|2026|02|ALL|UPLOAD_batch-A"));
assertEquals("APPEND_SNAPSHOT",sourceMode("black|2026|02|ALL|UPLOAD_batch-B"));
assertWeight(appendRelease,21);
assertThrows(ResponseStatusException.class,()->configure("same-content.xlsx","black","APPEND","batch-C",0,"duplicate bytes",appendRelease,appendB.bytes()));
AppUserPrincipal limitedActor=limitedActor();
var unauthorized=new SourceUploadService.ConfigureRequest("unused","black",2026,2,"报废明细",1,fieldMap("black"),null,"APPEND",appendRelease,null,null,"out of range","ALL","limited-batch");
ResponseStatusException forbidden=assertThrows(ResponseStatusException.class,()->uploads.configure(unauthorized,limitedActor));
assertEquals(HttpStatus.FORBIDDEN,forbidden.getStatusCode());
Map<String,Object> target=targets().stream().filter(t->Objects.equals("batch-A",t.get("fileId"))).findFirst().orElseThrow();
assertEquals("APPEND_SNAPSHOT",target.get("sourceMode"));
String aDocument=String.valueOf(target.get("documentId")),aRevision=String.valueOf(target.get("revisionId"));
ConfiguredSource correctedA=configureReplace("append-a-corrected.xlsx","black","batch-A",8,"replace batch A only",appendRelease,aDocument,aRevision,blackWorkbook(8));
String replacementCandidate=importSources(List.of(correctedA.sourceId()),appendRelease);
publish(replacementCandidate,appendRelease,"publish-replace-a");
String replacementRelease=currentRelease();
Map<String,Member> replacedMembers=members(replacementRelease);
assertEquals(4,replacedMembers.size());
assertEquals(appendedMembers.get("black|2026|02|ALL"),replacedMembers.get("black|2026|02|ALL"));
assertEquals(appendedMembers.get("bright|2026|02|ALL"),replacedMembers.get("bright|2026|02|ALL"));
assertEquals(appendedMembers.get("black|2026|02|ALL|UPLOAD_batch-B"),replacedMembers.get("black|2026|02|ALL|UPLOAD_batch-B"));
assertEquals(aDocument,replacedMembers.get("black|2026|02|ALL|UPLOAD_batch-A").documentId());
assertNotEquals(aRevision,replacedMembers.get("black|2026|02|ALL|UPLOAD_batch-A").revisionId());
assertWeight(replacementRelease,25);
assertWeight(appendRelease,21);
assertEquals(originalMembers, members(mainRelease));
Map<String,Object> requested=tx.execute(status->governance.request(new SourceGovernanceService.ChangeRequest("WITHDRAW",aDocument,null,"withdraw appended batch A","withdraw-batch-a"),adminActor));
String requestId=String.valueOf(((Map<?,?>)requested.get("request")).get("id"));
tx.executeWithoutResult(status->governance.review(requestId,new SourceGovernanceService.ReviewRequest(true,"approve batch A withdrawal",0L),adminActor));
Map<String,Object> built=tx.execute(status->governance.build(requestId,new SourceGovernanceService.BuildRequest(replacementRelease,"build-withdraw-a",null),adminActor));
String withdrawalCandidate=String.valueOf(built.get("releaseId"));
assertTrue(Boolean.TRUE.equals(jdbc.queryForObject("SELECT complete FROM data_release WHERE id=?",Boolean.class,withdrawalCandidate)));
publish(withdrawalCandidate,replacementRelease,"publish-withdraw-a");
String withdrawalRelease=currentRelease();
Map<String,Member> withdrawnMembers=members(withdrawalRelease);
assertEquals(3,withdrawnMembers.size());
assertFalse(withdrawnMembers.containsKey("black|2026|02|ALL|UPLOAD_batch-A"));
assertEquals(replacedMembers.get("black|2026|02|ALL"),withdrawnMembers.get("black|2026|02|ALL"));
assertEquals(replacedMembers.get("bright|2026|02|ALL"),withdrawnMembers.get("bright|2026|02|ALL"));
assertEquals(replacedMembers.get("black|2026|02|ALL|UPLOAD_batch-B"),withdrawnMembers.get("black|2026|02|ALL|UPLOAD_batch-B"));
assertWeight(withdrawalRelease,17);
assertWeight(replacementRelease,25);
assertWeight(appendRelease,21);
assertEquals(originalMembers,members(mainRelease));
ConfiguredSource departmentAppend=configureScoped("department-append.xlsx","black","APPEND","dept-batch",7,"scope overlap regression",withdrawalRelease,"质量管理部",blackWorkbook(7));
String scopeOverlapCandidate=importSources(List.of(departmentAppend.sourceId()),withdrawalRelease);
assertFalse(Boolean.TRUE.equals(jdbc.queryForObject("SELECT complete FROM data_release WHERE id=?",Boolean.class,scopeOverlapCandidate)));
assertTrue(coverage.overlapCount(scopeOverlapCandidate)>0);
}
@Test void concurrentSameShaUploadsUseCurrentLockingReadsAndOnlyOneIsConfirmed() throws Exception {
byte[] bytes=blackWorkbook(11);
Map<String,Object> firstPreview=uploads.previewUpload(multipart("same-one.xlsx",bytes),adminActor);
Map<String,Object> secondPreview=uploads.previewUpload(multipart("same-two.xlsx",bytes),adminActor);
var first=request(firstPreview,"black","APPEND","concurrent-A",null,"same content A");
var second=request(secondPreview,"black","APPEND","concurrent-B",null,"same content B");
CountDownLatch ready=new CountDownLatch(2),start=new CountDownLatch(1);
try(var pool=Executors.newFixedThreadPool(2)){
var a=pool.submit(()->concurrentConfigure(first,ready,start));
var b=pool.submit(()->concurrentConfigure(second,ready,start));
assertTrue(ready.await(10,TimeUnit.SECONDS));start.countDown();
List<String> outcomes=List.of(a.get(20,TimeUnit.SECONDS),b.get(20,TimeUnit.SECONDS));
assertEquals(1,outcomes.stream().filter("CONFIRMED"::equals).count());
assertEquals(1,outcomes.stream().filter("DUPLICATE_SHA"::equals).count());
}
assertEquals(1,jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate WHERE status='DISCOVERED' AND resolution='APPEND_CONFIRMED'",Integer.class));
}
private String concurrentConfigure(SourceUploadService.ConfigureRequest request,CountDownLatch ready,CountDownLatch start){
try{return tx.execute(status->{
jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate",Integer.class); // Establish a repeatable-read snapshot before the shared pointer lock.
ready.countDown();try{if(!start.await(10,TimeUnit.SECONDS))throw new IllegalStateException("barrier timeout");}catch(InterruptedException e){Thread.currentThread().interrupt();throw new IllegalStateException(e);}
try{uploads.configure(request,adminActor);return "CONFIRMED";}catch(ResponseStatusException duplicate){if(duplicate.getReason()!=null&&duplicate.getReason().contains("请勿重复计入"))return "DUPLICATE_SHA";throw duplicate;}catch(java.io.IOException e){throw new IllegalStateException(e);}
});}catch(RuntimeException failure){throw failure;}
}
private ConfiguredSource configure(String filename,String stage,String mode,String fileId,int weight,String reason,String expected)throws Exception{
return configure(filename,stage,mode,fileId,weight,reason,expected,"black".equals(stage)?blackWorkbook(weight):brightWorkbook(weight));
}
private ConfiguredSource configure(String filename,String stage,String mode,String fileId,int weight,String reason,String expected,byte[] bytes)throws Exception{
return configureScoped(filename,stage,mode,fileId,weight,reason,expected,"ALL",bytes);
}
private ConfiguredSource configureScoped(String filename,String stage,String mode,String fileId,int weight,String reason,String expected,String scope,byte[] bytes)throws Exception{
Map<String,Object> preview=uploads.previewUpload(multipart(filename,bytes),adminActor);
var request=request(preview,stage,mode,fileId,expected,reason,scope);
Map<String,Object> configured=tx.execute(status->{try{return uploads.configure(request,adminActor);}catch(java.io.IOException e){throw new IllegalStateException(e);}});
return new ConfiguredSource(((SourceCatalogService.SourceView)configured.get("source")).id(),String.valueOf(preview.get("uploadId")),bytes);
}
private ConfiguredSource configureReplace(String filename,String stage,String fileId,int weight,String reason,String expected,String targetDocument,String targetRevision,byte[] bytes)throws Exception{
Map<String,Object> preview=uploads.previewUpload(multipart(filename,bytes),adminActor);
var request=new SourceUploadService.ConfigureRequest(String.valueOf(preview.get("uploadId")),stage,2026,2,"black".equals(stage)?"报废明细":"明细",1,fieldMap(stage),null,"REPLACE",expected,targetDocument,targetRevision,reason,"ALL",fileId);
Map<String,Object> configured=tx.execute(status->{try{return uploads.configure(request,adminActor);}catch(java.io.IOException e){throw new IllegalStateException(e);}});
return new ConfiguredSource(((SourceCatalogService.SourceView)configured.get("source")).id(),String.valueOf(preview.get("uploadId")),bytes);
}
private Map<String,Object> configureAgain(ConfiguredSource source,String stage,String mode,String fileId,int weight,String reason,String expected)throws Exception{
var request=new SourceUploadService.ConfigureRequest(source.uploadId(),stage,2026,2,"black".equals(stage)?"报废明细":"明细",1,fieldMap(stage),null,mode,expected,null,null,reason,"ALL",fileId);
return tx.execute(status->{try{return uploads.configure(request,adminActor);}catch(java.io.IOException e){throw new IllegalStateException(e);}});
}
private SourceUploadService.ConfigureRequest request(Map<String,Object> preview,String stage,String mode,String fileId,String expected,String reason){
return request(preview,stage,mode,fileId,expected,reason,"ALL");
}
private SourceUploadService.ConfigureRequest request(Map<String,Object> preview,String stage,String mode,String fileId,String expected,String reason,String scope){
return new SourceUploadService.ConfigureRequest(String.valueOf(preview.get("uploadId")),stage,2026,2,"black".equals(stage)?"报废明细":"明细",1,fieldMap(stage),null,mode,expected,null,null,reason,scope,fileId);
}
private Map<String,String> fieldMap(String stage){Map<String,String> result=new LinkedHashMap<>();SourceParserConfig.required(stage).forEach(header->result.put(header,header));return result;}
private String importSources(List<String> ids,String expected){
var run=tx.execute(status->imports.create(new ImportRunService.CreateImportRequest(ids,UUID.randomUUID().toString(),expected),adminId));
Map<String,Object> result=tx.execute(status->worker.execute(run.id()));
assertEquals("COMPLETED",result.get("status"),String.valueOf(result.get("errorMessage")));
return String.valueOf(result.get("resultReleaseId"));
}
private void publish(String releaseId,String expected,String key){
tx.execute(status->releases.transitionInTestWorkflow(releaseId,"PUBLISH",new ReleaseService.ReleaseActionRequest(expected,key,"multifile integration acceptance"),adminActor,false));
}
private String currentRelease(){return jdbc.query("SELECT release_id FROM current_release WHERE dataset_id='quality'",(rs,n)->rs.getString(1)).getFirst();}
private Map<String,Member> members(String releaseId){
return jdbc.query("SELECT d.source_slot,d.id,m.source_revision_id,m.transform_build_id FROM release_member m JOIN source_document d ON d.id=m.source_document_id WHERE m.release_id=? ORDER BY d.source_slot",(rs,n)->Map.entry(rs.getString(1),new Member(rs.getString(2),rs.getString(3),rs.getString(4))),releaseId).stream().collect(java.util.stream.Collectors.toMap(Map.Entry::getKey,Map.Entry::getValue, (a,b)->a,LinkedHashMap::new));
}
private String sourceMode(String slot){return jdbc.queryForObject("SELECT source_mode FROM source_document WHERE source_slot=?",String.class,slot);}
private void assertWeight(String releaseId,int expected){
var query=new MetricsService.MetricsQuery(2026,2,null,null,null,null,null,"overall",false,releaseId,"month",Map.of());
BigDecimal total=metrics.calculate(query,adminActor).summary().black().totalWeight();
assertNotNull(total);assertEquals(0,new BigDecimal(expected).compareTo(total),"black-stage accumulated total for "+releaseId);
}
private List<Map<String,Object>> targets(){return (List<Map<String,Object>>)uploads.options(adminActor).get("targets");}
private String sourceId(Map<String,Object> response){return ((SourceCatalogService.SourceView)response.get("source")).id();}
private AppUserPrincipal limitedActor(){
UUID id=UUID.randomUUID();jdbc.update("INSERT INTO app_user(id,username,display_name,password_hash,active) VALUES(?,?,?,?,TRUE)",id.toString(),"limited-upload-user","Limited upload user","fixture-hash");
jdbc.update("INSERT INTO app_user_role(user_id,role_code) VALUES(?,'DATA_ADMIN')",id.toString());
jdbc.update("INSERT INTO app_user_scope(user_id,scope_type,scope_value) VALUES(?,'DEPARTMENT','DEPT_A')",id.toString());
return new AppUserPrincipal(id,"limited-upload-user","fixture-hash","Limited upload user",true,false,List.of("DATA_ADMIN"),List.of("DEPT_A"),List.of(),AppRoles.authorities(List.of("DATA_ADMIN")));
}
private MockMultipartFile multipart(String filename,byte[] bytes){return new MockMultipartFile("file",filename,"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",bytes);}
private byte[] blackWorkbook(int weight)throws Exception{return workbook("报废明细","black",weight);}
private byte[] brightWorkbook(int weight)throws Exception{return workbook("明细","bright",weight);}
private byte[] workbook(String sheetName,String stage,int weight)throws Exception{
List<String> headers=SourceParserConfig.required(stage);Map<String,String> values=new HashMap<>();
if("black".equals(stage)){
values.put("序号",Integer.toString(weight));values.put("PDT代码","P002");values.put("销售片区","中国区");values.put("事业部","质量管理部");values.put("成品规格","P");values.put("送电镀重量",Integer.toString(weight));values.put("黑片出货重量","0");values.put("责废总重量","0");values.put("料废重量","0");values.put("外废重量","0");values.put("发料地点","常规");values.put("检验备注","");
}else{
values.put("序号",Integer.toString(weight));values.put("PDT代码","P002");values.put("销售片区","中国区");values.put("事业部","质量管理部");values.put("成品规格","P");values.put("是否电镀来料/二等品库来料","是");values.put("合格重量",Integer.toString(weight));values.put("检包入废品库重量","1");values.put("本部报废重量","0");values.put("镀废重量合计","1");values.put("材料废重量合计","0");values.put("加工废重量合计","0");values.put("晶界扩散报废合计重量","0");
}
try(XSSFWorkbook workbook=new XSSFWorkbook();ByteArrayOutputStream out=new ByteArrayOutputStream()){
Sheet sheet=workbook.createSheet(sheetName);Row header=sheet.createRow(0);for(int i=0;i<headers.size();i++)header.createCell(i).setCellValue(headers.get(i));
Row row=sheet.createRow(1);for(int i=0;i<headers.size();i++)row.createCell(i).setCellValue(values.getOrDefault(headers.get(i),""));
workbook.write(out);return out.toByteArray();
}
}
private record ConfiguredSource(String sourceId,String uploadId,byte[] bytes){}
private record Member(String documentId,String revisionId,String buildId){}
}
@@ -27,6 +27,59 @@ class SourceUploadServiceTest {
assertConfiguredWithoutTemplate("REPLACE"); assertConfiguredWithoutTemplate("REPLACE");
} }
@Test void appendUsesStablePerFileSlotAndAppendProtocol() throws Exception {
Fixture f=fixture("APPEND");
f.service.configure(f.request(null),f.actor);
ArgumentCaptor<Object[]> candidate=ArgumentCaptor.forClass(Object[].class);
verify(f.jdbc).update(contains("INSERT INTO source_candidate"),candidate.capture());
assertEquals("black|2026|02|ALL|UPLOAD_file-1",candidate.getValue()[2]);
assertEquals("APPEND_CONFIRMED",candidate.getValue()[12]);
ArgumentCaptor<Object[]> protocol=ArgumentCaptor.forClass(Object[].class);
verify(f.jdbc).update(contains("INSERT INTO source_protocol"),protocol.capture());
assertEquals("UPLOAD_file-1",protocol.getValue()[7]);
assertEquals("APPEND_SNAPSHOT",protocol.getValue()[8]);
}
@Test void sameWorkbookHashInCurrentOverlappingScopeIsRejected() throws Exception {
Fixture f=fixture("APPEND");
when(f.jdbc.queryForList(startsWith("SELECT d.id,d.scope_code FROM release_member"),any(Object[].class)))
.thenReturn(List.of(Map.of("id","monthly-document","scope_code","ALL")));
var error=assertThrows(org.springframework.web.server.ResponseStatusException.class,()->f.service.configure(f.request(null),f.actor));
assertEquals(org.springframework.http.HttpStatus.CONFLICT,error.getStatusCode());
assertTrue(error.getReason().contains("请勿重复计入"));
verify(f.jdbc,never()).update(contains("INSERT INTO source_candidate"),any(Object[].class));
}
@Test void replacementTargetsOneAppendFileByItsStableFileIdAndRevision() throws Exception {
Fixture f=fixture("REPLACE");
when(f.jdbc.queryForList(startsWith("SELECT d.id,r.source_revision_id,d.source_mode"),any(Object[].class)))
.thenReturn(List.of(Map.of("id","document-current","source_revision_id","revision-current","source_mode","APPEND_SNAPSHOT")));
SourceUploadService.ConfigureRequest base=f.request(null);
var request=new SourceUploadService.ConfigureRequest(base.uploadId(),base.stage(),base.year(),base.month(),base.sheetName(),base.headerRow(),base.fieldMap(),base.templateId(),base.mode(),base.expectedCurrentReleaseId(),base.targetDocumentId(),base.targetRevisionId(),base.reason(),base.scopeCode(),"file-1");
f.service.configure(request,f.actor);
ArgumentCaptor<Object[]> candidate=ArgumentCaptor.forClass(Object[].class);
verify(f.jdbc).update(contains("INSERT INTO source_candidate"),candidate.capture());
assertEquals("black|2026|02|ALL|UPLOAD_file-1",candidate.getValue()[2]);
assertEquals("document-current",candidate.getValue()[17]);
assertEquals("revision-current",candidate.getValue()[18]);
}
@Test void sameAppendFileIdCannotBeRegisteredAsAnotherNewBatch() throws Exception {
Fixture f=fixture("APPEND");
when(f.jdbc.queryForList(startsWith("SELECT id FROM source_candidate WHERE"),any(Object[].class)))
.thenReturn(List.of(Map.of("id","prior-candidate")));
var error=assertThrows(org.springframework.web.server.ResponseStatusException.class,()->f.service.configure(f.request(null),f.actor));
assertTrue(error.getReason().contains("该fileId已登记"));
verify(f.jdbc,never()).update(contains("INSERT INTO source_candidate"),any(Object[].class));
}
@Test void uploadOptionsDoNotExposeOrReadTemplateCatalog() throws Exception { @Test void uploadOptionsDoNotExposeOrReadTemplateCatalog() throws Exception {
Fixture f = fixture("ADD"); Fixture f = fixture("ADD");
when(f.jdbc.queryForList(anyString(), any(Object[].class))).thenReturn(List.of()); when(f.jdbc.queryForList(anyString(), any(Object[].class))).thenReturn(List.of());
@@ -134,7 +187,7 @@ class SourceUploadServiceTest {
boolean replace = "REPLACE".equals(mode); boolean replace = "REPLACE".equals(mode);
return new SourceUploadService.ConfigureRequest("upload-draft", "black", 2026, 2, "工作表", 2, return new SourceUploadService.ConfigureRequest("upload-draft", "black", 2026, 2, "工作表", 2,
fieldMap, templateId, mode, "release-current", replace ? "document-current" : null, fieldMap, templateId, mode, "release-current", replace ? "document-current" : null,
replace ? "revision-current" : null, "测试直接字段映射", "ALL"); replace ? "revision-current" : null, "测试直接字段映射", "ALL", "APPEND".equals(mode)?"file-1":null);
} }
} }
} }
@@ -0,0 +1,49 @@
package cn.yunsheng.quality.update;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.jupiter.api.Test;
import org.springframework.mock.web.MockHttpServletRequest;
import org.springframework.mock.web.MockHttpServletResponse;
class SystemUpdateMaintenanceFilterTest {
@Test
void blocksBusinessWritesDuringUpdateButKeepsReadsAuthAndUpdateStatusAvailable() throws Exception {
var updates = mock(SystemUpdateService.class);
when(updates.maintenanceActive()).thenReturn(true);
var filter = new SystemUpdateMaintenanceFilter(updates);
for (String method : List.of("POST", "PUT", "PATCH", "DELETE")) {
var request = new MockHttpServletRequest(method, "/api/sources/test");
var response = new MockHttpServletResponse();
var continued = new AtomicBoolean();
filter.doFilter(request, response, (req, res) -> continued.set(true));
assertFalse(continued.get());
assertEquals(503, response.getStatus());
assertEquals("10", response.getHeader("Retry-After"));
}
var password = new MockHttpServletResponse();
filter.doFilter(new MockHttpServletRequest("POST", "/api/auth/change-password"), password,
(req, res) -> fail("Password changes must remain blocked during update"));
assertEquals(503, password.getStatus());
for (var route : List.of(new String[]{"GET", "/api/sources"}, new String[]{"GET", "/api/system-update"},
new String[]{"POST", "/api/system-update/check"}, new String[]{"POST", "/api/auth/login"},
new String[]{"GET", "/actuator/health"})) {
var continued = new AtomicBoolean();
filter.doFilter(new MockHttpServletRequest(route[0], route[1]), new MockHttpServletResponse(),
(req, res) -> continued.set(true));
assertTrue(continued.get());
}
}
@Test
void normalAndSourceDeploymentsContinueToAllowWrites() throws Exception {
var updates = mock(SystemUpdateService.class);
when(updates.maintenanceActive()).thenReturn(false);
var continued = new AtomicBoolean();
new SystemUpdateMaintenanceFilter(updates).doFilter(new MockHttpServletRequest("POST", "/api/sources"),
new MockHttpServletResponse(), (req, res) -> continued.set(true));
assertTrue(continued.get());
}
}
@@ -0,0 +1,88 @@
package cn.yunsheng.quality.update;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import cn.yunsheng.quality.config.SecurityConfig;
import cn.yunsheng.quality.governance.GovernanceAccessService;
import cn.yunsheng.quality.identity.JdbcAppUserDetailsService;
import jakarta.servlet.Filter;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.mock.web.MockHttpSession;
import org.springframework.mock.web.MockServletContext;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.web.csrf.CsrfToken;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.support.AnnotationConfigWebApplicationContext;
import org.springframework.web.servlet.config.annotation.EnableWebMvc;
/** Exercises the real application security chain, including CSRF and method roles. */
class SystemUpdateSecurityTest {
@Configuration
@EnableWebMvc
@Import(SecurityConfig.class)
static class TestConfig {
@Bean JdbcAppUserDetailsService users() { return mock(JdbcAppUserDetailsService.class); }
@Bean GovernanceAccessService access() { return mock(GovernanceAccessService.class); }
@Bean SystemUpdateService updates() {
var service = mock(SystemUpdateService.class);
var result = Map.<String, Object>of("enabled", true, "state", "idle");
when(service.status()).thenReturn(result);
when(service.check()).thenReturn(result);
when(service.apply()).thenReturn(result);
return service;
}
@Bean SystemUpdateController controller(SystemUpdateService updates) { return new SystemUpdateController(updates); }
}
private MockHttpSession session(String role) {
var context = SecurityContextHolder.createEmptyContext();
context.setAuthentication(UsernamePasswordAuthenticationToken.authenticated("test-user", null,
List.of(new SimpleGrantedAuthority("ROLE_" + role))));
var session = new MockHttpSession();
session.setAttribute("SPRING_SECURITY_CONTEXT", context);
return session;
}
@Test
void requiresAdminAndCsrfForBothUpdateMutations() throws Exception {
try (var context = new AnnotationConfigWebApplicationContext()) {
context.setServletContext(new MockServletContext());
context.register(TestConfig.class);
context.refresh();
var mvc = MockMvcBuilders.webAppContextSetup(context)
.addFilters(context.getBean("springSecurityFilterChain", Filter.class)).build();
mvc.perform(get("/api/system-update")).andExpect(status().isUnauthorized());
mvc.perform(get("/api/system-update").session(session("USER"))).andExpect(status().isForbidden());
var admin = session("SYSTEM_ADMIN");
var initial = mvc.perform(get("/api/system-update").session(admin)).andExpect(status().isOk()).andReturn();
var csrf = (CsrfToken) initial.getRequest().getAttribute(CsrfToken.class.getName());
assertNotNull(csrf);
String token = csrf.getToken();
var cookie = initial.getResponse().getCookie("XSRF-TOKEN");
assertNotNull(cookie);
for (String path : List.of("/api/system-update/check", "/api/system-update/apply")) {
mvc.perform(post(path).session(admin)).andExpect(status().isForbidden());
mvc.perform(post(path).session(admin).cookie(cookie).header(csrf.getHeaderName(), "bad-token"))
.andExpect(status().isForbidden());
mvc.perform(post(path).session(session("DATA_ADMIN")).cookie(cookie).header(csrf.getHeaderName(), token))
.andExpect(status().isForbidden());
mvc.perform(post(path).session(admin).cookie(cookie).header(csrf.getHeaderName(), token))
.andExpect(status().isOk());
}
verify(context.getBean(SystemUpdateService.class), times(1)).check();
verify(context.getBean(SystemUpdateService.class), times(1)).apply();
} finally {
SecurityContextHolder.clearContext();
}
}
}
@@ -0,0 +1,63 @@
package cn.yunsheng.quality.update;
import static org.junit.jupiter.api.Assertions.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.server.ResponseStatusException;
import tools.jackson.databind.json.JsonMapper;
class SystemUpdateServiceTest {
@Test
void sourceDeploymentIsDisabledEvenIfFlagIsAccidentallyEnabled() {
var service = new SystemUpdateService(JsonMapper.builder().build(), "", true);
assertEquals(false, service.status().get("enabled"));
assertThrows(ResponseStatusException.class, service::check);
assertThrows(ResponseStatusException.class, service::apply);
}
@Test
void updaterReceiptsNeverExposePrivateConfigurationOrPaths() throws Exception {
var result = SystemUpdateService.safeResponse(Map.of("enabled", true, "state", "available",
"version", "2026.11.04", "rootPassword", "secret", "backupPath", "C:/private"));
assertEquals(Map.of("enabled", true, "state", "available", "version", "2026.11.04"), result);
assertThrows(IOException.class, () -> SystemUpdateService.safeResponse(Map.of("enabled", "true", "state", "idle")));
assertThrows(IOException.class, () -> SystemUpdateService.safeResponse(Map.of("enabled", true, "state", Map.of())));
}
@Test
void allUpdateRoutesRequireSystemAdministrator() {
assertEquals("hasRole('SYSTEM_ADMIN')", SystemUpdateController.class.getAnnotation(PreAuthorize.class).value());
}
@Test
void maintenanceStartsBeforeBackupAndPersistsAcrossAnInterruptedInstall(@TempDir Path root) throws Exception {
var service = new SystemUpdateService(JsonMapper.builder().build(), root.toString(), false);
// Exercise host-independent metadata handling without changing the
// production Windows/runtime enablement guard.
var enabled = SystemUpdateService.class.getDeclaredField("enabled");
enabled.setAccessible(true);
enabled.setBoolean(service, true);
assertFalse(service.maintenanceActive());
Path directory = Files.createDirectories(root.resolve(".runtime/update"));
for (String phase : new String[]{"queued", "downloading", "completed", "failed"}) {
Files.writeString(directory.resolve("status.json"), "{\"state\":\"" + phase + "\"}");
assertFalse(service.maintenanceActive(), phase);
}
for (String phase : new String[]{"backingUp", "installing", "restarting", "rollingBack"}) {
Files.writeString(directory.resolve("status.json"), "{\"state\":\"" + phase + "\"}");
assertTrue(service.maintenanceActive(), phase);
}
Files.writeString(directory.resolve("status.json"), "{\"state\":\"failed\"}");
Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"installing\"}");
assertTrue(service.maintenanceActive());
Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"rolledBack\"}");
assertFalse(service.maintenanceActive());
Files.writeString(directory.resolve("status.json"), "broken JSON");
assertTrue(service.maintenanceActive());
}
}
@@ -0,0 +1,79 @@
package cn.yunsheng.quality.update;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import cn.yunsheng.quality.analysis.AnalysisReportService;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
import org.springframework.core.env.MapPropertySource;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.ScheduledAnnotationBeanPostProcessor;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.EnableTransactionManagement;
/** Actual Spring event/scheduling registration, including transaction proxies. */
class SystemUpdateSpringLifecycleTest {
private static final AtomicBoolean MAINTENANCE = new AtomicBoolean();
private static AnalysisReportService originalReports;
@Configuration
@EnableScheduling
@EnableTransactionManagement
static class Config {
@Bean static SystemUpdateStartupBarrier barrier(ObjectProvider<SystemUpdateService> updates, Environment environment) {
return new SystemUpdateStartupBarrier(updates, environment);
}
@Bean SystemUpdateService updates() {
var updates = mock(SystemUpdateService.class);
when(updates.maintenanceActive()).thenAnswer(call -> MAINTENANCE.get());
return updates;
}
@Bean AnalysisReportService reports() { originalReports = mock(AnalysisReportService.class); return originalReports; }
@Bean PlatformTransactionManager transactionManager() { return mock(PlatformTransactionManager.class); }
}
@Test
void actualSpringReadyEventAndScheduledTaskRemainPausedUntilCommit() {
// Initialize Mockito before temporarily simulating the target OS flag;
// no external process or filesystem/runtime guard is changed.
mock(SystemUpdateService.class);
String originalOs = System.getProperty("os.name");
MAINTENANCE.set(true);
try (var context = new AnnotationConfigApplicationContext()) {
context.getEnvironment().getPropertySources().addFirst(new MapPropertySource("update-test",
Map.of("QUALITY_UPDATE_ENABLED", "true")));
context.register(Config.class);
try {
System.setProperty("os.name", "Windows 11");
context.refresh();
} finally {
System.setProperty("os.name", originalOs);
}
var event = new ApplicationReadyEvent(new SpringApplication(Config.class), new String[0], context, Duration.ZERO);
context.publishEvent(event);
verify(originalReports, never()).recoverPendingReports(any(ApplicationReadyEvent.class));
var schedules = context.getBean(ScheduledAnnotationBeanPostProcessor.class).getScheduledTasks();
assertEquals(1, schedules.size());
var task = schedules.iterator().next().getTask().getRunnable();
task.run();
verify(originalReports, never()).scheduledReports();
MAINTENANCE.set(false);
context.getBean(SystemUpdateStartupBarrier.class).drain();
verify(originalReports, times(1)).recoverPendingReports(event);
task.run();
verify(originalReports, times(1)).scheduledReports();
} finally {
System.setProperty("os.name", originalOs);
MAINTENANCE.set(false);
}
}
}
@@ -0,0 +1,58 @@
package cn.yunsheng.quality.update;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import cn.yunsheng.quality.analysis.AnalysisTurnService;
import cn.yunsheng.quality.ingestion.ImportRecoveryRunner;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.core.env.Environment;
import org.springframework.scheduling.annotation.Scheduled;
class SystemUpdateStartupBarrierTest {
public static class ScheduledProbe {
int ticks;
@Scheduled(fixedDelay = 1000) public void tick() { ticks++; }
public void ordinaryRead() { ticks += 10; }
}
@Test
@SuppressWarnings("unchecked")
void defersBusinessStartupExactlyOnceUntilCommitAndSkipsScheduledWrites() {
var maintenance = new AtomicBoolean(true);
var updates = mock(SystemUpdateService.class);
when(updates.maintenanceActive()).thenAnswer(call -> maintenance.get());
var provider = (ObjectProvider<SystemUpdateService>) mock(ObjectProvider.class);
when(provider.getObject()).thenReturn(updates);
try (var scope = new BarrierScope(new SystemUpdateStartupBarrier(provider, mock(Environment.class)))) {
var barrier = scope.barrier;
var imports = mock(ImportRecoveryRunner.class);
var importProxy = (ImportRecoveryRunner) barrier.wrap(imports, ImportRecoveryRunner.class);
var turns = mock(AnalysisTurnService.class);
var turnProxy = (AnalysisTurnService) barrier.wrap(turns, AnalysisTurnService.class);
var raw = new ScheduledProbe();
var scheduled = (ScheduledProbe) barrier.wrap(raw, ScheduledProbe.class);
importProxy.run(null);
turnProxy.recoverQueuedTurns();
scheduled.tick();
scheduled.ordinaryRead();
assertEquals(10, raw.ticks);
verify(imports, never()).run(null);
verify(turns, never()).recoverQueuedTurns();
barrier.drain();
verify(imports, never()).run(null);
maintenance.set(false);
barrier.drain();
barrier.drain();
verify(imports, times(1)).run(null);
verify(turns, times(1)).recoverQueuedTurns();
scheduled.tick();
assertEquals(11, raw.ticks);
}
}
private record BarrierScope(SystemUpdateStartupBarrier barrier) implements AutoCloseable {
@Override public void close() { barrier.close(); }
}
}
@@ -62,6 +62,41 @@ class DataWorkflowServiceTest {
verifyNoMoreInteractions(jdbc); verifyNoMoreInteractions(jdbc);
} }
@Test
void candidateDataErrorsAreNotReportedAsVersionMismatches() {
when(jdbc.queryForMap(contains("SELECT mapping_version"), eq("candidate")))
.thenReturn(Map.of("mappingVersion", "mapping", "ruleVersion", "rule"));
when(jdbc.queryForObject(contains("rejected_count"), eq(Integer.class), any(Object[].class))).thenReturn(2);
when(jdbc.queryForList(contains("FROM import_issue"), any(Object[].class))).thenReturn(List.of(
Map.of("issueCode", "INVALID_NUMBER", "fieldName", "合格重量", "issueCount", 2L)));
RuntimeException error = assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(service,
"assertWorkflowCandidate", "candidate", "mapping", "rule"));
assertTrue(error.getMessage().contains("合格重量"), error.getMessage());
assertTrue(error.getMessage().contains("数值"), error.getMessage());
assertFalse(error.getMessage().contains("未使用本次冻结版本"));
}
@Test
void incompleteCandidateBuildHasItsOwnFailureReason() {
when(jdbc.queryForMap(contains("SELECT mapping_version"), eq("candidate")))
.thenReturn(Map.of("mappingVersion", "mapping", "ruleVersion", "rule"));
when(jdbc.queryForObject(contains("b.status<>'COMPLETED'"), eq(Integer.class), any(Object[].class))).thenReturn(1);
RuntimeException error = assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(service,
"assertWorkflowCandidate", "candidate", "mapping", "rule"));
assertTrue(error.getMessage().contains("尚未完成"), error.getMessage());
assertFalse(error.getMessage().contains("未使用本次冻结版本"));
}
@Test
void actualCandidateVersionMismatchStillBlocksPublication() {
when(jdbc.queryForMap(contains("SELECT mapping_version"), eq("candidate")))
.thenReturn(Map.of("mappingVersion", "mapping", "ruleVersion", "rule"));
when(jdbc.queryForObject(contains("b.mapping_version<>? OR b.rule_version<>?"), eq(Integer.class), any(Object[].class))).thenReturn(1);
RuntimeException error = assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(service,
"assertWorkflowCandidate", "candidate", "mapping", "rule"));
assertTrue(error.getMessage().contains("未使用本次冻结版本"));
}
@Test @Test
void disabledModeKeepsTheLegacyPathAndDoesNotQueueMalformedRequests() { void disabledModeKeepsTheLegacyPathAndDoesNotQueueMalformedRequests() {
ReflectionTestUtils.setField(service, "testMode", false); ReflectionTestUtils.setField(service, "testMode", false);
@@ -72,55 +107,35 @@ class DataWorkflowServiceTest {
} }
@Test @Test
void writeActionsStillRequirePublisherEvenWhenApprovalsAreSkipped() { void usersCannotQueueDataWrites() {
ResponseStatusException error = assertThrows(ResponseStatusException.class, ResponseStatusException error = assertThrows(ResponseStatusException.class,
() -> service.recalculate(user("DATA_MAINTAINER"))); () -> service.recalculate(user("USER")));
assertEquals(HttpStatus.FORBIDDEN, error.getStatusCode()); assertEquals(HttpStatus.FORBIDDEN, error.getStatusCode());
verify(access).requireUserActivePolicy(any()); verify(access).requireUserActivePolicy(any());
verifyNoInteractions(jdbc); verifyNoInteractions(jdbc);
} }
@Test @Test
void sourceCapabilitiesReflectManagerAndPublisherIntersection() { void threeRolesInheritDataCapabilitiesButKeepRuleEditingRestricted() {
Map<String, Object> maintainer = DataWorkflowService.capabilities(true, user("DATA_MAINTAINER")); Map<String, Object> reader = DataWorkflowService.capabilities(true, user("USER"));
assertEquals(true, maintainer.get("canDeleteSource")); for (String key : List.of("canApply", "canImport", "canPublish", "canDeleteSource", "canRestoreSource"))
assertEquals(false, maintainer.get("canDeleteCurrentSource")); assertEquals(false, reader.get(key), key);
assertEquals(false, maintainer.get("canRestoreSource")); Map<String, Object> dataAdmin = DataWorkflowService.capabilities(true, user("DATA_ADMIN"));
assertEquals(false, maintainer.get("canApplyMapping")); for (String key : List.of("canApplyMapping", "canImport", "canPublish", "canRecalculate", "canDeleteSource", "canDeleteCurrentSource", "canRestoreSource"))
assertEquals(false, maintainer.get("canImport")); assertEquals(true, dataAdmin.get(key), key);
assertEquals(false, dataAdmin.get("canApplyRule"));
Map<String, Object> publisher = DataWorkflowService.capabilities(true, user("DATA_PUBLISHER")); assertEquals(true, DataWorkflowService.capabilities(true, admin()).get("canApplyRule"));
assertEquals(false, publisher.get("canDeleteSource"));
assertEquals(false, publisher.get("canRestoreSource"));
assertEquals(false, publisher.get("canApplyMapping"));
assertEquals(false, publisher.get("canApplyRule"));
assertEquals(false, publisher.get("canImport"));
assertEquals(true, publisher.get("canRecalculate"));
Map<String, Object> publisherAndDataMaintainer = DataWorkflowService.capabilities(true,
user("DATA_PUBLISHER", "DATA_MAINTAINER"));
assertEquals(true, publisherAndDataMaintainer.get("canApplyMapping"));
assertEquals(false, publisherAndDataMaintainer.get("canApplyRule"));
assertEquals(true, publisherAndDataMaintainer.get("canImport"));
Map<String, Object> publisherAndRuleMaintainer = DataWorkflowService.capabilities(true,
user("DATA_PUBLISHER", "RULE_MAINTAINER"));
assertEquals(true, publisherAndRuleMaintainer.get("canApplyMapping"));
assertEquals(true, publisherAndRuleMaintainer.get("canApplyRule"));
assertEquals(false, publisherAndRuleMaintainer.get("canImport"));
Map<String, Object> rulesMaintainer = DataWorkflowService.capabilities(true, user("RULE_MAINTAINER"));
assertEquals(false, rulesMaintainer.get("canApply"));
assertEquals(false, rulesMaintainer.get("canApplyRule"));
assertEquals(false, rulesMaintainer.get("canDeleteSource"));
assertEquals(false, rulesMaintainer.get("canRestoreSource"));
Map<String, Object> disabled = DataWorkflowService.capabilities(false, admin()); Map<String, Object> disabled = DataWorkflowService.capabilities(false, admin());
assertEquals(false, disabled.get("canApply")); for (String key : List.of("canApply", "canApplyMapping", "canApplyRule", "canImport", "canPublish"))
assertEquals(false, disabled.get("canApplyMapping")); assertEquals(false, disabled.get(key), key);
assertEquals(false, disabled.get("canApplyRule")); }
assertEquals(false, disabled.get("canImport"));
assertEquals(false, disabled.get("canPublish")); @Test
void dataAdministratorCannotQueueGlobalRuleChanges() {
var error = assertThrows(ResponseStatusException.class, () -> service.apply(
new DataWorkflowController.ApplyRequest("rule", "rules", Map.of(), null), user("DATA_ADMIN")));
assertEquals(HttpStatus.FORBIDDEN, error.getStatusCode());
verifyNoInteractions(jdbc);
} }
@Test @Test
@@ -192,13 +207,13 @@ class DataWorkflowServiceTest {
} }
@Test @Test
void sourceMutationEndpointsRequireTheUnderlyingManagerAndPublisherRoles() { void usersCannotDeleteOrRestoreDataSources() {
ResponseStatusException deleteError = assertThrows(ResponseStatusException.class, ResponseStatusException deleteError = assertThrows(ResponseStatusException.class,
() -> service.deleteSource("candidate", user("DATA_PUBLISHER"))); () -> service.deleteSource("candidate", user("USER")));
assertEquals(HttpStatus.FORBIDDEN, deleteError.getStatusCode()); assertEquals(HttpStatus.FORBIDDEN, deleteError.getStatusCode());
ResponseStatusException restoreError = assertThrows(ResponseStatusException.class, ResponseStatusException restoreError = assertThrows(ResponseStatusException.class,
() -> service.restoreSource("candidate", user("DATA_MAINTAINER"))); () -> service.restoreSource("candidate", user("USER")));
assertEquals(HttpStatus.FORBIDDEN, restoreError.getStatusCode()); assertEquals(HttpStatus.FORBIDDEN, restoreError.getStatusCode());
verifyNoInteractions(jdbc); verifyNoInteractions(jdbc);
} }
@@ -333,18 +348,18 @@ class DataWorkflowServiceTest {
} }
@Test @Test
void batchDeletePreflightsAllSourcesAndEnforcesCurrentSourcePublishingPermission() { void dataAdministratorPreflightsAllSourcesBeforeQueueingBatchDeletion() {
when(jdbc.query(eq("SELECT release_id FROM current_release WHERE dataset_id=?"), any(RowMapper.class), eq("quality"))) when(jdbc.query(eq("SELECT release_id FROM current_release WHERE dataset_id=?"), any(RowMapper.class), eq("quality")))
.thenReturn(List.of("current")); .thenReturn(List.of("current"));
when(deletion.preview(eq("a"), any())).thenReturn(Map.of("hidden", false, "currentUsed", false)); when(deletion.preview(eq("a"), any())).thenReturn(Map.of("hidden", false, "currentUsed", false));
when(deletion.preview(eq("b"), any())).thenReturn(Map.of("hidden", false, "currentUsed", true)); when(deletion.preview(eq("b"), any())).thenReturn(Map.of("hidden", false, "currentUsed", true));
var error = assertThrows(ResponseStatusException.class, () -> service.deleteSources( var queued = service.deleteSources(
new DataWorkflowController.DeleteSourcesRequest(List.of("a", "b"), "current"), user("DATA_MAINTAINER"))); new DataWorkflowController.DeleteSourcesRequest(List.of("a", "b"), "current"), user("DATA_ADMIN"));
assertEquals(HttpStatus.FORBIDDEN, error.getStatusCode()); assertEquals("DELETE_SOURCES", queued.get("operationType"));
verify(deletion).preview(eq("a"), any()); verify(deletion).preview(eq("a"), any());
verify(deletion).preview(eq("b"), any()); verify(deletion).preview(eq("b"), any());
verify(deletion, never()).delete(anyString(), any(), any()); verify(deletion, never()).delete(anyString(), any(), any());
verify(jdbc, never()).update(anyString(), any(Object[].class)); verify(jdbc).update(anyString(), any(Object[].class));
} }
@Test @Test
@@ -357,13 +372,13 @@ class DataWorkflowServiceTest {
@Test @Test
void workflowWriteEndpointsKeepRoleGuards() throws Exception { void workflowWriteEndpointsKeepRoleGuards() throws Exception {
assertPreAuthorize("apply", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); assertPreAuthorize("apply", "SYSTEM_ADMIN", "DATA_ADMIN");
assertPreAuthorize("recalculate", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); assertPreAuthorize("recalculate", "SYSTEM_ADMIN", "DATA_ADMIN");
assertPreAuthorize("importSources", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); assertPreAuthorize("importSources", "SYSTEM_ADMIN", "DATA_ADMIN");
assertPreAuthorize("publish", "SYSTEM_ADMIN", "DATA_PUBLISHER"); assertPreAuthorize("publish", "SYSTEM_ADMIN", "DATA_ADMIN");
assertPreAuthorize("deleteSource", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); assertPreAuthorize("deleteSource", "SYSTEM_ADMIN", "DATA_ADMIN");
assertPreAuthorize("deleteSources", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); assertPreAuthorize("deleteSources", "SYSTEM_ADMIN", "DATA_ADMIN");
assertPreAuthorize("restoreSource", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); assertPreAuthorize("restoreSource", "SYSTEM_ADMIN", "DATA_ADMIN");
} }
private void assertPreAuthorize(String method, String... roles) throws Exception { private void assertPreAuthorize(String method, String... roles) throws Exception {
+36
View File
@@ -0,0 +1,36 @@
# 质量中心离线部署
此目录是 Windows / Linux 原生离线包的打包与启动工具。编译后的前端、Spring Boot fat JAR、Java 21、MySQL 8.4、Python 3.11.15 和固定版本的 AI 依赖全部随包交付;目标机不执行 pip、pnpm、Maven 或联网下载。不需要 Docker。
Windows 推荐将离线包解压到 `%LOCALAPPDATA%\QualityCenter`,或本机固定 NTFS 磁盘上更短的当前用户可写目录。启动器会读取包内 `manifest.json`,要求部署根路径长度加最长相对文件路径不超过 248 个字符;若用户名较长导致 `%LOCALAPPDATA%\QualityCenter` 超限,请选择更短的用户可写目录。请用当前用户运行 `windows-start.ps1`;服务进程在当前用户会话中启动,运行环境随包提供,不安装系统组件、不注册全局服务,也不修改 Windows 防火墙。网页服务会监听内网网卡并显示本机及内网访问链接;MySQL 和后端接口只监听 `127.0.0.1`。其他设备能否访问网页取决于系统现有防火墙和网络策略,需要放行时由用户或管理员按组织策略处理。
Linux 默认只允许本机访问,使用 `./start.sh --lan` 开启内网监听;已有防火墙策略由部署者按实际网段配置。使用普通用户启动,不能以 root 运行 MySQL。
`build_bundle.py --work output/offline-20261104` 使用隔离构建目录中的 JAR / 前端产物及已下载的官方发行文件组装包。`downloads/assets.json` 记录来源及摘要,`requirements-*.lock` 固定各平台依赖,`wheels-*` 是已通过 pip 哈希验证的平台 wheel。该命令不会读取本机数据库、用户配置、API 密钥或 AI 会话。
为保留本机运行环境,跨平台适配只应用在包内的 Python 脚本副本上;原有 macOS 脚本不被改写。适配包括 Windows 的文件锁、进程创建时间校验和 NTFS 继承权限,以及随包 Python / nao-core 的相对位置。
包内 `manifest.json` 记录每个只读文件的 SHA-256 和符号链接目标。首次及后续启动都会检查文件完整性和平台原生程序;运行数据单独保存在 `.runtime/` 和 `runtimedata/`。初始管理员密码在目标机随机生成,不使用公共默认密码。
包内不包含业务数据库、工作簿、现有用户、模型密钥或本机私有配置。全离线时可使用常规查询、导入和分析功能;AI 调用在线模型时需要网络,或自行配置可达的内网模型服务。代码执行沙箱和开发测试工具不在业务部署范围内。
不能在服务运行时移动安装目录。迁移业务数据应使用逻辑备份与对应版本的恢复流程,不得直接将 macOS 的 MySQL 数据目录复制到其他系统。
## Windows 增量更新包(维护人员)
离线签名工具读取 Windows x64 部署包的 `manifest.json`,只打包 `update_manager.program_path` 认可的完整程序清单,并使用包内运行环境和 wheel 信息计算运行时指纹。私有配置、业务数据、迁移文件、启动引导和密钥不会进入更新包。普通发布的序号必须大于部署包当前的 `updateSequence`。首次通道种子可显式加 `--allow-baseline`,但版本和序号必须分别与 `updateVersion`、`updateSequence` 完全相同;低序号始终拒绝。私钥必须是与包内更新公钥匹配的 Ed25519 PEM 文件,并保存在部署包及产物目录之外。
```sh
python3 deploy/offline/publish_update.py \
--root /path/to/QualityCenter-Windows-x64 \
--version 2026.11.04.2 \
--sequence 2 \
--private-key /secure/path/update-private-key.pem \
--output-dir /path/to/new-update-output
```
若需要为 `updateVersion=2026.11.04.1`、`updateSequence=1` 的原始包签署首个通道基线,使用与该包完全相同的版本和序号,并显式传入 `--allow-baseline`。不要修改包内 manifest 来绕过检查。
工具只在本地产生 `quality-update-<version>.zip` 和签名 `latest.json`,不会上传或发布文件。每个版本请使用新的输出目录;发布到更新服务器由维护人员另行完成。
参考:[Java 21 官方发行](https://adoptium.net/temurin/releases?version=21)、[MySQL Windows 安装要求](https://dev.mysql.com/doc/refman/8.4/en/windows-installation.html)、[MySQL Linux 二进制安装](https://dev.mysql.com/doc/refman/8.4/en/binary-installation.html)、[Python standalone](https://github.com/astral-sh/python-build-standalone)、[nao-core 0.3.26](https://pypi.org/project/nao-core/0.3.26/)、[Microsoft 运行库](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist/)。
+330
View File
@@ -0,0 +1,330 @@
#!/usr/bin/env python3
"""Assemble native offline packages from reviewed builds and verified downloads."""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import hashlib
import io
import json
from pathlib import Path
import re
import shutil
import stat
import subprocess
import tarfile
import tempfile
import zipfile
ROOT = Path(__file__).resolve().parents[2]
TEMPLATES = Path(__file__).resolve().parent
def digest(path, algorithm='sha256'):
with path.open('rb') as stream:
return hashlib.file_digest(stream, algorithm).hexdigest()
def extract(archive, target):
target.mkdir(parents=True)
with tempfile.TemporaryDirectory(dir=target.parent) as directory:
stage = Path(directory)
if archive.suffix == '.zip':
with zipfile.ZipFile(archive) as source:
source.extractall(stage)
else:
with tarfile.open(archive) as source:
# Terminfo contains case-distinct names. They alias on a Mac's
# default filesystem and are unused by this noninteractive service.
members = [member for member in source.getmembers()
if '/share/terminfo/' not in member.name and not member.name.endswith('/share/terminfo')]
source.extractall(stage, members=members, filter='data')
roots = list(stage.iterdir())
if len(roots) != 1 or not roots[0].is_dir():
raise RuntimeError('Expected one archive root: ' + archive.name)
shutil.copytree(roots[0], target, symlinks=True, dirs_exist_ok=True)
def extract_deb(archive, target):
raw = archive.read_bytes()
if raw[:8] != b'!<arch>\n':
raise RuntimeError('Invalid Debian archive')
position = 8
while position + 60 <= len(raw):
header = raw[position:position + 60]
name = header[:16].decode().strip().rstrip('/')
length = int(header[48:58].decode().strip())
position += 60
data = raw[position:position + length]
position += length + length % 2
if name.startswith('data.tar'):
with tempfile.TemporaryDirectory(dir=target.parent) as directory:
if name.endswith('.zst'):
import zstandard
with zstandard.ZstdDecompressor().stream_reader(io.BytesIO(data)) as reader:
with tarfile.open(fileobj=reader, mode='r|') as source:
source.extractall(directory, filter='data')
else:
with tarfile.open(fileobj=io.BytesIO(data)) as source:
source.extractall(directory, filter='data')
for path in Path(directory).rglob('*'):
if '.so' in path.name and (path.is_file() or path.is_symlink()):
dest = target / path.name
if path.is_symlink():
dest.symlink_to(Path(path.readlink()).name)
else:
shutil.copy2(path, dest)
docs = Path(directory) / 'usr/share/doc'
if docs.exists():
shutil.copytree(docs, target / 'licenses', dirs_exist_ok=True, symlinks=True)
return
raise RuntimeError('No data member in ' + archive.name)
def install_wheel(wheel, python_root, windows):
site = python_root / ('Lib/site-packages' if windows else 'lib/python3.11/site-packages')
site.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(wheel) as source:
for member in source.infolist():
if member.is_dir():
continue
relative = Path(member.filename)
if relative.is_absolute() or '..' in relative.parts:
raise RuntimeError('Wheel path escape')
target = site / relative
if relative.parts[0].endswith('.data'):
scheme = relative.parts[1]
prefix = {'purelib': site, 'platlib': site, 'data': python_root,
'scripts': python_root / ('Scripts' if windows else 'bin'),
'headers': python_root / 'include'}[scheme]
target = prefix.joinpath(*relative.parts[2:])
target.parent.mkdir(parents=True, exist_ok=True)
with source.open(member) as reader, target.open('wb') as writer:
shutil.copyfileobj(reader, writer)
mode = member.external_attr >> 16
target.chmod((mode & 0o777) or 0o644)
def adapt_runtime(source):
def replace(old, new, count=None):
nonlocal source
actual = source.count(old)
if not actual or (count is not None and actual != count):
raise RuntimeError('Runtime source changed; inspect the portability patch: ' + old)
source = source.replace(old, new)
replace('import fcntl\n', 'import runtime_platform as _host\nfrom runtime_platform import fcntl\n', 1)
replace('PYTHON = VENV / "bin" / "python"',
'PYTHON = Path(os.environ.get("QUALITY_ASSISTANT_PYTHON", str(VENV / "bin" / "python")))', 1)
old = 'VENV / "lib" / "python3.11" / "site-packages" / "nao_core"'
replace(old, 'Path(os.environ.get("QUALITY_ASSISTANT_PACKAGE", str(' + old + ')))', 2)
replace('/ "nao-chat-server"', '/ ("nao-chat-server.exe" if os.name == "nt" else "nao-chat-server")', 1)
source, count = re.subn(r'\b(info|directory_info)\.st_mode & 0o077',
lambda match: '_host.private_bits(' + match[1] + '.st_mode)', source)
if count != 3:
raise RuntimeError('Runtime private-permission checks changed; inspect before packaging')
replace('os.fchmod(', '_host.fchmod(')
replace('os.getpgid(pid) != pid', 'not _host.group_leader(pid)', 1)
replace('os.killpg(', '_host.killpg(')
replace('signal.SIGKILL', 'getattr(signal, "SIGKILL", 9)')
replace('def _ps_value(pid: int, field: str) -> str | None:\n',
'def _ps_value(pid: int, field: str) -> str | None:\n'
' if os.name == "nt":\n return _host.ps_value(pid, field)\n', 1)
compile(source, 'nao_quality_runtime.py', 'exec')
return source
def adapt_script(name, source):
if name == 'nao_quality_runtime.py':
return adapt_runtime(source)
if name == 'analysis_visual_evidence.py':
source = source.replace('import stat\n', 'import stat\nimport runtime_platform as _host\n', 1)
source = source.replace('os.O_NOFOLLOW', 'getattr(os, "O_NOFOLLOW", 0)')
source = source.replace('os.O_NONBLOCK', 'getattr(os, "O_NONBLOCK", 0)')
source = source.replace('before.st_mode & 0o077', '_host.private_bits(before.st_mode)')
source = source.replace('before.st_uid != os.getuid()', '_host.owner_mismatch(before)')
source = source.replace(' fd = os.open(path,',
' if path.is_symlink():\n raise ValueError("invalid evidence link")\n fd = os.open(path,', 1)
elif name == 'nao_evidence_finalizer.py':
source = source.replace('import copy\n', 'import copy\nimport runtime_platform as _host\n', 1)
source = source.replace('path.stat().st_mode & 0o077', '_host.private_bits(path.stat().st_mode)')
compile(source, name, 'exec')
return source
def windows_local_runtime(root):
"""Use the signed vendor JRE's redistributable runtime; never install globally."""
libraries = ('msvcp140.dll', 'vcruntime140.dll', 'vcruntime140_1.dll')
for name in libraries:
source = root / 'runtimes/java/bin' / name
if not source.is_file():
raise RuntimeError('The official JRE is missing an app-local Microsoft runtime: ' + name)
for folder in ('runtimes/mysql/bin', 'runtimes/mysql/lib'):
shutil.copy2(source, root / folder / name)
mysql = root / 'runtimes/mysql'
# Debug-only tools depend on non-redistributable Microsoft debug runtimes.
for path in sorted(mysql.rglob('*'), reverse=True):
if path.is_file() and (any(part.lower() == 'debug' for part in path.relative_to(mysql).parts)
or '-debug.' in path.name.lower()):
path.unlink()
elif path.is_dir() and path.name.lower() == 'debug':
shutil.rmtree(path)
return {'name': 'app-local-msvc', 'os': 'windows', 'source': 'bundled-official-temurin-jre',
'libraries': {name: digest(root / 'runtimes/java/bin' / name) for name in libraries}}
def payload(root, work, osname, assets):
windows = osname == 'windows'
app = root / 'app'
app.mkdir(parents=True)
jars = list((work / 'source/backend/target').glob('*.jar'))
if len(jars) != 1:
raise RuntimeError('Expected one freshly built backend JAR')
with zipfile.ZipFile(jars[0]) as archive:
if 'BOOT-INF/classes/verification/original-workbook-acceptance.json' in archive.namelist():
raise RuntimeError('Exclude the private workbook acceptance input before building a distributable JAR')
shutil.copy2(jars[0], app / 'backend.jar')
shutil.copytree(work / 'source/web/dist', app / 'web')
for name in ('java', 'mysql', 'python'):
kind = 'jre' if name == 'java' else name
item = next(a for a in assets if a['name'] == kind and a['os'] == osname)
archive = work / 'downloads' / item['file']
if digest(archive) != item['sha256']:
raise RuntimeError('Download checksum mismatch: ' + item['file'])
extract(archive, root / 'runtimes' / name)
wheels = sorted((work / ('wheels-' + osname)).glob('*.whl'))
if len(wheels) < 60:
raise RuntimeError('The offline Python dependency set is incomplete')
lock_text = (work / f'requirements-{osname}.lock').read_text()
for wheel in wheels:
if '--hash=sha256:' + digest(wheel) not in lock_text:
raise RuntimeError('Wheel absent from hash lock: ' + wheel.name)
install_wheel(wheel, root / 'runtimes/python', windows)
scripts = root / 'scripts'
scripts.mkdir()
for pattern in ('nao_*.py', 'analysis_visual*.py'):
for path in (ROOT / 'scripts').glob(pattern):
(scripts / path.name).write_text(adapt_script(path.name, path.read_text()), encoding='utf-8')
for name in ('dbgpt_trial_data.py', 'local_api.py', 'project.py'):
shutil.copy2(ROOT / 'scripts' / name, scripts / name)
shutil.copy2(TEMPLATES / 'runtime_platform.py', scripts / 'runtime_platform.py')
policy = root / 'backend/src/main/resources/analysis-runtime-policy.json'
policy.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(ROOT / 'backend/src/main/resources/analysis-runtime-policy.json', policy)
shutil.copytree(ROOT / 'ai/nao-quality', root / 'ai/nao-quality', ignore=shutil.ignore_patterns('.DS_Store'))
deploy = root / 'deploy/offline'
deploy.mkdir(parents=True)
for name in ('control.py', 'gateway.py', 'runtime_platform.py', 'migration_common.py',
'restore_migration.py', 'update_manager.py', 'update-public-key.pem',
'windows-start.ps1', 'windows-prepare.ps1', 'README.md'):
if not windows and name.endswith('.ps1'):
continue
source = TEMPLATES / name
if name.endswith('.ps1'):
(deploy / name).write_text(source.read_text(), encoding='utf-8-sig')
else:
shutil.copy2(source, deploy / name)
shutil.copy2(work / f'requirements-{osname}.lock', root / 'requirements.lock')
if windows:
local_runtime = windows_local_runtime(root)
shutil.copy2(root / 'runtimes/python/python.exe', root / 'runtimes/python/python3.exe')
bootstrap = root / 'deploy/bootstrap'
bootstrap.mkdir()
shutil.copy2(TEMPLATES / 'update_bootstrap.py', bootstrap / 'update-recovery.py')
for name in ('windows-start.ps1', 'windows-prepare.ps1'):
(bootstrap / name).write_text((TEMPLATES / name).read_text(), encoding='utf-8-sig')
(root / '一键部署启动.cmd').write_bytes(
b'@echo off\r\nchcp 65001 >nul\r\npowershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0deploy\\bootstrap\\windows-start.ps1"\r\n'
b'if errorlevel 1 (echo Startup failed. See the message above.)\r\npause\r\n')
for name, action in (('停止服务.cmd', 'stop'), ('查看状态.cmd', 'status'), ('检查环境.cmd', 'verify')):
(root / name).write_bytes(('@echo off\r\nchcp 65001 >nul\r\nset PYTHONUTF8=1\r\nset PYTHONIOENCODING=utf-8\r\n'
'"%~dp0runtimes\\python\\python.exe" "%~dp0deploy\\offline\\control.py" ' + action + '\r\npause\r\n').encode())
else:
libs = root / 'runtimes/linux-libs'
libs.mkdir()
for item in assets:
if item.get('kind') == 'deb':
archive = work / 'downloads' / item['file']
if digest(archive) != item['sha256']:
raise RuntimeError('Debian library checksum mismatch')
extract_deb(archive, libs)
for name, action in (('start.sh', 'start'), ('stop.sh', 'stop'), ('status.sh', 'status'), ('verify.sh', 'verify')):
(root / name).write_text('#!/bin/sh\nset -eu\nROOT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)\n'
'if [ "$(id -u)" -eq 0 ]; then echo "Run as a normal user, not root." >&2; exit 1; fi\n'
'export PYTHONUTF8=1 PYTHONIOENCODING=utf-8 PYTHONDONTWRITEBYTECODE=1\n'
'export LD_LIBRARY_PATH="$ROOT/runtimes/linux-libs:$ROOT/runtimes/mysql/lib:$ROOT/runtimes/mysql/lib/private"\n'
'exec "$ROOT/runtimes/python/bin/python3" "$ROOT/deploy/offline/control.py" ' + action + ' "$@"\n')
(root / name).chmod(0o755)
system = ('Windows 10/11 或 Windows Server 2019/2022/2025,Intel/AMD x64。仅当前用户运行,无需管理员权限。'
if windows else 'Ubuntu 22.04/24.04、Debian 12/13 的 Intel/AMD x86_64。需要 glibc 2.35 及以上;不支持 Alpine/musl、ARM 或旧版 CentOS。')
usage = ('解压到当前用户的 %LOCALAPPDATA%\\QualityCenter 等较短的本地 NTFS 路径,双击“一键部署启动.cmd”。不要在压缩包中直接打开。\n'
'运行库随包提供;不安装系统服务,不修改系统环境变量、注册表或防火墙,不申请管理员权限。\n'
'启动后自动打开浏览器,命令窗口显示本机和内网地址。关闭此窗口不会停止服务。\n'
'其他电脑连接同一内网后,用浏览器打开显示的 http://内网IP:5280 即可。\n'
'停止服务请双击“停止服务.cmd”;重新启动仍双击原入口。\n' if windows else
'解压后以普通用户运行 ./start.sh;内网共享运行 ./start.sh --lan。\n'
'同事用浏览器打开显示的 http://内网IP:5280;如系统有防火墙,仅对可信内网放行该端口。\n'
'停止运行 ./stop.sh,查看状态运行 ./status.sh,检查完整性运行 ./verify.sh。\n')
(root / '使用说明.txt').write_text('质量中心离线部署包\n\n系统要求\n' + system + '\n建议内存 8GB 以上,至少预留 5GB 磁盘及后续业务数据空间。\n\n'
+ usage + '\n登录信息保存在 runtimedata/登录信息.txt,初始管理员密码在本机随机生成。登录后请修改密码。\n'
'首次数据库为空,不包含原电脑的用户、业务数据、来源文件和模型密钥。通过网页上传授权的数据文件。\n'
'普通功能和安装不需要外网;AI 在线模型调用需要网络,或自行配置内网模型。\n'
'不要在服务运行时移动目录,不要覆盖 .runtime 和 runtimedata。数据迁移使用逻辑备份,不能复制 macOS MySQL 原始数据目录。\n'
'内网访问需要主机保持开机、服务运行、网络互通;本机 MySQL 和内部 API 端口不对内网开放。\n'
'默认网页端口 5280。更改 .runtime/deployment.json 中的 webPort 后重新运行入口。内网连接受现有防火墙规则限制,程序不会自动更改这些规则。\n'
'详细技术说明见 deploy/offline/README.md。\n\n'
'验证范围:前后端隔离构建及本机自动检查已完成;Windows/Linux 原生首次启动和跨电脑内网访问需要在目标机实测。\n', encoding='utf-8-sig')
metadata = {'platform': osname, 'architecture': 'x86_64', 'builtAtUtc': datetime.now(timezone.utc).isoformat(),
'updateVersion': '2026.11.04.1', 'updateSequence': 1, 'installationScope': 'current-user', 'requiresAdministrator': False,
'databaseIncluded': False, 'credentialsIncluded': False, 'requiresDownloadAtInstall': False,
'nativeTargetSmokeTest': 'not-run-on-macos-host',
'runtimes': [{k: v for k, v in a.items() if k not in ('kind',)} for a in assets if a['os'] == osname and a['name'] != 'vcredist'] + ([local_runtime] if windows else []),
'pythonWheels': [{'file': p.name, 'sha256': digest(p)} for p in wheels], 'files': []}
for path in sorted(root.rglob('*')):
if path.is_symlink():
metadata['files'].append({'path': path.relative_to(root).as_posix(), 'symlink': str(path.readlink())})
elif path.is_file():
metadata['files'].append({'path': path.relative_to(root).as_posix(), 'sha256': digest(path)})
(root / 'manifest.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
return metadata
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--work', type=Path, required=True)
parser.add_argument('--platform', choices=('windows', 'linux', 'both'), default='both')
parser.add_argument('--assemble-only', action='store_true')
args = parser.parse_args()
work = args.work.resolve()
assets = json.loads((work / 'downloads/assets.json').read_text())
platforms = ('windows', 'linux') if args.platform == 'both' else (args.platform,)
summaries = []
for osname in platforms:
name = 'QualityCenter-' + ('Windows' if osname == 'windows' else 'Linux') + '-x64'
directory = work / name
if directory.exists():
raise RuntimeError('Preserving an existing bundle; choose an empty work directory: ' + str(directory))
print('Assembling ' + name, flush=True)
metadata = payload(directory, work, osname, assets)
print('Files verified: ' + str(len(metadata['files'])), flush=True)
if args.assemble_only:
continue
if osname == 'windows':
archive = work / (name + '.zip')
with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as target:
for path in sorted(directory.rglob('*')):
if path.is_file():
target.write(path, path.relative_to(work).as_posix())
else:
archive = work / (name + '.tar.gz')
with tarfile.open(archive, 'w:gz', compresslevel=6) as target:
target.add(directory, arcname=name)
summaries.append({'file': archive.name, 'bytes': archive.stat().st_size, 'sha256': digest(archive)})
print(json.dumps(summaries[-1]), flush=True)
if summaries:
(work / 'SHA256SUMS.txt').write_text(''.join(a['sha256'] + ' ' + a['file'] + '\n' for a in summaries))
(work / 'packages.json').write_text(json.dumps(summaries, indent=2) + '\n')
if __name__ == '__main__':
main()
+486
View File
@@ -0,0 +1,486 @@
#!/usr/bin/env python3
"""Offline, unprivileged service supervisor for the standalone distribution."""
from __future__ import annotations
import argparse
import contextlib
import hashlib
import json
import os
from pathlib import Path
import platform
import re
import secrets
import signal
import socket
import subprocess
import sys
import threading
import time
from urllib.request import ProxyHandler, build_opener
import psutil
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import runtime_platform as host
from gateway import make_server
PRIVATE = ROOT / '.runtime'
DATA = ROOT / 'runtimedata'
CONFIG = PRIVATE / 'deployment.json'
STATE = PRIVATE / 'service.json'
STOP = PRIVATE / 'stop-request.json'
IS_WINDOWS = os.name == 'nt'
EXE = '.exe' if IS_WINDOWS else ''
PYTHON = ROOT / 'runtimes/python' / ('python.exe' if IS_WINDOWS else 'bin/python3')
JAVA = ROOT / 'runtimes/java/bin' / ('java' + EXE)
MYSQL = ROOT / 'runtimes/mysql/bin' / ('mysql' + EXE)
MYSQLD = ROOT / 'runtimes/mysql/bin' / ('mysqld' + EXE)
ADMIN = ROOT / 'runtimes/mysql/bin' / ('mysqladmin' + EXE)
OPENER = build_opener(ProxyHandler({}))
def atomic(path, value):
temporary = path.with_name(path.name + '.' + secrets.token_hex(5) + '.tmp')
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
temporary.replace(path)
def private_directory(path):
if path.is_symlink():
raise RuntimeError('运行目录不能是符号链接。')
path.mkdir(parents=True, exist_ok=True, mode=0o700)
if IS_WINDOWS:
result = subprocess.run(['whoami', '/user', '/fo', 'csv', '/nh'], capture_output=True, check=True)
sid = re.search(rb'S-1-\d+(?:-\d+)+', result.stdout)
if not sid:
raise RuntimeError('无法确定当前 Windows 用户。')
subprocess.run(['icacls', str(path), '/inheritance:r', '/grant:r',
'*' + sid.group().decode() + ':(OI)(CI)F', '*S-1-5-18:(OI)(CI)F', '/Q'],
stdout=subprocess.DEVNULL, check=True)
else:
path.chmod(0o700)
def configuration():
if CONFIG.exists():
value = json.loads(CONFIG.read_text(encoding='utf-8'))
else:
value = {'mysqlPort': 53306, 'backendPort': 5281, 'webPort': 5280,
'webBind': '0.0.0.0' if IS_WINDOWS else '127.0.0.1',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)}
atomic(CONFIG, value)
value.setdefault('mysqlDatabase', 'quality_platform')
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', value['mysqlDatabase']):
raise RuntimeError('部署数据库名称无效。')
relative_archive = Path(value.get('archiveDirectory', 'runtimedata/archive'))
if relative_archive.is_absolute() or '..' in relative_archive.parts or not (ROOT / relative_archive).resolve().is_relative_to(DATA.resolve()):
raise RuntimeError('来源归档目录无效。')
for key in ('mysqlPort', 'backendPort', 'webPort'):
if type(value.get(key)) is not int or not 1024 <= value[key] <= 65535:
raise RuntimeError('部署端口配置无效。')
if value['backendPort'] not in (5281, 5283, 5284, 5285):
raise RuntimeError('接口端口必须为 5281、5283、5284 或 5285。')
if len({value[k] for k in ('mysqlPort', 'backendPort', 'webPort')}) != 3:
raise RuntimeError('数据库、接口和网页端口不能重复。')
if value.get('webBind') not in ('127.0.0.1', '0.0.0.0'):
raise RuntimeError('网页监听配置无效。')
for key in ('mysqlPassword', 'rootPassword', 'adminPassword'):
if not re.fullmatch(r'[A-Za-z0-9_-]{20,100}', value.get(key, '')):
raise RuntimeError('本机随机凭据配置无效,原配置已保留。')
return value
def environment(value):
env = os.environ.copy()
package = ROOT / 'runtimes/python' / ('Lib/site-packages' if IS_WINDOWS else 'lib/python3.11/site-packages') / 'nao_core'
env.update({
'JAVA_HOME': str(JAVA.parent.parent),
'PATH': os.pathsep.join([str(PYTHON.parent), str(JAVA.parent), str(MYSQL.parent), env.get('PATH', '')]),
'PYTHONUTF8': '1', 'PYTHONIOENCODING': 'utf-8', 'PYTHONDONTWRITEBYTECODE': '1',
'QUALITY_PROJECT_ROOT': str(ROOT), 'QUALITY_ASSISTANT_PYTHON': str(PYTHON),
'QUALITY_ASSISTANT_PACKAGE': str(package),
'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{value["mysqlPort"]}/{value.get("mysqlDatabase", "quality_platform")}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true',
'MYSQL_USER': 'quality_app', 'MYSQL_PASSWORD': value['mysqlPassword'],
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': '' if value.get('restoredFromMigration') else value['adminUsername'],
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': '' if value.get('restoredFromMigration') else value['adminPassword'],
'QUALITY_ARCHIVE_ROOT': str(ROOT / value.get('archiveDirectory', 'runtimedata/archive')), 'QUALITY_ALLOWED_ROOTS': str(DATA / 'sources'),
'QUALITY_ANALYSIS_RUNTIME_QUERY_PORT': str(value['backendPort']),
'PORT': str(value['backendPort']), 'SERVER_ADDRESS': '127.0.0.1',
'QUALITY_DATA_WORKFLOW_TEST_MODE': 'false',
'MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS': 'UNCONFIRMED',
})
env['QUALITY_DATA_WORKFLOW_TEST_MODE'] = str(value.get('dataWorkflowTestMode', False)).lower()
env['QUALITY_ANALYSIS_AUTONOMOUS_ENABLED'] = str(value.get('analysisAutonomousEnabled', False)).lower()
env['QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED'] = str(value.get('analysisProtocolV1Enabled', False)).lower()
env['QUALITY_UPDATE_ENABLED'] = 'true' if IS_WINDOWS else 'false'
# Do not accidentally inherit a development or source-machine AI command.
env.pop('QUALITY_ANALYSIS_NAO_COMMAND', None)
env.pop('QUALITY_ANALYSIS_NAO_SCRIPT', None)
env.pop('PYTHONHOME', None)
env.pop('PYTHONPATH', None)
if not IS_WINDOWS:
env['LD_LIBRARY_PATH'] = os.pathsep.join([str(ROOT / 'runtimes/linux-libs'),
str(ROOT / 'runtimes/mysql/lib'),
str(ROOT / 'runtimes/mysql/lib/private')])
return env
def record(process, command):
observed = psutil.Process(process.pid)
return {'pid': process.pid, 'created': observed.create_time(),
'executable': str(Path(observed.exe()).resolve()), 'args': [str(a) for a in command[1:]]}
def matches(info):
try:
process = psutil.Process(int(info['pid']))
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
and abs(process.create_time() - info['created']) < .01
and os.path.normcase(str(Path(process.exe()).resolve())) == os.path.normcase(info['executable'])
and process.cmdline()[1:] == info['args'])
except (psutil.Error, KeyError, ValueError, TypeError):
return False
def read_state():
try:
return json.loads(STATE.read_text(encoding='utf-8'))
except FileNotFoundError:
return {}
def free_port(port, bind='127.0.0.1'):
# A recently closed TCP connection may remain in TIME_WAIT. Reject a live
# listener, but allow a clean restart without waiting for that state to expire.
try:
connection = socket.create_connection(('127.0.0.1', port), timeout=.3)
except OSError:
pass
else:
connection.close()
raise RuntimeError(f'端口 {port} 已被其他服务使用。')
with socket.socket() as test:
test.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
test.bind((bind, port))
def mysql_command(value, binary=MYSQL):
return [str(binary), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
'--port=' + str(value['mysqlPort']), '--user=root', '--connect-timeout=3']
def sql(value, query):
env = environment(value)
env['MYSQL_PWD'] = value['rootPassword']
return subprocess.run(mysql_command(value) + ['--batch', '--skip-column-names'], input=query,
text=True, encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.PIPE,
env=env, timeout=10)
def verify(check_hashes=True):
manifest = json.loads((ROOT / 'manifest.json').read_text(encoding='utf-8'))
expected_os = 'windows' if IS_WINDOWS else platform.system().lower()
if manifest['platform'] != expected_os or platform.machine().lower() not in ('amd64', 'x86_64'):
raise RuntimeError('此包仅用于 ' + manifest['platform'] + ' 的 Intel/AMD 64 位电脑。')
if check_hashes:
for entry in manifest['files']:
path = ROOT / entry['path']
if entry.get('symlink') is not None:
valid = path.is_symlink() and os.readlink(path) == entry['symlink']
else:
valid = path.is_file() and hashlib.file_digest(path.open('rb'), 'sha256').hexdigest() == entry['sha256']
if not valid:
raise RuntimeError('文件缺失或校验失败:' + entry['path'])
value = configuration()
env = environment(value)
for command in ([JAVA, '-version'], [MYSQLD, '--no-defaults', '--version']):
result = subprocess.run([str(a) for a in command], capture_output=True, env=env, timeout=30)
if result.returncode:
raise RuntimeError('运行环境检查失败:' + Path(command[0]).name + '。请查看安装说明中的系统要求。')
probe = subprocess.run([str(PYTHON), '-c', 'import duckdb,yaml,fastapi,psutil,nao_core;import sys;assert sys.version_info[:3]==(3,11,15)'],
capture_output=True, env=env, timeout=45)
if probe.returncode:
raise RuntimeError('AI Python 运行环境检查失败。')
application_probe = subprocess.run([str(PYTHON), '-c',
'import sys;sys.path.insert(0,sys.argv[1]);import nao_quality_runtime as r;'
'import nao_native_run,nao_quality_runtime_mcp,nao_visual_recovery;'
'assert len(r._config_fingerprint("https://offline.invalid/v1","offline-check","offline-check"))==64',
str(ROOT / 'scripts')], capture_output=True, env=env, timeout=45)
if application_probe.returncode:
raise RuntimeError('AI 应用脚本或策略资源检查失败。')
package = Path(env['QUALITY_ASSISTANT_PACKAGE'])
native = subprocess.run([str(package / 'bin' / ('nao-chat-server' + EXE)), '--help'],
capture_output=True, env=env, timeout=30)
if native.returncode:
raise RuntimeError('AI 原生服务无法运行,请检查系统版本和 CPU 支持。')
return value
def addresses(value):
ips = set()
if value['webBind'] == '0.0.0.0':
for entries in psutil.net_if_addrs().values():
for entry in entries:
if entry.family == socket.AF_INET and not entry.address.startswith(('127.', '169.254.')):
ips.add(entry.address)
return ['http://127.0.0.1:' + str(value['webPort'])] + [f'http://{ip}:{value["webPort"]}' for ip in sorted(ips)]
def write_login(value):
path = DATA / '登录信息.txt'
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as stream:
if value.get('restoredFromMigration'):
stream.write('质量中心\n' + '\n'.join(addresses(value)) + '\n业务数据已迁移,请使用原有账号和密码登录。\n')
else:
stream.write('质量中心\n' + '\n'.join(addresses(value)) + '\n用户名:' + value['adminUsername']
+ '\n初始密码:' + value['adminPassword'] + '\n登录后请在个人账户修改密码。此文件仅记录初始密码,请勿分享。\n')
def serve(database_only=False):
lock = os.open(PRIVATE / 'supervisor.lock', os.O_RDWR | os.O_CREAT, 0o600)
try:
host.fcntl.flock(lock, host.fcntl.LOCK_EX | host.fcntl.LOCK_NB)
except BlockingIOError:
raise RuntimeError('此目录的服务已经启动。') from None
children = []
value = configuration()
state = {'supervisor': record(psutil.Process(), [sys.executable, *sys.argv]),
'token': secrets.token_hex(24), 'ready': False, 'databaseOnly': database_only, 'children': children}
stopped = threading.Event()
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, lambda *_: stopped.set())
def stopping():
if STOP.exists():
try:
if json.loads(STOP.read_text(encoding='utf-8')).get('token') == state['token']:
stopped.set()
except (OSError, ValueError):
pass
return stopped.is_set()
def spawn(name, command):
with (DATA / 'logs' / (name + '.log')).open('ab') as log:
child = subprocess.Popen([str(a) for a in command], cwd=ROOT, env=environment(value),
stdin=subprocess.DEVNULL, stdout=log, stderr=log)
info = record(child, command)
info['name'] = name
children.append(info)
atomic(STATE, state)
return child
def wait_ready(child, check, seconds):
end = time.monotonic() + seconds
while time.monotonic() < end:
if stopping():
raise RuntimeError('启动已停止。')
if child.poll() is not None:
raise RuntimeError('服务启动失败,请查看 runtimedata/logs。')
try:
if check():
return
except (OSError, ValueError, subprocess.TimeoutExpired):
pass
time.sleep(.5)
raise RuntimeError('服务启动超时,请查看 runtimedata/logs。')
try:
for key in (('mysqlPort',) if database_only else ('mysqlPort', 'backendPort', 'webPort')):
free_port(value[key], value['webBind'] if key == 'webPort' else '127.0.0.1')
atomic(STATE, state)
db = DATA / 'mysql'
if not (db / 'auto.cnf').exists():
if db.exists() and any(db.iterdir()):
raise RuntimeError('数据库目录含有未完成的初始化文件,已保留原目录;请查看初始化日志。')
db.mkdir(exist_ok=True)
with (DATA / 'logs/mysql-initialize.log').open('ab') as log:
result = subprocess.run([str(MYSQLD), '--no-defaults', '--initialize-insecure',
'--basedir=' + str(MYSQLD.parent.parent), '--datadir=' + str(db),
'--lower-case-table-names=1'], env=environment(value),
stdout=log, stderr=log, timeout=180)
if result.returncode:
raise RuntimeError('数据库初始化失败,请查看 runtimedata/logs/mysql-initialize.log。')
bootstrap = PRIVATE / 'bootstrap.sql'
fd = os.open(bootstrap, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as stream:
stream.write(f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{value['rootPassword']}';\n"
f"CREATE USER IF NOT EXISTS 'root'@'127.0.0.1' IDENTIFIED BY '{value['rootPassword']}';\n"
"GRANT ALL PRIVILEGES ON *.* TO 'root'@'127.0.0.1' WITH GRANT OPTION;\n"
f"CREATE DATABASE IF NOT EXISTS `{value['mysqlDatabase']}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;\n"
f"CREATE USER IF NOT EXISTS 'quality_app'@'127.0.0.1' IDENTIFIED BY '{value['mysqlPassword']}';\n"
f"GRANT ALL PRIVILEGES ON `{value['mysqlDatabase']}`.* TO 'quality_app'@'127.0.0.1';\n")
mysql_args = [MYSQLD, '--no-defaults', '--basedir=' + str(MYSQLD.parent.parent), '--datadir=' + str(db),
'--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1', '--mysqlx=OFF',
'--lower-case-table-names=1', '--skip-name-resolve', '--skip-log-bin',
'--max-connections=60', '--innodb-buffer-pool-size=' + ('1024M' if database_only else '256M'), '--init-file=' + bootstrap.as_posix()]
if database_only:
# The restored database is unpublished until validation and a clean
# MySQL shutdown finish. Normal startup restores durable defaults.
mysql_args += ['--innodb-redo-log-capacity=1G', '--innodb-flush-log-at-trx-commit=2']
if not IS_WINDOWS:
mysql_args.append('--socket=' + str(PRIVATE / 'mysql.sock'))
database = spawn('mysql', mysql_args)
wait_ready(database, lambda: sql(value, 'SELECT 1;').returncode == 0, 120)
# The server does not reread this file after startup.
bootstrap.unlink(missing_ok=True)
if database_only:
state['ready'] = True
atomic(STATE, state)
while not stopping():
if database.poll() is not None:
raise RuntimeError('恢复数据库服务已退出,请查看日志。')
time.sleep(.25)
return
backend = spawn('backend', [JAVA, '-Djava.awt.headless=true', '-Xms128m', '-Xmx2048m', '-jar', ROOT / 'app/backend.jar'])
def backend_ready():
with OPENER.open(f'http://127.0.0.1:{value["backendPort"]}/actuator/health', timeout=3) as response:
return json.load(response).get('status') == 'UP'
wait_ready(backend, backend_ready, 900)
server = make_server(ROOT / 'app/web', value['webBind'], value['webPort'], value['backendPort'])
server.timeout = .5
state['ready'] = True
write_login(value)
atomic(STATE, state)
print('质量中心已启动:' + ' '.join(addresses(value)), flush=True)
try:
while not stopping():
if database.poll() is not None or backend.poll() is not None:
raise RuntimeError('后台服务已退出,请查看 runtimedata/logs。')
server.handle_request()
finally:
server.server_close()
finally:
state['ready'] = False
atomic(STATE, state)
# Stop backend first, then its isolated AI processes, then the database.
for info in reversed(children):
if info['name'] != 'backend' or not matches(info):
continue
process = psutil.Process(info['pid'])
descendants = process.children(recursive=True)
process.terminate()
try:
process.wait(35)
except psutil.TimeoutExpired:
if matches(info):
process.kill()
for child in descendants:
with contextlib.suppress(psutil.NoSuchProcess):
child.terminate()
with contextlib.suppress(OSError, subprocess.SubprocessError):
subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'],
env=environment(value), cwd=ROOT, capture_output=True, timeout=25)
if any(info['name'] == 'mysql' and matches(info) for info in children):
env = environment(value)
env['MYSQL_PWD'] = value['rootPassword']
with contextlib.suppress(OSError, subprocess.SubprocessError):
subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env,
capture_output=True, timeout=60)
# mysqladmin can return before InnoDB has finished closing files.
for info in children:
if info['name'] == 'mysql' and matches(info):
with contextlib.suppress(psutil.NoSuchProcess, psutil.TimeoutExpired):
psutil.Process(info['pid']).wait(60)
survivors = [info for info in children if matches(info)]
if survivors:
state['children'] = survivors
atomic(STATE, state)
else:
STATE.unlink(missing_ok=True)
STOP.unlink(missing_ok=True)
(PRIVATE / 'bootstrap.sql').unlink(missing_ok=True)
os.close(lock)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'serve'))
parser.add_argument('--lan', action='store_true')
parser.add_argument('--database-only', action='store_true', help=argparse.SUPPRESS)
args = parser.parse_args()
if args.action in ('start', 'serve', 'verify'):
private_directory(PRIVATE)
private_directory(DATA)
(DATA / 'logs').mkdir(exist_ok=True)
(DATA / 'sources').mkdir(exist_ok=True)
(DATA / 'archive').mkdir(exist_ok=True)
if args.action == 'serve':
return serve(args.database_only)
updater = ROOT / 'deploy/offline/update_manager.py'
if args.action == 'start' and IS_WINDOWS and updater.is_file():
recovered = subprocess.run([str(PYTHON), str(updater), 'recover'], cwd=ROOT,
env=environment(configuration()), capture_output=True, text=True,
encoding='utf-8')
if recovered.returncode:
try:
message = json.loads(recovered.stdout).get('error')
except (ValueError, AttributeError):
message = None
raise RuntimeError(message or '程序更新正在进行或尚未安全恢复,请查看更新记录。')
state = read_state()
running = matches(state.get('supervisor', {}))
if args.action == 'status':
print('已启动' if running and state.get('ready') else '正在启动或停止' if running else '已停止')
return
if args.action == 'stop':
if not running:
if any(matches(info) for info in state.get('children', [])):
raise RuntimeError('发现上次退出后遗留的进程,请根据日志检查;未结束任何不明进程。')
print('服务已停止。')
return
atomic(STOP, {'token': state['token']})
for _ in range(240):
if not matches(state['supervisor']):
print('服务已停止。')
return
time.sleep(.5)
raise RuntimeError('服务仍在安全关闭,请稍后检查状态。')
if running:
if args.lan and configuration()['webBind'] != '0.0.0.0':
raise RuntimeError('请先停止服务,再使用 --lan 启动。')
print('服务已启动。' if state.get('ready') else '服务正在启动,请稍候。')
for url in addresses(configuration()):
print(url)
return
if any(matches(info) for info in state.get('children', [])):
raise RuntimeError('发现上次退出后的服务进程,原数据已保留;请检查日志。')
value = verify()
if args.action == 'verify':
print('文件完整性、Java、MySQL、AI Python 与 AI 原生程序检查通过。')
return
if args.lan and value['webBind'] != '0.0.0.0':
value['webBind'] = '0.0.0.0'
atomic(CONFIG, value)
with (DATA / 'logs/supervisor.log').open('ab') as log:
options = {'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP} if IS_WINDOWS else {'start_new_session': True}
process = subprocess.Popen([str(PYTHON), str(Path(__file__).resolve()), 'serve'], cwd=ROOT,
env=environment(value), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **options)
for _ in range(2000):
state = read_state()
if state.get('ready') and state.get('supervisor', {}).get('pid') == process.pid and matches(state['supervisor']):
for url in addresses(value):
print(url)
print('管理员初始登录信息:' + str(DATA / '登录信息.txt'))
if IS_WINDOWS:
os.startfile(addresses(value)[0])
return
if process.poll() is not None:
raise RuntimeError('启动未完成,请查看 runtimedata/logs/supervisor.log。')
time.sleep(.5)
raise RuntimeError('服务仍在启动,请检查状态与日志。')
if __name__ == '__main__':
try:
main()
except (RuntimeError, OSError, subprocess.SubprocessError) as error:
print(str(error), file=sys.stderr)
raise SystemExit(1)
+102
View File
@@ -0,0 +1,102 @@
"""Serve built assets and stream the authenticated API on one LAN port."""
from __future__ import annotations
import functools
import http.client
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import urlsplit
HOP_HEADERS = {'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization',
'te', 'trailer', 'transfer-encoding', 'upgrade'}
class Gateway(SimpleHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
backend_port = 5281
extensions_map = {**SimpleHTTPRequestHandler.extensions_map, '.js': 'text/javascript',
'.mjs': 'text/javascript', '.css': 'text/css', '.svg': 'image/svg+xml',
'.wasm': 'application/wasm'}
def log_message(self, *_):
# Do not record query strings, authentication data or business values.
pass
def _proxy(self):
upstream = http.client.HTTPConnection('127.0.0.1', self.backend_port, timeout=1800)
try:
if self.headers.get('Transfer-Encoding'):
self.send_error(400, 'Content-Length is required')
return
length = int(self.headers.get('Content-Length', '0'))
if length < 0 or length > 273 * 1024 * 1024:
self.send_error(413)
return
upstream.putrequest(self.command, self.path, skip_host=True, skip_accept_encoding=True)
connection_headers = {s.strip().lower() for s in self.headers.get('Connection', '').split(',')}
for name, value in self.headers.items():
if name.lower() not in HOP_HEADERS | connection_headers:
upstream.putheader(name, value)
upstream.putheader('Connection', 'close')
upstream.endheaders()
remaining = length
while remaining:
chunk = self.rfile.read(min(remaining, 1024 * 1024))
if not chunk:
raise ConnectionError('Incomplete upload')
upstream.send(chunk)
remaining -= len(chunk)
response = upstream.getresponse()
self.send_response(response.status)
response_connection = {s.strip().lower() for s in response.getheader('Connection', '').split(',')}
for name, value in response.getheaders():
if name.lower() not in HOP_HEADERS | response_connection:
self.send_header(name, value)
self.send_header('Connection', 'close')
self.send_header('X-Content-Type-Options', 'nosniff')
self.end_headers()
self.close_connection = True
if self.command != 'HEAD':
while chunk := response.read1(64 * 1024):
self.wfile.write(chunk)
self.wfile.flush() # SSE events must arrive without buffering.
except (BrokenPipeError, ConnectionResetError):
self.close_connection = True
except (OSError, http.client.HTTPException, ValueError):
self.close_connection = True
# Never expose upstream exception text or an internal file path.
if not self.wfile.closed:
try:
self.send_error(502, 'Backend unavailable')
except OSError:
pass
finally:
upstream.close()
def _dispatch(self):
path = urlsplit(self.path).path
if path == '/api' or path.startswith('/api/') or path == '/actuator/health':
return self._proxy()
if self.command not in ('GET', 'HEAD'):
self.send_error(405)
return
candidate = Path(self.translate_path(self.path)).resolve()
root = Path(self.directory).resolve()
if not candidate.is_relative_to(root):
self.send_error(403)
return
if candidate.is_dir() and not (candidate / 'index.html').is_file():
self.send_error(404)
return
if not candidate.exists() and not Path(path).suffix:
self.path = '/index.html'
return super().do_HEAD() if self.command == 'HEAD' else super().do_GET()
do_GET = do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = _dispatch
def make_server(root, address, port, backend_port=5281):
handler = type('ConfiguredGateway', (Gateway,), {'backend_port': backend_port})
server = ThreadingHTTPServer((address, port), functools.partial(handler, directory=str(root)))
server.daemon_threads = True
return server
+154
View File
@@ -0,0 +1,154 @@
"""Portable migration validation; no source-machine paths or credentials are logged."""
from __future__ import annotations
import gzip
import hashlib
import json
from pathlib import Path, PurePosixPath, PureWindowsPath
import re
import tarfile
import uuid
def sha256(path):
with Path(path).open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def sql_text(value):
return 'CONVERT(0x' + value.encode('utf-8').hex() + ' USING utf8mb4)'
def relative_name(name):
if not isinstance(name, str) or not name or '\\' in name or ':' in name:
raise ValueError('迁移文件路径无效。')
path = PurePosixPath(name)
if not path.parts or path.is_absolute() or path.as_posix() != name or any(part in ('.', '..', '') for part in path.parts):
raise ValueError('迁移文件路径无效。')
for part in path.parts:
if part.endswith((' ', '.')) or re.fullmatch(r'(?i)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(?:\..*)?', part):
raise ValueError('迁移文件名不兼容 Windows。')
return path
def path_under_source(stored, source_root):
# A Windows restore must interpret the old Mac path as POSIX, not Path().
path, root = PurePosixPath(stored), PurePosixPath(source_root)
if not path.is_absolute() or '..' in path.parts or not path.is_relative_to(root):
raise ValueError('来源文件引用不在已验证的归档目录内。')
return relative_name(path.relative_to(root).as_posix()).as_posix()
def extract_checked(archive, destination, expected):
destination = Path(destination)
destination.mkdir(parents=True, exist_ok=False, mode=0o700)
observed = set()
for name in expected:
relative_name(name)
folded = [name.casefold() for name in expected]
if len(folded) != len(set(folded)):
raise ValueError('迁移文件在 Windows 上存在重名。')
with tarfile.open(archive, 'r:gz') as source:
for member in source:
rel = relative_name(member.name)
if not member.isfile() or member.name in observed or member.name not in expected:
raise ValueError('迁移归档包含未登记或无效的文件。')
entry = expected[member.name]
target = destination.joinpath(*rel.parts)
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
if not target.resolve().is_relative_to(destination.resolve()):
raise ValueError('迁移文件路径越界。')
content = source.extractfile(member)
if content is None:
raise ValueError('迁移文件无法读取。')
digest = hashlib.sha256()
with content, target.open('xb') as output:
while chunk := content.read(1024 * 1024):
digest.update(chunk)
output.write(chunk)
target.chmod(0o600)
if target.stat().st_size != entry['bytes'] or digest.hexdigest() != entry['sha256']:
raise ValueError('迁移文件完整性检查失败。')
observed.add(member.name)
if observed != set(expected):
raise ValueError('迁移归档缺少文件。')
def table_counts(query):
result = {}
for name in query('SHOW TABLES;').splitlines():
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', name):
raise ValueError('数据库表名无效。')
result[name] = int(query('SELECT COUNT(*) FROM `' + name + '`;').strip())
return result
def query_evidence(query):
current = query("SELECT release_id FROM current_release WHERE dataset_id='quality';").strip()
if not current:
return {'releaseId': None, 'months': []}
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
raise ValueError('正式数据版本无效。')
result = {'releaseId': current, 'months': []}
components = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap',
'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap',
'bright_processing_scrap', 'bright_diffusion_scrap']
for stage in ('black', 'bright'):
pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*),'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)"
for key in components:
pairs += ", '" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))"
result['months'].extend(json.loads(line) for line in query("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;").splitlines())
result['members'] = [json.loads(line) for line in query("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;").splitlines()]
return result
_VALUES_TOKENS = re.compile(rb"'(?:[^'\\]+|\\.)*'|(\),\()")
def inserted_rows(line):
if not line.startswith(b'INSERT INTO `') or b' VALUES (' not in line:
raise ValueError('不支持的数据库导出行。')
values = line.split(b' VALUES ', 1)[1]
if not values.rstrip().endswith(b';'):
raise ValueError('数据库导出行不完整。')
return 1 + sum(match[1] is not None for match in _VALUES_TOKENS.finditer(values))
def dump_counts(path):
counts, uncompressed = {}, 0
with gzip.open(path, 'rb') as source:
for line in source:
uncompressed += len(line)
create = re.match(rb'^CREATE TABLE `([A-Za-z0-9_]+)`', line)
if create:
counts[create[1].decode()] = 0
insert = re.match(rb'^INSERT INTO `([A-Za-z0-9_]+)`', line)
if insert:
name = insert[1].decode()
if name not in counts:
raise ValueError('导出数据缺少表结构。')
counts[name] += inserted_rows(line)
return counts, uncompressed
def filter_models(value):
profiles = value.get('profiles')
if not isinstance(profiles, list):
profiles = [{**value, 'id': 'migrated-profile', 'name': '迁移模型'}]
def excluded(profile):
return any('mimo' in str(profile.get(key, '')).lower() for key in ('model', 'name', 'baseUrl'))
kept = [profile.copy() for profile in profiles if not excluded(profile)]
removed = [profile for profile in profiles if excluded(profile)]
if not kept:
return None, removed
selected = next((profile for profile in kept if profile.get('id') == value.get('activeProfileId')), kept[0])
required = ('baseUrl', 'apiKey', 'model')
if any(not isinstance(selected.get(key), str) or not selected[key] for key in required):
raise ValueError('需要迁移的模型配置不完整。')
filtered = {key: value[key] for key in ('schemaVersion',) if key in value}
filtered.update(selected)
filtered.pop('id', None)
filtered.pop('name', None)
filtered.update({'schemaVersion': 3, 'profiles': kept, 'activeProfileId': selected['id']})
filtered['revision'] = value.get('revision') or str(uuid.uuid4())
return filtered, removed
+200
View File
@@ -0,0 +1,200 @@
"""Capture verified source files and private model settings for a Windows migration."""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import sys
import tarfile
import uuid
from migration_common import filter_models, relative_name, sha256
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import backup
def private_json(path, data):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as output:
json.dump(data, output, ensure_ascii=False, indent=2)
output.write('\n')
def native_copy(source, target):
source_db = sqlite3.connect(source.resolve().as_uri() + '?mode=ro', uri=True)
destination = sqlite3.connect(target)
try:
source_db.backup(destination)
destination.execute('PRAGMA foreign_keys=OFF')
destination.execute('PRAGMA secure_delete=ON')
tables = {row[0] for row in destination.execute("SELECT name FROM sqlite_master WHERE type='table'")}
# Access sessions, external integrations and cached provider keys are
# recreated on Windows. Historical chats and their ownership stay intact.
for name in ('session', 'verification', 'api_key', 'jwks', 'oauth_access_token',
'oauth_refresh_token', 'oauth_consent', 'oauth_client', 'mcp_user_token',
'mcp_oauth_client', 'sandbox_secret', 'log', 'mcp_call_log'):
if name in tables:
destination.execute('DELETE FROM "' + name + '"')
if 'project_llm_config' in tables:
destination.execute('DELETE FROM project_llm_config')
for name in ('user', 'account', 'organization'):
if name not in tables:
continue
columns = list(destination.execute('PRAGMA table_info("' + name + '")'))
for column in columns:
key = column[1]
if key in ('github_access_token', 'gitlab_access_token', 'access_token', 'refresh_token',
'id_token', 'google_client_secret'):
replacement = "''" if column[3] else 'NULL'
destination.execute('UPDATE "' + name + '" SET "' + key + '"=' + replacement)
destination.commit()
destination.execute('VACUUM')
assert destination.execute('PRAGMA integrity_check').fetchone()[0] == 'ok'
finally:
destination.close()
source_db.close()
target.chmod(0o600)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--work', type=Path, required=True)
args = parser.parse_args()
work = args.work.resolve()
payload = work / 'migration'
manifest = json.loads((payload / 'source-snapshot.json').read_text())
database = manifest['sourceDatabase']
query = lambda sql: backup.client(sql, database)
archive_root = ROOT / 'runtimedata/archive'
files, sources, digest_index = {}, {}, {}
for path in sorted(archive_root.rglob('*')):
if path.is_symlink():
raise RuntimeError('来源归档包含符号链接。')
if not path.is_file():
continue
rel = relative_name(path.relative_to(archive_root).as_posix()).as_posix()
digest = sha256(path)
files[rel] = {'sha256': digest, 'bytes': path.stat().st_size}
sources[rel] = path
digest_index.setdefault(digest, rel)
bindings = []
missing_optional = 0
for table, column, state in (
('source_revision', 'archive_path', "'REQUIRED'"),
('source_candidate', 'source_path', 'status'),
('source_candidate', 'archive_path', 'status'),
('source_upload_draft', 'source_path', "IF(expires_at<CURRENT_TIMESTAMP(6),'EXPIRED','ACTIVE')"),
):
rows = query("SELECT JSON_OBJECT('id',id,'path'," + column + ",'sha256',sha256,'size',file_size,'state'," + state + ') FROM ' + table + ' WHERE ' + column + ' IS NOT NULL;')
for line in rows.splitlines():
row = json.loads(line)
old = Path(row['path'])
digest = row['sha256']
relative = None
if old.is_relative_to(archive_root) and old.is_file():
relative = old.relative_to(archive_root).as_posix()
elif digest in digest_index:
relative = digest_index[digest]
elif old.is_file() and not old.is_symlink():
if sha256(old) != digest or old.stat().st_size != int(row['size']):
raise RuntimeError('需要迁移的来源文件与登记内容不一致。')
relative = 'migration-inputs/' + digest + '.xlsx'
files[relative] = {'sha256': digest, 'bytes': old.stat().st_size}
sources[relative] = old
digest_index[digest] = relative
optional = False
if relative is None:
if table != 'source_upload_draft' or row['state'] != 'EXPIRED':
raise RuntimeError('需要迁移的来源文件缺失。')
relative = 'expired-upload-drafts/' + row['id'] + '.xlsx'
optional = True
missing_optional += 1
else:
info = files[relative]
if info['sha256'] != digest or info['bytes'] != int(row['size']):
raise RuntimeError('来源引用与归档校验不一致。')
bindings.append({'table': table, 'column': column, 'id': row['id'], 'oldPath': row['path'],
'relative': relative, 'sha256': digest, 'bytes': int(row['size']), 'expiredMissing': optional})
archive = payload / 'archive.tar.gz'
with tarfile.open(archive, 'w:gz', compresslevel=1) as output:
for rel, path in sorted(sources.items()):
output.add(path, arcname=rel, recursive=False)
if sha256(path) != files[rel]['sha256']:
raise RuntimeError('打包期间来源文件发生变化。')
archive.chmod(0o600)
print('Source files captured:', len(files), 'references:', len(bindings), flush=True)
config = json.loads((ROOT / '.runtime/nao-model-config.json').read_text())
filtered, excluded = filter_models(config)
if filtered:
private_json(payload / 'model-config.json', filtered)
kept_keys = {p.get('apiKey') for p in (filtered or {}).get('profiles', [])}
forbidden_keys = {p.get('apiKey') for p in excluded if p.get('apiKey') and p.get('apiKey') not in kept_keys}
# Used only by the local privacy verifier; never shipped or printed.
private_json(work / 'excluded-model-audit-private.json', {'keys': sorted(forbidden_keys)})
session_rows = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'userId',user_id) FROM analysis_session;").splitlines()]
users = {row['id'] for row in manifest['sourceUsers']}
native_source = ROOT / '.runtime/nao-users'
native_stage = work / 'native-snapshot'
native_stage.mkdir(mode=0o700, exist_ok=False)
runtime_roots = []
for actor in users:
if not re.fullmatch(r'[a-fA-F0-9-]{36}', actor):
raise RuntimeError('用户标识无效。')
if (native_source / actor / 'private/nao.sqlite').is_file():
runtime_roots.append(native_source / actor)
for row in session_rows:
source = native_source / row['userId'] / 'sessions' / row['id']
if (source / 'private/nao.sqlite').is_file():
runtime_roots.append(source)
selected = set()
for runtime in runtime_roots:
for name in ('credentials.json', 'nao.sqlite', 'native-chat.json', 'current-session.json', 'sessions.json'):
source = runtime / 'private' / name
if source.is_file() and not source.is_symlink():
selected.add(source)
evidence = runtime / 'evidence'
if evidence.is_dir():
selected.update(p for p in evidence.rglob('*') if p.is_file() and not p.is_symlink())
native_files = {}
for source in sorted(selected):
rel = source.relative_to(native_source).as_posix()
relative_name(rel)
target = native_stage / rel
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
if source.name == 'nao.sqlite':
native_copy(source, target)
else:
shutil.copy2(source, target)
target.chmod(0o600)
content = target.read_bytes()
if any(key.encode() in content for key in forbidden_keys):
raise RuntimeError('原生会话中检测到被排除模型的密钥,已停止打包。')
native_files[rel] = {'sha256': sha256(target), 'bytes': target.stat().st_size}
with tarfile.open(payload / 'native-history.tar.gz', 'w:gz', compresslevel=1) as output:
for rel in sorted(native_files):
output.add(native_stage / rel, arcname=rel, recursive=False)
(payload / 'native-history.tar.gz').chmod(0o600)
manifest.update({'migrationId': uuid.uuid4().hex, 'archiveFiles': files, 'pathBindings': bindings,
'archiveSha256': sha256(archive), 'nativeHistoryFiles': native_files,
'nativeHistorySha256': sha256(payload / 'native-history.tar.gz'),
'sourceProjectRoot': str(ROOT), 'modelProfilesMigrated': len((filtered or {}).get('profiles', [])),
'modelProfilesExcluded': len(excluded), 'modelConfigSha256': sha256(payload / 'model-config.json') if filtered else None,
'expiredMissingDrafts': missing_optional,
'sourceDatabaseBytes': int(backup.client("SELECT SUM(data_length+index_length) FROM information_schema.tables WHERE table_schema='" + database + "';").strip()),
'nativeHistoryDatabases': len(runtime_roots)})
private_json(work / 'migration-file-snapshot.json', manifest)
print('Model profiles migrated:', manifest['modelProfilesMigrated'], '; MiMo excluded:', len(excluded), flush=True)
print('Native business-history databases:', len(runtime_roots), '; files:', len(native_files), flush=True)
if __name__ == '__main__':
main()
+288
View File
@@ -0,0 +1,288 @@
#!/usr/bin/env python3
"""Build a signed, program-only Windows update bundle without network access."""
from __future__ import annotations
import argparse
import base64
from dataclasses import dataclass
import hashlib
import os
from pathlib import Path
import re
import stat
import tempfile
import zipfile
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
import update_manager
VERSION = re.compile(r'[A-Za-z0-9][A-Za-z0-9._-]{0,79}')
ARCHIVE_VERSION = re.compile(r'[A-Za-z0-9][A-Za-z0-9._-]{0,100}')
REQUIRED_PROGRAM_FILES = {
'app/backend.jar',
'deploy/offline/control.py',
'deploy/offline/update_manager.py',
'backend/src/main/resources/analysis-runtime-policy.json',
}
SENSITIVE_PARTS = {'.runtime', 'runtimedata', 'migration', 'private', 'secrets', 'credentials'}
SENSITIVE_SUFFIXES = {'.pem', '.key', '.p12', '.pfx', '.jks'}
class PublishError(RuntimeError):
"""Release inputs cannot produce a safe update package."""
@dataclass(frozen=True)
class PublishedArtifacts:
archive: Path
latest: Path
file_count: int
archive_sha256: str
def _manifest(root: Path) -> dict:
try:
value = update_manager.read(root / 'manifest.json')
except Exception as error:
raise PublishError('无法读取部署包 manifest.json。') from error
if not isinstance(value, dict) or value.get('platform') != 'windows' or value.get('architecture') != 'x86_64':
raise PublishError('输入目录必须是 Windows x64 离线部署包。')
if not isinstance(value.get('files'), list) or len(value['files']) > 50000:
raise PublishError('部署包文件清单无效。')
for key in ('runtimes', 'pythonWheels'):
if not isinstance(value.get(key), list):
raise PublishError('部署包缺少运行环境指纹信息。')
current_sequence = value.get('updateSequence', 0)
if type(current_sequence) is not int or current_sequence < 0:
raise PublishError('部署包更新序号无效。')
return value
def _sensitive_path(name: str) -> bool:
path = update_manager.safe_path(name)
parts = [part.casefold() for part in path.parts]
return (name == update_manager.KEY
or name.startswith('deploy/bootstrap/')
or any(part in SENSITIVE_PARTS for part in parts)
or path.suffix.casefold() in SENSITIVE_SUFFIXES)
def _program_inventory(root: Path, manifest: dict) -> list[dict]:
entries: dict[str, dict] = {}
folded: set[str] = set()
for entry in manifest['files']:
if not isinstance(entry, dict):
raise PublishError('部署包文件清单无效。')
name = entry.get('path')
try:
update_manager.safe_path(name)
except update_manager.UpdateError as error:
raise PublishError('部署包文件清单含有无效路径。') from error
if name.casefold() in folded:
raise PublishError('部署包文件清单含有重复或大小写冲突路径。')
folded.add(name.casefold())
if not update_manager.program_path(name):
continue
if _sensitive_path(name):
raise PublishError('程序清单中包含受保护的私有、迁移、引导或密钥文件路径。')
if entry.get('symlink') is not None or not update_manager.SHA.fullmatch(str(entry.get('sha256', ''))):
raise PublishError('程序文件清单缺少有效摘要或包含符号链接。')
path = update_manager.destination(root, name)
if not path.is_file() or path.is_symlink():
raise PublishError('程序清单文件缺失或不是普通文件。')
try:
digest = update_manager.digest(path)
except OSError as error:
raise PublishError('读取程序清单文件失败。') from error
if digest != entry['sha256']:
raise PublishError('部署包程序文件与 manifest.json 摘要不一致。')
entries[name] = {'path': name, 'sha256': digest, 'size': path.stat().st_size}
if not REQUIRED_PROGRAM_FILES.issubset(entries) or not any(name.startswith('app/web/') for name in entries):
raise PublishError('程序清单缺少更新所需的后端、网页、策略或更新管理程序。')
# The updater replaces program files as a complete snapshot. Refuse to
# sign an incomplete manifest if a program file exists outside its list.
present: set[str] = set()
for path in root.rglob('*'):
name = path.relative_to(root).as_posix()
if not update_manager.program_path(name):
continue
if path.is_symlink():
raise PublishError('程序目录含有符号链接或特殊文件。')
if path.is_dir():
continue
if not path.is_file():
raise PublishError('程序目录含有符号链接或特殊文件。')
present.add(name)
if present != set(entries):
raise PublishError('manifest.json 与磁盘上的完整程序文件清单不一致。')
return [entries[name] for name in sorted(entries)]
def _load_private_key(path: Path, root: Path, output_dir: Path) -> Ed25519PrivateKey:
try:
resolved = path.expanduser().resolve(strict=True)
if resolved.is_relative_to(root) or resolved.is_relative_to(output_dir):
raise PublishError('签名私钥必须保存在部署包和更新产物目录之外。')
key = serialization.load_pem_private_key(resolved.read_bytes(), password=None)
except PublishError:
raise
except Exception as error:
raise PublishError('无法读取未加密 PEM 格式的 Ed25519 签名私钥。') from error
if not isinstance(key, Ed25519PrivateKey):
raise PublishError('签名私钥必须是 Ed25519。')
return key
def _write_archive(root: Path, archive_temp: Path, files: list[dict]) -> None:
with zipfile.ZipFile(archive_temp, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=9, allowZip64=True) as output:
for entry in files:
name = entry['path']
source = update_manager.destination(root, name)
info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
info.compress_type = zipfile.ZIP_DEFLATED
info.create_system = 3
info.external_attr = (stat.S_IFREG | 0o644) << 16
digest = hashlib.sha256()
size = 0
with source.open('rb') as reader, output.open(info, 'w') as writer:
while block := reader.read(1024 * 1024):
size += len(block)
if size > entry['size'] or size > update_manager.MAX_EXPANDED:
raise PublishError('程序文件在打包期间发生变化。')
digest.update(block)
writer.write(block)
if size != entry['size'] or digest.hexdigest() != entry['sha256']:
raise PublishError('程序文件在打包期间发生变化,已停止发布。')
def publish(root: Path, version: str, sequence: int, private_key_path: Path, output_dir: Path,
allow_baseline: bool = False) -> PublishedArtifacts:
"""Create quality-update-<version>.zip and signed latest.json locally."""
root = root.expanduser().resolve(strict=True)
if not root.is_dir():
raise PublishError('部署包根路径必须是目录。')
if not isinstance(version, str) or not VERSION.fullmatch(version):
raise PublishError('版本号只能包含字母、数字、点、下划线和连字符。')
if not ARCHIVE_VERSION.fullmatch(version):
raise PublishError('版本号不能用于更新文件名。')
if type(sequence) is not int or sequence <= 0:
raise PublishError('更新序号必须是正整数。')
manifest = _manifest(root)
current_sequence = manifest['updateSequence']
if sequence < current_sequence:
raise PublishError('更新序号不能低于部署包当前序号。')
if sequence == current_sequence and not allow_baseline:
raise PublishError('更新序号必须大于部署包当前序号。')
if sequence == current_sequence and version != manifest.get('updateVersion'):
raise PublishError('基线版本必须与部署包当前 updateVersion 完全一致。')
files = _program_inventory(root, manifest)
if sum(entry['size'] for entry in files) > update_manager.MAX_EXPANDED:
raise PublishError('更新程序文件总大小超过限制。')
output_dir = output_dir.expanduser().resolve()
if output_dir.is_relative_to(root):
raise PublishError('更新产物目录必须位于部署包目录之外。')
private_key = _load_private_key(Path(private_key_path), root, output_dir)
public_key_path = update_manager.destination(root, update_manager.KEY)
try:
public_key = serialization.load_pem_public_key(public_key_path.read_bytes())
except Exception as error:
raise PublishError('无法读取部署包中的更新公钥。') from error
if not isinstance(public_key, Ed25519PublicKey) or public_key.public_bytes(
serialization.Encoding.Raw, serialization.PublicFormat.Raw) != private_key.public_key().public_bytes(
serialization.Encoding.Raw, serialization.PublicFormat.Raw):
raise PublishError('签名私钥与部署包中的更新公钥不匹配。')
output_dir.mkdir(parents=True, exist_ok=True)
archive_name = f'quality-update-{version}.zip'
archive_path = output_dir / archive_name
latest_path = output_dir / 'latest.json'
if archive_path.exists() or latest_path.exists():
raise PublishError('输出目录已包含同名发布文件;请使用新的空输出目录。')
archive_temp: Path | None = None
latest_temp: Path | None = None
published_archive = False
try:
with tempfile.NamedTemporaryFile(prefix='.quality-update-', suffix='.tmp', dir=output_dir, delete=False) as stream:
archive_temp = Path(stream.name)
_write_archive(root, archive_temp, files)
archive_size = archive_temp.stat().st_size
if not 0 < archive_size <= update_manager.MAX_ARCHIVE:
raise PublishError('更新压缩包大小超过限制。')
archive_hash = update_manager.digest(archive_temp)
release = {
'schema': 1,
'version': version,
'sequence': sequence,
'platform': 'windows-x64',
'runtimeFingerprint': update_manager.runtime_fingerprint(manifest),
'archive': {'file': archive_name, 'size': archive_size, 'sha256': archive_hash},
'files': files,
}
raw = update_manager.canonical(release)
if len(raw) > update_manager.MAX_MANIFEST:
raise PublishError('更新文件清单超过大小限制。')
envelope = {
'payload': base64.b64encode(raw).decode('ascii'),
'signature': base64.b64encode(private_key.sign(raw)).decode('ascii'),
}
try:
update_manager.validate_release(envelope, root)
except update_manager.UpdateError as error:
raise PublishError('生成的更新包未通过客户端契约校验:' + str(error)) from error
latest_bytes = update_manager.canonical(envelope)
with tempfile.NamedTemporaryFile(prefix='.latest-', suffix='.tmp', dir=output_dir, delete=False) as stream:
latest_temp = Path(stream.name)
stream.write(latest_bytes)
stream.flush()
os.fsync(stream.fileno())
os.link(archive_temp, archive_path)
archive_temp.unlink()
archive_temp = None
published_archive = True
os.link(latest_temp, latest_path)
latest_temp.unlink()
latest_temp = None
return PublishedArtifacts(archive_path, latest_path, len(files), archive_hash)
except FileExistsError as error:
raise PublishError('输出目录出现同名发布文件;请使用新的空输出目录。') from error
except OSError as error:
raise PublishError('写入更新产物失败。') from error
finally:
if archive_temp is not None:
archive_temp.unlink(missing_ok=True)
if latest_temp is not None:
latest_temp.unlink(missing_ok=True)
if published_archive and not latest_path.exists():
archive_path.unlink(missing_ok=True)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--root', type=Path, required=True, help='Windows x64 offline bundle directory')
parser.add_argument('--version', required=True, help='Release version, for example 2026.11.04.2')
parser.add_argument('--sequence', type=int, required=True, help='Monotonically increasing update sequence')
parser.add_argument('--private-key', type=Path, required=True, help='Ed25519 PEM private key kept outside the bundle')
parser.add_argument('--output-dir', type=Path, required=True, help='New output directory for the ZIP and latest.json')
parser.add_argument('--allow-baseline', action='store_true',
help='Allow a seed release only when version and sequence exactly match the bundle manifest')
args = parser.parse_args(argv)
try:
result = publish(args.root, args.version, args.sequence, args.private_key, args.output_dir, args.allow_baseline)
except (PublishError, update_manager.UpdateError) as error:
parser.error(str(error))
print(f'已生成更新包:{result.archive}')
print(f'已生成签名清单:{result.latest}')
print(f'版本 {args.version},序号 {args.sequence},程序文件 {result.file_count} 个,SHA-256 {result.archive_sha256}')
return 0
if __name__ == '__main__':
raise SystemExit(main())
+327
View File
@@ -0,0 +1,327 @@
"""Restore into a new database, validate, then switch the Windows deployment."""
from __future__ import annotations
from datetime import datetime, timezone
import gzip
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import sqlite3
import subprocess
import sys
import time
import uuid
import control
from migration_common import extract_checked, query_evidence, relative_name, sha256, sql_text, table_counts
ROOT = control.ROOT
PAYLOAD = ROOT / 'migration'
JOURNAL = control.PRIVATE / 'migration-progress.json'
RECEIPT = control.PRIVATE / 'migration-receipt.json'
def checked_local(relative):
rel = relative_name(relative)
path = ROOT.joinpath(*rel.parts)
if not path.resolve().is_relative_to(ROOT.resolve()):
raise RuntimeError('恢复暂存路径越界。')
return path
def run_query(value, statement, database=None):
command = control.mysql_command(value)
command += ['--batch', '--skip-column-names', '--default-character-set=utf8mb4', '--max-allowed-packet=1G']
if database:
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', database):
raise RuntimeError('恢复数据库名称无效。')
command += ['--database=' + database]
env = control.environment(value)
env['MYSQL_PWD'] = value['rootPassword']
result = subprocess.run(command, input=statement, text=True, encoding='utf-8',
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=1800)
if result.returncode:
raise RuntimeError('恢复数据库校验未完成;已有数据库保留。')
return result.stdout
def stop_owned_service():
state = control.read_state()
if control.matches(state.get('supervisor', {})):
control.atomic(control.STOP, {'token': state['token']})
for _ in range(300):
if not control.matches(state['supervisor']):
break
time.sleep(.5)
else:
raise RuntimeError('原服务仍在关闭,请稍后重新运行。')
if any(control.matches(info) for info in state.get('children', [])):
raise RuntimeError('发现本目录尚未关闭的后台进程,请检查服务日志。')
def start_database(value):
with (control.DATA / 'logs/migration-database-supervisor.log').open('ab') as log:
options = ({'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP}
if control.IS_WINDOWS else {'start_new_session': True})
process = subprocess.Popen([str(control.PYTHON), str(Path(control.__file__).resolve()),
'serve', '--database-only'], cwd=ROOT, env=control.environment(value),
stdin=subprocess.DEVNULL, stdout=log, stderr=log, **options)
for _ in range(360):
state = control.read_state()
if (state.get('ready') and state.get('databaseOnly') and
state.get('supervisor', {}).get('pid') == process.pid and control.matches(state['supervisor'])):
return
if process.poll() is not None:
raise RuntimeError('恢复数据库无法启动,请查看 runtimedata/logs。')
time.sleep(.5)
raise RuntimeError('恢复数据库仍未就绪,请查看日志后重试。')
def restore_dump(value, database, dump):
env = control.environment(value)
env['MYSQL_PWD'] = value['rootPassword']
command = control.mysql_command(value) + ['--database=' + database,
'--default-character-set=utf8mb4', '--max-allowed-packet=1G',
'--init-command=SET SESSION autocommit=0']
with (control.DATA / 'logs/migration-import.log').open('ab') as errors:
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors, env=env)
try:
total, next_progress = dump.stat().st_size, 0
with dump.open('rb') as raw, gzip.GzipFile(fileobj=raw) as source:
while chunk := source.read(1024 * 1024):
process.stdin.write(chunk)
percent = min(99, raw.tell() * 100 // total)
if percent >= next_progress:
print(f'数据库导入进度约 {percent}%……', flush=True)
next_progress = percent + 10
process.stdin.write(b'\nCOMMIT;\n')
process.stdin.close()
if process.wait(timeout=1800):
raise RuntimeError('业务数据导入失败,请查看私有恢复日志;原数据库未覆盖。')
except BaseException:
if process.poll() is None:
process.kill()
process.wait()
raise
def rebind_sources(query, manifest, archive):
counts = {'source_revision': 0, 'source_candidate': 0, 'source_upload_draft': 0}
for entry in manifest['pathBindings']:
pair = (entry['table'], entry['column'])
if pair not in {('source_revision', 'archive_path'), ('source_candidate', 'source_path'),
('source_candidate', 'archive_path'), ('source_upload_draft', 'source_path')}:
raise RuntimeError('来源路径绑定类型无效。')
rel = relative_name(entry['relative'])
target = archive.joinpath(*rel.parts)
if not entry['expiredMissing']:
if not target.is_file() or target.stat().st_size != entry['bytes'] or sha256(target) != entry['sha256']:
raise RuntimeError('来源文件校验失败,恢复未切换。')
query('UPDATE `' + entry['table'] + '` SET `' + entry['column'] + '`=' + sql_text(str(target))
+ ' WHERE id=' + sql_text(entry['id']) + ' AND `' + entry['column'] + '`=' + sql_text(entry['oldPath']) + ';')
actual = query('SELECT `' + entry['column'] + '` FROM `' + entry['table'] + '` WHERE id=' + sql_text(entry['id']) + ';').strip()
if actual != str(target):
raise RuntimeError('来源文件路径重绑定失败。')
counts[entry['table']] += 1
return counts
def relocate_native(stage, source_root):
source_root = PurePosixPath(source_root)
def relocated(value):
if isinstance(value, str) and value.startswith(str(source_root) + '/'):
relative = PurePosixPath(value).relative_to(source_root)
if '..' in relative.parts:
raise RuntimeError('原生历史路径无效。')
return str(ROOT.joinpath(*relative.parts))
if isinstance(value, dict):
return {key: relocated(child) for key, child in value.items()}
if isinstance(value, list):
return [relocated(child) for child in value]
return value
for path in stage.rglob('*.json'):
control.atomic(path, relocated(json.loads(path.read_text(encoding='utf-8'))))
databases = 0
for path in stage.rglob('nao.sqlite'):
connection = sqlite3.connect(path)
try:
connection.execute('PRAGMA secure_delete=ON')
for table, column in (('project', 'path'), ('message_part', 'storage_path')):
exists = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone()
if not exists:
continue
for rowid, original in connection.execute('SELECT rowid,"' + column + '" FROM "' + table + '" WHERE "' + column + '" IS NOT NULL').fetchall():
replacement = relocated(original)
if replacement != original:
connection.execute('UPDATE "' + table + '" SET "' + column + '"=? WHERE rowid=?', (replacement, rowid))
connection.commit()
connection.execute('VACUUM')
if connection.execute('PRAGMA integrity_check').fetchone()[0] != 'ok':
raise RuntimeError('原生分析历史数据库校验失败。')
databases += 1
finally:
connection.close()
return databases
def rollback_plan(plan):
# Only paths generated by this migration are eligible. Databases and source
# archive trees are retained even after an interrupted restore.
attempt = plan['attempt']
if not re.fullmatch(r'[a-f0-9]{16}', attempt):
raise RuntimeError('恢复日志无效。')
for item in plan['privateSwitches']:
target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged'))
failed = checked_local(item['failed'])
if previous.exists():
if target.exists():
target.rename(failed)
previous.rename(target)
elif not item['existed'] and not staged.exists() and target.exists():
target.rename(failed)
control.atomic(control.CONFIG, plan['previousConfig'])
JOURNAL.rename(control.PRIVATE / ('migration-rolled-back-' + attempt + '.json'))
def commit_plan(plan, receipt):
control.atomic(JOURNAL, plan)
try:
for item in plan['privateSwitches']:
target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged'))
if target.is_symlink() or staged.is_symlink() or previous.exists():
raise RuntimeError('私有配置切换路径无效。')
if target.exists():
target.rename(previous)
staged.rename(target)
control.atomic(control.CONFIG, plan['newConfig'])
control.atomic(RECEIPT, receipt)
except BaseException:
rollback_plan(plan)
raise
JOURNAL.unlink(missing_ok=True)
def restore(check_runtime=True):
control.private_directory(control.PRIVATE)
control.private_directory(control.DATA)
(control.DATA / 'logs').mkdir(exist_ok=True)
(control.DATA / 'sources').mkdir(exist_ok=True)
(control.DATA / 'archive').mkdir(exist_ok=True)
lock = os.open(control.PRIVATE / 'migration.lock', os.O_CREAT | os.O_RDWR, 0o600)
try:
control.host.fcntl.flock(lock, control.host.fcntl.LOCK_EX | control.host.fcntl.LOCK_NB)
except BlockingIOError:
os.close(lock)
raise RuntimeError('另一项迁移正在进行,请等待原窗口完成。') from None
try:
return restore_locked(check_runtime)
finally:
os.close(lock)
def restore_locked(check_runtime):
manifest_path = PAYLOAD / 'manifest.json'
manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
if manifest.get('formatVersion') != 1 or not re.fullmatch(r'[a-f0-9]{32}', manifest.get('migrationId', '')):
raise RuntimeError('迁移包格式无效。')
migration_id = manifest['migrationId']
if RECEIPT.exists():
receipt = json.loads(RECEIPT.read_text(encoding='utf-8'))
current = control.configuration()
if receipt.get('migrationId') == migration_id and current.get('restoredFromMigration') == migration_id and current.get('mysqlDatabase') == receipt.get('database'):
print('此迁移已完成,保留 Windows 上的后续数据,直接启动。', flush=True)
JOURNAL.unlink(missing_ok=True)
return receipt
raise RuntimeError('此目录已有其他迁移记录,请使用新的部署目录。')
stop_owned_service()
if JOURNAL.exists():
rollback_plan(json.loads(JOURNAL.read_text(encoding='utf-8')))
print('上次未完成的切换已撤回,原配置已保留。', flush=True)
value = control.verify() if check_runtime else control.configuration()
print('正在校验迁移包……', flush=True)
for name, key in (('database.sql.gz', 'databaseDumpSha256'), ('archive.tar.gz', 'archiveSha256'),
('native-history.tar.gz', 'nativeHistorySha256')):
if sha256(PAYLOAD / name) != manifest[key]:
raise RuntimeError('迁移包文件不完整,已停止恢复。')
if manifest.get('modelConfigSha256') and sha256(PAYLOAD / 'model-config.json') != manifest['modelConfigSha256']:
raise RuntimeError('模型配置校验失败。')
if shutil.disk_usage(ROOT).free < manifest['minimumFreeBytes']:
raise RuntimeError('可用磁盘空间不足,请至少预留 ' + str((manifest['minimumFreeBytes'] + 1024**3 - 1) // 1024**3) + ' GB。')
attempt = uuid.uuid4().hex[:16]
stage = control.PRIVATE / ('migration-stage-' + attempt)
stage.mkdir(mode=0o700)
archive = control.DATA / 'migrations' / attempt / 'archive'
archive.parent.mkdir(parents=True, mode=0o700)
native = stage / 'nao-users'
print('正在恢复来源文件和分析历史……', flush=True)
extract_checked(PAYLOAD / 'archive.tar.gz', archive, manifest['archiveFiles'])
extract_checked(PAYLOAD / 'native-history.tar.gz', native, manifest['nativeHistoryFiles'])
native_count = relocate_native(native, manifest['sourceProjectRoot'])
if native_count != manifest['nativeHistoryDatabases']:
raise RuntimeError('分析历史文件数量不一致。')
model = stage / 'model-config.json'
if manifest.get('modelConfigSha256'):
shutil.copy2(PAYLOAD / 'model-config.json', model)
model.chmod(0o600)
settings = json.loads(model.read_text(encoding='utf-8'))
if any('mimo' in str(profile.get(key, '')).lower() for profile in settings['profiles'] for key in ('model', 'name', 'baseUrl')):
raise RuntimeError('迁移模型配置中仍含 MiMo。')
database = 'quality_restore_' + uuid.uuid4().hex
try:
start_database(value)
run_query(value, 'CREATE DATABASE `' + database + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
restore_dump(value, database, PAYLOAD / 'database.sql.gz')
query = lambda statement: run_query(value, statement, database)
print('正在核对全部数据表、原账号和正式统计结果……', flush=True)
actual_counts = table_counts(query)
if actual_counts != manifest['tableCounts']:
raise RuntimeError('恢复后的数据表行数不一致,未切换正式使用库。')
actual_users = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'username',username,'passwordHashSha256',SHA2(password_hash,256),'active',active) FROM app_user ORDER BY id;").splitlines()]
if sorted(actual_users, key=lambda row: row['id']) != sorted(manifest['sourceUsers'], key=lambda row: row['id']):
raise RuntimeError('原用户账号校验失败,未切换使用库。')
if query_evidence(query) != manifest['queryEvidence']:
raise RuntimeError('正式版本和月度统计结果不一致,未切换使用库。')
rebound = rebind_sources(query, manifest, archive)
run_query(value, 'GRANT ALL PRIVILEGES ON `' + database + "`.* TO 'quality_app'@'127.0.0.1';")
run_query(value, 'FLUSH TABLES;')
finally:
stop_owned_service()
new_config = {**value, 'mysqlDatabase': database, 'archiveDirectory': archive.relative_to(ROOT).as_posix(),
'restoredFromMigration': migration_id, 'webBind': '0.0.0.0'}
for name in ('dataWorkflowTestMode', 'analysisAutonomousEnabled', 'analysisProtocolV1Enabled'):
if name in manifest.get('applicationSettings', {}):
new_config[name] = bool(manifest['applicationSettings'][name])
switches = []
for staged, target in [(native, control.PRIVATE / 'nao-users')] + ([(model, control.PRIVATE / 'nao-model-config.json')] if model.exists() else []):
switches.append({'staged': staged.relative_to(ROOT).as_posix(), 'target': target.relative_to(ROOT).as_posix(),
'previous': (control.PRIVATE / ('before-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(),
'failed': (control.PRIVATE / ('failed-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(),
'existed': target.exists()})
receipt = {'migrationId': migration_id, 'migrationManifestSha256': sha256(manifest_path), 'database': database,
'restoredAtUtc': datetime.now(timezone.utc).isoformat(), 'tables': len(actual_counts),
'totalRows': sum(actual_counts.values()), 'sourceReferencesRebound': rebound,
'modelProfilesMigrated': manifest['modelProfilesMigrated'], 'modelProfilesExcluded': manifest['modelProfilesExcluded'],
'nativeHistoryDatabases': native_count, 'originalDatabasesPreserved': True,
'archiveDirectory': new_config['archiveDirectory']}
plan = {'attempt': attempt, 'previousConfig': value, 'newConfig': new_config, 'privateSwitches': switches}
commit_plan(plan, receipt)
print('迁移恢复完成。请使用原来的账号和密码登录;MiMo 配置未迁移。', flush=True)
return receipt
if __name__ == '__main__':
try:
restore()
except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error:
import traceback
log = control.DATA / 'logs/migration-validation.log'
log.parent.mkdir(parents=True, exist_ok=True)
fd = os.open(log, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as output:
traceback.print_exc(file=output)
if isinstance(error, RuntimeError):
print(str(error), file=sys.stderr)
print('迁移未完成。已有数据库保留;原配置在切换失败时自动恢复。请查看私有恢复日志后重试。', file=sys.stderr)
raise SystemExit(1)
+87
View File
@@ -0,0 +1,87 @@
"""Platform helpers for the standalone distribution; POSIX behavior is preserved."""
from __future__ import annotations
import errno
import os
import signal
import time
IS_WINDOWS = os.name == 'nt'
if os.name != 'nt':
import fcntl
else:
import msvcrt
class _FileLocks:
LOCK_EX, LOCK_NB, LOCK_UN = 2, 4, 8
@staticmethod
def flock(fd, operation):
os.lseek(fd, 0, os.SEEK_SET)
if operation & _FileLocks.LOCK_UN:
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
return
while True:
try:
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
return
except OSError as error:
if error.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
raise
if operation & _FileLocks.LOCK_NB:
raise BlockingIOError(errno.EAGAIN, 'File is locked') from None
time.sleep(.05)
fcntl = _FileLocks()
def private_bits(mode):
# NTFS access is protected by the inherited user-only ACL on .runtime.
# Windows' st_mode has no information about that ACL.
return not IS_WINDOWS and bool(mode & 0o077)
def owner_mismatch(info):
return not IS_WINDOWS and info.st_uid != os.getuid()
def fchmod(fd, mode):
if os.name != 'nt':
os.fchmod(fd, mode)
def ps_value(pid, field):
import psutil
try:
process = psutil.Process(int(pid))
if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE:
return None
if field == 'lstart':
return repr(process.create_time())
if field == 'command':
return ' '.join(process.cmdline())
return None
except (psutil.Error, ValueError):
return None
def group_leader(pid):
return True if os.name == 'nt' else os.getpgid(pid) == pid
def killpg(pid, sig):
if os.name != 'nt':
return os.killpg(pid, sig)
import psutil
try:
process = psutil.Process(pid)
# Caller verifies creation time and executable before reaching this point.
children = process.children(recursive=True)
for child in reversed(children):
try:
child.kill() if sig == getattr(signal, 'SIGKILL', 9) else child.terminate()
except psutil.NoSuchProcess:
pass
process.kill() if sig == getattr(signal, 'SIGKILL', 9) else process.terminate()
except psutil.NoSuchProcess:
raise ProcessLookupError(pid) from None
+117
View File
@@ -0,0 +1,117 @@
"""Exercise an isolated empty MySQL database with the host's native test runtimes."""
import argparse
import http.cookiejar
import json
import os
from pathlib import Path
import secrets
import shutil
import socket
import subprocess
import sys
import tempfile
import time
import urllib.request
from build_bundle import ROOT, TEMPLATES, adapt_runtime
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--work', type=Path, required=True)
args = parser.parse_args()
work = args.work.resolve()
stage = Path(tempfile.mkdtemp(prefix='qc-offline-', dir='/tmp')).resolve()
(work / 'smoke-root.txt').write_text(str(stage))
for folder in ('deploy/offline', 'scripts', 'runtimes', 'app', '.runtime'):
(stage / folder).mkdir(parents=True, exist_ok=True, mode=0o700)
for name in ('control.py', 'gateway.py'):
shutil.copy2(TEMPLATES / name, stage / 'deploy/offline' / name)
shutil.copy2(TEMPLATES / 'runtime_platform.py', stage / 'scripts/runtime_platform.py')
for pattern in ('nao_*.py', 'analysis_visual*.py'):
for path in (ROOT / 'scripts').glob(pattern):
(stage / 'scripts' / path.name).write_text(adapt_runtime(path.read_text()) if path.name == 'nao_quality_runtime.py' else path.read_text())
shutil.copytree(ROOT / 'ai/nao-quality', stage / 'ai/nao-quality')
shutil.copy2(next((work / 'source/backend/target').glob('*.jar')), stage / 'app/backend.jar')
shutil.copytree(work / 'source/web/dist', stage / 'app/web')
for name, target in (('java', ROOT / '.runtime/jdk/Contents/Home'), ('mysql', ROOT / '.runtime/mysql'),
('python', ROOT / '.runtime/nao-trial/venv')):
(stage / 'runtimes' / name).symlink_to(target, target_is_directory=True)
def available(port):
try:
with socket.socket() as test:
test.bind(('127.0.0.1', port))
return True
except OSError:
return False
port = next(p for p in (5284, 5283, 5285) if available(p))
def free():
with socket.socket() as test:
test.bind(('127.0.0.1', 0))
return test.getsockname()[1]
value = {'mysqlPort': free(), 'backendPort': port, 'webPort': free(), 'webBind': '127.0.0.1',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)}
config = stage / '.runtime/deployment.json'
config.write_text(json.dumps(value))
config.chmod(0o600)
report = {'environment': 'macOS isolated empty database using host-native runtimes',
'targetWindowsLinuxNativeValidated': False, 'checks': []}
process = None
try:
for cycle in range(2):
with (stage / f'cycle-{cycle}.log').open('w') as log:
process = subprocess.Popen([sys.executable, str(stage / 'deploy/offline/control.py'), 'serve'], stdout=log, stderr=subprocess.STDOUT)
for _ in range(180):
if process.poll() is not None:
raise RuntimeError('Supervisor exited; inspect ' + str(stage / f'cycle-{cycle}.log'))
try:
state = json.loads((stage / '.runtime/service.json').read_text())
if state.get('ready'):
break
except (FileNotFoundError, ValueError):
pass
time.sleep(1)
else:
raise RuntimeError('Isolated startup timed out')
cookies = http.cookiejar.CookieJar()
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(cookies))
base = f'http://127.0.0.1:{value["webPort"]}'
with opener.open(base + '/', timeout=5) as response:
assert response.status == 200 and b'<div id="root">' in response.read()
with opener.open(base + '/actuator/health', timeout=5) as response:
assert json.load(response)['status'] == 'UP'
with opener.open(base + '/api/auth/csrf', timeout=5) as response:
token = json.load(response)['token']
request = urllib.request.Request(base + '/api/auth/login',
data=json.dumps({'username': value['adminUsername'], 'password': value['adminPassword']}).encode(),
headers={'Content-Type': 'application/json', 'X-XSRF-TOKEN': token})
with opener.open(request, timeout=10) as response:
assert response.status == 200
for path in ('/api/auth/me', '/api/options'):
with opener.open(base + path, timeout=10) as response:
assert response.status == 200
report['checks'].append({'cycle': cycle + 1, 'web': True, 'databaseMigration': True,
'csrfAndLogin': True, 'authenticatedOptions': True})
result = subprocess.run([sys.executable, str(stage / 'deploy/offline/control.py'), 'stop'], capture_output=True, text=True, timeout=150)
if result.returncode:
raise RuntimeError('Stop failed: ' + result.stderr)
process.wait(timeout=10)
process = None
assert not (stage / '.runtime/service.json').exists()
print('Isolated startup/login/stop cycle', cycle + 1, 'passed', flush=True)
report['passed'] = True
except Exception as error:
report['passed'] = False
report['error'] = str(error)
print(str(error), flush=True)
finally:
if process is not None and process.poll() is None:
subprocess.run([sys.executable, str(stage / 'deploy/offline/control.py'), 'stop'], capture_output=True, timeout=150)
(work / 'smoke-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
print(json.dumps(report, ensure_ascii=False), flush=True)
return 0 if report['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+148
View File
@@ -0,0 +1,148 @@
import gzip
import hashlib
import io
import json
import os
from pathlib import Path
import sqlite3
import sys
import tarfile
import tempfile
import unittest
from unittest.mock import patch
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import control
import restore_migration as restore
from migration_common import dump_counts, extract_checked, filter_models, inserted_rows, path_under_source, relative_name
from prepare_migration import native_copy
class MigrationValidationTest(unittest.TestCase):
def test_extended_insert_counts_ignore_strings_and_escaped_quotes(self):
cases = [
(b"INSERT INTO `x` VALUES (1,'a'),(2,'b');\n", 2),
(b"INSERT INTO `x` VALUES (1,'fake ),( delimiter'),(2,'quote \\' ),( escaped'),(3,0x001122);\n", 3),
(b"INSERT INTO `x` VALUES ('a\\\\',NULL,'(nested)'),(NULL,'\\n',4);\n", 2),
(b"INSERT INTO `x` VALUES ('\xe4\xb8\xad\xe6\x96\x87', '{\\\"text\\\":\\\"),(\\\"}');\n", 1),
]
for statement, count in cases:
with self.subTest(statement=statement):
self.assertEqual(inserted_rows(statement), count)
def test_dump_counts_include_empty_tables(self):
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / 'dump.gz'
with gzip.open(path, 'wb') as stream:
stream.write(b"CREATE TABLE `empty` (id INT);\nCREATE TABLE `x` (id INT);\nINSERT INTO `x` VALUES (1),(2);\n")
counts, size = dump_counts(path)
self.assertEqual(counts, {'empty': 0, 'x': 2})
self.assertGreater(size, 0)
def test_mimo_profiles_and_root_selection_are_excluded(self):
config = {'revision': 'original', 'activeProfileId': 'mimo', 'profiles': [
{'id': 'mimo', 'name': 'MiMo', 'model': 'mimo-v2.6', 'apiKey': 'exclude-this', 'baseUrl': 'https://api.example.test'},
{'id': 'other', 'name': 'Qwen', 'model': 'qwen-test', 'apiKey': 'retain-this', 'baseUrl': 'https://other.example.test'}]}
filtered, removed = filter_models(config)
self.assertEqual(len(removed), 1)
self.assertEqual(filtered['activeProfileId'], 'other')
self.assertEqual(filtered['apiKey'], 'retain-this')
self.assertEqual(filtered['revision'], 'original')
self.assertNotIn('exclude-this', json.dumps(filtered))
self.assertEqual(len(filtered['profiles']), 1)
def test_mac_path_is_parsed_portably(self):
self.assertEqual(path_under_source('/Users/a/archive/source/文件.xlsx', '/Users/a/archive'), 'source/文件.xlsx')
for name in ('../outside', '/absolute', 'C:/data', 'a\\b', '.', 'a/./b', 'CON.txt', 'x/name.'):
with self.subTest(name=name), self.assertRaises(ValueError):
relative_name(name)
with self.assertRaises(ValueError):
path_under_source('/Users/a/archive-other/secret', '/Users/a/archive')
def test_archive_rejects_traversal_links_duplicates_and_bad_hash(self):
for kind in ('traversal', 'link', 'duplicate', 'checksum'):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory:
folder = Path(directory)
archive = folder / 'archive.gz'
body = b'contents'
with tarfile.open(archive, 'w:gz') as output:
info = tarfile.TarInfo('../outside' if kind == 'traversal' else 'file')
if kind == 'link':
info.type, info.linkname = tarfile.SYMTYPE, '/outside'
output.addfile(info)
else:
info.size = len(body)
output.addfile(info, io.BytesIO(body))
if kind == 'duplicate':
output.addfile(info, io.BytesIO(body))
expected = {'file': {'bytes': len(body), 'sha256': '0' * 64 if kind == 'checksum' else hashlib.sha256(body).hexdigest()}}
with self.assertRaises(ValueError):
extract_checked(archive, folder / 'target', expected)
self.assertFalse((folder / 'outside').exists())
def test_native_history_preserves_chats_and_scrubs_credentials(self):
with tempfile.TemporaryDirectory() as directory:
folder = Path(directory)
source = folder / 'source.sqlite'
db = sqlite3.connect(source)
db.executescript("CREATE TABLE chat(id TEXT,body TEXT);INSERT INTO chat VALUES('a','history');"
"CREATE TABLE project_llm_config(api_key TEXT);INSERT INTO project_llm_config VALUES('mimo-exclusive-secret');"
"CREATE TABLE session(token TEXT);INSERT INTO session VALUES('transient-token');")
db.commit()
db.close()
target = folder / 'target.sqlite'
native_copy(source, target)
migrated = sqlite3.connect(target)
self.assertEqual(migrated.execute('SELECT body FROM chat').fetchone()[0], 'history')
self.assertEqual(migrated.execute('SELECT COUNT(*) FROM project_llm_config').fetchone()[0], 0)
self.assertEqual(migrated.execute('SELECT COUNT(*) FROM session').fetchone()[0], 0)
migrated.close()
self.assertNotIn(b'mimo-exclusive-secret', target.read_bytes())
class MigrationCommitTest(unittest.TestCase):
def test_partial_file_switch_rolls_back_existing_files_and_configuration(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
private = root / '.runtime'
private.mkdir()
target = private / 'nao-users'
target.mkdir()
(target / 'old').write_text('original')
staged = private / 'stage-native'
staged.mkdir()
(staged / 'new').write_text('new')
model = private / 'nao-model-config.json'
model.write_text('original model')
attempt = 'a' * 16
plan = {'attempt': attempt, 'previousConfig': {'previous': True}, 'newConfig': {'new': True}, 'privateSwitches': [
{'target': '.runtime/nao-users', 'previous': '.runtime/previous-native', 'staged': '.runtime/stage-native', 'failed': '.runtime/failed-native', 'existed': True},
{'target': '.runtime/nao-model-config.json', 'previous': '.runtime/previous-model', 'staged': '.runtime/missing-stage-model', 'failed': '.runtime/failed-model', 'existed': True}]}
with patch.multiple(restore, ROOT=root, JOURNAL=private / 'journal', RECEIPT=private / 'receipt'), patch.object(control, 'CONFIG', private / 'deployment.json'):
with self.assertRaises(FileNotFoundError):
restore.commit_plan(plan, {})
self.assertEqual((target / 'old').read_text(), 'original')
self.assertEqual(model.read_text(), 'original model')
self.assertEqual(json.loads((private / 'deployment.json').read_text()), {'previous': True})
self.assertTrue((private / 'failed-native/new').exists())
def test_repeated_restore_does_not_reimport_old_snapshot(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
payload = root / 'migration'
payload.mkdir()
migration_id = 'a' * 32
(payload / 'manifest.json').write_text(json.dumps({'formatVersion': 1, 'migrationId': migration_id}))
receipt = root / 'receipt'
value = {'migrationId': migration_id, 'database': 'quality_restore_' + 'b' * 32}
receipt.write_text(json.dumps(value))
config = {'restoredFromMigration': migration_id, 'mysqlDatabase': value['database']}
with patch.multiple(restore, PAYLOAD=payload, RECEIPT=receipt, JOURNAL=root / 'journal'), patch.object(control, 'configuration', return_value=config), patch.object(restore, 'stop_owned_service') as stop, patch.object(restore, 'run_query') as query:
self.assertEqual(restore.restore_locked(False), value)
stop.assert_not_called()
query.assert_not_called()
if __name__ == '__main__':
unittest.main()
+179
View File
@@ -0,0 +1,179 @@
import functools
import http.client
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import importlib.util
import json
import os
from pathlib import Path
import socket
import subprocess
import sys
import tempfile
import threading
import time
import unittest
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
from gateway import make_server
from build_bundle import adapt_runtime
import runtime_platform
class Upstream(BaseHTTPRequestHandler):
def log_message(self, *_):
pass
def do_GET(self):
if self.path == '/api/events':
self.send_response(200)
self.send_header('Content-Type', 'text/event-stream')
self.end_headers()
self.wfile.write(b'data: first\n\n')
self.wfile.flush()
time.sleep(.8)
self.wfile.write(b'data: second\n\n')
self.wfile.flush()
else:
body = json.dumps({'cookie': self.headers.get('Cookie'), 'host': self.headers.get('Host')}).encode()
self.send_response(401)
self.send_header('Set-Cookie', 'one=1; HttpOnly')
self.send_header('Set-Cookie', 'two=2; SameSite=Strict')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_POST(self):
body = self.rfile.read(int(self.headers['Content-Length']))
self.send_response(201)
self.send_header('Content-Length', str(len(body)))
self.send_header('X-CSRF-Echo', self.headers.get('X-XSRF-TOKEN', ''))
self.end_headers()
self.wfile.write(body)
class GatewayTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.temp = tempfile.TemporaryDirectory()
root = Path(cls.temp.name)
(root / 'index.html').write_text('QUALITY APP', encoding='utf-8')
(root / 'assets').mkdir()
(root / 'assets/file.js').write_text('const x=1;')
cls.backend = ThreadingHTTPServer(('127.0.0.1', 0), Upstream)
cls.frontend = make_server(root, '127.0.0.1', 0, cls.backend.server_port)
for server in (cls.backend, cls.frontend):
threading.Thread(target=server.serve_forever, daemon=True).start()
@classmethod
def tearDownClass(cls):
for server in (cls.frontend, cls.backend):
server.shutdown()
server.server_close()
cls.temp.cleanup()
def connection(self):
return http.client.HTTPConnection('127.0.0.1', self.frontend.server_port, timeout=3)
def test_static_and_spa(self):
for url, body in [('/', b'QUALITY APP'), ('/?page=home', b'QUALITY APP'), ('/analysis', b'QUALITY APP'), ('/assets/file.js', b'const x=1;')]:
connection = self.connection()
connection.request('GET', url)
response = connection.getresponse()
self.assertEqual(response.status, 200)
self.assertEqual(response.read(), body)
connection.close()
def test_no_directory_listing_or_private_files(self):
for url in ('/assets/', '/.runtime/deployment.json', '/../.runtime/deployment.json', '/missing.js'):
connection = self.connection()
connection.request('GET', url)
response = connection.getresponse()
self.assertEqual(response.status, 404)
response.read()
connection.close()
def test_auth_cookies_and_host_are_preserved(self):
connection = self.connection()
connection.request('GET', '/api/auth/me', headers={'Cookie': 'SESSION=abc', 'Host': '192.168.1.22:5280'})
response = connection.getresponse()
self.assertEqual(response.status, 401)
self.assertEqual(len(response.headers.get_all('Set-Cookie')), 2)
self.assertEqual(json.loads(response.read()), {'cookie': 'SESSION=abc', 'host': '192.168.1.22:5280'})
connection.close()
def test_upload_and_csrf(self):
connection = self.connection()
data = os.urandom(2 * 1024 * 1024 + 19)
connection.request('POST', '/api/sources/upload', data, {'X-XSRF-TOKEN': 'csrf-value'})
response = connection.getresponse()
self.assertEqual(response.status, 201)
self.assertEqual(response.getheader('X-CSRF-Echo'), 'csrf-value')
self.assertEqual(response.read(), data)
connection.close()
def test_sse_is_streamed_before_upstream_finishes(self):
connection = self.connection()
started = time.monotonic()
connection.request('GET', '/api/events')
response = connection.getresponse()
self.assertEqual(response.readline(), b'data: first\n')
self.assertLess(time.monotonic() - started, .65)
self.assertIn(b'data: second', response.read())
connection.close()
class PortabilityTest(unittest.TestCase):
def test_port_probe_rejects_live_listener_and_allows_restart(self):
import control
with socket.socket() as listener:
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind(('127.0.0.1', 0))
port = listener.getsockname()[1]
listener.listen()
with self.assertRaises(RuntimeError):
control.free_port(port)
accepted, _ = listener.accept()
accepted.close()
control.free_port(port)
def test_process_identity_requires_start_time_executable_and_arguments(self):
import psutil
import control
process = psutil.Process()
info = control.record(process, process.cmdline())
self.assertTrue(control.matches(info))
self.assertFalse(control.matches({**info, 'created': info['created'] - 10}))
self.assertFalse(control.matches({**info, 'executable': '/unrelated/program'}))
self.assertFalse(control.matches({**info, 'args': ['unrelated-command']}))
def test_posix_permissions_are_not_relaxed(self):
if os.name != 'nt':
self.assertFalse(runtime_platform.private_bits(0o100600))
self.assertTrue(runtime_platform.private_bits(0o100640))
def test_lock_excludes_another_process(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / 'lock'
fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
runtime_platform.fcntl.flock(fd, runtime_platform.fcntl.LOCK_EX | runtime_platform.fcntl.LOCK_NB)
code = ('import sys,os;sys.path.insert(0,sys.argv[1]);import runtime_platform as h;'
'fd=os.open(sys.argv[2],os.O_RDWR);h.fcntl.flock(fd,h.fcntl.LOCK_EX|h.fcntl.LOCK_NB)')
result = subprocess.run([sys.executable, '-c', code, str(HERE), str(path)], capture_output=True)
self.assertNotEqual(result.returncode, 0)
os.close(fd)
def test_source_adapter_preserves_mac_source(self):
path = HERE.parents[1] / 'scripts/nao_quality_runtime.py'
source = path.read_text()
adapted = adapt_runtime(source)
self.assertIn('import fcntl', source)
self.assertIn('from runtime_platform import fcntl', adapted)
self.assertIn('_host.private_bits(directory_info.st_mode)', adapted)
self.assertIn('_host.killpg', adapted)
compile(adapted, str(path), 'exec')
self.assertEqual(path.read_text(), source)
if __name__ == '__main__':
unittest.main()
+175
View File
@@ -0,0 +1,175 @@
"""Offline tests for the signed Windows update publisher."""
import base64
import hashlib
import json
from pathlib import Path
import tempfile
import sys
import unittest
import zipfile
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
sys.path.insert(0, str(Path(__file__).resolve().parent))
import publish_update
import update_manager
class PublishUpdateTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.base = Path(self.temp.name)
self.root = self.base / 'windows-bundle'
self.root.mkdir()
self.output = self.base / 'published'
self.key_path = self.base / 'release-private-key.pem'
self.private = Ed25519PrivateKey.generate()
self.key_path.write_bytes(self.private.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.PKCS8,
serialization.NoEncryption(),
))
public_key_path = self.root / update_manager.KEY
public_key_path.parent.mkdir(parents=True, exist_ok=True)
public_key_path.write_bytes(self.private.public_key().public_bytes(
serialization.Encoding.PEM,
serialization.PublicFormat.SubjectPublicKeyInfo,
))
self.data = {
'app/backend.jar': b'backend program',
'app/web/index.html': b'<html>program</html>',
'deploy/offline/control.py': b'control program',
'deploy/offline/update_manager.py': b'update manager program',
'deploy/offline/publish_update.py': b'publisher program',
'deploy/offline/windows-start.ps1': b'start script',
'deploy/offline/README.md': b'package instructions',
'backend/src/main/resources/analysis-runtime-policy.json': b'{"enabled":false}',
'scripts/runtime.py': b'helper program',
'ai/nao-quality/app.py': b'ai program',
'runtimes/python/python.exe': b'local runtime must not ship in an update',
'migration/manifest.json': b'private migration data',
'deploy/bootstrap/update-recovery.py': b'immutable bootstrap',
'.runtime/deployment.json': b'{"mysqlPassword":"private"}',
'runtimedata/archive/source.xlsx': b'user business data',
update_manager.KEY: public_key_path.read_bytes(),
}
for name, value in self.data.items():
target = self.root.joinpath(*name.split('/'))
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(value)
self.manifest = {
'platform': 'windows',
'architecture': 'x86_64',
'updateVersion': '2026.11.04.1',
'updateSequence': 1,
'runtimes': [{'name': 'python', 'version': '3.11.15'}],
'pythonWheels': [{'file': 'cryptography.whl', 'sha256': 'a' * 64}],
'files': [
{'path': name, 'sha256': hashlib.sha256(value).hexdigest()}
for name, value in self.data.items()
],
}
(self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8')
def build(self, **changes):
args = {
'root': self.root,
'version': '2026.11.04.2',
'sequence': 2,
'private_key_path': self.key_path,
'output_dir': self.output,
}
args.update(changes)
return publish_update.publish(**args)
def test_release_is_signed_program_snapshot_with_runtime_fingerprint(self):
artifacts = self.build()
self.assertTrue(artifacts.archive.is_file())
self.assertTrue(artifacts.latest.is_file())
envelope = update_manager.read(artifacts.latest)
release = update_manager.validate_release(envelope, self.root)
raw = base64.b64decode(envelope['payload'], validate=True)
self.private.public_key().verify(base64.b64decode(envelope['signature'], validate=True), raw)
self.assertEqual('2026.11.04.2', release['version'])
self.assertEqual(2, release['sequence'])
self.assertEqual(update_manager.runtime_fingerprint(self.manifest), release['runtimeFingerprint'])
self.assertNotIn('bundleManifest', release)
included = {entry['path'] for entry in release['files']}
self.assertEqual({name for name in self.data if update_manager.program_path(name)}, included)
self.assertFalse(included & {
'runtimes/python/python.exe',
'migration/manifest.json',
'deploy/bootstrap/update-recovery.py',
'.runtime/deployment.json',
'runtimedata/archive/source.xlsx',
update_manager.KEY,
})
with zipfile.ZipFile(artifacts.archive) as archive:
self.assertEqual(included, set(archive.namelist()))
self.assertEqual(self.data['app/backend.jar'], archive.read('app/backend.jar'))
self.assertEqual(hashlib.sha256(artifacts.archive.read_bytes()).hexdigest(), release['archive']['sha256'])
def test_rejects_wrong_key_stale_sequence_and_package_mutation(self):
with self.assertRaisesRegex(publish_update.PublishError, '不匹配'):
wrong_path = self.base / 'wrong-private-key.pem'
wrong_path.write_bytes(Ed25519PrivateKey.generate().private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.PKCS8,
serialization.NoEncryption(),
))
self.build(private_key_path=wrong_path)
with self.assertRaisesRegex(publish_update.PublishError, '大于部署包当前序号'):
self.build(sequence=1)
(self.root / 'app/backend.jar').write_bytes(b'changed after manifest')
with self.assertRaisesRegex(publish_update.PublishError, '摘要不一致'):
self.build()
self.assertFalse(self.output.exists())
def test_allows_only_explicit_exact_baseline_and_still_rejects_lower_sequence(self):
with self.assertRaisesRegex(publish_update.PublishError, '大于部署包当前序号'):
self.build(version='2026.11.04.1', sequence=1)
with self.assertRaisesRegex(publish_update.PublishError, '完全一致'):
self.build(version='2026.11.04.2', sequence=1, allow_baseline=True)
baseline = self.build(version='2026.11.04.1', sequence=1, allow_baseline=True)
release = update_manager.validate_release(update_manager.read(baseline.latest), self.root)
self.assertEqual('2026.11.04.1', release['version'])
self.assertEqual(1, release['sequence'])
self.manifest['updateVersion'] = '2026.11.04.1'
self.manifest['updateSequence'] = 2
(self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8')
self.output = self.base / 'lower-sequence-output'
with self.assertRaisesRegex(publish_update.PublishError, '不能低于'):
self.build(version='2026.11.04.1', sequence=1, allow_baseline=True)
def test_rejects_sensitive_program_path_and_missing_program_inventory(self):
sensitive_name = 'ai/nao-quality/private/model.json'
sensitive_path = self.root.joinpath(*sensitive_name.split('/'))
sensitive_path.parent.mkdir(parents=True, exist_ok=True)
sensitive_path.write_bytes(b'private model config')
self.manifest['files'].append({'path': sensitive_name, 'sha256': hashlib.sha256(sensitive_path.read_bytes()).hexdigest()})
(self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8')
with self.assertRaisesRegex(publish_update.PublishError, '受保护'):
self.build()
self.manifest['files'].pop()
self.manifest['files'] = [entry for entry in self.manifest['files'] if entry['path'] != 'scripts/runtime.py']
(self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8')
with self.assertRaisesRegex(publish_update.PublishError, '完整程序文件清单'):
self.build()
def test_refuses_key_inside_source_or_output_and_refuses_overwrite(self):
with self.assertRaisesRegex(publish_update.PublishError, '部署包和更新产物目录之外'):
self.build(private_key_path=self.root / 'manifest.json')
with self.assertRaisesRegex(publish_update.PublishError, '部署包目录之外'):
self.build(output_dir=self.root / 'updates')
self.build()
original_latest = (self.output / 'latest.json').read_bytes()
with self.assertRaisesRegex(publish_update.PublishError, '同名发布文件'):
self.build()
self.assertEqual(original_latest, (self.output / 'latest.json').read_bytes())
if __name__ == '__main__':
unittest.main()
+339
View File
@@ -0,0 +1,339 @@
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import textwrap
import time
import unittest
from unittest.mock import patch
import psutil
sys.path.insert(0, str(Path(__file__).resolve().parent))
import update_bootstrap as bootstrap
class ImmutableRecoveryTest(unittest.TestCase):
def fixture(self, folder):
root = Path(folder)
directory = root / '.runtime/update'
backup = directory / 'backup-20261104T120000Z-abcdef12'
(backup / 'program/deploy/offline').mkdir(parents=True)
(root / 'deploy/offline').mkdir(parents=True)
old = {
'app/backend.jar': b'verified old backend jar',
'deploy/offline/control.py': b'VALID_OLD_VERSION = True\n',
'deploy/offline/update_manager.py': b'def recover_locked(root):\n (root / ".runtime/restored").write_text("done")\n return {"recovered": True}\n',
}
entries = []
for name, content in old.items():
path = backup / 'program' / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
(root / name).parent.mkdir(parents=True, exist_ok=True)
(root / name).write_bytes(b'def broken syntax\n')
entries.append({'path': name, 'sha256': hashlib.sha256(content).hexdigest()})
(backup / 'manifest.json').write_text(json.dumps({'files': entries}))
(backup / 'program.json').write_text(json.dumps({'all': list(old), 'present': list(old)}))
journal = {'phase': 'installing', 'databaseBackedUp': True, 'backup': backup.name,
'backupManifestSha256': bootstrap.digest(backup / 'manifest.json'),
'programInventorySha256': bootstrap.digest(backup / 'program.json')}
(directory / 'journal.json').write_text(json.dumps(journal))
return root, directory, backup, journal, old
def runtime_aliases(self, root):
for role in ('supervisor', 'backend'):
executable = bootstrap.runtime_executable(root, role)
executable.parent.mkdir(parents=True, exist_ok=True)
executable.symlink_to(psutil.Process().exe())
def record(self, process, name=None):
observed = psutil.Process(process.pid)
value = {'pid': process.pid, 'created': observed.create_time(),
'executable': str(Path(observed.exe()).resolve()), 'args': observed.cmdline()[1:]}
if name:
value['name'] = name
return value
def stop_test_child(self, process):
if process.poll() is not None:
return
descendants = []
try:
descendants = psutil.Process(process.pid).children(recursive=True)
except psutil.Error:
pass
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=5)
for child in descendants:
try:
if child.is_running() and child.status() != psutil.STATUS_ZOMBIE:
child.kill()
except psutil.Error:
pass
def wait_file(self, path, process):
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
if path.exists():
return
self.assertIsNone(process.poll(), 'Owned test child exited before readiness')
time.sleep(.02)
self.fail('Owned test child did not become ready')
def backend_script(self, root):
path = root / 'backend-test.py'
path.write_text(textwrap.dedent('''
import hashlib, os, signal, sys, time
from pathlib import Path
root = Path(__file__).parent
jar = root / 'app/backend.jar'
held = jar.open('rb')
def stop(*ignored):
(root / '.runtime/backend-stopped-before-copy').write_text(hashlib.sha256(jar.read_bytes()).hexdigest())
held.close()
raise SystemExit(0)
signal.signal(signal.SIGTERM, stop)
(root / '.runtime/backend-ready').write_text(str(os.getpid()))
while True:
time.sleep(.02)
'''))
return path
def start_backend(self, root):
self.runtime_aliases(root)
script = self.backend_script(root)
process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'backend')), str(script), '-jar', str(root / 'app/backend.jar')],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.addCleanup(self.stop_test_child, process)
self.wait_file(root / '.runtime/backend-ready', process)
return process
def start_supervisor(self, root):
self.runtime_aliases(root)
self.backend_script(root)
script = root / 'deploy/offline/control.py'
script.write_text(textwrap.dedent('''
import json, os, signal, subprocess, sys, time
from pathlib import Path
import psutil
root = Path(__file__).resolve().parents[2]
token = 'a' * 48
java = root / 'runtimes/java/bin' / ('java.exe' if os.name == 'nt' else 'java')
child = subprocess.Popen([str(java), str(root / 'backend-test.py'), '-jar', str(root / 'app/backend.jar')])
def record(pid):
process = psutil.Process(pid)
return {'pid': pid, 'created': process.create_time(), 'executable': str(Path(process.exe()).resolve()), 'args': process.cmdline()[1:]}
state = {'supervisor': record(os.getpid()), 'token': token, 'children': [{**record(child.pid), 'name': 'backend'}], 'ready': True}
def stop(*ignored):
child.terminate()
child.wait(timeout=5)
(root / '.runtime/supervisor-stopped').write_text('done')
(root / '.runtime/service.json').unlink(missing_ok=True)
raise SystemExit(0)
signal.signal(signal.SIGTERM, stop)
(root / '.runtime/service.json').write_text(json.dumps(state))
(root / '.runtime/supervisor-ready').write_text('ready')
while True:
try:
request = json.loads((root / '.runtime/stop-request.json').read_text())
if request.get('token') == token:
(root / '.runtime/token-observed').write_text(token)
stop()
except (FileNotFoundError, ValueError):
pass
time.sleep(.02)
'''))
process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'supervisor')), str(script), 'serve'],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.addCleanup(self.stop_test_child, process)
self.wait_file(root / '.runtime/supervisor-ready', process)
self.wait_file(root / '.runtime/backend-ready', process)
return process
def test_recovers_before_importing_broken_control_and_updater(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, backup, journal, old = self.fixture(folder)
result = bootstrap.recover(root)
self.assertTrue(result['recovered'])
self.assertEqual((root / '.runtime/restored').read_text(), 'done')
for name, content in old.items():
self.assertEqual((root / name).read_bytes(), content)
self.assertFalse((directory / 'worker.json').exists())
def test_corrupt_backup_is_rejected_before_any_replacement(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, backup, journal, old = self.fixture(folder)
(backup / 'program/deploy/offline/control.py').write_text('corruption')
with self.assertRaises(RuntimeError):
bootstrap.recover(root)
self.assertEqual((root / 'deploy/offline/control.py').read_text(), 'def broken syntax\n')
def test_active_worker_cannot_be_overridden(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, backup, journal, old = self.fixture(folder)
(directory / 'worker.json').write_text(json.dumps({'pid': os.getpid(), 'created': psutil.Process().create_time()}))
with self.assertRaises(RuntimeError):
bootstrap.recover(root)
self.assertFalse((root / '.runtime/restored').exists())
def test_program_protection_and_no_update_are_read_only(self):
for name in ('deploy/bootstrap/windows-start.ps1', 'deploy/offline/update-public-key.pem', '.runtime/deployment.json', 'runtimedata/mysql/file'):
self.assertFalse(bootstrap.program(name))
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
self.assertEqual(bootstrap.recover(root), {'recovered': False})
self.assertFalse((root / '.runtime').exists())
def test_orphaned_backend_stops_before_jar_copy_after_worker_crash(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, _, _, old = self.fixture(folder)
child = self.start_backend(root)
state = {'supervisor': {'pid': 999999999, 'created': 1, 'executable': str(sys.executable),
'args': [str(root / 'deploy/offline/control.py'), 'serve']}, 'token': 'b' * 48,
'children': [self.record(child, 'backend')]}
(root / '.runtime/service.json').write_text(json.dumps(state))
(directory / 'worker.json').write_text(json.dumps({'pid': 999999998, 'created': 1}))
before = bootstrap.digest(root / 'app/backend.jar')
real_copy = bootstrap.atomic_copy
def checked_copy(source, target):
self.assertIsNotNone(child.poll(), 'A mutable program was copied while the old backend still held its JAR')
return real_copy(source, target)
with patch.object(bootstrap, 'atomic_copy', side_effect=checked_copy):
self.assertTrue(bootstrap.recover(root)['recovered'])
self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text())
self.assertEqual(old['app/backend.jar'], (root / 'app/backend.jar').read_bytes())
def test_live_supervisor_receives_matching_stop_token_and_children_exit_before_copy(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, _, _, _ = self.fixture(folder)
supervisor = self.start_supervisor(root)
(root / 'deploy/offline/control.py').write_text('def broken syntax\n')
before = bootstrap.digest(root / 'app/backend.jar')
result = bootstrap.recover(root)
self.assertTrue(result['recovered'])
self.assertIsNotNone(supervisor.poll())
self.assertEqual('a' * 48, (root / '.runtime/token-observed').read_text())
self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text())
self.assertFalse((root / '.runtime/stop-request.json').exists())
def test_stale_record_pointing_at_an_unrelated_live_pid_never_kills_or_replaces(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
self.runtime_aliases(root)
child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.addCleanup(self.stop_test_child, child)
info = self.record(child, 'backend')
info['created'] -= 60
(root / '.runtime/service.json').write_text(json.dumps({'children': [info]}))
with self.assertRaisesRegex(RuntimeError, '归属'):
bootstrap.recover(root)
self.assertIsNone(child.poll())
self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text())
self.assertFalse((root / '.runtime/restored').exists())
def test_unregistered_root_backend_is_refused_without_signalling(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
child = self.start_backend(root)
with self.assertRaisesRegex(RuntimeError, '未登记'):
bootstrap.recover(root)
self.assertIsNone(child.poll())
self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text())
def test_other_deployment_root_cannot_be_claimed_by_a_valid_live_pid_record(self):
with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second:
root, _, _, _, _ = self.fixture(first)
other, _, _, _, _ = self.fixture(second)
self.runtime_aliases(root)
child = self.start_backend(other)
(root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]}))
with self.assertRaisesRegex(RuntimeError, '归属'):
bootstrap.recover(root)
self.assertIsNone(child.poll())
self.assertFalse((root / '.runtime/restored').exists())
def test_invalid_supervisor_token_does_not_signal_any_owned_process(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
supervisor = self.start_supervisor(root)
state = json.loads((root / '.runtime/service.json').read_text())
state['token'] = 'invalid-token'
(root / '.runtime/service.json').write_text(json.dumps(state))
with self.assertRaisesRegex(RuntimeError, '令牌'):
bootstrap.recover(root)
self.assertIsNone(supervisor.poll())
self.assertFalse((root / '.runtime/stop-request.json').exists())
def test_dead_pid_records_do_not_block_safe_recovery(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
state = {'children': [{'pid': 999999999, 'created': 1, 'executable': str(sys.executable),
'args': ['missing'], 'name': 'backend'}]}
(root / '.runtime/service.json').write_text(json.dumps(state))
self.assertTrue(bootstrap.recover(root)['recovered'])
def test_a_process_owned_by_another_user_is_never_stopped(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
child = self.start_backend(root)
(root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]}))
with patch.object(bootstrap, 'current_user', return_value=False), self.assertRaisesRegex(RuntimeError, '归属'):
bootstrap.recover(root)
self.assertIsNone(child.poll())
self.assertFalse((root / '.runtime/restored').exists())
@unittest.skipIf(os.name == 'nt', 'Host-side immutable-client fixture uses a POSIX shebang; real Windows binaries need native acceptance')
def test_orphaned_mysql_uses_immutable_shutdown_client_and_is_not_force_killed(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
executable = bootstrap.runtime_executable(root, 'mysql')
executable.parent.mkdir(parents=True)
executable.symlink_to(psutil.Process().exe())
server = root / 'mysql-test.py'
server.write_text(textwrap.dedent('''
import os, signal, time
from pathlib import Path
root = Path(__file__).parent
def stop(*ignored):
(root / '.runtime/mysql-stopped-cleanly').write_text('done')
raise SystemExit(0)
signal.signal(signal.SIGTERM, stop)
(root / '.runtime/mysql-test-pid').write_text(str(os.getpid()))
while True:
time.sleep(.02)
'''))
child = subprocess.Popen([str(executable), str(server), '--datadir=' + str(root / 'runtimedata/mysql'), '--port=54489'],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.addCleanup(self.stop_test_child, child)
self.wait_file(root / '.runtime/mysql-test-pid', child)
admin = executable.parent / 'mysqladmin'
admin.write_text('#!' + psutil.Process().exe() + '\n' + textwrap.dedent('''
import os, signal, sys
from pathlib import Path
root = Path(__file__).resolve().parents[3]
assert os.environ['MYSQL_PWD'] == 'isolated_password_value_1234'
assert '--no-defaults' in sys.argv and '--port=54489' in sys.argv and sys.argv[-1] == 'shutdown'
assert not any('password' in arg.lower() for arg in sys.argv)
(root / '.runtime/immutable-mysqladmin-called').write_text('done')
os.kill(int((root / '.runtime/mysql-test-pid').read_text()), signal.SIGTERM)
'''))
admin.chmod(0o700)
(root / '.runtime/deployment.json').write_text(json.dumps({'mysqlPort': 54489, 'rootPassword': 'isolated_password_value_1234'}))
(root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'mysql')]}))
with patch.object(psutil.Process, 'kill', side_effect=AssertionError('MySQL must not be force killed')):
self.assertTrue(bootstrap.recover(root)['recovered'])
self.assertIsNotNone(child.poll())
self.assertTrue((root / '.runtime/immutable-mysqladmin-called').exists())
self.assertTrue((root / '.runtime/mysql-stopped-cleanly').exists())
if __name__ == '__main__':
unittest.main()
+303
View File
@@ -0,0 +1,303 @@
"""Offline security and rollback tests. No network, production DB or services."""
import base64
import copy
import hashlib
import importlib.util
import json
from pathlib import Path
import shutil
import stat
import sys
import tempfile
import types
import unittest
from unittest.mock import patch
import zipfile
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
from cryptography.hazmat.primitives import serialization
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import update_manager as update
class UpdateTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name) / 'bundle'
self.root.mkdir()
self.private = Ed25519PrivateKey.generate()
self.files = {
'app/backend.jar': b'old backend',
'app/web/index.html': b'old web',
'app/web/obsolete.js': b'old asset',
'deploy/offline/control.py': b'old supervisor',
'deploy/offline/update_manager.py': b'old updater',
'backend/src/main/resources/analysis-runtime-policy.json': b'{}',
update.KEY: self.private.public_key().public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo),
'runtimes/python/python.exe': b'unchanged runtime',
'migration/snapshot.sql': b'private migration data',
}
for name, data in self.files.items():
self.write(name, data)
self.manifest = {
'platform': 'windows', 'architecture': 'x86_64',
'runtimes': [{'name': 'python', 'version': '3.11.15'}],
'pythonWheels': [{'file': 'cryptography.whl', 'sha256': 'a' * 64}],
'files': [{'path': k, 'sha256': hashlib.sha256(v).hexdigest()} for k, v in self.files.items()],
'updateSequence': 5, 'updateVersion': '2026.11.03',
}
update.atomic(self.root / 'manifest.json', self.manifest)
self.config = {'mysqlDatabase': 'quality_restore_' + 'a' * 24, 'archiveDirectory': 'runtimedata/archive/restore-old',
'mysqlPort': 53306, 'rootPassword': 'privateRootPasswordValue', 'mysqlPassword': 'privateAppPasswordValue'}
update.atomic(self.root / '.runtime/deployment.json', self.config)
self.write('.runtime/analysis-api-settings.json', b'{"apiKey":"keep-private"}')
self.program = {k: b'new ' + v for k, v in self.files.items() if update.program_path(k) and k != 'app/web/obsolete.js'}
self.release = {'schema': 1, 'version': '2026.11.04', 'sequence': 6, 'platform': 'windows-x64',
'runtimeFingerprint': update.runtime_fingerprint(self.manifest),
'archive': {'file': 'quality-update-2026.11.04.zip', 'size': 123, 'sha256': 'a' * 64},
'files': [{'path': k, 'size': len(v), 'sha256': hashlib.sha256(v).hexdigest()} for k, v in self.program.items()]}
def write(self, name, data):
path = self.root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
def envelope(self, release=None, private=None):
raw = update.canonical(release or self.release)
return {'payload': base64.b64encode(raw).decode(), 'signature': base64.b64encode((private or self.private).sign(raw)).decode()}
def validate(self, release=None):
return update.validate_release(self.envelope(release), self.root)
def archive(self, entries=None):
archive = Path(self.temp.name) / 'update.zip'
with zipfile.ZipFile(archive, 'w') as out:
for name, data in (entries or self.program).items():
out.writestr(name, data)
return archive
def test_valid_signature_and_runtime_identity(self):
self.assertEqual('2026.11.04', self.validate()['version'])
self.assertEqual(update.runtime_fingerprint(self.manifest), update.runtime_fingerprint(copy.deepcopy(self.manifest)))
def test_wrong_signature_key_and_modified_payload_are_rejected(self):
with self.assertRaises(update.UpdateError):
update.validate_release(self.envelope(private=Ed25519PrivateKey.generate()), self.root)
envelope = self.envelope()
envelope['payload'] = base64.b64encode(update.canonical({**self.release, 'sequence': 999})).decode()
with self.assertRaises(update.UpdateError):
update.validate_release(envelope, self.root)
def test_immutable_trust_anchor_checked_against_local_manifest(self):
self.write(update.KEY, Ed25519PrivateKey.generate().public_key().public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo))
with self.assertRaises(update.UpdateError):
self.validate()
def test_runtime_mismatch_requires_full_bundle(self):
release = {**self.release, 'runtimeFingerprint': 'b' * 64}
with self.assertRaisesRegex(update.UpdateError, '完整安装包'):
self.validate(release)
def test_forbidden_destinations_and_trust_key_are_never_update_files(self):
for name in ['.runtime/deployment.json', 'runtimedata/mysql/table.ibd', 'migration/snapshot.sql',
'runtimes/python/python.exe', update.KEY, 'manifest.json', 'backend/src/private.key']:
with self.subTest(name=name):
release = copy.deepcopy(self.release)
release['files'].append({'path': name, 'size': 0, 'sha256': 'a' * 64})
with self.assertRaises(update.UpdateError):
self.validate(release)
def test_windows_aliases_traversal_and_case_collisions_rejected(self):
for name in ['../evil', 'app/web/../evil', 'app/web/C:evil', 'app\\web\\evil', '/absolute',
'app/web/NUL.txt', 'app/web/trailing.', 'app/web/double//slash', 'app/web/control\x01']:
with self.subTest(name=name), self.assertRaises(update.UpdateError):
update.safe_path(name)
release = copy.deepcopy(self.release)
release['files'].append({**release['files'][0], 'path': release['files'][0]['path'].upper()})
with self.assertRaises(update.UpdateError):
self.validate(release)
def test_destination_symlink_rejected(self):
target = self.root / 'app/web/link'
target.symlink_to(Path(self.temp.name), target_is_directory=True)
with self.assertRaises(update.UpdateError):
update.destination(self.root, 'app/web/link/escape.html')
def test_missing_required_program_or_deployment_manifest_rejected(self):
release = copy.deepcopy(self.release)
release['files'] = [e for e in release['files'] if e['path'] != 'backend/src/main/resources/analysis-runtime-policy.json']
with self.assertRaises(update.UpdateError):
self.validate(release)
with self.assertRaisesRegex(update.UpdateError, '部署数据'):
self.validate({**self.release, 'bundleManifest': self.manifest})
def test_zip_inventory_and_hash_enforced(self):
stage = Path(self.temp.name) / 'good'
update.unpack(self.archive(), stage, self.release)
self.assertEqual(self.program['app/backend.jar'], (stage / 'app/backend.jar').read_bytes())
bad = dict(self.program)
bad['../evil'] = b'escape'
with self.assertRaises(update.UpdateError):
update.unpack(self.archive(bad), Path(self.temp.name) / 'escape', self.release)
bad = dict(self.program)
bad['app/backend.jar'] = b'x' * len(bad['app/backend.jar'])
with self.assertRaisesRegex(update.UpdateError, '校验失败'):
update.unpack(self.archive(bad), Path(self.temp.name) / 'hash', self.release)
def test_zip_symlink_rejected(self):
archive = Path(self.temp.name) / 'symlink.zip'
info = zipfile.ZipInfo('app/backend.jar')
info.external_attr = (stat.S_IFLNK | 0o777) << 16
with zipfile.ZipFile(archive, 'w') as out:
out.writestr(info, self.program['app/backend.jar'])
with self.assertRaises(update.UpdateError):
update.unpack(archive, Path(self.temp.name) / 'links', self.release)
def test_snapshot_replace_preserves_local_data_and_manifest_entries(self):
backup = self.root / '.runtime/update/backup-test'
backup.mkdir(parents=True)
stage = Path(self.temp.name) / 'stage'
update.unpack(self.archive(), stage, self.release)
update.backup_program(self.root, backup, self.release)
update.replace_program(self.root, stage, backup, self.release)
result = update.read(self.root / 'manifest.json')
self.assertEqual(6, result['updateSequence'])
self.assertEqual(self.config, update.read(self.root / '.runtime/deployment.json'))
self.assertFalse((self.root / 'app/web/obsolete.js').exists())
for name in (update.KEY, 'runtimes/python/python.exe', 'migration/snapshot.sql'):
self.assertEqual(self.files[name], (self.root / name).read_bytes())
self.assertIn(next(e for e in self.manifest['files'] if e['path'] == name), result['files'])
update.restore_program(self.root, backup)
self.assertEqual(self.manifest, update.read(self.root / 'manifest.json'))
self.assertEqual(self.files['app/web/obsolete.js'], (self.root / 'app/web/obsolete.js').read_bytes())
def worker_harness(self, fail_verify=False):
directory = self.root / '.runtime/update'
update.atomic(directory / 'request.json', {'token': 'a' * 32})
archive = self.archive()
envelope = self.envelope()
original_download = update.download
calls, restores, queries = [], [], []
def fake_download(url, target=None, *args):
if target:
shutil.copyfile(archive, target)
return b''
return json.dumps(envelope).encode()
def fake_control(root, action, database_only=False, timeout=0):
calls.append((action, database_only))
if action == 'verify' and fail_verify:
raise update.UpdateError('模拟新版启动校验失败。')
return types.SimpleNamespace(pid=1234)
def fake_mysql(root, config, binary='mysql', database=None, stdin=None, stdout=None):
if binary == 'mysqldump':
stdout.write(b'-- database snapshot\n' + b'SELECT 1;\n' * 30)
else:
restores.append((database, stdin.read()))
process = types.SimpleNamespace(parents=lambda: [], create_time=lambda: 1,
is_running=lambda: True, status=lambda: 'running')
with patch.object(update.psutil, 'pid_exists', return_value=False), \
patch.object(update.psutil, 'Process', return_value=process), \
patch.object(update, 'download', side_effect=fake_download), \
patch.object(update, 'controller', side_effect=fake_control), \
patch.object(update, 'wait_ready'), \
patch.object(update, 'mysql', side_effect=fake_mysql), \
patch.object(update, 'sql', side_effect=lambda root, config, query: queries.append(query)):
update.worker(987654, 'a' * 32, self.root)
return calls, restores, queries
def test_complete_update_orchestration_with_isolated_database_backup(self):
calls, restores, queries = self.worker_harness()
self.assertEqual([('stop', False), ('serve', True), ('stop', False), ('verify', False), ('serve', False)], calls)
self.assertFalse(restores)
self.assertEqual('completed', update.read(self.root / '.runtime/update/status.json')['state'])
self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase'])
self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence'])
def test_failed_update_restores_old_program_and_new_rollback_database(self):
self.config['restoredFromMigration'] = 'isolated-test-migration'
update.atomic(self.root / '.runtime/deployment.json', self.config)
update.atomic(self.root / '.runtime/migration-receipt.json', {'migrationId': 'isolated-test-migration',
'database': self.config['mysqlDatabase'], 'migrationManifestSha256': 'b' * 64})
calls, restores, queries = self.worker_harness(fail_verify=True)
self.assertEqual(self.manifest, update.read(self.root / 'manifest.json'))
config = update.read(self.root / '.runtime/deployment.json')
self.assertRegex(config['mysqlDatabase'], r'^quality_restore_[a-f0-9]{24}$')
self.assertNotEqual(self.config['mysqlDatabase'], config['mysqlDatabase'])
self.assertEqual(self.config['archiveDirectory'], config['archiveDirectory'])
self.assertEqual(config['mysqlDatabase'], restores[0][0])
self.assertEqual(config['mysqlDatabase'], update.read(self.root / '.runtime/migration-receipt.json')['database'])
self.assertTrue(any('CREATE DATABASE' in q for q in queries))
self.assertFalse(any('DROP' in q for q in queries))
self.assertEqual('failed', update.read(self.root / '.runtime/update/status.json')['state'])
self.assertIn('已恢复原版本', update.read(self.root / '.runtime/update/status.json')['message'])
self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase'])
def test_recovery_blocks_live_worker_and_rejects_untrusted_backup_path(self):
directory = self.root / '.runtime/update'
update.atomic(directory / 'journal.json', {'phase': 'installing', 'backup': '../../escape'})
with patch.object(update, 'live', return_value=False), self.assertRaises(update.UpdateError):
update.recover(self.root)
with patch.object(update, 'live', return_value=True), self.assertRaisesRegex(update.UpdateError, '正在进行'):
update.recover(self.root)
def test_arbitrary_download_url_is_rejected_before_network(self):
for url in ['http://quality.lyunlab.me/quality-updates/windows-x64/latest.json', 'https://evil.example/update.zip',
update.BASE + '../private.zip', update.BASE + 'quality-update-x.zip?token=x']:
with self.subTest(url=url), self.assertRaises(update.UpdateError):
update.download(url)
def test_source_is_disabled_and_status_never_exposes_private_fields(self):
update.atomic(self.root / '.runtime/update/status.json', {'state': 'idle', 'rootPassword': 'secret', 'backupPath': '/private'})
with patch.object(update.os, 'name', 'posix'):
self.assertFalse(update.enabled(self.root))
result = update.status(self.root)
self.assertNotIn('rootPassword', result)
self.assertNotIn('backupPath', result)
def test_recovery_and_worker_share_lock_and_live_identity(self):
with update.exclusive_update(self.root):
self.assertTrue(update.live(update.read(self.root / '.runtime/update/worker.json')))
with self.assertRaises(update.BusyUpdate):
update.recover(self.root)
# The lock remains authoritative if the identity file disappears.
(self.root / '.runtime/update/worker.json').unlink()
with self.assertRaises(update.BusyUpdate):
with update.exclusive_update(self.root):
self.fail('Concurrent update lock was acquired')
def test_atomic_copy_failure_preserves_existing_control_and_configuration(self):
source = Path(self.temp.name) / 'new-content'
source.write_bytes(b'new file content')
for name in ('deploy/offline/control.py', '.runtime/deployment.json', 'manifest.json'):
target = self.root / name
previous = target.read_bytes()
with self.subTest(name=name), patch.object(update.os, 'replace', side_effect=OSError('simulated interruption')):
with self.assertRaises(OSError):
update.copy_atomic(source, target)
self.assertEqual(previous, target.read_bytes())
self.assertFalse(list(target.parent.glob(target.name + '.update-*')))
def test_mysqldump_does_not_receive_mysql_only_connect_timeout_option(self):
with patch.object(update.subprocess, 'run', return_value=types.SimpleNamespace(returncode=0)) as run:
update.mysql(self.root, self.config, 'mysqldump', stdout=update.subprocess.DEVNULL)
self.assertFalse(any(arg.startswith('--connect-timeout') for arg in run.call_args.args[0]))
update.mysql(self.root, self.config, 'mysql')
self.assertIn('--connect-timeout=10', run.call_args.args[0])
def test_actual_dump_is_followed_by_a_second_recovery_space_check(self):
with patch.object(update.shutil, 'disk_usage', side_effect=[types.SimpleNamespace(free=10**15), types.SimpleNamespace(free=0)]):
calls, _, _ = self.worker_harness()
self.assertNotIn(('verify', False), calls)
self.assertEqual(self.manifest, update.read(self.root / 'manifest.json'))
self.assertIn('剩余磁盘空间不足', update.read(self.root / '.runtime/update/status.json')['message'])
if __name__ == '__main__':
unittest.main()
+3
View File
@@ -0,0 +1,3 @@
-----BEGIN PUBLIC KEY-----
MCowBQYDK2VwAyEAMPNTCeYpWygXdSxtaJzijXu0mKZF64lP7NcUuYZ4NcU=
-----END PUBLIC KEY-----
+357
View File
@@ -0,0 +1,357 @@
"""Immutable stage-zero recovery. Installed under deploy/bootstrap, never updated."""
from __future__ import annotations
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import re
import secrets
import shutil
import subprocess
import sys
import time
import psutil # Bundled immutable Python environment, not an application module.
ROOT = Path(__file__).resolve().parents[2]
def read(path, default=None):
return json.loads(path.read_text(encoding='utf-8')) if path.exists() else default
def digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def atomic_copy(source, target):
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_name(target.name + '.recover-' + secrets.token_hex(6))
try:
with source.open('rb') as reader, temporary.open('xb') as writer:
shutil.copyfileobj(reader, writer)
writer.flush()
os.fsync(writer.fileno())
temporary.replace(target)
finally:
temporary.unlink(missing_ok=True)
def safe(root, name):
if (not isinstance(name, str) or not name or '\\' in name or ':' in name or name.startswith('/')
or any(part in ('', '.', '..') or part.endswith((' ', '.')) for part in name.split('/'))):
raise RuntimeError('更新恢复路径无效。')
target = root.joinpath(*name.split('/'))
for path in (target, *target.parents):
if path == root:
break
if path.is_symlink() or (path.exists() and getattr(path.lstat(), 'st_file_attributes', 0) & 0x400):
raise RuntimeError('更新恢复目录包含无效链接。')
if not target.resolve().is_relative_to(root.resolve()):
raise RuntimeError('更新恢复路径越界。')
return target
def program(name):
if name == 'deploy/offline/update-public-key.pem' or name.startswith('deploy/bootstrap/'):
return False
path = Path(name)
return (name == 'app/backend.jar' or name.startswith('app/web/')
or name == 'backend/src/main/resources/analysis-runtime-policy.json'
or name.startswith('scripts/') and path.suffix == '.py'
or name.startswith('ai/nao-quality/')
or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md'))
def atomic_json(path, value):
temporary = path.with_name(path.name + '.stop-' + secrets.token_hex(6))
try:
with temporary.open('x', encoding='utf-8') as stream:
json.dump(value, stream)
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
finally:
temporary.unlink(missing_ok=True)
def same_path(left, right):
return os.path.normcase(str(Path(left).resolve())) == os.path.normcase(str(Path(right).resolve()))
def runtime_executable(root, role):
suffix = '.exe' if os.name == 'nt' else ''
relative = {'supervisor': 'runtimes/python/' + ('python.exe' if os.name == 'nt' else 'bin/python3'),
'backend': 'runtimes/java/bin/java' + suffix, 'mysql': 'runtimes/mysql/bin/mysqld' + suffix}[role]
return root / relative
def bound_to_root(root, role, args):
if not isinstance(args, list) or not all(isinstance(arg, str) for arg in args):
return False
if role == 'supervisor':
return (len(args) in (2, 3) and same_path(args[0], root / 'deploy/offline/control.py')
and args[1] == 'serve' and (len(args) == 2 or args[2] == '--database-only'))
if role == 'backend':
return len(args) >= 2 and args[-2] == '-jar' and same_path(args[-1], root / 'app/backend.jar')
datadirs = [arg.split('=', 1)[1] for arg in args if arg.startswith('--datadir=')]
return len(datadirs) == 1 and same_path(datadirs[0], root / 'runtimedata/mysql')
def current_user(process):
if os.name == 'nt':
return process.username().casefold() == psutil.Process().username().casefold()
return process.uids().real == os.getuid()
def owned_process(root, role, info):
"""Return a fully verified owned PID, None for a dead PID, never guess."""
if (not isinstance(info, dict) or type(info.get('pid')) is not int or info['pid'] <= 0
or not isinstance(info.get('created'), (int, float))
or not isinstance(info.get('executable'), str) or not isinstance(info.get('args'), list)):
raise RuntimeError('本包服务进程记录无效,未停止任何不明进程。')
try:
process = psutil.Process(info['pid'])
if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE:
return None
forbidden = {os.getpid(), *(parent.pid for parent in psutil.Process().parents())}
if (process.pid in forbidden or abs(process.create_time() - info['created']) >= .01
or not current_user(process) or not same_path(process.exe(), info['executable'])
or process.cmdline()[1:] != info['args']
or not same_path(info['executable'], runtime_executable(root, role))
or not bound_to_root(root, role, info['args'])):
raise RuntimeError('无法确认本包服务进程归属,未停止任何不明进程。')
return process
except psutil.NoSuchProcess:
return None
except psutil.Error:
raise RuntimeError('当前用户无法核验本包服务进程,恢复已安全停止。') from None
def root_markers(root, args):
"""Detect unregistered package processes without inspecting unrelated data."""
for arg in args:
if arg.startswith('--datadir='):
if same_path(arg.split('=', 1)[1], root / 'runtimedata/mysql'):
return True
elif Path(arg).is_absolute() and (same_path(arg, root / 'app/backend.jar')
or same_path(arg, root / 'deploy/offline/control.py')):
return True
return False
def refuse_unregistered(root, expected):
for process in psutil.process_iter():
try:
if process.pid == os.getpid() or process.status() == psutil.STATUS_ZOMBIE:
continue
args = process.cmdline()[1:]
if not root_markers(root, args):
continue
if process.pid not in expected:
raise RuntimeError('发现未登记的本包服务进程,已保留现场,请先检查该进程。')
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
continue
def wait_owned(root, role, info, seconds):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
if owned_process(root, role, info) is None:
return True
time.sleep(.05)
return owned_process(root, role, info) is None
def terminate_owned(root, role, info, seconds):
process = owned_process(root, role, info)
if process is None:
return
try:
process.terminate()
if wait_owned(root, role, info, seconds):
return
process = owned_process(root, role, info)
if process is not None:
process.kill()
if not wait_owned(root, role, info, 10):
raise RuntimeError('本包服务尚未退出,未替换任何程序文件。')
except psutil.NoSuchProcess:
return
except psutil.Error:
raise RuntimeError('当前用户无法停止本包服务,未替换任何程序文件。') from None
def shutdown_owned_mysql(root, info):
process = owned_process(root, 'mysql', info)
if process is None:
return
value = read(safe(root, '.runtime/deployment.json'), {})
port = value.get('mysqlPort')
if (type(port) is not int or not 1024 <= port <= 65535
or not re.fullmatch(r'[A-Za-z0-9_-]{20,100}', str(value.get('rootPassword', '')))
or info['args'].count('--port=' + str(port)) != 1):
raise RuntimeError('无法核验本包数据库关闭配置,未替换任何程序文件。')
executable = root / 'runtimes/mysql/bin' / ('mysqladmin.exe' if os.name == 'nt' else 'mysqladmin')
env = os.environ.copy()
env['MYSQL_PWD'] = value['rootPassword']
# Only the immutable bundled MySQL client is executed; no application code
# is loaded, and the current user's privileges are never elevated.
command = [str(executable), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
'--port=' + str(port), '--user=root', '--connect-timeout=3', 'shutdown']
try:
subprocess.run(command, env=env, cwd=root, stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20)
except (OSError, subprocess.SubprocessError):
raise RuntimeError('本包数据库无法安全关闭,未替换任何程序文件。') from None
if not wait_owned(root, 'mysql', info, 120):
raise RuntimeError('本包数据库仍在关闭,未替换任何程序文件,请稍后重新启动。')
def stop_owned_services(root, grace_seconds=200):
"""Immutable implementation of service.json + stop-request token contract.
A crashed worker can leave a healthy new supervisor/Java behind. Validate
every recorded identity before requesting shutdown; a live stale/reused PID
is an error rather than permission to terminate it. Java must be gone before
a JAR is replaced. An orphaned MySQL is shut down through its immutable
client, never forcibly killed on Windows.
"""
state = read(safe(root, '.runtime/service.json'), {})
if not isinstance(state, dict) or not isinstance(state.get('children', []), list):
raise RuntimeError('本包服务状态无效,未替换任何程序文件。')
entries = []
supervisor = state.get('supervisor')
if supervisor:
entries.append(('supervisor', supervisor))
for info in state.get('children', []):
if not isinstance(info, dict) or info.get('name') not in ('backend', 'mysql'):
raise RuntimeError('本包服务子进程记录无效,未替换任何程序文件。')
entries.append((info['name'], info))
if len(entries) > 20 or len({info.get('pid') for _, info in entries}) != len(entries):
raise RuntimeError('本包服务进程记录重复,未替换任何程序文件。')
# Validate all owners before the first stop request or signal is sent.
active = {(role, info['pid']): owned_process(root, role, info) for role, info in entries}
refuse_unregistered(root, {pid for (_, pid), process in active.items() if process is not None})
if supervisor and active.get(('supervisor', supervisor['pid'])) is not None:
token = state.get('token')
if not isinstance(token, str) or not re.fullmatch(r'[a-f0-9]{48}', token):
raise RuntimeError('本包服务停止令牌无效,未停止任何进程。')
atomic_json(safe(root, '.runtime/stop-request.json'), {'token': token})
if not wait_owned(root, 'supervisor', supervisor, grace_seconds):
terminate_owned(root, 'supervisor', supervisor, 10)
# A dead supervisor cannot finish its own finally block. Only independently
# verified package children may be stopped; unrelated/reused PIDs are refused.
for role, info in entries:
if role == 'backend':
terminate_owned(root, role, info, 35)
for role, info in entries:
if role == 'mysql':
shutdown_owned_mysql(root, info)
for role, info in entries:
if owned_process(root, role, info) is not None:
raise RuntimeError('本包服务尚未完全退出,未替换任何程序文件。')
refuse_unregistered(root, set())
stop = safe(root, '.runtime/stop-request.json')
if isinstance(state.get('token'), str) and read(stop, {}).get('token') == state['token']:
stop.unlink(missing_ok=True)
def restore_program_before_import(root, directory, journal):
name = journal.get('backup', '')
if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name):
raise RuntimeError('更新恢复记录无效。')
backup = safe(directory, name)
manifest_file, inventory_file = backup / 'manifest.json', backup / 'program.json'
if (digest(manifest_file) != journal.get('backupManifestSha256')
or digest(inventory_file) != journal.get('programInventorySha256')):
raise RuntimeError('本机更新备份校验失败,请保留目录并联系管理员。')
manifest = read(manifest_file)
old = {entry['path']: entry for entry in manifest['files'] if program(entry['path'])}
inventory = read(inventory_file)
all_names, present = inventory['all'], set(inventory['present'])
if (not isinstance(all_names, list) or len(all_names) > 50000
or len(set(all_names)) != len(all_names) or not present.issubset(all_names)):
raise RuntimeError('更新恢复清单无效。')
for name in all_names:
if not program(name):
raise RuntimeError('更新恢复清单包含受保护路径。')
target = safe(root, name)
if name in present:
source = safe(backup / 'program', name)
if name not in old or not source.is_file() or digest(source) != old[name]['sha256']:
raise RuntimeError('原程序备份缺失或损坏。')
elif name in old:
raise RuntimeError('原程序备份不完整。')
# All backups are validated before any application file is replaced.
stop_owned_services(root)
for name in all_names:
target = safe(root, name)
if name in present:
atomic_copy(safe(backup / 'program', name), target)
else:
target.unlink(missing_ok=True)
atomic_copy(manifest_file, root / 'manifest.json')
def live(info):
try:
process = psutil.Process(info['pid'])
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
and abs(process.create_time() - info['created']) < .01)
except (psutil.Error, ValueError, TypeError, KeyError):
return False
def recover(root=ROOT):
directory = safe(root, '.runtime/update')
journal = read(directory / 'journal.json', {})
if journal.get('phase') in (None, 'complete', 'rolledBack'):
return {'recovered': False}
if live(read(directory / 'worker.json', {})):
raise RuntimeError('更新正在进行,请等待原窗口完成。')
with (directory / 'worker.lock').open('a+b') as lock:
try:
if os.name == 'nt':
import msvcrt
lock.seek(0)
msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
raise RuntimeError('已有更新恢复正在进行,请等待完成。') from None
identity = {'pid': os.getpid(), 'created': psutil.Process().create_time()}
worker = directory / 'worker.json'
temporary = worker.with_suffix('.bootstrap.tmp')
temporary.write_text(json.dumps(identity), encoding='utf-8')
temporary.replace(worker)
try:
journal = read(directory / 'journal.json', {})
if journal.get('phase') == 'installing':
restore_program_before_import(root, directory, journal)
elif journal.get('phase') == 'stopping':
stop_owned_services(root)
else:
raise RuntimeError('更新恢复阶段无效,未替换任何程序文件。')
# This import occurs only after restoring a complete, verified old
# program set; no mutable application module was imported above.
path = safe(root, 'deploy/offline/update_manager.py')
spec = importlib.util.spec_from_file_location('quality_recovered_updater', path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.recover_locked(root)
finally:
if read(worker, {}) == identity:
worker.unlink(missing_ok=True)
if __name__ == '__main__':
try:
print(json.dumps(recover(), ensure_ascii=False))
except Exception as error:
print(json.dumps({'error': str(error) if isinstance(error, RuntimeError) else '更新恢复未完成,请保留本机备份并联系管理员。'}, ensure_ascii=False))
raise SystemExit(1)
+328
View File
@@ -0,0 +1,328 @@
#!/usr/bin/env python3
"""Real, isolated MySQL application-update acceptance on the macOS build host.
Never reads the source deployment configuration or connects to its MySQL port.
The test copies the application and creates its own database server and ports.
Only HTTP transport and the target-runtime verify action are adapted for the
host; signature verification, backup, file replacement, startup and rollback
are the production updater implementation.
"""
from __future__ import annotations
import argparse
import base64
from datetime import datetime, timezone
import hashlib
import http.cookiejar
import json
import os
from pathlib import Path
import secrets
import shutil
import socket
import subprocess
import sys
import tempfile
import time
import traceback
from unittest.mock import patch
import urllib.request
import zipfile
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
import update_manager as update
SOURCE = Path(__file__).resolve().parents[2]
OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def free_port():
with socket.socket() as test:
test.bind(('127.0.0.1', 0))
return test.getsockname()[1]
def backend_port():
for port in (5285, 5284, 5283):
try:
with socket.socket() as test:
test.bind(('127.0.0.1', port))
return port
except OSError:
continue
raise RuntimeError('No unused standalone test backend port is available; no existing instance was touched.')
def query(root, sql, database=None):
config = update.read(root / '.runtime/deployment.json')
command = [str(root / 'runtimes/mysql/bin/mysql'), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
'--port=' + str(config['mysqlPort']), '--user=root', '--connect-timeout=5', '--batch', '--skip-column-names',
'--default-character-set=utf8mb4']
if database:
command.append(database)
env = os.environ.copy()
env['MYSQL_PWD'] = config['rootPassword']
result = subprocess.run(command, input=sql, encoding='utf-8', capture_output=True, env=env, timeout=60)
if result.returncode:
(root / '.runtime/update/test-sql-error.log').write_text(result.stderr)
raise RuntimeError('SQL failed in the isolated test server; see its private test log.')
return result.stdout.strip()
def web(root, path='/'):
config = update.read(root / '.runtime/deployment.json')
with OPENER.open(f'http://127.0.0.1:{config["webPort"]}' + path, timeout=10) as response:
return response.read()
def login(root, username, password):
config = update.read(root / '.runtime/deployment.json')
base = f'http://127.0.0.1:{config["webPort"]}'
cookies = http.cookiejar.CookieJar()
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(cookies))
with opener.open(base + '/api/auth/csrf', timeout=10) as response:
token = json.load(response)['token']
request = urllib.request.Request(base + '/api/auth/login', data=json.dumps({'username': username, 'password': password}).encode(),
headers={'Content-Type': 'application/json', 'X-XSRF-TOKEN': token})
with opener.open(request, timeout=15) as response:
assert response.status == 200
with opener.open(base + '/api/auth/me', timeout=10) as response:
assert response.status == 200
def prepare(work):
# MySQL's Unix socket has a 103-byte path limit on this host.
root = Path(tempfile.mkdtemp(prefix='qc-upd-', dir='/tmp')).resolve()
(work / 'isolated-root.txt').write_text(str(root) + '\n')
for name in ('app/web', 'scripts', 'deploy/offline', 'runtimes/python/bin', '.runtime/update',
'runtimedata/logs', 'runtimedata/sources', 'runtimedata/archive', 'backend/src/main/resources'):
(root / name).mkdir(parents=True, exist_ok=True)
for name in ('control.py', 'gateway.py', 'update_manager.py'):
shutil.copy2(SOURCE / 'deploy/offline' / name, root / 'deploy/offline' / name)
shutil.copy2(SOURCE / 'deploy/offline/runtime_platform.py', root / 'scripts/runtime_platform.py')
# No AI processes run in this acceptance test; preserve supervisor cleanup's
# normal subprocess path while keeping it confined to the disposable root.
(root / 'scripts/nao_quality_runtime.py').write_text('"""Isolated test: no AI processes have been started."""\n')
shutil.copy2(SOURCE / 'backend/target/quality-platform-backend-0.1.0-SNAPSHOT.jar', root / 'app/backend.jar')
shutil.copy2(SOURCE / 'backend/src/main/resources/analysis-runtime-policy.json', root / 'backend/src/main/resources/analysis-runtime-policy.json')
(root / 'app/web/index.html').write_text('<!doctype html><html><body><div id="root">isolated-update-v1</div></body></html>')
(root / 'app/web/obsolete.js').write_text('/* obsolete test application asset */')
(root / 'runtimes/java').symlink_to(SOURCE / '.runtime/jdk/Contents/Home', target_is_directory=True)
(root / 'runtimes/mysql').symlink_to(SOURCE / '.runtime/mysql', target_is_directory=True)
(root / 'runtimes/python/bin/python3').symlink_to(sys.executable)
key = Ed25519PrivateKey.generate()
(root / update.KEY).write_bytes(key.public_key().public_bytes(Encoding.PEM, PublicFormat.SubjectPublicKeyInfo))
config = {'mysqlPort': free_port(), 'backendPort': backend_port(), 'webPort': free_port(), 'webBind': '127.0.0.1',
'mysqlDatabase': 'quality_platform', 'archiveDirectory': 'runtimedata/archive',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'update-test-admin', 'adminPassword': secrets.token_urlsafe(24)}
assert config['mysqlPort'] != 53306
update.atomic(root / '.runtime/deployment.json', config)
(root / '.runtime/deployment.json').chmod(0o600)
private_marker = secrets.token_hex(20)
update.atomic(root / '.runtime/analysis-api-settings.json', {'testPrivateMarker': private_marker})
manifest = {'platform': 'darwin', 'architecture': 'arm64', 'updateVersion': 'host-initial', 'updateSequence': 1,
'runtimes': [{'name': 'java', 'hostTestRuntime': True}, {'name': 'mysql', 'hostTestRuntime': True},
{'name': 'python', 'version': sys.version.split()[0], 'hostTestRuntime': True}],
'pythonWheels': [], 'files': []}
for path in sorted(root.rglob('*')):
if path.is_file() and not path.is_symlink() and not path.relative_to(root).as_posix().startswith(('.runtime/', 'runtimedata/', 'runtimes/')):
manifest['files'].append({'path': path.relative_to(root).as_posix(), 'sha256': update.digest(path)})
update.atomic(root / 'manifest.json', manifest)
return root, key, config, private_marker
def release_fixture(root, work, key, version, sequence, bad=False):
folder = work / ('fixture-' + version)
folder.mkdir()
program = {}
for entry in update.read(root / 'manifest.json')['files']:
name = entry['path']
if update.program_path(name) and name != 'app/web/obsolete.js':
program[name] = root / name
index = folder / 'index.html'
index.write_text(f'<!doctype html><html><body><div id="root">isolated-update-{version}</div></body></html>')
program['app/web/index.html'] = index
if bad:
bad_jar = folder / 'bad-backend.jar'
# The real new Java process runs Flyway, changes data, then fails. This
# proves rollback restores the dump, rather than merely restarting Java.
migration = ("UPDATE platform_metadata SET metadata_value='FAILED-UPDATE-DATA' WHERE metadata_key='update_integration_sentinel';\n"
"UPDATE app_user SET password_hash='FAILED-UPDATE-HASH' WHERE username='update-test-admin';\n"
"CREATE TABLE isolated_update_failure_marker(id INT PRIMARY KEY);\n"
"SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Intentional isolated updater integration failure';\n")
with zipfile.ZipFile(root / 'app/backend.jar') as source, zipfile.ZipFile(bad_jar, 'w') as target:
for member in source.infolist():
target.writestr(member, source.read(member))
target.writestr('BOOT-INF/classes/db/migration/V999__isolated_update_failure.sql', migration)
program['app/backend.jar'] = bad_jar
archive = folder / ('quality-update-' + version + '.zip')
with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED, compresslevel=1) as target:
for name, path in sorted(program.items()):
target.write(path, name)
release = {'schema': 1, 'platform': 'windows-x64', 'version': version, 'sequence': sequence,
'runtimeFingerprint': update.runtime_fingerprint(update.read(root / 'manifest.json')),
'archive': {'file': archive.name, 'size': archive.stat().st_size, 'sha256': update.digest(archive)},
'files': [{'path': name, 'size': path.stat().st_size, 'sha256': update.digest(path)} for name, path in sorted(program.items())]}
payload = update.canonical(release)
envelope = {'payload': base64.b64encode(payload).decode(), 'signature': base64.b64encode(key.sign(payload)).decode()}
(folder / 'latest.json').write_bytes(update.canonical(envelope))
return release, envelope, archive
def run_update(root, release, envelope, archive):
real_controller = update.controller
verify_checks = []
def host_controller(target, action, database_only=False, timeout=1100):
assert target == root
if action != 'verify':
return real_controller(target, action, database_only, timeout)
# Host adaptation: this ARM Mac cannot satisfy Windows/x64 runtime probes.
# All installed program hashes and the native Java/MySQL executables are
# still checked. Real full application startup is checked below by serve.
for entry in update.read(root / 'manifest.json')['files']:
assert update.digest(root / entry['path']) == entry['sha256'], entry['path']
for command in ([root / 'runtimes/java/bin/java', '-version'],
[root / 'runtimes/mysql/bin/mysqld', '--no-defaults', '--version']):
result = subprocess.run([str(a) for a in command], capture_output=True, timeout=30)
assert result.returncode == 0
verify_checks.append(True)
def transport(url, target=None, expected_size=None, expected_hash=None):
if url == update.LATEST and target is None:
return update.canonical(envelope)
assert url == update.BASE + archive.name and target is not None
assert expected_size == archive.stat().st_size and expected_hash == update.digest(archive)
with archive.open('rb') as reader, target.open('xb') as writer:
shutil.copyfileobj(reader, writer)
return b''
launcher = subprocess.Popen([sys.executable, '-c', 'pass'])
launcher.wait(timeout=10)
token = secrets.token_hex(16)
update.atomic(root / '.runtime/update/request.json', {'token': token, 'created': time.time()})
# Patch only this test process. Production Windows/source enablement remains
# unchanged, including in the real backend processes started by control.py.
with patch.object(update, 'enabled', return_value=True), patch.object(update, 'download', side_effect=transport), \
patch.object(update, 'controller', side_effect=host_controller):
update.worker(launcher.pid, token, root)
assert verify_checks, update.read(root / '.runtime/update/status.json').get('message')
return update.read(root / '.runtime/update/status.json')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--work', type=Path, required=True)
parser.add_argument('--expected-tables', type=int, default=83)
args = parser.parse_args()
work = args.work.resolve()
work.mkdir(parents=True, exist_ok=False)
root = None
report = {'startedAt': datetime.now(timezone.utc).isoformat(), 'passed': False,
'environment': 'macOS ARM host, disposable native MySQL + Java + Python runtimes',
'productionServicesTouched': False, 'productionDatabasePortUsed': False,
'networkTransport': 'local signed fixtures substituted only in test process',
'targetRuntimeVerify': 'host adapter: all manifest hashes plus native Java/MySQL probes; actual backend restarts',
'windowsNativeProcessDetachmentValidated': False, 'windowsNativeRuntimeCompatibilityValidated': False,
'checks': {}}
try:
print('Preparing isolated update acceptance root.', flush=True)
root, key, config, private_marker = prepare(work)
report['isolatedRoot'] = str(root)
report['ports'] = {k: config[k] for k in ('mysqlPort', 'backendPort', 'webPort')}
print('Starting isolated MySQL and Java; applying real Flyway schema.', flush=True)
process = update.controller(root, 'serve')
update.wait_ready(root, process)
database = config['mysqlDatabase']
count = int(query(root, f"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='{database}';"))
assert count == args.expected_tables, f'Expected {args.expected_tables} tables; actual isolated schema has {count}.'
report['checks']['tableCount'] = count
sentinel = secrets.token_hex(24)
query(root, f"INSERT INTO platform_metadata(metadata_key,metadata_value) VALUES ('update_integration_sentinel','{sentinel}');", database)
account_hash = query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database)
assert account_hash.startswith('$2')
login(root, config['adminUsername'], config['adminPassword'])
report['checks']['initialLogin'] = True
config['restoredFromMigration'] = 'isolated-update-migration-fixture'
update.atomic(root / '.runtime/deployment.json', config)
update.atomic(root / '.runtime/migration-receipt.json', {'migrationId': config['restoredFromMigration'],
'database': database, 'migrationManifestSha256': 'a' * 64})
before_pid = update.read(root / '.runtime/service.json')['supervisor']['pid']
print('Applying signed success release with actual dump, file replacement and service restart.', flush=True)
good = release_fixture(root, work, key, 'host-success', 2)
result = run_update(root, *good)
assert result['state'] == 'completed', result.get('message')
assert update.read(root / 'manifest.json')['updateVersion'] == 'host-success'
assert b'isolated-update-host-success' in web(root)
assert not (root / 'app/web/obsolete.js').exists()
assert update.read(root / '.runtime/service.json')['supervisor']['pid'] != before_pid
assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", database) == sentinel
assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database) == account_hash
login(root, config['adminUsername'], config['adminPassword'])
first_journal = update.read(root / '.runtime/update/journal.json')
first_dump = root / '.runtime/update' / first_journal['backup'] / 'database.sql'
assert first_dump.is_file() and first_dump.stat().st_size > 10000
report['checks'].update(realMysqldump=True, successProgramSwitch=True, realServiceRestart=True,
obsoleteProgramRemoved=True, successSentinelPreserved=True,
successAccountHashPreserved=True, successLogin=True,
successDumpBytes=first_dump.stat().st_size)
print('Applying signed bad Java release: real Flyway mutates sentinel/hash then intentionally fails.', flush=True)
bad = release_fixture(root, work, key, 'host-bad', 3, bad=True)
result = run_update(root, *bad)
assert result['state'] == 'failed' and '已恢复原版本' in result['message'], result
restored = update.read(root / '.runtime/deployment.json')
restored_db = restored['mysqlDatabase']
assert restored_db != database and restored_db.startswith('quality_restore_')
assert update.read(root / 'manifest.json')['updateVersion'] == 'host-success'
assert b'isolated-update-host-success' in web(root)
assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", restored_db) == sentinel
assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", restored_db) == account_hash
assert int(query(root, f"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='{restored_db}';")) == count
assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", database) == 'FAILED-UPDATE-DATA', 'Failed original sentinel did not retain the migration write.'
assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database) == 'FAILED-UPDATE-HASH', 'Failed original account hash did not retain the migration write.'
assert query(root, "SELECT COUNT(*) FROM flyway_schema_history WHERE version='999' AND success=0;", database) == '1'
assert query(root, "SELECT COUNT(*) FROM flyway_schema_history WHERE version='999';", restored_db) == '0'
assert update.read(root / '.runtime/analysis-api-settings.json')['testPrivateMarker'] == private_marker
assert restored['archiveDirectory'] == config['archiveDirectory']
assert restored['rootPassword'] == config['rootPassword'] and restored['mysqlPassword'] == config['mysqlPassword']
assert update.read(root / '.runtime/migration-receipt.json')['database'] == restored_db
login(root, config['adminUsername'], config['adminPassword'])
failed_journal = update.read(root / '.runtime/update/journal.json')
assert failed_journal['phase'] == 'rolledBack'
failed_dump = root / '.runtime/update' / failed_journal['backup'] / 'database.sql'
report['checks'].update(badReleaseActuallyMutatedDatabase=True, failedOriginalDatabasePreserved=True,
restoredIntoNewDatabase=True, restoredTableCount=count, rollbackSentinelRestored=True,
rollbackAccountHashRestored=True, rollbackLogin=True, oldProgramRestarted=True,
privateConfigurationPreserved=True, archiveDirectoryPreserved=True,
migrationReceiptDatabaseAligned=True,
failedFlywayHistoryPreserved=True, restoredFlywayHistoryClean=True,
rollbackDumpBytes=failed_dump.stat().st_size,
originalDatabase=database, rollbackDatabase=restored_db)
report['passed'] = True
print('Real MySQL success and failed-migration rollback acceptance passed.', flush=True)
except Exception as error:
report['error'] = str(error)
(work / 'test-failure-trace.txt').write_text(traceback.format_exc())
print('Isolated acceptance failed: ' + str(error), flush=True)
finally:
if root is not None:
try:
update.controller(root, 'stop', timeout=180)
state = update.read(root / '.runtime/service.json', {})
assert not update.live(state.get('supervisor', {}))
report['isolatedServicesStopped'] = True
except Exception as error:
report['isolatedServicesStopped'] = False
report['cleanupError'] = str(error)
report['passed'] = False
report['finishedAt'] = datetime.now(timezone.utc).isoformat()
(work / 'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
print(json.dumps(report, ensure_ascii=False), flush=True)
return 0 if report['passed'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+646
View File
@@ -0,0 +1,646 @@
#!/usr/bin/env python3
"""Signed Windows standalone application updates. No caller-supplied URLs or paths.
The release envelope signs the *decoded payload bytes* with Ed25519. Runtime
identity is SHA256(canonical({runtimes, pythonWheels})). Program files are a full
snapshot; private data and runtimes are never update destinations.
"""
from __future__ import annotations
import argparse
import base64
import contextlib
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import secrets
import shutil
import stat
import subprocess
import sys
import time
import zipfile
from urllib.request import ProxyHandler, HTTPRedirectHandler, build_opener
import psutil
from cryptography.hazmat.primitives.serialization import load_pem_public_key
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
ROOT = Path(__file__).resolve().parents[2]
BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/'
LATEST = BASE + 'latest.json'
KEY = 'deploy/offline/update-public-key.pem'
MAX_MANIFEST = 16 * 1024 * 1024
MAX_ARCHIVE = 2 * 1024**3
MAX_EXPANDED = 4 * 1024**3
ACTIVE = {'queued', 'downloading', 'backingUp', 'installing', 'restarting', 'rollingBack'}
SHA = re.compile(r'[a-f0-9]{64}')
class UpdateError(RuntimeError):
pass
class BusyUpdate(UpdateError):
pass
def canonical(value):
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8')
def unique_object(pairs):
value = {}
for key, item in pairs:
if key in value:
raise UpdateError('更新清单包含重复字段。')
value[key] = item
return value
def decode(raw):
return json.loads(raw, object_pairs_hook=unique_object)
def digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def atomic(path, value):
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
temporary = path.with_name(path.name + '.' + secrets.token_hex(6) + '.tmp')
with temporary.open('xb') as stream:
stream.write(canonical(value))
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
def copy_atomic(source, target):
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_name(target.name + '.update-' + secrets.token_hex(6))
try:
with source.open('rb') as reader, temporary.open('xb') as writer:
shutil.copyfileobj(reader, writer)
writer.flush()
os.fsync(writer.fileno())
shutil.copystat(source, temporary)
temporary.replace(target)
finally:
temporary.unlink(missing_ok=True)
def read(path, default=None):
return decode(path.read_bytes()) if path.exists() else default
def runtime_fingerprint(manifest):
return hashlib.sha256(canonical({key: manifest.get(key, []) for key in ('runtimes', 'pythonWheels')})).hexdigest()
def safe_path(name):
if (not isinstance(name, str) or not name or len(name) > 230 or '\\' in name
or ':' in name or '\x00' in name or name.startswith('/')
or any(ord(c) < 32 for c in name)):
raise UpdateError('更新文件路径无效。')
parts = name.split('/')
if any(p in ('', '.', '..') or p.endswith((' ', '.'))
or re.fullmatch(r'(?i)(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?', p) for p in parts):
raise UpdateError('更新文件路径无效。')
return PurePosixPath(name)
def program_path(name):
path = safe_path(name)
if name == KEY or name.startswith('deploy/bootstrap/'):
return False # immutable trust anchor, even in a correctly signed release
return (name == 'app/backend.jar' or name.startswith('app/web/')
or name == 'backend/src/main/resources/analysis-runtime-policy.json'
or name.startswith('scripts/') and path.suffix == '.py'
or name.startswith('ai/nao-quality/')
or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md'))
def destination(root, name):
safe_path(name)
target = root.joinpath(*name.split('/'))
for path in (target, *target.parents):
if path == root:
break
if path.is_symlink():
raise UpdateError('更新目录包含符号链接,已停止更新。')
if os.name == 'nt' and path.exists() and getattr(path.lstat(), 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT:
raise UpdateError('更新目录包含重解析点,已停止更新。')
if not target.resolve().is_relative_to(root.resolve()):
raise UpdateError('更新文件超出程序目录。')
return target
def entry_map(entries, program_only=False):
if not isinstance(entries, list) or len(entries) > 50000:
raise UpdateError('更新文件清单无效。')
result, folded = {}, set()
for entry in entries:
if not isinstance(entry, dict):
raise UpdateError('更新文件清单无效。')
name = entry.get('path')
safe_path(name)
if name.casefold() in folded:
raise UpdateError('更新文件路径重复。')
folded.add(name.casefold())
if program_only and not program_path(name):
raise UpdateError('更新包含不允许替换的文件。')
if entry.get('symlink') is not None or not SHA.fullmatch(str(entry.get('sha256', ''))):
raise UpdateError('更新文件校验信息无效。')
result[name] = entry
return result
def validate_release(envelope, root=ROOT):
try:
raw = base64.b64decode(envelope['payload'], validate=True)
signature = base64.b64decode(envelope['signature'], validate=True)
key = load_pem_public_key(destination(root, KEY).read_bytes())
if not isinstance(key, Ed25519PublicKey):
raise ValueError('key type')
key.verify(signature, raw)
release = decode(raw)
except Exception as error:
raise UpdateError('更新签名验证失败。') from error
current = read(root / 'manifest.json')
if release.get('schema') != 1 or release.get('platform') != 'windows-x64':
raise UpdateError('更新包不适用于此系统。')
if not isinstance(release.get('version'), str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]{0,79}', release['version']):
raise UpdateError('更新版本无效。')
if type(release.get('sequence')) is not int or release['sequence'] <= 0:
raise UpdateError('更新版本序号无效。')
if release.get('runtimeFingerprint') != runtime_fingerprint(current):
raise UpdateError('运行环境版本不同,请使用完整安装包更新。')
archive = release.get('archive', {})
if (not re.fullmatch(r'quality-update-[A-Za-z0-9][A-Za-z0-9._-]{0,100}\.zip', str(archive.get('file', '')))
or type(archive.get('size')) is not int or not 0 < archive['size'] <= MAX_ARCHIVE
or not SHA.fullmatch(str(archive.get('sha256', '')))):
raise UpdateError('更新下载信息无效。')
files = entry_map(release.get('files'), True)
if not files or any(type(e.get('size')) is not int or e['size'] < 0 for e in files.values()) or sum(e['size'] for e in files.values()) > MAX_EXPANDED:
raise UpdateError('更新文件大小无效。')
if 'bundleManifest' in release:
raise UpdateError('更新清单不能包含部署数据。')
old_files = entry_map(current.get('files'))
required = {'app/backend.jar', 'deploy/offline/control.py', 'deploy/offline/update_manager.py',
'backend/src/main/resources/analysis-runtime-policy.json'}
if not required.issubset(files) or not any(name.startswith('app/web/') for name in files):
raise UpdateError('更新缺少必要程序文件。')
if KEY not in old_files or digest(destination(root, KEY)) != old_files[KEY]['sha256']:
raise UpdateError('更新信任公钥校验失败。')
for name in set(old_files) | set(files):
destination(root, name)
return release
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
raise UpdateError('更新服务器返回了不允许的跳转。')
def download(url, target=None, expected_size=None, expected_hash=None):
if url != LATEST and not (url.startswith(BASE) and re.fullmatch(r'quality-update-[A-Za-z0-9][A-Za-z0-9._-]{0,100}\.zip', url[len(BASE):])):
raise UpdateError('更新下载地址无效。')
limit = expected_size if expected_size is not None else MAX_MANIFEST
hasher, count, chunks = hashlib.sha256(), 0, []
with build_opener(ProxyHandler({}), NoRedirect()).open(url, timeout=30) as response:
if response.geturl() != url or response.status != 200:
raise UpdateError('更新服务器响应无效。')
with target.open('xb') if target is not None else contextlib.nullcontext() as stream:
while block := response.read(1024 * 1024):
count += len(block)
if count > limit:
raise UpdateError('更新下载大小不符合清单。')
hasher.update(block)
if stream is not None:
stream.write(block)
else:
chunks.append(block)
if expected_size is not None and (count != expected_size or hasher.hexdigest() != expected_hash):
raise UpdateError('更新下载校验失败。')
return b''.join(chunks)
def unpack(archive, stage, release):
entries = {e['path']: e for e in release['files']}
with zipfile.ZipFile(archive) as source:
seen = set()
for member in source.infolist():
name = member.filename
if name not in entries or name in seen or member.is_dir() or member.flag_bits & 1:
raise UpdateError('更新压缩包文件清单不符。')
seen.add(name)
if stat.S_ISLNK(member.external_attr >> 16) or member.file_size != entries[name]['size']:
raise UpdateError('更新压缩包文件类型或大小不符。')
target = destination(stage, name)
target.parent.mkdir(parents=True, exist_ok=True)
count = 0
with source.open(member) as reader, target.open('xb') as writer:
while block := reader.read(1024 * 1024):
count += len(block)
if count > entries[name]['size']:
raise UpdateError('更新解压大小不符。')
writer.write(block)
if digest(target) != entries[name]['sha256']:
raise UpdateError('更新解压校验失败。')
if seen != set(entries):
raise UpdateError('更新压缩包缺少文件。')
def enabled(root=ROOT):
return os.name == 'nt' and os.environ.get('QUALITY_UPDATE_ENABLED') == 'true' and (root / 'runtimes/python/python.exe').is_file() and (root / 'manifest.json').is_file()
def status(root=ROOT):
manifest = read(root / 'manifest.json', {})
value = read(root / '.runtime/update/status.json', {})
allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'}
result = {k: v for k, v in value.items() if k in allowed}
result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本')))
result.setdefault('state', 'idle')
if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})):
pending = read(root / '.runtime/update/request.json', {})
journal = read(root / '.runtime/update/journal.json', {})
if time.time() - pending.get('created', 0) > 120:
result.update(state='failed', available=False, message='更新进程已中断,请重新运行软件启动入口恢复。'
if journal.get('phase') not in (None, 'complete', 'rolledBack') else '更新进程已结束,请重新检查更新。')
if not result['enabled']:
result['message'] = '此部署不支持软件内更新。'
return result
def write_status(state, message, root=ROOT, **extra):
destination(root, '.runtime/update/status.json')
value = read(root / '.runtime/update/status.json', {})
value.update(extra, state=state, message=message)
atomic(root / '.runtime/update/status.json', value)
return status(root)
def check(root=ROOT):
release = validate_release(decode(download(LATEST)), root)
available = release['sequence'] > read(root / 'manifest.json').get('updateSequence', 0)
atomic(root / '.runtime/update/checked.json', release)
return write_status('available' if available else 'current', '发现新版本。' if available else '已是最新版本。', root,
version=release['version'], available=available, checkedAt=datetime.now(timezone.utc).isoformat())
def process_options():
return {'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_BREAKAWAY_FROM_JOB} if os.name == 'nt' else {'start_new_session': True}
@contextlib.contextmanager
def exclusive_update(root=ROOT):
directory = root / '.runtime/update'
destination(root, '.runtime/update')
directory.mkdir(parents=True, exist_ok=True)
if live(read(directory / 'worker.json', {})):
raise BusyUpdate('已有更新或恢复正在进行。')
sys.path.insert(0, str(root / 'scripts'))
from runtime_platform import fcntl
with (directory / 'worker.lock').open('a+b') as lock:
try:
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except (BlockingIOError, OSError):
raise BusyUpdate('已有更新或恢复正在进行。') from None
identity = {'pid': os.getpid(), 'created': psutil.Process().create_time()}
atomic(directory / 'worker.json', identity)
try:
yield
finally:
if read(directory / 'worker.json', {}) == identity:
(directory / 'worker.json').unlink(missing_ok=True)
def launch(root=ROOT):
with exclusive_update(root):
return launch_locked(root)
def launch_locked(root=ROOT):
directory = root / '.runtime/update'
destination(root, '.runtime/update')
directory.mkdir(parents=True, exist_ok=True)
# This short launcher MUST exit before its worker may stop Java. The worker
# verifies launcher disappearance; DETACHED alone does not remove ancestry.
pending = read(directory / 'request.json', {})
if pending and time.time() - pending.get('created', 0) < 120:
raise BusyUpdate('已有更新正在启动。')
check(root) # refetch and verify; do not trust browser or cached JSON
if not status(root).get('available'):
return status(root)
token = secrets.token_hex(16)
atomic(directory / 'request.json', {'token': token, 'created': time.time()})
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False)
try:
with (directory / 'worker.log').open('ab') as log:
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token],
cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
except Exception:
(directory / 'request.json').unlink(missing_ok=True)
raise UpdateError('独立更新进程无法启动,服务仍在运行。') from None
return status(root)
def live(info):
try:
process = psutil.Process(info['pid'])
return process.is_running() and process.status() != psutil.STATUS_ZOMBIE and abs(process.create_time() - info['created']) < .01
except (psutil.Error, KeyError, TypeError):
return False
def controller(root, action, database_only=False, timeout=1100):
python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3')
command = [str(python), str(root / 'deploy/offline/control.py'), action]
if database_only:
command += ['--database-only']
if action == 'serve':
with (root / '.runtime/update/service.log').open('ab') as log:
return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
result = subprocess.run(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
if result.returncode:
raise UpdateError('服务启停失败,请查看本机更新日志。')
return result
def wait_ready(root, process, database_only=False):
deadline = time.monotonic() + (180 if database_only else 1000)
while time.monotonic() < deadline:
state = read(root / '.runtime/service.json', {})
if state.get('ready') and state.get('databaseOnly', False) == database_only and state.get('supervisor', {}).get('pid') == process.pid:
return
if process.poll() is not None:
raise UpdateError('更新后服务启动失败,正在恢复旧版本。')
time.sleep(.5)
raise UpdateError('更新后服务启动超时,正在恢复旧版本。')
def mysql(root, config, binary='mysql', database=None, stdin=None, stdout=subprocess.PIPE):
exe = '.exe' if os.name == 'nt' else ''
command = [str(root / 'runtimes/mysql/bin' / (binary + exe)), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
'--port=' + str(config['mysqlPort']), '--user=root', '--default-character-set=utf8mb4']
if binary == 'mysqldump':
command += ['--single-transaction', '--routines', '--events', '--triggers', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', config.get('mysqlDatabase', 'quality_platform')]
elif database:
command.append(database)
if binary == 'mysql':
command.append('--connect-timeout=10')
env = os.environ.copy()
env['MYSQL_PWD'] = config['rootPassword']
result = subprocess.run(command, cwd=root, env=env, stdin=stdin, stdout=stdout, stderr=subprocess.PIPE, timeout=3600)
if result.returncode:
error_log = destination(root, '.runtime/update/database-error.log')
with error_log.open('ab') as log:
log.write((binary + ' failed\n').encode() + result.stderr[-16384:] + b'\n')
raise UpdateError('数据库备份或恢复失败,已保留本机备份和原数据库。')
def sql(root, config, query):
# stdin must be a real file on Windows; do not place passwords in arguments.
path = root / '.runtime/update/query.sql'
try:
path.write_text(query, encoding='utf-8')
with path.open('rb') as source:
mysql(root, config, stdin=source)
finally:
path.unlink(missing_ok=True)
def backup_program(root, backup, release):
current = read(root / 'manifest.json')
names = sorted({e['path'] for e in current['files'] if program_path(e['path'])} | {e['path'] for e in release['files']})
present = []
for name in names:
source = destination(root, name)
if source.exists():
target = destination(backup / 'program', name)
target.parent.mkdir(parents=True, exist_ok=True)
copy_atomic(source, target)
present.append(name)
copy_atomic(root / 'manifest.json', backup / 'manifest.json')
# All private configuration (AI keys, deployment credentials and bootstrap
# flags) is backed up locally, but never becomes a release payload.
shutil.copytree(root / '.runtime', backup / 'private', ignore=shutil.ignore_patterns('update', '*.lock', '*.log', 'service.json', 'stop-request.json'), symlinks=False)
for path in (backup / 'private').rglob('*'):
if path.is_file():
with path.open('rb') as stream:
os.fsync(stream.fileno())
atomic(backup / 'program.json', {'all': names, 'present': present})
def replace_program(root, stage, backup, release):
current = read(root / 'manifest.json')
names = read(backup / 'program.json')['all']
wanted = {e['path'] for e in release['files']}
for name in names:
target = destination(root, name)
if name in wanted:
copy_atomic(stage / name, target)
else:
target.unlink(missing_ok=True)
current['files'] = ([e for e in current['files'] if not program_path(e['path'])]
+ [{'path': e['path'], 'sha256': e['sha256']} for e in release['files']])
current.update(updateSequence=release['sequence'], updateVersion=release['version'])
atomic(root / 'manifest.json', current)
def restore_program(root, backup):
names = read(backup / 'program.json')
for name in names['all']:
target = destination(root, name)
if name in names['present']:
copy_atomic(backup / 'program' / name, target)
else:
target.unlink(missing_ok=True)
copy_atomic(backup / 'manifest.json', destination(root, 'manifest.json'))
for source in (backup / 'private').rglob('*'):
if source.is_file():
relative = source.relative_to(backup / 'private').as_posix()
copy_atomic(source, destination(root, '.runtime/' + relative))
def rollback(root, backup, database_backed_up):
write_status('rollingBack', '更新未完成,正在恢复原版本。', root)
controller(root, 'stop', timeout=180)
restore_program(root, backup)
if database_backed_up:
config = read(root / '.runtime/deployment.json')
process = controller(root, 'serve', True)
wait_ready(root, process, True)
database = 'quality_restore_' + secrets.token_hex(12)
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
with (backup / 'database.sql').open('rb') as source:
mysql(root, config, database=database, stdin=source)
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
controller(root, 'stop', timeout=180)
config['mysqlDatabase'] = database
atomic(root / '.runtime/deployment.json', config)
receipt_path = root / '.runtime/migration-receipt.json'
receipt = read(receipt_path, {})
if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'):
receipt['database'] = database
atomic(receipt_path, receipt)
process = controller(root, 'serve')
wait_ready(root, process)
def worker(launcher, token, root=ROOT):
directory = root / '.runtime/update'
for _ in range(300):
if not psutil.pid_exists(launcher):
break
time.sleep(.1)
else:
raise UpdateError('更新启动器未退出,已安全停止。')
if any('java' in p.name().lower() for p in psutil.Process().parents()):
raise UpdateError('更新进程未独立启动,已安全停止。')
if read(directory / 'request.json', {}).get('token') != token:
raise UpdateError('更新请求已失效。')
with exclusive_update(root):
backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4))
stopped = backed_up = program_saved = changed = False
try:
envelope = decode(download(LATEST))
release = validate_release(envelope, root)
if release['sequence'] <= read(root / 'manifest.json').get('updateSequence', 0):
raise UpdateError('此更新版本已安装。')
backup.mkdir()
archive, stage = backup / 'update.zip', backup / 'stage'
expanded = sum(entry['size'] for entry in release['files'])
# Keep enough headroom for extraction, old program files and a
# logical dump; the dump itself must complete before replacement.
db_size = sum(p.stat().st_size for p in (root / 'runtimedata/mysql').rglob('*') if p.is_file())
old_program_size = sum((root / entry['path']).stat().st_size for entry in read(root / 'manifest.json')['files'] if program_path(entry['path']) and (root / entry['path']).is_file())
private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory))
if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2:
raise UpdateError('磁盘剩余空间不足,请释放空间后重试。')
write_status('downloading', '正在下载并校验更新。', root)
info = release['archive']
download(BASE + info['file'], archive, info['size'], info['sha256'])
unpack(archive, stage, release)
# Revalidate immediately before any service or disk mutation.
validate_release(envelope, root)
write_status('backingUp', '正在备份,服务即将重新启动。', root)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False})
controller(root, 'stop', timeout=180)
stopped = True
backup_program(root, backup, release)
program_saved = True
config = read(root / '.runtime/deployment.json')
database = controller(root, 'serve', True)
wait_ready(root, database, True)
with (backup / 'database.sql').open('xb') as output:
mysql(root, config, 'mysqldump', stdout=output)
output.flush()
os.fsync(output.fileno())
if (backup / 'database.sql').stat().st_size < 100:
raise UpdateError('数据库备份为空,已停止更新。')
backed_up = True
controller(root, 'stop', timeout=180)
# The actual dump can exceed the source's physical storage. Do not
# replace anything unless a separate rollback database still fits.
required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2
if shutil.disk_usage(root).free < required_recovery:
raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。')
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True,
'backupManifestSha256': digest(backup / 'manifest.json'),
'programInventorySha256': digest(backup / 'program.json')})
write_status('installing', '正在安装更新。', root)
changed = True
replace_program(root, stage, backup, release)
controller(root, 'verify', timeout=180)
write_status('restarting', '正在重新启动,请稍候。', root)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'complete', 'databaseBackedUp': True})
write_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
except Exception as error:
message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。'
if stopped:
try:
if program_saved and changed:
rollback(root, backup, backed_up)
else:
controller(root, 'stop', timeout=180)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'rolledBack', 'databaseBackedUp': backed_up})
message += ' 已恢复原版本。'
except Exception:
message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。'
write_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
finally:
(directory / 'request.json').unlink(missing_ok=True)
def recover(root=ROOT):
with exclusive_update(root):
return recover_locked(root)
def recover_locked(root=ROOT):
directory = root / '.runtime/update'
journal = read(directory / 'journal.json', {})
if journal.get('phase') in (None, 'complete', 'rolledBack'):
return status(root)
name = journal.get('backup', '')
if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name):
raise UpdateError('更新恢复记录无效。')
backup = directory / name
if journal.get('phase') == 'installing':
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
raise UpdateError('更新恢复备份校验失败,已保留现场。')
rollback(root, backup, journal.get('databaseBackedUp') is True)
else:
controller(root, 'stop', timeout=180)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'})
return write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('status', 'check', 'launch', 'worker', 'recover'))
parser.add_argument('--launcher', type=int)
parser.add_argument('--token')
args = parser.parse_args()
if args.action == 'status':
return status()
if not enabled():
raise UpdateError('此部署不支持软件内更新。')
if args.action == 'worker':
if not args.launcher or not re.fullmatch(r'[a-f0-9]{32}', args.token or ''):
raise UpdateError('更新请求无效。')
worker(args.launcher, args.token)
return status()
if args.action == 'recover':
return recover()
journal = read(ROOT / '.runtime/update/journal.json', {})
if journal.get('phase') not in (None, 'complete', 'rolledBack'):
raise UpdateError('上次更新中断,请重新运行软件启动入口恢复。')
if live(read(ROOT / '.runtime/update/worker.json', {})):
raise BusyUpdate('已有更新正在进行。')
return check() if args.action == 'check' else launch()
if __name__ == '__main__':
try:
print(json.dumps(main(), ensure_ascii=False))
except Exception as error:
message = str(error) if isinstance(error, UpdateError) else '更新服务暂不可用,请稍后重试。'
if enabled() and not isinstance(error, BusyUpdate) and sys.argv[1:2] != ['status'] and not live(read(ROOT / '.runtime/update/worker.json', {})):
write_status('failed', message)
print(json.dumps({'error': message}, ensure_ascii=False))
raise SystemExit(1)
+7
View File
@@ -0,0 +1,7 @@
param([ValidateRange(1024,65535)][int]$Port = 5280)
$ErrorActionPreference = 'Stop'
Write-Host '此兼容脚本不执行任何系统级准备操作,也不需要管理员权限。'
Write-Host '运行库随离线包提供。请使用 windows-start.ps1 启动应用。'
Write-Host "网页端口为 $Port;其他设备能否访问取决于 Windows 当前防火墙和网络策略。"
exit 0
+64
View File
@@ -0,0 +1,64 @@
$ErrorActionPreference = 'Stop'
$root = [System.IO.Path]::GetFullPath((Split-Path (Split-Path $PSScriptRoot -Parent) -Parent))
try {
if (-not [Environment]::Is64BitOperatingSystem -or $env:PROCESSOR_ARCHITECTURE -eq 'ARM64') {
throw '此离线包需要 Intel/AMD 64 位 Windows 电脑。'
}
$manifestPath = Join-Path $root 'manifest.json'
if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
throw '未找到离线包清单 manifest.json。请解压完整离线包后再启动。'
}
$manifest = Get-Content -Raw -Encoding UTF8 $manifestPath | ConvertFrom-Json
$maxRelativePath = 0
foreach ($entry in $manifest.files) {
if ($null -eq $entry.path -or $entry.path -isnot [string] -or [string]::IsNullOrWhiteSpace($entry.path)) {
throw '离线包清单包含无效文件路径。请重新解压完整离线包。'
}
if ($entry.path.Length -gt $maxRelativePath) { $maxRelativePath = $entry.path.Length }
}
if ($maxRelativePath -eq 0) { throw '离线包清单没有文件路径。请重新解压完整离线包。' }
$maxRootLength = 248 - $maxRelativePath
if ($root.Length -gt $maxRootLength) {
$recommended = if ($env:LOCALAPPDATA) { Join-Path $env:LOCALAPPDATA 'QualityCenter' } else { '当前用户可写的较短本地 NTFS 目录' }
throw "部署目录路径过长:根路径和清单最长相对路径不能超过 248 个字符。请尝试 $recommended;若该路径仍过长,请选择更短的当前用户可写 NTFS 目录。"
}
$driveRoot = [System.IO.Path]::GetPathRoot($root)
$drive = [System.IO.DriveInfo]::new($driveRoot)
if ($drive.DriveType -ne [System.IO.DriveType]::Fixed -or $drive.DriveFormat -ne 'NTFS') {
throw '请将离线包解压到本机固定 NTFS 磁盘上的当前用户可写目录。'
}
$python = Join-Path $root 'runtimes\python\python.exe'
if (-not (Test-Path -LiteralPath $python -PathType Leaf)) {
throw '未找到随包提供的 Python 运行环境。请重新解压完整离线包。'
}
$env:PYTHONUTF8 = '1'
$env:PYTHONIOENCODING = 'utf-8'
$env:QUALITY_UPDATE_ENABLED = 'true'
# Recover an interrupted app-local update before reading mutable deployment
# configuration or starting the local service processes.
& $python (Join-Path $root 'deploy\bootstrap\update-recovery.py')
if ($LASTEXITCODE -ne 0) { throw '更新恢复仍未完成。请保留本地更新备份后重试。' }
$port = 5280
$config = Join-Path $root '.runtime\deployment.json'
if (Test-Path -LiteralPath $config -PathType Leaf) {
$port = [int](Get-Content -Raw -Encoding UTF8 $config | ConvertFrom-Json).webPort
}
if ($port -lt 1024 -or $port -gt 65535) { throw '网页端口配置无效。' }
if (Test-Path (Join-Path $root 'migration\manifest.json')) {
& $python (Join-Path $root 'deploy\offline\restore_migration.py')
if ($LASTEXITCODE -ne 0) { throw '数据恢复未完成。现有数据库已保留,请查看 runtimedata\logs。' }
}
Write-Host '正在以当前用户启动本机应用进程;不会安装系统组件、创建 Windows 服务或更改防火墙规则。'
Write-Host "网页服务使用内网监听,端口为 $port。其他设备能否访问取决于 Windows 当前防火墙和网络策略。"
& $python (Join-Path $root 'deploy\offline\control.py') start --lan
if ($LASTEXITCODE -ne 0) { throw '启动未完成。请查看 runtimedata\logs\supervisor.log。' }
} catch {
Write-Host $_ -ForegroundColor Red
exit 1
}
+2 -2
View File
@@ -291,8 +291,8 @@ function SourcesPage({ onUnauthorized, user }: { onUnauthorized: (error: unknown
return <div className="management-page"> return <div className="management-page">
{error && <Alert showIcon type="error" message="来源信息读取失败" description={sourcesLoaded || releaseLoaded ? `${error};已成功读取的内容仍保留。` : error} action={<Button size="small" onClick={() => void load()}>重试</Button>} />} {error && <Alert showIcon type="error" message="来源信息读取失败" description={sourcesLoaded || releaseLoaded ? `${error};已成功读取的内容仍保留。` : error} action={<Button size="small" onClick={() => void load()}>重试</Button>} />}
<ManagementSections initialKey={historyOpen ? 'history' : 'intake'} items={[ <ManagementSections initialKey={historyOpen ? 'history' : 'intake'} items={[
{ key: 'intake', label: '数据接入', children: <DataWorkflowHub user={user} sources={sources} currentReleaseId={currentReleaseId} pendingDeletion={pendingDeletion} onDeleted={onDeleted} onBusyChange={setMaintenanceBusy} onUnauthorized={onUnauthorized} onRefresh={() => void load()} /> }, { key: 'intake', label: '数据接入', children: <DataWorkflowHub user={user} sources={sources} currentReleaseId={currentReleaseId} pendingDeletion={pendingDeletion} onBusyChange={setMaintenanceBusy} onUnauthorized={onUnauthorized} onRefresh={() => void load()} /> },
{ key: 'files', label: '来源文件', children: <SourceFilesPanel user={user} sources={sources} sourcesLoaded={sourcesLoaded} loading={sourceRead.busy} pendingIds={pendingDeletion.ids} maintenanceBusy={maintenanceBusy} onToggleDelete={toggleDelete} onClearDeletes={() => setPendingDeletion({ ids: [], releaseId: null })} onUnauthorized={onUnauthorized} onRefresh={() => void load()} /> }, { key: 'files', label: '来源文件', children: <SourceFilesPanel user={user} sources={sources} sourcesLoaded={sourcesLoaded} loading={sourceRead.busy} pendingIds={pendingDeletion.ids} pendingReleaseId={pendingDeletion.releaseId} maintenanceBusy={maintenanceBusy} onDeleted={onDeleted} onBusyChange={setMaintenanceBusy} onToggleDelete={toggleDelete} onClearDeletes={() => setPendingDeletion({ ids: [], releaseId: null })} onUnauthorized={onUnauthorized} onRefresh={() => void load()} /> },
{ key: 'history', label: '导入记录', children: <ImportHistoryPanel user={user} onUnauthorized={onUnauthorized} /> }, { key: 'history', label: '导入记录', children: <ImportHistoryPanel user={user} onUnauthorized={onUnauthorized} /> },
{ key: 'audit', label: '来源审计', children: <SourceGovernancePanel user={user} currentReleaseId={currentReleaseId} onUnauthorized={onUnauthorized} /> }, { key: 'audit', label: '来源审计', children: <SourceGovernancePanel user={user} currentReleaseId={currentReleaseId} onUnauthorized={onUnauthorized} /> },
]} /> ]} />
+2 -2
View File
@@ -7,9 +7,9 @@ export type WorkbookPreview = {
preview: { sheetName: string; headerRow: number; headers: string[]; sampleRows: { rowNumber: number; values: Record<string, unknown> }[] }; preview: { sheetName: string; headerRow: number; headers: string[]; sampleRows: { rowNumber: number; values: Record<string, unknown> }[] };
fields: Record<DataStage, DataField[]>; fields: Record<DataStage, DataField[]>;
}; };
export type PublishedSourceTarget = { documentId: string; revisionId: string; sourceSlot: string; stage: DataStage; year: number; month: number; scopeCode: string; sourceFile: string }; export type PublishedSourceTarget = { documentId: string; revisionId: string; sourceSlot: string; stage: DataStage; year: number; month: number; scopeCode: string; sourceFile: string; fileId?: string | null; sourceMode?: 'MAIN' | 'SHARD_SNAPSHOT' | string };
export type SourceUploadOptions = { currentReleaseId: string | null; targets: PublishedSourceTarget[]; fields: Record<DataStage, DataField[]>; scopes: string[]; currentSourceIds?: string[]; currentSourcesComplete?: boolean }; export type SourceUploadOptions = { currentReleaseId: string | null; targets: PublishedSourceTarget[]; fields: Record<DataStage, DataField[]>; scopes: string[]; currentSourceIds?: string[]; currentSourcesComplete?: boolean };
export type ConfigureSourceBody = { uploadId: string; stage: DataStage; year: number; month: number; sheetName: string; headerRow: number; fieldMap: Record<string, string>; mode: 'ADD' | 'REPLACE'; expectedCurrentReleaseId: string | null; targetDocumentId?: string; targetRevisionId?: string; reason: string; scopeCode: string }; export type ConfigureSourceBody = { uploadId: string; stage: DataStage; year: number; month: number; sheetName: string; headerRow: number; fieldMap: Record<string, string>; mode: 'ADD' | 'APPEND' | 'REPLACE'; fileId?: string; expectedCurrentReleaseId: string | null; targetDocumentId?: string; targetRevisionId?: string; reason: string; scopeCode: string };
export type MasterWorkbookKind = 'departments' | 'groups' | 'products' | 'projectGroups' | 'categoryLookup' | 'projectGroupLookup' | 'departmentLookup' | 'productLookup' | 'standardBundle'; export type MasterWorkbookKind = 'departments' | 'groups' | 'products' | 'projectGroups' | 'categoryLookup' | 'projectGroupLookup' | 'departmentLookup' | 'productLookup' | 'standardBundle';
export type MasterWorkbookKeyFormat = 'SEPARATE_COLUMNS' | 'PDT_REGION_COMBINED'; export type MasterWorkbookKeyFormat = 'SEPARATE_COLUMNS' | 'PDT_REGION_COMBINED';
export type MasterWorkbookPreview = { export type MasterWorkbookPreview = {
+109 -65
View File
@@ -14,6 +14,7 @@ import { usePageCache } from './pageCache';
import { useDelayedBusy, useDelayedIndicator } from './useDelayedBusy'; import { useDelayedBusy, useDelayedIndicator } from './useDelayedBusy';
import { useVisiblePolling } from './useVisiblePolling'; import { useVisiblePolling } from './useVisiblePolling';
import { runImportPollFollowup } from './importLifecycleGuards'; import { runImportPollFollowup } from './importLifecycleGuards';
import { configureSourceBatch } from './sourceBatchConfigure';
import './no-code-data-workflow.css'; import './no-code-data-workflow.css';
type Row = Record<string, unknown>; type Row = Record<string, unknown>;
@@ -43,28 +44,28 @@ function WorkflowOperationFeedback({ operation, error, busy, onResume }: { opera
const status = String(operation?.status || '').toUpperCase(); const status = String(operation?.status || '').toUpperCase();
const errors = Array.isArray(operation?.errors) ? operation.errors.map(item => item.message).filter(Boolean) : []; const errors = Array.isArray(operation?.errors) ? operation.errors.map(item => item.message).filter(Boolean) : [];
return <section className="nc-operation-status panel"> return <section className="nc-operation-status panel">
{error && <Alert showIcon type="error" message={status === 'FAILED' ? '操作未完成' : '进度读取失败'} description={[...errors, error].filter(Boolean).join(';')} action={operation?.operationId ? <Button size="small" onClick={onResume}>重试读取进度</Button> : undefined} />} {error && <Alert showIcon type="error" message={status === 'FAILED' || !operation ? '操作未完成' : '进度读取失败'} description={[...new Set([...errors, error].filter(Boolean))].join(';')} action={operation?.operationId ? <Button size="small" onClick={onResume}>重试读取进度</Button> : undefined} />}
<div><strong>{workflowStageLabel[String(operation?.stage || operation?.status)] || (busy ? '正在处理' : status === 'COMPLETED' ? '报表已更新' : '操作状态')}</strong></div> <div><strong>{workflowStageLabel[String(operation?.stage || operation?.status)] || (busy ? '正在处理' : status === 'COMPLETED' ? '报表已更新' : '操作状态')}</strong></div>
<Progress percent={busy ? undefined : status === 'COMPLETED' ? 100 : 0} status={status === 'FAILED' ? 'exception' : status === 'COMPLETED' ? 'success' : 'active'} /> <Progress percent={busy ? undefined : status === 'COMPLETED' ? 100 : 0} status={status === 'FAILED' ? 'exception' : status === 'COMPLETED' ? 'success' : 'active'} />
</section>; </section>;
} }
export type PendingSourceDeletion = { ids: string[]; releaseId: string | null }; export type PendingSourceDeletion = { ids: string[]; releaseId: string | null };
type SourceDraft = { id: string; file: File; workbook: WorkbookPreview; stage: DataStage; year: number; month: number; sheetName: string; headerRow: number; fieldMap: Record<string, string>; mode: 'ADD' | 'REPLACE'; targetId: string; scopeCode: string }; type SourceDraft = { id: string; file: File; workbook: WorkbookPreview; stage: DataStage; year: number; month: number; sheetName: string; headerRow: number; fieldMap: Record<string, string>; mode: 'APPEND' | 'REPLACE'; fileId: string; targetId: string; scopeCode: string };
type WizardFileDraft = { id: string; file: File; workbook: WorkbookPreview; sheetName: string; headerRow: number; fieldMap: Record<string, string>; fileId: string };
function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onDeleted, onBusyChange, onUnauthorized, onRefresh }: { options: SourceUploadOptions; workflowStatus: DataWorkflowStatus; pendingDeletion: PendingSourceDeletion; onDeleted: (ids: string[]) => void; onBusyChange: (busy: boolean) => void; onUnauthorized: (error: unknown) => boolean; onRefresh: () => void }) { function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onBusyChange, onUnauthorized, onRefresh }: { options: SourceUploadOptions; workflowStatus: DataWorkflowStatus; pendingDeletion: PendingSourceDeletion; onBusyChange: (busy: boolean) => void; onUnauthorized: (error: unknown) => boolean; onRefresh: () => void }) {
const confirmDelete = useDeleteConfirmation(); const confirmDelete = useDeleteConfirmation();
const { message } = App.useApp(); const { message } = App.useApp();
const input = useRef<HTMLInputElement>(null); const input = useRef<HTMLInputElement>(null);
const configured = useRef(new Map<string, { signature: string; sourceId: string }>()); const configured = useRef(new Map<string, { signature: string; sourceId: string }>());
const consumed = useRef(new Set<string>());
const [drafts, setDrafts] = useState<SourceDraft[]>([]); const [drafts, setDrafts] = useState<SourceDraft[]>([]);
const [activeId, setActiveId] = useState(''); const [activeId, setActiveId] = useState('');
const [reading, setReading] = useState<{ done: number; total: number } | null>(null); const [reading, setReading] = useState<{ done: number; total: number } | null>(null);
const [busy, setBusy] = useState(false); const [error, setError] = useState(''); const [busy, setBusy] = useState(false); const [error, setError] = useState('');
const actionLock = useRef(false); const actionLock = useRef(false);
const uploadOperation = useDataWorkflowOperation('source-import', () => { setDrafts([]); setActiveId(''); configured.current.clear(); consumed.current.clear(); onRefresh(); message.success('所选来源已统一导入,报表已更新'); }); const uploadOperation = useDataWorkflowOperation('source-import', () => { setDrafts([]); setActiveId(''); configured.current.clear(); onRefresh(); message.success('所选来源已统一导入,报表已更新'); });
const recalcOperation = useDataWorkflowOperation('recalculate', value => { if (Array.isArray(value.sourceIds)) onDeleted(value.sourceIds.map(String)); onRefresh(); message.success('报表已更新'); }); const recalcOperation = useDataWorkflowOperation('recalculate', () => { onRefresh(); message.success('报表已更新'); });
const active = drafts.find(item => item.id === activeId) || drafts[0]; const active = drafts.find(item => item.id === activeId) || drafts[0];
const workbook = active?.workbook || null; const workbook = active?.workbook || null;
const stage = active?.stage || 'black'; const year = active?.year || new Date().getFullYear(); const month = active?.month || 1; const stage = active?.stage || 'black'; const year = active?.year || new Date().getFullYear(); const month = active?.month || 1;
@@ -72,19 +73,28 @@ function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onDe
const mode = active?.mode || 'ADD'; const targetId = active?.targetId || ''; const scopeCode = active?.scopeCode || options.scopes[0] || 'ALL'; const mode = active?.mode || 'ADD'; const targetId = active?.targetId || ''; const scopeCode = active?.scopeCode || options.scopes[0] || 'ALL';
const patch = (values: Partial<SourceDraft>) => { if (active) setDrafts(previous => previous.map(item => item.id === active.id ? { ...item, ...values } : item)); }; const patch = (values: Partial<SourceDraft>) => { if (active) setDrafts(previous => previous.map(item => item.id === active.id ? { ...item, ...values } : item)); };
const setWorkbook = (value: WorkbookPreview) => patch({ workbook: value }); const setWorkbook = (value: WorkbookPreview) => patch({ workbook: value });
const setStage = (value: DataStage) => patch({ stage: value }); const setYear = (value: number) => patch({ year: value }); const setMonth = (value: number) => patch({ month: value }); const setStage = (value: DataStage) => patch({ stage: value, targetId: '' }); const setYear = (value: number) => patch({ year: value, targetId: '' }); const setMonth = (value: number) => patch({ month: value, targetId: '' });
const setSheetName = (value: string) => patch({ sheetName: value }); const setHeaderRow = (value: number) => patch({ headerRow: value }); const setSheetName = (value: string) => patch({ sheetName: value }); const setHeaderRow = (value: number) => patch({ headerRow: value });
const setFieldMap = (value: Record<string, string>) => patch({ fieldMap: value }); const setMode = (value: 'ADD' | 'REPLACE') => patch({ mode: value }); const setFieldMap = (value: Record<string, string>) => patch({ fieldMap: value }); const setMode = (value: 'APPEND' | 'REPLACE') => patch({ mode: value, targetId: '' });
const setTargetId = (value: string) => patch({ targetId: value }); const setScopeCode = (value: string) => patch({ scopeCode: value }); const setTargetId = (value: string) => patch({ targetId: value }); const setScopeCode = (value: string) => patch({ scopeCode: value, targetId: '' });
const fields = options.fields[stage] || []; const headers = workbook?.preview.headers || []; const fields = options.fields[stage] || []; const headers = workbook?.preview.headers || [];
const missing = fields.filter(field => field.required && !fieldMap[field.name]); const duplicate = Object.values(fieldMap).length !== new Set(Object.values(fieldMap)).size; const missing = fields.filter(field => field.required && !fieldMap[field.name]); const duplicate = Object.values(fieldMap).length !== new Set(Object.values(fieldMap)).size;
const importBusy = busy || uploadOperation.busy || recalcOperation.busy; const importBusy = busy || uploadOperation.busy || recalcOperation.busy;
useEffect(() => { onBusyChange(importBusy); return () => onBusyChange(false); }, [importBusy, onBusyChange]); useEffect(() => { onBusyChange(importBusy); return () => onBusyChange(false); }, [importBusy, onBusyChange]);
const hasImportCapability = workflowStatus.capabilities?.canImport !== false; const hasImportCapability = workflowStatus.capabilities?.canImport !== false;
const draftReady = (item: SourceDraft) => (options.fields[item.stage] || []).filter(field => field.required).every(field => item.fieldMap[field.name]) && Object.values(item.fieldMap).length === new Set(Object.values(item.fieldMap)).size && (item.mode !== 'REPLACE' || options.targets.some(target => target.documentId === item.targetId)); const targetFor = (item: SourceDraft) => options.targets.find(target => target.documentId === item.targetId && target.year === item.year && target.month === item.month && target.stage === item.stage && target.scopeCode === item.scopeCode);
const slots = drafts.map(item => `${item.year}:${item.month}:${item.stage}:${item.scopeCode}`); const hasMainSource = (item: Pick<SourceDraft, 'year' | 'month' | 'stage' | 'scopeCode'>) => options.targets.some(target => !target.fileId && target.sourceMode !== 'MONTHLY_SNAPSHOT' && target.year === item.year && target.month === item.month && target.stage === item.stage && target.scopeCode === item.scopeCode);
const duplicateSlots = new Set(slots).size !== slots.length; const draftReady = (item: SourceDraft) => {
const canImport = hasImportCapability && drafts.length > 0 && drafts.every(draftReady) && !duplicateSlots && !pendingDeletion.ids.length; const target = targetFor(item);
const matched = (options.fields[item.stage] || []).filter(field => field.required).every(field => item.fieldMap[field.name]) && Object.values(item.fieldMap).length === new Set(Object.values(item.fieldMap)).size;
return matched && (item.mode === 'APPEND' ? !hasMainSource(item) : Boolean(target) && (target?.sourceMode !== 'APPEND_SNAPSHOT' || Boolean(target.fileId)));
};
const replacementTargets = active ? options.targets.filter(item => item.year === year && item.month === month && item.stage === stage && item.scopeCode === scopeCode) : [];
const activeHasMainSource = active ? hasMainSource(active) : false;
const configuredActive = Boolean(active && configured.current.has(active.id));
const configuredReplacement = configuredActive && mode === 'REPLACE';
const repeatedReplacement = drafts.filter(item => item.mode === 'REPLACE' && item.targetId).length !== new Set(drafts.filter(item => item.mode === 'REPLACE' && item.targetId).map(item => item.targetId)).size;
const canImport = hasImportCapability && drafts.length > 0 && drafts.every(draftReady) && !repeatedReplacement && !pendingDeletion.ids.length;
const reportError = (value: unknown) => { if (!onUnauthorized(value)) setError(failure(value)); }; const reportError = (value: unknown) => { if (!onUnauthorized(value)) setError(failure(value)); };
const removeDraft = (id: string) => confirmDelete({title:`移除文件“${drafts.find(item=>item.id===id)?.workbook.sourceFile || "选中的文件"}”?`,description:"仅从本次导入列表移除,此文件的字段对应设置也将移除。电脑上的原文件及已接收来源保留。",confirmText:"确认移除",onConfirm:()=>{setDrafts(previous => previous.filter(item => item.id !== id));configured.current.delete(id);}}); const removeDraft = (id: string) => confirmDelete({title:`移除文件“${drafts.find(item=>item.id===id)?.workbook.sourceFile || "选中的文件"}”?`,description:"仅从本次导入列表移除,此文件的字段对应设置也将移除。电脑上的原文件及已接收来源保留。",confirmText:"确认移除",onConfirm:()=>{setDrafts(previous => previous.filter(item => item.id !== id));configured.current.delete(id);}});
const readFiles = async (files: File[]) => { const readFiles = async (files: File[]) => {
@@ -102,7 +112,7 @@ function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onDe
const { workbook: result, stage: detected } = await detectSourceWorkbook(uploaded, options.fields, api.sourceWorkbookPreview); const { workbook: result, stage: detected } = await detectSourceWorkbook(uploaded, options.fields, api.sourceWorkbookPreview);
const nextMonth = Number(file.name.match(/(?:^|\D)(0?[1-9]|1[0-2])月/)?.[1] || new Date().getMonth() + 1); const nextMonth = Number(file.name.match(/(?:^|\D)(0?[1-9]|1[0-2])月/)?.[1] || new Date().getMonth() + 1);
const nextYear = Number(file.name.match(/(20\d{2})年/)?.[1] || new Date().getFullYear()); const nextYear = Number(file.name.match(/(20\d{2})年/)?.[1] || new Date().getFullYear());
return { id: crypto.randomUUID(), file, workbook: result, stage: detected, year: nextYear, month: nextMonth, sheetName: result.preview.sheetName, headerRow: result.preview.headerRow, fieldMap: matchFields(options.fields[detected] || [], result.preview.headers), mode: 'ADD' as const, targetId: '', scopeCode: options.scopes[0] || 'ALL' }; return { id: crypto.randomUUID(), file, workbook: result, stage: detected, year: nextYear, month: nextMonth, sheetName: result.preview.sheetName, headerRow: result.preview.headerRow, fieldMap: matchFields(options.fields[detected] || [], result.preview.headers), mode: 'APPEND' as const, fileId: crypto.randomUUID(), targetId: '', scopeCode: options.scopes[0] || 'ALL' };
} catch (value) { if (!onUnauthorized(value)) failures.push(`${file.name}:${failure(value)}`); return null; } } catch (value) { if (!onUnauthorized(value)) failures.push(`${file.name}:${failure(value)}`); return null; }
})); }));
const ready = results.filter(item => item !== null); const ready = results.filter(item => item !== null);
@@ -124,58 +134,50 @@ function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onDe
const importAndUpdate = async () => { const importAndUpdate = async () => {
if (!canImport || importBusy || actionLock.current) return; actionLock.current = true; setBusy(true); setError(''); if (!canImport || importBusy || actionLock.current) return; actionLock.current = true; setBusy(true); setError('');
try { try {
const sourceIds: string[] = []; const items = drafts.map(draft => {
for (const draft of drafts) { const target = targetFor(draft);
const target = options.targets.find(item => item.documentId === draft.targetId); return { id: draft.id, body: { uploadId: draft.workbook.uploadId, stage: draft.stage, year: draft.year, month: draft.month, sheetName: draft.sheetName, headerRow: draft.headerRow, fieldMap: draft.fieldMap, mode: draft.mode, ...(draft.mode === 'APPEND' ? { fileId: draft.fileId } : target?.fileId ? { fileId: target.fileId } : {}), expectedCurrentReleaseId: workflowStatus.currentReleaseId ?? options.currentReleaseId, ...(draft.mode === 'REPLACE' && target ? { targetDocumentId: target.documentId, targetRevisionId: target.revisionId } : {}), reason: draft.mode === 'APPEND' ? '按独立批次追加,不覆盖旧文件' : '替换所选来源文件', scopeCode: draft.scopeCode } };
const body = { stage: draft.stage, year: draft.year, month: draft.month, sheetName: draft.sheetName, headerRow: draft.headerRow, fieldMap: draft.fieldMap, mode: draft.mode, expectedCurrentReleaseId: workflowStatus.currentReleaseId ?? options.currentReleaseId, ...(draft.mode === 'REPLACE' && target ? { targetDocumentId: target.documentId, targetRevisionId: target.revisionId } : {}), reason: '批量导入来源数据', scopeCode: draft.scopeCode }; });
const signature = JSON.stringify(body); const prior = configured.current.get(draft.id); const sourceIds = await configureSourceBatch(items, configured.current, body => api.configureSourceWorkbook(body));
if (prior?.signature === signature) { sourceIds.push(prior.sourceId); continue; }
let uploadId = draft.workbook.uploadId;
if (consumed.current.has(uploadId)) uploadId = (await api.previewSourceWorkbook(draft.file)).uploadId;
const result = await api.configureSourceWorkbook({ ...body, uploadId });
consumed.current.add(uploadId); configured.current.set(draft.id, { signature, sourceId: result.source.id }); sourceIds.push(result.source.id);
}
await uploadOperation.start(() => api.importDataWorkflow({ sourceIds })); await uploadOperation.start(() => api.importDataWorkflow({ sourceIds }));
} catch (value) { reportError(value); } finally { actionLock.current = false; setBusy(false); } } catch (value) { reportError(value); } finally { actionLock.current = false; setBusy(false); }
}; };
const performRecalculation = async () => { const performRecalculation = async () => {
if (actionLock.current) return; actionLock.current = true; if (actionLock.current || pendingDeletion.ids.length) return; actionLock.current = true;
setError(''); setError('');
try { await recalcOperation.start(() => pendingDeletion.ids.length ? api.deleteDataWorkflowSources({ sourceIds: pendingDeletion.ids, expectedCurrentReleaseId: pendingDeletion.releaseId }) : api.recalculateDataWorkflow()); } try { await recalcOperation.start(() => api.recalculateDataWorkflow()); }
catch (value) { reportError(value); } catch (value) { reportError(value); }
finally { actionLock.current = false; } finally { actionLock.current = false; }
}; };
const recalculate = () => {
if (actionLock.current) return;
if (!pendingDeletion.ids.length) return performRecalculation();
confirmDelete({title:`删除 ${pendingDeletion.ids.length} 个来源并更新报表?`,description:<><p>将从当前查询结果移除以下来源并更新报表,原文件和处理记录保留。</p><ul>{pendingDeletion.ids.slice(0,10).map((id,index)=><li key={id}>{options.targets.find(item=>item.documentId===id)?.sourceFile || `第 ${index+1} 个待删除来源`}</li>)}</ul>{pendingDeletion.ids.length>10&&<p>另有 {pendingDeletion.ids.length-10} 个待删除来源。</p>}</>,confirmText:'确认删除并更新',onConfirm:performRecalculation});
};
return <div className="nc-simplified-intake"> return <div className="nc-simplified-intake">
<section className="panel nc-simple-upload-panel"> <section className="panel nc-simple-upload-panel">
<div className="nc-simple-upload-heading"><div><span className="nc-eyebrow">测试数据维护</span><h2>导入来源并更新报表</h2><p>支持一次选择多个 Excel 文件,分别核对年月与阶段后统一导入。</p></div><div className="nc-simple-upload-actions">{workflowStatus.capabilities?.canRecalculate !== false && <Button icon={<ReloadOutlined />} disabled={importBusy} loading={recalcOperation.busy} onClick={() => void recalculate()}>{pendingDeletion.ids.length ? `更新报表(${pendingDeletion.ids.length}项删除)` : '更新报表'}</Button>}</div></div> <div className="nc-simple-upload-heading"><div><span className="nc-eyebrow">测试数据维护</span><h2>导入来源并更新报表</h2><p>同一月份可分批添加多个文件;相同文件内容不会重复计入。</p></div><div className="nc-simple-upload-actions">{workflowStatus.capabilities?.canRecalculate !== false && <Button icon={<ReloadOutlined />} disabled={importBusy || pendingDeletion.ids.length > 0} loading={recalcOperation.busy} onClick={() => void performRecalculation()}>更新报表</Button>}</div></div>
{!hasImportCapability && <Alert showIcon type="info" message="当前账号没有导入权限" description="可查看来源;导入和更新报表需要发布权限。" />} {!hasImportCapability && <Alert showIcon type="info" message="当前账号没有导入权限" description="可查看来源;导入和更新报表需要发布权限。" />}
{error && <Alert showIcon type="error" message="操作未完成,输入内容已保留" description={error} />} {error && <Alert showIcon type="error" message="操作未完成,输入内容已保留" description={error} />}
<input className="visually-hidden-file" ref={input} type="file" multiple accept=".xlsx" onChange={event => void readFiles(Array.from(event.target.files || []))} aria-label="选择质量数据Excel文件" /> <input className="visually-hidden-file" ref={input} type="file" multiple accept=".xlsx" onChange={event => void readFiles(Array.from(event.target.files || []))} aria-label="选择质量数据Excel文件" />
{pendingDeletion.ids.length > 0 && <Alert type="warning" showIcon message={`有 ${pendingDeletion.ids.length} 个来源待删除,报表尚未改变`} description="可继续标记或撤销删除,完成后点击“更新报表”。请先处理待删除文件,再导入新文件。" />} {pendingDeletion.ids.length > 0 && <Alert type="warning" showIcon message={`有 ${pendingDeletion.ids.length} 个来源待删除,报表尚未改变`} description="请到“来源文件”页确认或撤销删除,再导入新文件。" />}
{reading && <div className="nc-section-title"><Spin size="small" /><span>正在上传并识别文件 {reading.done} / {reading.total}</span></div>} {reading && <div className="nc-section-title"><Spin size="small" /><span>正在上传并识别文件 {reading.done} / {reading.total}</span></div>}
{drafts.length > 0 && <><div className="nc-section-title"><strong>待导入文件 · {drafts.length} 个</strong><Button disabled={importBusy} icon={<UploadOutlined />} onClick={() => input.current?.click()}>继续添加文件</Button></div><Table size="small" rowKey="id" pagination={false} scroll={{ x: 700 }} dataSource={drafts} columns={[ {drafts.length > 0 && <><div className="nc-section-title"><strong>待导入文件 · {drafts.length} 个</strong><Button disabled={importBusy} icon={<UploadOutlined />} onClick={() => input.current?.click()}>继续添加文件</Button></div><Table size="small" rowKey="id" pagination={false} scroll={{ x: 700 }} dataSource={drafts} columns={[
{ title: '文件', key: 'file', render: (_, item) => <Button type="link" onClick={() => setActiveId(item.id)}>{item.workbook.sourceFile}</Button> }, { title: '文件', key: 'file', render: (_, item) => <Button type="link" onClick={() => setActiveId(item.id)}>{item.workbook.sourceFile} · {item.id.slice(0, 8)}</Button> },
{ title: '阶段', key: 'stage', render: (_, item) => stageName(item.stage) }, { title: '阶段', key: 'stage', render: (_, item) => stageName(item.stage) },
{ title: '统计期间', key: 'period', render: (_, item) => `${item.year}年${item.month}月` }, { title: '统计期间', key: 'period', render: (_, item) => `${item.year}年${item.month}月` },
{ title: '识别结果', key: 'validation', render: (_, item) => draftReady(item) ? <Tag color="green">已识别</Tag> : <Tag color="orange">需核对列或替换对象</Tag> }, { title: '识别结果', key: 'validation', render: (_, item) => draftReady(item) ? <Tag color="green">已识别</Tag> : <Tag color="orange">需核对列或替换对象</Tag> },
{ title: '操作', key: 'action', render: (_, item) => <Button type="link" disabled={importBusy} onClick={() => removeDraft(item.id)}>移除</Button> }, { title: '操作', key: 'action', render: (_, item) => configured.current.has(item.id) ? <Tag color="blue">已接收待导入</Tag> : <Button type="link" disabled={importBusy} onClick={() => removeDraft(item.id)}>移除</Button> },
]} /></>} ]} /></>}
{duplicateSlots && <Alert type="warning" showIcon message="多个文件对应同一期间、阶段和范围,请调整或移除重复文件后再导入" />}
{!workbook ? <div className="nc-file-drop nc-simple-file-drop"><FileExcelOutlined /><h3>选择 Excel 来源文件</h3><p>可多选文件;系统自动识别阶段和月份,每个文件都可单独调整。</p>{hasImportCapability && <Button type="primary" icon={<UploadOutlined />} loading={busy} onClick={() => input.current?.click()}>选择 Excel 文件</Button>}</div> : <> {!workbook ? <div className="nc-file-drop nc-simple-file-drop"><FileExcelOutlined /><h3>选择 Excel 来源文件</h3><p>可多选文件;系统自动识别阶段和月份,每个文件都可单独调整。</p>{hasImportCapability && <Button type="primary" icon={<UploadOutlined />} loading={busy} onClick={() => input.current?.click()}>选择 Excel 文件</Button>}</div> : <>
<div className="nc-simple-file-line"><FileExcelOutlined /><strong>{workbook.sourceFile}</strong>{hasImportCapability && <Button type="link" disabled={importBusy} onClick={() => { if (active) removeDraft(active.id); }}>移除此文件</Button>}</div> <div className="nc-simple-file-line"><FileExcelOutlined /><strong>{workbook.sourceFile}</strong>{hasImportCapability && <Button type="link" disabled={importBusy} onClick={() => { if (active) removeDraft(active.id); }}>移除此文件</Button>}</div>
<div className="nc-simple-upload-fields"> <div className="nc-simple-upload-fields">
<label>导入方式<Select disabled={importBusy} value={mode} options={[{ value: 'ADD', label: '新增数据' }, { value: 'REPLACE', label: '替换已有数据' }]} onChange={value => { setMode(value); }} /></label> <label>导入方式<Select disabled={importBusy || configuredActive} value={mode} options={[{ value: 'APPEND', label: '追加文件并合并统计' }, { value: 'REPLACE', label: '替换一份已有文件' }]} onChange={setMode} /></label>
<label>数据阶段<Select disabled={importBusy} value={stage} options={[{ value: 'black', label: '黑片' }, { value: 'bright', label: '亮片' }]} onChange={value => { setStage(value); setFieldMap(matchFields(options.fields[value] || [], headers)); }} /></label> <label>数据阶段<Select disabled={importBusy || configuredReplacement} value={stage} options={[{ value: 'black', label: '黑片' }, { value: 'bright', label: '亮片' }]} onChange={value => { setStage(value); setFieldMap(matchFields(options.fields[value] || [], headers)); }} /></label>
<label>统计年份<InputNumber disabled={importBusy} min={2000} max={2100} value={year} onChange={value => { if (value) setYear(value); }} /></label> <label>统计年份<InputNumber disabled={importBusy || configuredReplacement} min={2000} max={2100} value={year} onChange={value => { if (value) setYear(value); }} /></label>
<label>统计月份<Select disabled={importBusy} value={month} options={Array.from({ length: 12 }, (_, index) => ({ value: index + 1, label: `${index + 1}月` }))} onChange={value => { setMonth(value); }} /></label> <label>统计月份<Select disabled={importBusy || configuredReplacement} value={month} options={Array.from({ length: 12 }, (_, index) => ({ value: index + 1, label: `${index + 1}月` }))} onChange={value => { setMonth(value); }} /></label>
{options.scopes.length > 1 && <label>数据范围<Select disabled={importBusy} value={scopeCode} options={options.scopes.map(value => ({ value, label: value === 'ALL' ? '全部范围' : value }))} onChange={value => { setScopeCode(value); }} /></label>} {options.scopes.length > 1 && <label>数据范围<Select disabled={importBusy || configuredReplacement} value={scopeCode} options={options.scopes.map(value => ({ value, label: value === 'ALL' ? '全部范围' : value }))} onChange={value => { setScopeCode(value); }} /></label>}
{mode === 'REPLACE' && <label className="nc-simple-replace-target">替换对象<Select {...readableSelectPopup} disabled={importBusy} showSearch optionFilterProp="label" value={targetId || undefined} placeholder="选择已发布期间和阶段" options={options.targets.map(item => ({ value: item.documentId, label: `${item.year}年${item.month}月 · ${stageName(item.stage)} · ${item.sourceFile}` }))} onChange={value => { setTargetId(value); }} /></label>} {mode === 'REPLACE' && <label className="nc-simple-replace-target">替换对象<Select {...readableSelectPopup} disabled={importBusy || configuredReplacement} showSearch optionFilterProp="label" value={targetId || undefined} placeholder="选择同期间的一份文件" options={replacementTargets.map(item => ({ value: item.documentId, label: `${item.sourceFile} · ${item.fileId ? `分批文件 ${item.fileId.slice(0, 8)}` : '整月文件'}` }))} onChange={setTargetId} /></label>}
</div> </div>
{mode === 'APPEND' && activeHasMainSource && <Alert type="warning" showIcon message="该月份目前是整月文件,不能与分批文件混合累计" description="如需修订,请改用“替换一份已有文件”;整月文件替换会替换该月份的完整数据。" />}
{mode === 'REPLACE' && replacementTargets.length === 0 && <Alert type="info" showIcon message="这个年月、阶段和范围没有可替换的已发布文件" />}
{repeatedReplacement && <Alert type="warning" showIcon message="同一份已有文件只能替换一次,请为每个待替换文件选择不同目标。" />}
<p className="nc-note">请确保文件范围内的记录不重叠;相同文件内容会被系统阻止,不同文件里的重复业务行无法自动识别。</p>
<details className="nc-advanced-import-settings"><summary>列识别与预览</summary> <details className="nc-advanced-import-settings"><summary>列识别与预览</summary>
<div className="nc-form-grid"><label>工作表<Select disabled={importBusy} value={sheetName} options={workbook.sheets.map(item => ({ value: item.sheetName, label: item.sheetName }))} onChange={value => void changePreview(value)} /></label><label>表头所在行<InputNumber disabled={importBusy} min={1} max={100} value={headerRow} onChange={value => { if (value) setHeaderRow(value); }} onBlur={() => { if (workbook.preview.headerRow !== headerRow) void changePreview(sheetName, headerRow); }} /></label></div> <div className="nc-form-grid"><label>工作表<Select disabled={importBusy} value={sheetName} options={workbook.sheets.map(item => ({ value: item.sheetName, label: item.sheetName }))} onChange={value => void changePreview(value)} /></label><label>表头所在行<InputNumber disabled={importBusy} min={1} max={100} value={headerRow} onChange={value => { if (value) setHeaderRow(value); }} onBlur={() => { if (workbook.preview.headerRow !== headerRow) void changePreview(sheetName, headerRow); }} /></label></div>
<div className="nc-section-title"><h3>业务字段对应 Excel 列</h3><span>{missing.length ? `还有 ${missing.length} 个必填字段待匹配` : duplicate ? '同一 Excel 列不能重复对应' : '已匹配'}</span></div><ColumnMatching disabled={importBusy} fields={fields} headers={headers} value={fieldMap} onChange={value => { setFieldMap(value); }} /> <div className="nc-section-title"><h3>业务字段对应 Excel 列</h3><span>{missing.length ? `还有 ${missing.length} 个必填字段待匹配` : duplicate ? '同一 Excel 列不能重复对应' : '已匹配'}</span></div><ColumnMatching disabled={importBusy} fields={fields} headers={headers} value={fieldMap} onChange={value => { setFieldMap(value); }} />
@@ -196,7 +198,7 @@ function ValidationNotice({ value }: { value: unknown }) {
return errors.length ? <Alert showIcon type="warning" message="校验仍有需要处理的事项" description={<ul className="nc-issue-list">{errors.map((item, index) => <li key={index}>{txt(item)}</li>)}</ul>} /> : result.complete === true ? <Alert showIcon type="success" message="来源及对应关系校验通过" /> : result.complete === false ? <Alert showIcon type="warning" message="候选校验尚未通过" description="请核对来源是否齐备,以及基础数据和对应关系是否完整。" /> : null; return errors.length ? <Alert showIcon type="warning" message="校验仍有需要处理的事项" description={<ul className="nc-issue-list">{errors.map((item, index) => <li key={index}>{txt(item)}</li>)}</ul>} /> : result.complete === true ? <Alert showIcon type="success" message="来源及对应关系校验通过" /> : result.complete === false ? <Alert showIcon type="warning" message="候选校验尚未通过" description="请核对来源是否齐备,以及基础数据和对应关系是否完整。" /> : null;
} }
export function DataWorkflowHub({ user, sources, currentReleaseId, pendingDeletion, onDeleted, onBusyChange, onUnauthorized, onRefresh }: { user: AuthUser; sources: SourceItem[]; currentReleaseId: string | null; pendingDeletion: PendingSourceDeletion; onDeleted: (ids: string[]) => void; onBusyChange: (busy: boolean) => void; onUnauthorized: (error: unknown) => boolean; onRefresh: () => void }) { export function DataWorkflowHub({ user, sources, currentReleaseId, pendingDeletion, onBusyChange, onUnauthorized, onRefresh }: { user: AuthUser; sources: SourceItem[]; currentReleaseId: string | null; pendingDeletion: PendingSourceDeletion; onBusyChange: (busy: boolean) => void; onUnauthorized: (error: unknown) => boolean; onRefresh: () => void }) {
const { message } = App.useApp(); const { message } = App.useApp();
const { active } = usePageCache(); const { active } = usePageCache();
const [mode, setMode] = useState<'ADD' | 'REPLACE' | 'REBUILD' | null>(null); const [mode, setMode] = useState<'ADD' | 'REPLACE' | 'REBUILD' | null>(null);
@@ -238,7 +240,7 @@ export function DataWorkflowHub({ user, sources, currentReleaseId, pendingDeleti
const opened = mode !== null || Boolean(resumeId); const opened = mode !== null || Boolean(resumeId);
const open = (next: 'ADD' | 'REPLACE' | 'REBUILD') => { setMode(next); setResumeId(''); if (!options) { optionsAttempted.current = true; void loadOptions(); } }; const open = (next: 'ADD' | 'REPLACE' | 'REBUILD') => { setMode(next); setResumeId(''); if (!options) { optionsAttempted.current = true; void loadOptions(); } };
if (!workflowStatusLoaded) return <section className="panel nc-operation-status">{statusRead.busy ? <div><strong>读取当前维护模式</strong><span>正在加载可执行操作</span><Spin /></div> : workflowStatusError ? <Alert showIcon type="error" message="维护模式读取失败" description={workflowStatusError} action={<Button size="small" onClick={() => void loadWorkflowStatus()}>重试</Button>} /> : null}</section>; if (!workflowStatusLoaded) return <section className="panel nc-operation-status">{statusRead.busy ? <div><strong>读取当前维护模式</strong><span>正在加载可执行操作</span><Spin /></div> : workflowStatusError ? <Alert showIcon type="error" message="维护模式读取失败" description={workflowStatusError} action={<Button size="small" onClick={() => void loadWorkflowStatus()}>重试</Button>} /> : null}</section>;
if (workflowStatus?.simplified === true && workflowStatus.mode === 'TEST' && options) return <SimplifiedDataWorkflow options={options} workflowStatus={workflowStatus} pendingDeletion={pendingDeletion} onDeleted={onDeleted} onBusyChange={onBusyChange} onUnauthorized={onUnauthorized} onRefresh={onRefresh} />; if (workflowStatus?.simplified === true && workflowStatus.mode === 'TEST' && options) return <SimplifiedDataWorkflow options={options} workflowStatus={workflowStatus} pendingDeletion={pendingDeletion} onBusyChange={onBusyChange} onUnauthorized={onUnauthorized} onRefresh={onRefresh} />;
return <> return <>
<section className="nc-intake-start panel"> <section className="nc-intake-start panel">
<div className="nc-intake-intro"><span className="nc-eyebrow">日常数据维护</span><h2>选择文件,核对后发布</h2><p>文件名可以保持原样。黑片或亮片、统计年月和表头均在下一步选择。</p></div> <div className="nc-intake-intro"><span className="nc-eyebrow">日常数据维护</span><h2>选择文件,核对后发布</h2><p>文件名可以保持原样。黑片或亮片、统计年月和表头均在下一步选择。</p></div>
@@ -255,7 +257,7 @@ export function DataWorkflowHub({ user, sources, currentReleaseId, pendingDeleti
</>; </>;
} }
export function SourceFilesPanel({ user, sources, sourcesLoaded, loading, pendingIds, maintenanceBusy, onToggleDelete, onClearDeletes, onUnauthorized, onRefresh }: { user: AuthUser; sources: SourceItem[]; sourcesLoaded: boolean; loading: boolean; pendingIds: string[]; maintenanceBusy: boolean; onToggleDelete: (id: string) => void; onClearDeletes: () => void; onUnauthorized: (error: unknown) => boolean; onRefresh: () => void }) { export function SourceFilesPanel({ user, sources, sourcesLoaded, loading, pendingIds, pendingReleaseId, maintenanceBusy, onDeleted, onBusyChange, onToggleDelete, onClearDeletes, onUnauthorized, onRefresh }: { user: AuthUser; sources: SourceItem[]; sourcesLoaded: boolean; loading: boolean; pendingIds: string[]; pendingReleaseId: string | null; maintenanceBusy: boolean; onDeleted: (ids: string[]) => void; onBusyChange: (busy: boolean) => void; onToggleDelete: (id: string) => void; onClearDeletes: () => void; onUnauthorized: (error: unknown) => boolean; onRefresh: () => void }) {
const confirmDelete = useDeleteConfirmation(); const confirmDelete = useDeleteConfirmation();
const { message } = App.useApp(); const { message } = App.useApp();
const { active } = usePageCache(); const { active } = usePageCache();
@@ -276,9 +278,11 @@ export function SourceFilesPanel({ user, sources, sourcesLoaded, loading, pendin
catch (value) { if (!onUnauthorized(value)) setDeletedError(failure(value)); } catch (value) { if (!onUnauthorized(value)) setDeletedError(failure(value)); }
}), [deletedRead.run, onUnauthorized]); }), [deletedRead.run, onUnauthorized]);
const sourceOperation = useDataWorkflowOperation('source-maintenance', value => { const sourceOperation = useDataWorkflowOperation('source-maintenance', value => {
if (Array.isArray(value.sourceIds)) onDeleted(value.sourceIds.map(String));
onRefresh(); if (showDeleted) void loadDeleted(); onRefresh(); if (showDeleted) void loadDeleted();
message.success(String(value.operationType || '').toUpperCase().includes('RESTORE') ? '来源已恢复,报表已更新' : '来源已删除,报表已更新'); message.success(String(value.operationType || '').toUpperCase().includes('RESTORE') ? '来源已恢复,报表已更新' : '来源已删除,报表已更新');
}); });
useEffect(() => { onBusyChange(sourceOperation.busy); return () => onBusyChange(false); }, [sourceOperation.busy, onBusyChange]);
const loadMetadata = useCallback(async () => { const loadMetadata = useCallback(async () => {
const [status, options] = await Promise.allSettled([api.dataWorkflowStatus(), api.sourceUploadOptions()]); const [status, options] = await Promise.allSettled([api.dataWorkflowStatus(), api.sourceUploadOptions()]);
const errors: string[] = []; const errors: string[] = [];
@@ -301,14 +305,24 @@ export function SourceFilesPanel({ user, sources, sourcesLoaded, loading, pendin
try { await sourceOperation.start(() => restore ? api.restoreDataWorkflowSource(id) : api.deleteDataWorkflowSource(id)); } try { await sourceOperation.start(() => restore ? api.restoreDataWorkflowSource(id) : api.deleteDataWorkflowSource(id)); }
catch (value) { if (!onUnauthorized(value)) setDeletedError(failure(value)); } catch (value) { if (!onUnauthorized(value)) setDeletedError(failure(value)); }
}; };
const confirmPendingDeletion = () => {
if (!pendingIds.length || sourceOperation.busy || maintenanceBusy || loading) return;
const sourceIds = [...pendingIds];
confirmDelete({
title: `确认删除 ${sourceIds.length} 个来源并更新报表?`,
description: <><p>将从当前查询结果移除以下来源,原文件和处理记录保留。</p><ul>{sourceIds.slice(0, 10).map(id => <li key={id}>{sources.find(source => source.id === id)?.sourceFile || id}</li>)}</ul>{sourceIds.length > 10 && <p>另有 {sourceIds.length - 10} 个待删除来源。</p>}</>,
confirmText: '确认删除并更新',
onConfirm: () => sourceOperation.start(() => api.deleteDataWorkflowSources({ sourceIds, expectedCurrentReleaseId: pendingReleaseId })),
});
};
return <> return <>
<section className="panel section-panel nc-source-files"> <section className="panel section-panel nc-source-files">
<div className="resource-toolbar"><strong>{showDeleted ? '已删除的来源文件' : '已接收的来源文件'}</strong><span>{showDeleted ? `${deleted.length} 个文件 · 原件和处理记录保留` : simplified ? `${sources.length} 个文件${pendingIds.length ? ` · ${pendingIds.length} 个待删除` : ''}` : `${sources.length} 个文件 · 上传或接收不等于已经发布`}</span>{canOperate && <Checkbox checked={showDeleted} onChange={event => setShowDeleted(event.target.checked)}>查看已删除来源</Checkbox>}</div> <div className="resource-toolbar"><strong>{showDeleted ? '已删除的来源文件' : '已接收的来源文件'}</strong><span>{showDeleted ? `${deleted.length} 个文件 · 原件和处理记录保留` : simplified ? `${sources.length} 个文件${pendingIds.length ? ` · ${pendingIds.length} 个待删除` : ''}` : `${sources.length} 个文件 · 上传或接收不等于已经发布`}</span>{canOperate && <Checkbox checked={showDeleted} onChange={event => setShowDeleted(event.target.checked)}>查看已删除来源</Checkbox>}</div>
{simplified && <p className="nc-source-maintenance-note">删除先标记为待删除,可连续操作和撤销,点击上方“更新报表”后统一生效。原文件和处理记录保留。</p>} {simplified && <p className="nc-source-maintenance-note">删除先标记为待删除,可连续操作和撤销,点击“确认删除并更新报表”后统一生效。原文件和处理记录保留。</p>}
{simplified && pendingIds.length > 0 && <div className="nc-source-maintenance-note"><Button disabled={maintenanceBusy} onClick={onClearDeletes}>撤销全部待删除({pendingIds.length})</Button></div>} {simplified && !showDeleted && canOperate && pendingIds.length > 0 && <div className="nc-source-maintenance-actions"><Button type="primary" danger icon={<DeleteOutlined />} disabled={maintenanceBusy || loading || workflowStatus?.capabilities?.canDeleteSource === false} loading={sourceOperation.busy} onClick={confirmPendingDeletion}>确认删除并更新报表({pendingIds.length})</Button><Button disabled={sourceOperation.busy || maintenanceBusy || loading} onClick={onClearDeletes}>撤销全部待删除({pendingIds.length})</Button></div>}
{deletedError && <Alert showIcon type="error" message="已删除来源读取失败" description={deletedLoaded ? `${deletedError};上次成功读取的来源仍保留。` : deletedError} action={<Button size="small" onClick={() => void loadDeleted()}>重试</Button>} />} {deletedError && <Alert showIcon type="error" message="已删除来源读取失败" description={deletedLoaded ? `${deletedError};上次成功读取的来源仍保留。` : deletedError} action={<Button size="small" onClick={() => void loadDeleted()}>重试</Button>} />}
{metadataError && <Alert showIcon type="error" message="来源操作信息读取失败" description={workflowStatusLoaded || currentSourceIds !== null ? `${metadataError};已成功读取的状态仍保留。` : metadataError} action={<Button size="small" onClick={() => { metadataAttempted.current = true; setMetadataError(''); void loadMetadata(); }}>重试</Button>} />} {metadataError && <Alert showIcon type="error" message="来源操作信息读取失败" description={workflowStatusLoaded || currentSourceIds !== null ? `${metadataError};已成功读取的状态仍保留。` : metadataError} action={<Button size="small" onClick={() => { metadataAttempted.current = true; setMetadataError(''); void loadMetadata(); }}>重试</Button>} />}
{sourceOperation.operation && <WorkflowOperationFeedback operation={sourceOperation.operation} error={sourceOperation.error} busy={sourceOperation.busy} onResume={sourceOperation.resume} />} {(sourceOperation.operation || sourceOperation.error) && <WorkflowOperationFeedback operation={sourceOperation.operation} error={sourceOperation.error} busy={sourceOperation.busy} onResume={sourceOperation.resume} />}
{(!showDeleted && sourcesLoaded) || (showDeleted && (deletedLoaded || deletedRead.busy)) ? <Table<DeletionSource> size="small" bordered loading={showDeleted ? deletedRead.busy : loading} dataSource={showDeleted ? deleted.map(item => item.source) : sources} rowKey="id" pagination={{ pageSize: 15, showSizeChanger: false }} scroll={{ x: 980 }} columns={[ {(!showDeleted && sourcesLoaded) || (showDeleted && (deletedLoaded || deletedRead.busy)) ? <Table<DeletionSource> size="small" bordered loading={showDeleted ? deletedRead.busy : loading} dataSource={showDeleted ? deleted.map(item => item.source) : sources} rowKey="id" pagination={{ pageSize: 15, showSizeChanger: false }} scroll={{ x: 980 }} columns={[
{ title: '来源文件', dataIndex: 'sourceFile', key: 'sourceFile', width: 340, ellipsis: true }, { title: '来源文件', dataIndex: 'sourceFile', key: 'sourceFile', width: 340, ellipsis: true },
{ title: '阶段', dataIndex: 'stage', key: 'stage', align: 'center', width: 90, render: value => <Tag>{stageName(value)}</Tag> }, { title: '阶段', dataIndex: 'stage', key: 'stage', align: 'center', width: 90, render: value => <Tag>{stageName(value)}</Tag> },
@@ -319,7 +333,7 @@ export function SourceFilesPanel({ user, sources, sourcesLoaded, loading, pendin
const onCurrentRelease = currentSourceIds?.includes(row.id) ?? false; const onCurrentRelease = currentSourceIds?.includes(row.id) ?? false;
const permitted = !simplified || (showDeleted ? workflowStatus?.capabilities?.canRestoreSource !== false : currentSourceIds !== null && workflowStatus?.capabilities?.canDeleteSource !== false && (!onCurrentRelease || workflowStatus?.capabilities?.canDeleteCurrentSource !== false)); const permitted = !simplified || (showDeleted ? workflowStatus?.capabilities?.canRestoreSource !== false : currentSourceIds !== null && workflowStatus?.capabilities?.canDeleteSource !== false && (!onCurrentRelease || workflowStatus?.capabilities?.canDeleteCurrentSource !== false));
if (!workflowStatusLoaded || !canOperate || !permitted) return null; if (!workflowStatusLoaded || !canOperate || !permitted) return null;
return <Button type="link" danger={!showDeleted} icon={showDeleted ? <UndoOutlined /> : <DeleteOutlined />} disabled={sourceOperation.busy || maintenanceBusy || loading} onClick={() => {if(!simplified){setSelected({id:row.id,restore:showDeleted});return;}if(showDeleted){void runSourceOperation(row.id,true);return;}if(pendingIds.includes(row.id)){onToggleDelete(row.id);return;}confirmDelete({title:`删除来源“${row.sourceFile}”?`,description:"将标记为待删除,当前报表暂不改变。点击“更新报表”后才生效,原文件和处理记录保留。",confirmText:"标记删除",onConfirm:()=>onToggleDelete(row.id)});}}>{simplified ? showDeleted ? '恢复并更新报表' : pendingIds.includes(row.id) ? '撤销删除' : '删除' : showDeleted ? '恢复' : '删除'}</Button>; return <Button type="link" danger={!showDeleted} icon={showDeleted ? <UndoOutlined /> : <DeleteOutlined />} disabled={sourceOperation.busy || maintenanceBusy || loading} onClick={() => {if(!simplified){setSelected({id:row.id,restore:showDeleted});return;}if(showDeleted){void runSourceOperation(row.id,true);return;}if(pendingIds.includes(row.id)){onToggleDelete(row.id);return;}confirmDelete({title:`删除来源“${row.sourceFile}”?`,description:"将标记为待删除,当前报表暂不改变。在本页点击“确认删除并更新报表”后才生效,原文件和处理记录保留。",confirmText:"标记删除",onConfirm:()=>onToggleDelete(row.id)});}}>{simplified ? showDeleted ? '恢复并更新报表' : pendingIds.includes(row.id) ? '撤销删除' : '删除' : showDeleted ? '恢复' : '删除'}</Button>;
} }, } },
]} locale={{ emptyText: showDeleted ? '没有已删除的来源。' : '尚无来源文件,点击“上传新数据”开始。' }} /> : null} ]} locale={{ emptyText: showDeleted ? '没有已删除的来源。' : '尚无来源文件,点击“上传新数据”开始。' }} /> : null}
</section> </section>
@@ -416,11 +430,15 @@ function DataUploadWizard({ user, mode, purpose, handoffSourceId, initialReason,
const runStorageKey = `quality-import-run:${user.id}`; const runStorageKey = `quality-import-run:${user.id}`;
const [storedRunId] = useState(() => { try { return localStorage.getItem(runStorageKey) || ''; } catch { return ''; } }); const [storedRunId] = useState(() => { try { return localStorage.getItem(runStorageKey) || ''; } catch { return ''; } });
const [step, setStep] = useState(resumeId ? 4 : storedRunId ? 3 : 0); const [busy, setBusy] = useState(Boolean(storedRunId)); const [error, setError] = useState(''); const [step, setStep] = useState(resumeId ? 4 : storedRunId ? 3 : 0); const [busy, setBusy] = useState(Boolean(storedRunId)); const [error, setError] = useState('');
const [workbook, setWorkbook] = useState<WorkbookPreview | null>(null); const [stage, setStage] = useState<DataStage>('black'); const [wizardFiles, setWizardFiles] = useState<WizardFileDraft[]>([]); const [wizardFileId, setWizardFileId] = useState(''); const [stage, setStage] = useState<DataStage>('black');
const [year, setYear] = useState(new Date().getFullYear()); const [month, setMonth] = useState(new Date().getMonth() + 1); const [year, setYear] = useState(new Date().getFullYear()); const [month, setMonth] = useState(new Date().getMonth() + 1);
const [scopeCode, setScopeCode] = useState(options.scopes[0] || 'ALL'); const [targetId, setTargetId] = useState(''); const [scopeCode, setScopeCode] = useState(options.scopes[0] || 'ALL'); const [targetId, setTargetId] = useState('');
const [sheetName, setSheetName] = useState(''); const [headerRow, setHeaderRow] = useState(1); const [fieldMap, setFieldMap] = useState<Record<string, string>>({}); const activeWizardFile = wizardFiles.find(item => item.id === wizardFileId) || wizardFiles[0];
const [configuredSourceId, setConfiguredSourceId] = useState(''); const workbook = activeWizardFile?.workbook || null;
const sheetName = activeWizardFile?.sheetName || ''; const headerRow = activeWizardFile?.headerRow || 1; const fieldMap = activeWizardFile?.fieldMap || {};
const patchWizardFile = (values: Partial<WizardFileDraft>, id = activeWizardFile?.id) => { if (id) setWizardFiles(previous => previous.map(item => item.id === id ? { ...item, ...values } : item)); };
const setHeaderRow = (value: number) => patchWizardFile({ headerRow: value }); const setFieldMap = (value: Record<string, string>) => patchWizardFile({ fieldMap: value });
const configuredByFile = useRef(new Map<string, { signature: string; sourceId: string }>());
const [reason, setReason] = useState(initialReason || ''); const [run, setRun] = useState<Row | null>(() => storedRunId ? { id: storedRunId, status: 'RUNNING' } : null); const [release, setRelease] = useState<Row | null>(null); const [reason, setReason] = useState(initialReason || ''); const [run, setRun] = useState<Row | null>(() => storedRunId ? { id: storedRunId, status: 'RUNNING' } : null); const [release, setRelease] = useState<Row | null>(null);
const [runTerminalId, setRunTerminalId] = useState(''); const [runTerminalId, setRunTerminalId] = useState('');
const completionNotified = useRef(new Set<string>()); const completionNotified = useRef(new Set<string>());
@@ -492,25 +510,49 @@ function DataUploadWizard({ user, mode, purpose, handoffSourceId, initialReason,
} }
}, [loadCandidate, onComplete, reportError, runId, runStorageKey]); }, [loadCandidate, onComplete, reportError, runId, runStorageKey]);
useVisiblePolling(pollRun, 2000, active && Boolean(runId) && runTerminalId !== runId, true); useVisiblePolling(pollRun, 2000, active && Boolean(runId) && runTerminalId !== runId, true);
const readFile = async (file?: File) => { const readFiles = async (selected: File[]) => {
if (!file || actionLock.current) return; actionLock.current = true; setBusy(true); setError(''); const ownGeneration = generation.current; if (!selected.length || actionLock.current) return; actionLock.current = true; setBusy(true); setError(''); const ownGeneration = generation.current;
try { const result = await api.previewSourceWorkbook(file); if (ownGeneration !== generation.current) return; setWorkbook(result); setSheetName(result.preview.sheetName); setHeaderRow(result.preview.headerRow); setFieldMap(matchFields(options.fields[stage] || [], result.preview.headers)); setStep(1); } try {
if (mode === 'REPLACE' && (selected.length > 1 || wizardFiles.length > 0)) throw new Error('替换时请选择一份文件;要追加多份文件,请返回“上传新数据”。');
if (selected.length + wizardFiles.length > 50) throw new Error('一次最多选择50个文件。');
const known = new Set(wizardFiles.map(item => `${item.file.name}:${item.file.size}:${item.file.lastModified}`));
const files = selected.filter(file => { const key = `${file.name}:${file.size}:${file.lastModified}`; if (known.has(key)) return false; known.add(key); return true; });
const ready: WizardFileDraft[] = []; const failures: string[] = []; let detectedStage: DataStage | null = wizardFiles.length ? stage : null;
for (const file of files) {
try {
const uploaded = await api.previewSourceWorkbook(file);
const detected = await detectSourceWorkbook(uploaded, options.fields, api.sourceWorkbookPreview);
if (!detectedStage) detectedStage = detected.stage;
ready.push({ id: crypto.randomUUID(), file, workbook: detected.workbook, sheetName: detected.workbook.preview.sheetName, headerRow: detected.workbook.preview.headerRow, fieldMap: matchFields(options.fields[detectedStage] || [], detected.workbook.preview.headers), fileId: crypto.randomUUID() });
} catch (value) { if (!onUnauthorized(value)) failures.push(`${file.name}:${failure(value)}`); }
}
if (ownGeneration !== generation.current) return;
if (ready.length) {
setWizardFiles(previous => [...previous, ...ready]);
setWizardFileId(previous => previous || ready[0].id);
if (detectedStage) setStage(detectedStage);
setStep(1);
}
if (failures.length) setError(`以下文件未加入本次导入,可重新选择:${failures.join(';')}`);
else if (!ready.length && !wizardFiles.length) setError('没有可加入的文件。');
}
catch (value) { reportError(value); } finally { actionLock.current = false; setBusy(false); if (input.current) input.current.value = ''; } catch (value) { reportError(value); } finally { actionLock.current = false; setBusy(false); if (input.current) input.current.value = ''; }
}; };
const changePreview = async (nextSheet: string, nextHeader: number, nextStage = stage, preferredMap?: Record<string, string>) => { const changePreview = async (nextSheet: string, nextHeader: number, nextStage = stage, preferredMap?: Record<string, string>) => {
if (!workbook || actionLock.current) return; actionLock.current = true; setBusy(true); setError(''); if (!workbook || actionLock.current) return; actionLock.current = true; setBusy(true); setError('');
try { const result = await api.sourceWorkbookPreview(workbook.uploadId, { sheetName: nextSheet, headerRow: nextHeader, stage: nextStage }); setWorkbook(result); setSheetName(result.preview.sheetName); setHeaderRow(result.preview.headerRow); const chosen = preferredMap || fieldMap; setFieldMap({ ...matchFields(options.fields[nextStage] || [], result.preview.headers), ...Object.fromEntries(Object.entries(chosen).filter(([, header]) => result.preview.headers.includes(header))) }); } try { const result = await api.sourceWorkbookPreview(workbook.uploadId, { sheetName: nextSheet, headerRow: nextHeader, stage: nextStage }); const chosen = preferredMap || fieldMap; patchWizardFile({ workbook: result, sheetName: result.preview.sheetName, headerRow: result.preview.headerRow, fieldMap: { ...matchFields(options.fields[nextStage] || [], result.preview.headers), ...Object.fromEntries(Object.entries(chosen).filter(([, header]) => result.preview.headers.includes(header))) } }); }
catch (value) { reportError(value); } finally { actionLock.current = false; setBusy(false); } catch (value) { reportError(value); } finally { actionLock.current = false; setBusy(false); }
}; };
const chooseTarget = (id: string) => { const selected = options.targets.find(item => item.documentId === id); setTargetId(id); if (selected) { setStage(selected.stage); setYear(selected.year); setMonth(selected.month); setScopeCode(selected.scopeCode); setFieldMap(matchFields(options.fields[selected.stage] || [], headers)); } }; const removeWizardFile = (id: string) => { const next = wizardFiles.filter(item => item.id !== id); setWizardFiles(next); if (wizardFileId === id) setWizardFileId(next[0]?.id || ''); if (!next.length) setStep(0); configuredByFile.current.delete(id); };
const chooseTarget = (id: string) => { const selected = options.targets.find(item => item.documentId === id); setTargetId(id); if (selected) { setStage(selected.stage); setYear(selected.year); setMonth(selected.month); setScopeCode(selected.scopeCode); setWizardFiles(previous => previous.map(item => ({ ...item, fieldMap: matchFields(options.fields[selected.stage] || [], item.workbook.preview.headers) }))); } };
const beginImport = async () => { const beginImport = async () => {
if (!active || !workbook || missingFields.length || (mode === 'REPLACE' && !target) || actionLock.current) return; if (!active || !wizardFiles.length || wizardFiles.some(item => (options.fields[stage] || []).some(field => field.required && !item.fieldMap[field.name]) || Object.values(item.fieldMap).length !== new Set(Object.values(item.fieldMap)).size) || (mode === 'REPLACE' && (!target || wizardFiles.length !== 1)) || actionLock.current) return;
actionLock.current = true; actionLock.current = true;
setBusy(true); setError(''); setBusy(true); setError('');
try { try {
const configured = configuredSourceId ? { source: { id: configuredSourceId } } : await api.configureSourceWorkbook({ uploadId: workbook.uploadId, stage, year, month, sheetName, headerRow, fieldMap, mode: mode === 'REPLACE' ? 'REPLACE' : 'ADD', expectedCurrentReleaseId: options.currentReleaseId, ...(target ? { targetDocumentId: target.documentId, targetRevisionId: target.revisionId } : {}), reason: reason.trim() || `新增${year}年${month}月${stageName(stage)}数据`, scopeCode }); const items = wizardFiles.map(item => ({ id: item.id, body: { uploadId: item.workbook.uploadId, stage, year, month, sheetName: item.sheetName, headerRow: item.headerRow, fieldMap: item.fieldMap, mode: mode === 'REPLACE' ? 'REPLACE' as const : 'APPEND' as const, ...(mode === 'REPLACE' && target?.fileId ? { fileId: target.fileId } : mode === 'ADD' ? { fileId: item.fileId } : {}), expectedCurrentReleaseId: options.currentReleaseId, ...(mode === 'REPLACE' && target ? { targetDocumentId: target.documentId, targetRevisionId: target.revisionId } : {}), reason: reason.trim() || `${mode === 'REPLACE' ? '替换' : '追加'}${year}年${month}月${stageName(stage)}文件`, scopeCode } }));
setConfiguredSourceId(configured.source.id); const sourceIds = await configureSourceBatch(items, configuredByFile.current, body => api.configureSourceWorkbook(body));
const created = asRow(await api.startImport({ sourceIds: [configured.source.id], expectedCurrentReleaseId: options.currentReleaseId, idempotencyKey: importKey.current })); const nextRun = asRow(created.run || created); setRun(nextRun); const created = asRow(await api.startImport({ sourceIds, expectedCurrentReleaseId: options.currentReleaseId, idempotencyKey: importKey.current })); const nextRun = asRow(created.run || created); setRun(nextRun);
if (!nextRun.id) throw new Error('服务端未返回计算任务,已接收的来源可稍后重新校验'); if (!nextRun.id) throw new Error('服务端未返回计算任务,已接收的来源可稍后重新校验');
try { localStorage.setItem(runStorageKey, String(nextRun.id)); } catch { /* Server task history remains available if local storage is unavailable. */ } try { localStorage.setItem(runStorageKey, String(nextRun.id)); } catch { /* Server task history remains available if local storage is unavailable. */ }
} catch (value) { reportError(value); setBusy(false); } } catch (value) { reportError(value); setBusy(false); }
@@ -549,13 +591,15 @@ function DataUploadWizard({ user, mode, purpose, handoffSourceId, initialReason,
const publishReady = release?.complete === true && reconciliation?.publishReady === true; const publishReady = release?.complete === true && reconciliation?.publishReady === true;
const steps = purpose === 'WITHDRAW' ? ['确认来源', '提交申请', '审批与候选', '来源校验', '对账与发布'] : mode === 'REBUILD' ? ['确认重算', '选择期间', '表头匹配', '计算校验', '对账与发布'] : ['选择文件', '确认期间', '工作表与表头', '预览与校验', '对账与发布']; const steps = purpose === 'WITHDRAW' ? ['确认来源', '提交申请', '审批与候选', '来源校验', '对账与发布'] : mode === 'REBUILD' ? ['确认重算', '选择期间', '表头匹配', '计算校验', '对账与发布'] : ['选择文件', '确认期间', '工作表与表头', '预览与校验', '对账与发布'];
const close = () => { generation.current++; onClose(); }; const close = () => { generation.current++; onClose(); };
return <Modal className="management-modal nc-workflow-modal" open width={1120} title={published ? purpose === 'WITHDRAW' ? '来源删除已发布' : '数据已发布' : purpose === 'WITHDRAW' ? '核对并发布删除候选' : resumeId ? '继续处理候选数据' : mode === 'REPLACE' ? '替换已有数据' : mode === 'REBUILD' ? '按当前来源重新计算' : '上传新数据'} onCancel={close} maskClosable={!busy} footer={<div className="nc-modal-footer"><span>{runId ? '关闭后任务继续,可在“数据接入”的“导入记录”查看;完成后从候选继续。' : '确认发布前,当前查询结果保持不变。'}</span><div>{step > 0 && step < 4 && !runId && !configuredSourceId && !busy && <Button onClick={() => { setStep(step - 1); setError(''); }}>上一步</Button>}<Button onClick={close}>{published ? '完成' : busy ? '关闭并稍后查看' : '关闭'}</Button>{!published && step === 1 && <Button type="primary" disabled={mode === 'REPLACE' && !target} loading={busy} onClick={() => { setStep(2); void changePreview(sheetName, headerRow); }}>下一步:匹配表头</Button>}{!published && step === 2 && <Button type="primary" loading={busy} disabled={Boolean(missingFields.length) || duplicateColumns || !sheetName} onClick={() => { if (workbook && (workbook.preview.headerRow !== headerRow || workbook.preview.sheetName !== sheetName)) { void changePreview(sheetName, headerRow); return; } setStep(3); }}>下一步:预览验证</Button>}{!published && step === 3 && !runId && mode !== 'REBUILD' && <Button type="primary" loading={busy} disabled={mode === 'REPLACE' && !reason.trim()} onClick={() => void beginImport()}>确认导入并校验</Button>}</div></div>}> return <Modal className="management-modal nc-workflow-modal" open width={1120} title={published ? purpose === 'WITHDRAW' ? '来源删除已发布' : '数据已发布' : purpose === 'WITHDRAW' ? '核对并发布删除候选' : resumeId ? '继续处理候选数据' : mode === 'REPLACE' ? '替换已有数据' : mode === 'REBUILD' ? '按当前来源重新计算' : '上传新数据'} onCancel={close} maskClosable={!busy} footer={<div className="nc-modal-footer"><span>{runId ? '关闭后任务继续,可在“数据接入”的“导入记录”查看;完成后从候选继续。' : '确认发布前,当前查询结果保持不变。'}</span><div>{step > 0 && step < 4 && !runId && !busy && <Button onClick={() => { setStep(step - 1); setError(''); }}>上一步</Button>}<Button onClick={close}>{published ? '完成' : busy ? '关闭并稍后查看' : '关闭'}</Button>{!published && step === 1 && <Button type="primary" disabled={(mode === 'REPLACE' && !target) || !wizardFiles.length} loading={busy} onClick={() => { setStep(2); if (workbook) void changePreview(sheetName, headerRow); }}>下一步:匹配表头</Button>}{!published && step === 2 && <Button type="primary" loading={busy} disabled={wizardFiles.some(item => (options.fields[stage] || []).some(field => field.required && !item.fieldMap[field.name]) || Object.values(item.fieldMap).length !== new Set(Object.values(item.fieldMap)).size) || !sheetName} onClick={() => { if (workbook && (workbook.preview.headerRow !== headerRow || workbook.preview.sheetName !== sheetName)) { void changePreview(sheetName, headerRow); return; } setStep(3); }}>下一步:预览验证</Button>}{!published && step === 3 && !runId && mode !== 'REBUILD' && <Button type="primary" loading={busy} disabled={(mode === 'REPLACE' && !reason.trim()) || !wizardFiles.length} onClick={() => void beginImport()}>确认导入并校验</Button>}</div></div>}>
<Steps className="nc-steps" current={published ? 5 : step} size="small" items={steps.map(title => ({ title }))} /> <Steps className="nc-steps" current={published ? 5 : step} size="small" items={steps.map(title => ({ title }))} />
{mode !== 'REBUILD' && !purpose && <input className="visually-hidden-file" ref={input} type="file" multiple={mode === 'ADD'} accept=".xlsx" onChange={event => void readFiles(Array.from(event.target.files || []))} aria-label={mode === 'REPLACE' ? '选择要替换的质量数据Excel文件' : '选择要追加的质量数据Excel文件'} />}
{error && <Alert showIcon type="error" message="本次操作未完成" description={error} action={resumeId && step === 4 ? <Button size="small" disabled={!active || busy || candidateReadLock.current} onClick={() => { if (!activeRef.current || candidateReadLock.current) return; candidateReadLock.current = true; setBusy(true); loadCandidate(resumeId).catch(reportError).finally(() => { candidateReadLock.current = false; setBusy(false); }); }}>重试读取候选</Button> : runId ? <Button size="small" disabled={!active || busy || pollLock.current} onClick={() => { if (!activeRef.current || pollLock.current) return; setBusy(true); void pollRun(); }}>重新获取处理结果</Button> : undefined} />} {error && <Alert showIcon type="error" message="本次操作未完成" description={error} action={resumeId && step === 4 ? <Button size="small" disabled={!active || busy || candidateReadLock.current} onClick={() => { if (!activeRef.current || candidateReadLock.current) return; candidateReadLock.current = true; setBusy(true); loadCandidate(resumeId).catch(reportError).finally(() => { candidateReadLock.current = false; setBusy(false); }); }}>重试读取候选</Button> : runId ? <Button size="small" disabled={!active || busy || pollLock.current} onClick={() => { if (!activeRef.current || pollLock.current) return; setBusy(true); void pollRun(); }}>重新获取处理结果</Button> : undefined} />}
{published ? <div className="nc-success"><FileExcelOutlined /><h2>{purpose === 'WITHDRAW' ? '该来源已从当前查询结果移除' : '当前查询数据已更新'}</h2><p>来源原件和本次对账记录均已保留。</p></div> : <> {published ? <div className="nc-success"><FileExcelOutlined /><h2>{purpose === 'WITHDRAW' ? '该来源已从当前查询结果移除' : '当前查询数据已更新'}</h2><p>来源原件和本次对账记录均已保留。</p></div> : <>
{step === 0 && (mode === 'REBUILD' ? <div className="nc-rebuild-confirm"><h3>使用当前数据所用的全部来源重新计算</h3><p>计算使用当前启用的基础数据、对应关系和规则。完成后先生成候选,核对差异后再发布。</p><div className="nc-target-list">{options.targets.map(item => <div key={item.documentId}><Tag>{stageName(item.stage)}</Tag><strong>{item.year}年{item.month}月</strong><span>{item.sourceFile}</span></div>)}</div><Button type="primary" loading={busy} disabled={!options.targets.length} onClick={() => void rebuild()}>开始重新计算</Button></div> : <div className="nc-file-drop"><FileExcelOutlined /><h3>选择要{mode === 'REPLACE' ? '替换' : '上传'}的 Excel 文件</h3><p>支持 .xlsx;保留你的文件名,接下来选择数据期间和表头。</p><input className="visually-hidden-file" ref={input} type="file" accept=".xlsx" onChange={event => void readFile(event.target.files?.[0])} aria-label="选择质量数据Excel文件" /><Button type="primary" icon={<UploadOutlined />} loading={busy} onClick={() => input.current?.click()}>选择 Excel 文件</Button></div>)} {step === 0 && (mode === 'REBUILD' ? <div className="nc-rebuild-confirm"><h3>使用当前数据所用的全部来源重新计算</h3><p>计算使用当前启用的基础数据、对应关系和规则。完成后先生成候选,核对差异后再发布。</p><div className="nc-target-list">{options.targets.map(item => <div key={item.documentId}><Tag>{stageName(item.stage)}</Tag><strong>{item.year}年{item.month}月</strong><span>{item.sourceFile}</span></div>)}</div><Button type="primary" loading={busy} disabled={!options.targets.length} onClick={() => void rebuild()}>开始重新计算</Button></div> : <div className="nc-file-drop"><FileExcelOutlined /><h3>选择要{mode === 'REPLACE' ? '替换' : '追加'}的 Excel 文件</h3><p>{mode === 'REPLACE' ? '替换一份已发布文件,其他来源保留。' : '文件会作为独立来源加入所选月份,并与已有来源合并统计。'} 请确保业务记录不与其他文件重叠。</p><Button type="primary" icon={<UploadOutlined />} loading={busy} onClick={() => input.current?.click()}>{mode === 'REPLACE' ? '选择替换文件' : '选择文件(可多选)'}</Button></div>)}
{workbook && step > 0 && step < 4 && <div className="nc-file-summary"><FileExcelOutlined /><strong>{workbook.sourceFile}</strong><span>{stageName(stage)} · {year}年{month}月</span></div>} {workbook && step > 0 && step < 4 && <div className="nc-file-summary"><FileExcelOutlined /><strong>{workbook.sourceFile}</strong><span>{stageName(stage)} · {year}年{month}月</span></div>}
{step === 1 && <Form layout="vertical"><div className="nc-form-grid">{mode === 'REPLACE' ? <Form.Item className="nc-full-width" label="选择当前数据中的替换目标" required><Select {...readableSelectPopup} aria-label="选择当前数据中的替换目标" showSearch optionFilterProp="label" value={targetId || undefined} placeholder="选择年月、阶段和原文件" options={options.targets.map(item => ({ value: item.documentId, label: `${item.year}年${item.month}月 · ${stageName(item.stage)} · ${item.sourceFile}` }))} onChange={chooseTarget} /></Form.Item> : <><Form.Item label="数据阶段" required><Select aria-label="数据阶段" value={stage} options={[{ value: 'black', label: '黑片' }, { value: 'bright', label: '亮片' }]} onChange={value => { setStage(value); setFieldMap(matchFields(options.fields[value] || [], headers)); }} /></Form.Item><Form.Item label="统计年份" required><InputNumber aria-label="统计年份" min={2000} max={2100} value={year} onChange={value => { if (value) setYear(value); }} /></Form.Item><Form.Item label="统计月份" required><Select aria-label="统计月份" value={month} options={Array.from({ length: 12 }, (_, index) => ({ value: index + 1, label: `${index + 1}月` }))} onChange={setMonth} /></Form.Item>{options.scopes.length > 1 && <Form.Item label="数据范围" required><Select aria-label="数据范围" value={scopeCode} options={options.scopes.map(value => ({ value, label: value === 'ALL' ? '全部范围' : value }))} onChange={setScopeCode} /></Form.Item>}</>}</div>{target && <div className="nc-replacement"><span>原文件</span><strong>{target.sourceFile}</strong><ArrowRightOutlined /><span>新文件</span><strong>{workbook?.sourceFile}</strong></div>}<Form.Item label={mode === 'REPLACE' ? '替换理由' : '本次说明'} required={mode === 'REPLACE'}><Input.TextArea aria-label={mode === 'REPLACE' ? '替换理由' : '本次说明'} value={reason} rows={2} maxLength={1000} placeholder={mode === 'REPLACE' ? '例如:补齐遗漏记录,使用更新后的完整月表' : '可填写本次数据说明'} onChange={event => setReason(event.target.value)} /></Form.Item></Form>} {wizardFiles.length > 0 && step > 0 && step < 4 && <><div className="nc-section-title"><strong>本次文件 · {wizardFiles.length} 份</strong>{mode === 'ADD' && step === 1 && <Button disabled={busy} icon={<UploadOutlined />} onClick={() => input.current?.click()}>继续添加本月文件</Button>}</div><Table size="small" rowKey="id" pagination={false} scroll={{ x: 600 }} dataSource={wizardFiles} columns={[{ title: '文件', key: 'file', render: (_, item) => <Button type="link" onClick={() => setWizardFileId(item.id)}>{item.workbook.sourceFile} · {item.fileId.slice(0, 8)}</Button> }, { title: '字段匹配', key: 'mapping', render: (_, item) => (options.fields[stage] || []).filter(field => field.required).every(field => item.fieldMap[field.name]) ? <Tag color="green">已完成</Tag> : <Tag color="orange">需核对</Tag> }, { title: '操作', key: 'action', render: (_, item) => configuredByFile.current.has(item.id) ? <Tag color="blue">已接收待导入</Tag> : <Button type="link" disabled={busy || mode === 'REPLACE'} onClick={() => removeWizardFile(item.id)}>移除</Button> }]} /></>}
{step === 1 && <Form layout="vertical"><div className="nc-form-grid">{mode === 'REPLACE' ? <Form.Item className="nc-full-width" label="选择要替换的具体文件" required><Select {...readableSelectPopup} aria-label="选择要替换的具体文件" disabled={Boolean(activeWizardFile && configuredByFile.current.has(activeWizardFile.id))} showSearch optionFilterProp="label" value={targetId || undefined} placeholder="选择年月、阶段、范围和文件" options={options.targets.map(item => ({ value: item.documentId, label: `${item.year}年${item.month}月 · ${stageName(item.stage)} · ${item.scopeCode} · ${item.sourceFile} · ${item.fileId ? `分批 ${item.fileId.slice(0, 8)}` : '整月文件'} (${item.documentId.slice(-6)})` }))} onChange={chooseTarget} /></Form.Item> : <><Form.Item label="数据阶段" required><Select aria-label="数据阶段" value={stage} options={[{ value: 'black', label: '黑片' }, { value: 'bright', label: '亮片' }]} onChange={value => { setStage(value); setWizardFiles(previous => previous.map(item => ({ ...item, fieldMap: matchFields(options.fields[value] || [], item.workbook.preview.headers) }))); }} /></Form.Item><Form.Item label="统计年份" required><InputNumber aria-label="统计年份" min={2000} max={2100} value={year} onChange={value => { if (value) setYear(value); }} /></Form.Item><Form.Item label="统计月份" required><Select aria-label="统计月份" value={month} options={Array.from({ length: 12 }, (_, index) => ({ value: index + 1, label: `${index + 1}月` }))} onChange={setMonth} /></Form.Item>{options.scopes.length > 1 && <Form.Item label="数据范围" required><Select aria-label="数据范围" value={scopeCode} options={options.scopes.map(value => ({ value, label: value === 'ALL' ? '全部范围' : value }))} onChange={setScopeCode} /></Form.Item>}</>}</div>{target && <div className="nc-replacement"><span>原文件</span><strong>{target.sourceFile} · {target.fileId ? `分批 ${target.fileId.slice(0, 8)}` : '整月文件'}</strong><ArrowRightOutlined /><span>新文件</span><strong>{workbook?.sourceFile}</strong></div>}<Form.Item label={mode === 'REPLACE' ? '替换理由' : '本次说明'} required={mode === 'REPLACE'}><Input.TextArea aria-label={mode === 'REPLACE' ? '替换理由' : '本次说明'} value={reason} rows={2} maxLength={1000} placeholder={mode === 'REPLACE' ? '例如:修订这份文件中的记录,其他来源保留' : '可填写本次数据说明'} onChange={event => setReason(event.target.value)} /></Form.Item></Form>}
{step === 2 && workbook && <><div className="nc-form-grid"><Form.Item label="工作表" required><Select aria-label="数据工作表" value={sheetName} options={workbook.sheets.map(sheet => ({ value: sheet.sheetName, label: sheet.sheetName }))} onChange={value => void changePreview(value, headerRow)} /></Form.Item><Form.Item label="表头在第几行" required><InputNumber aria-label="表头所在行" min={1} max={100} value={headerRow} onChange={value => { if (value) setHeaderRow(value); }} onBlur={() => { if (headerRow !== workbook.preview.headerRow) void changePreview(sheetName, headerRow); }} onPressEnter={() => void changePreview(sheetName, headerRow)} /></Form.Item></div><div className="nc-section-title"><h3>业务字段对应 Excel 中哪一列</h3><span>{duplicateColumns ? '同一 Excel 列不能对应多个业务字段' : missingFields.length ? `还需选择 ${missingFields.length} 个必填字段` : '必填字段已全部匹配'}</span></div><Spin spinning={busy}><ColumnMatching fields={fields} headers={headers} value={fieldMap} onChange={setFieldMap} /></Spin><h3>原表前几行</h3><SampleTable headers={headers} rows={workbook.preview.sampleRows.map(row => row.values)} /></>} {step === 2 && workbook && <><div className="nc-form-grid"><Form.Item label="工作表" required><Select aria-label="数据工作表" value={sheetName} options={workbook.sheets.map(sheet => ({ value: sheet.sheetName, label: sheet.sheetName }))} onChange={value => void changePreview(value, headerRow)} /></Form.Item><Form.Item label="表头在第几行" required><InputNumber aria-label="表头所在行" min={1} max={100} value={headerRow} onChange={value => { if (value) setHeaderRow(value); }} onBlur={() => { if (headerRow !== workbook.preview.headerRow) void changePreview(sheetName, headerRow); }} onPressEnter={() => void changePreview(sheetName, headerRow)} /></Form.Item></div><div className="nc-section-title"><h3>业务字段对应 Excel 中哪一列</h3><span>{duplicateColumns ? '同一 Excel 列不能对应多个业务字段' : missingFields.length ? `还需选择 ${missingFields.length} 个必填字段` : '必填字段已全部匹配'}</span></div><Spin spinning={busy}><ColumnMatching fields={fields} headers={headers} value={fieldMap} onChange={setFieldMap} /></Spin><h3>原表前几行</h3><SampleTable headers={headers} rows={workbook.preview.sampleRows.map(row => row.values)} /></>}
{step === 3 && <>{!runId && workbook && <><div className="nc-review-facts"><div><small>统计期间</small><strong>{year}年{month}月 · {stageName(stage)}</strong></div><div><small>工作表与表头</small><strong>{sheetName} · 第{headerRow}行</strong></div><div><small>已匹配字段</small><strong>{Object.keys(fieldMap).length} 项</strong></div></div>{target && <Alert type="warning" showIcon message="即将使用新文件替换该期间的完整数据" description={`原文件:${target.sourceFile};新文件:${workbook.sourceFile}。本次先生成候选,最后发布后才更新查询结果。`} />}<SampleTable headers={Object.keys(fieldMap)} rows={workbook.preview.sampleRows.map(row => Object.fromEntries(Object.entries(fieldMap).map(([field, header]) => [field, row.values[header]])))} /><p className="nc-note">确认后由系统校验文件内容、数据范围及对应关系,并计算候选结果。</p></>}{(runId || (busy && mode === 'REBUILD')) && <div className="nc-processing"><Spin spinning={busy} /><h3>{statusLabel[String(run?.status)] || '正在创建计算任务'}</h3><Progress percent={Number(run?.total) > 0 ? Math.min(100, Math.round(Number(run?.processed || 0) / Number(run?.total) * 100)) : 0} status={run?.status === 'FAILED' ? 'exception' : 'active'} /><p>已处理 {txt(run?.processed || 0)} / {txt(run?.total)} 行</p>{run?.errorMessage ? <Alert type="error" showIcon message={txt(run.errorMessage)} /> : null}{!busy && runId && !['FAILED', 'CANCELLED'].includes(String(run?.status)) && <Button disabled={!active || pollLock.current} onClick={() => { if (!activeRef.current || pollLock.current) return; setBusy(true); void pollRun(); }}>重新获取处理结果</Button>}</div>}<ValidationNotice value={run?.validation} /></>} {step === 3 && <>{!runId && workbook && <><div className="nc-review-facts"><div><small>统计期间</small><strong>{year}年{month}月 · {stageName(stage)}</strong></div><div><small>工作表与表头</small><strong>{sheetName} · 第{headerRow}行</strong></div><div><small>已匹配字段</small><strong>{Object.keys(fieldMap).length} 项</strong></div></div>{target && <Alert type="warning" showIcon message="即将使用新文件替换该期间的完整数据" description={`原文件:${target.sourceFile};新文件:${workbook.sourceFile}。本次先生成候选,最后发布后才更新查询结果。`} />}<SampleTable headers={Object.keys(fieldMap)} rows={workbook.preview.sampleRows.map(row => Object.fromEntries(Object.entries(fieldMap).map(([field, header]) => [field, row.values[header]])))} /><p className="nc-note">确认后由系统校验文件内容、数据范围及对应关系,并计算候选结果。</p></>}{(runId || (busy && mode === 'REBUILD')) && <div className="nc-processing"><Spin spinning={busy} /><h3>{statusLabel[String(run?.status)] || '正在创建计算任务'}</h3><Progress percent={Number(run?.total) > 0 ? Math.min(100, Math.round(Number(run?.processed || 0) / Number(run?.total) * 100)) : 0} status={run?.status === 'FAILED' ? 'exception' : 'active'} /><p>已处理 {txt(run?.processed || 0)} / {txt(run?.total)} 行</p>{run?.errorMessage ? <Alert type="error" showIcon message={txt(run.errorMessage)} /> : null}{!busy && runId && !['FAILED', 'CANCELLED'].includes(String(run?.status)) && <Button disabled={!active || pollLock.current} onClick={() => { if (!activeRef.current || pollLock.current) return; setBusy(true); void pollRun(); }}>重新获取处理结果</Button>}</div>}<ValidationNotice value={run?.validation} /></>}
{step === 4 && <Spin spinning={busy}><div className="nc-review-facts"><div><small>候选数据</small><strong>待核对</strong></div><div><small>来源校验</small><strong>{release?.complete === true ? '通过' : '尚未通过'}</strong></div><div><small>基线对账</small><strong>{statusLabel[String(reconciliation?.status)] || '待执行'}</strong></div></div><ValidationNotice value={release?.validation || run?.validation} />{reconciliationErrors.length > 0 && <Alert showIcon type="error" message="对账存在阻断事项" description={reconciliationErrors.map(txt).join(';')} />}<div className="nc-section-title"><h3>与当前数据的差异</h3><Button icon={<ReloadOutlined />} loading={busy} disabled={!release?.id} onClick={() => void reconcile()}>{reconciliation ? '重新对账' : '执行基线对账'}</Button></div>{comparisons.length > 0 ? <Table size="small" bordered dataSource={comparisons} rowKey={(row, index) => `${row.dimension}:${row.key}:${index}`} pagination={{ pageSize: 15, showSizeChanger: false }} scroll={{ x: 900 }} columns={[{ title: '期间', width: 130, render: (_, row) => { const period = asRow(row.period); return `${txt(period.year)}年${period.month ? `${txt(period.month)}月` : period.quarter ? `第${txt(period.quarter)}季度` : '全年'}`; } }, { title: '维度 / 对象', width: 200, render: (_, row) => `${({ overall: '总体', department: '产业部', group: '业务分组', product: '产品', projectGroup: '项目组' } as Record<string, string>)[String(row.dimension)] || txt(row.dimension)} / ${txt(row.key)}` }, { title: '原综合率', width: 120, render: (_, row) => ratio(asRow(row.before).compositeRate) }, { title: '新综合率', width: 120, render: (_, row) => ratio(asRow(row.after).compositeRate) }, { title: '差异', render: (_, row) => asRows(row.differences).map(item => `${metricLabel(item.path)}:${txt(item.before)} → ${txt(item.after)}`).join(';') || '一致' }]} /> : <Empty image={Empty.PRESENTED_IMAGE_SIMPLE} description={reconciliation ? '本次没有可展示的期间比较;以对账结论为准' : '执行对账后在这里查看期间和指标差异'} />}{reconciliation?.status === 'REVIEW_REQUIRED' && <div className="nc-difference-confirm"><Alert type="warning" showIcon message={`发现 ${txt(evidence.differenceCount)} 项差异,需要发布者确认`} />{canPublish ? <><Input.TextArea aria-label="业务差异确认理由" rows={2} value={differenceReason} maxLength={1000} placeholder="填写差异原因及确认依据" onChange={event => setDifferenceReason(event.target.value)} /><Checkbox checked={confirmDifferences} onChange={event => setConfirmDifferences(event.target.checked)}>我已核对上方差异,并确认新数据正确</Checkbox><Button type="primary" disabled={!confirmDifferences || !differenceReason.trim() || busy} onClick={() => void confirm()}>确认差异并验收</Button></> : <p>候选已保存,请交由有数据发布权限的同事完成差异确认。</p>}</div>}{canPublish ? <div className="nc-publish-confirm"><Checkbox disabled={!publishReady} checked={publishConfirmed} onChange={event => setPublishConfirmed(event.target.checked)}>{purpose === 'WITHDRAW' ? '我确认发布此删除候选,从当前查询结果移除已批准的来源' : '我确认将此候选发布为当前查询数据'}{target ? `,替换${target.year}年${target.month}月${stageName(target.stage)}` : ''}</Checkbox><Button size="large" type="primary" disabled={!publishReady || !publishConfirmed || busy} onClick={() => void publish()}>确认发布</Button>{!publishReady && <span>来源校验和基线对账通过后可发布。</span>}</div> : <Alert showIcon type="info" message="候选数据已保存,等待有权限的同事发布" description={<><span>当前账号没有数据发布权限。可复制处理链接交接,正式查询数据尚未更新。</span><Button type="link" onClick={() => { const link = new URL(window.location.href); link.searchParams.set('page', 'sources'); if (handoffSourceId) { link.searchParams.set('sourceDeleteId', handoffSourceId); link.searchParams.delete('candidateReleaseId'); } else { link.searchParams.set('candidateReleaseId', String(release?.id || '')); link.searchParams.delete('sourceDeleteId'); } void navigator.clipboard.writeText(link.toString()).then(() => message.success('处理链接已复制')).catch(reportError); }}>复制处理链接</Button></>} />}</Spin>} {step === 4 && <Spin spinning={busy}><div className="nc-review-facts"><div><small>候选数据</small><strong>待核对</strong></div><div><small>来源校验</small><strong>{release?.complete === true ? '通过' : '尚未通过'}</strong></div><div><small>基线对账</small><strong>{statusLabel[String(reconciliation?.status)] || '待执行'}</strong></div></div><ValidationNotice value={release?.validation || run?.validation} />{reconciliationErrors.length > 0 && <Alert showIcon type="error" message="对账存在阻断事项" description={reconciliationErrors.map(txt).join(';')} />}<div className="nc-section-title"><h3>与当前数据的差异</h3><Button icon={<ReloadOutlined />} loading={busy} disabled={!release?.id} onClick={() => void reconcile()}>{reconciliation ? '重新对账' : '执行基线对账'}</Button></div>{comparisons.length > 0 ? <Table size="small" bordered dataSource={comparisons} rowKey={(row, index) => `${row.dimension}:${row.key}:${index}`} pagination={{ pageSize: 15, showSizeChanger: false }} scroll={{ x: 900 }} columns={[{ title: '期间', width: 130, render: (_, row) => { const period = asRow(row.period); return `${txt(period.year)}年${period.month ? `${txt(period.month)}月` : period.quarter ? `第${txt(period.quarter)}季度` : '全年'}`; } }, { title: '维度 / 对象', width: 200, render: (_, row) => `${({ overall: '总体', department: '产业部', group: '业务分组', product: '产品', projectGroup: '项目组' } as Record<string, string>)[String(row.dimension)] || txt(row.dimension)} / ${txt(row.key)}` }, { title: '原综合率', width: 120, render: (_, row) => ratio(asRow(row.before).compositeRate) }, { title: '新综合率', width: 120, render: (_, row) => ratio(asRow(row.after).compositeRate) }, { title: '差异', render: (_, row) => asRows(row.differences).map(item => `${metricLabel(item.path)}:${txt(item.before)} → ${txt(item.after)}`).join(';') || '一致' }]} /> : <Empty image={Empty.PRESENTED_IMAGE_SIMPLE} description={reconciliation ? '本次没有可展示的期间比较;以对账结论为准' : '执行对账后在这里查看期间和指标差异'} />}{reconciliation?.status === 'REVIEW_REQUIRED' && <div className="nc-difference-confirm"><Alert type="warning" showIcon message={`发现 ${txt(evidence.differenceCount)} 项差异,需要发布者确认`} />{canPublish ? <><Input.TextArea aria-label="业务差异确认理由" rows={2} value={differenceReason} maxLength={1000} placeholder="填写差异原因及确认依据" onChange={event => setDifferenceReason(event.target.value)} /><Checkbox checked={confirmDifferences} onChange={event => setConfirmDifferences(event.target.checked)}>我已核对上方差异,并确认新数据正确</Checkbox><Button type="primary" disabled={!confirmDifferences || !differenceReason.trim() || busy} onClick={() => void confirm()}>确认差异并验收</Button></> : <p>候选已保存,请交由有数据发布权限的同事完成差异确认。</p>}</div>}{canPublish ? <div className="nc-publish-confirm"><Checkbox disabled={!publishReady} checked={publishConfirmed} onChange={event => setPublishConfirmed(event.target.checked)}>{purpose === 'WITHDRAW' ? '我确认发布此删除候选,从当前查询结果移除已批准的来源' : '我确认将此候选发布为当前查询数据'}{target ? `,替换${target.year}年${target.month}月${stageName(target.stage)}` : ''}</Checkbox><Button size="large" type="primary" disabled={!publishReady || !publishConfirmed || busy} onClick={() => void publish()}>确认发布</Button>{!publishReady && <span>来源校验和基线对账通过后可发布。</span>}</div> : <Alert showIcon type="info" message="候选数据已保存,等待有权限的同事发布" description={<><span>当前账号没有数据发布权限。可复制处理链接交接,正式查询数据尚未更新。</span><Button type="link" onClick={() => { const link = new URL(window.location.href); link.searchParams.set('page', 'sources'); if (handoffSourceId) { link.searchParams.set('sourceDeleteId', handoffSourceId); link.searchParams.delete('candidateReleaseId'); } else { link.searchParams.set('candidateReleaseId', String(release?.id || '')); link.searchParams.delete('sourceDeleteId'); } void navigator.clipboard.writeText(link.toString()).then(() => message.success('处理链接已复制')).catch(reportError); }}>复制处理链接</Button></>} />}</Spin>}
+107
View File
@@ -0,0 +1,107 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import { Alert, Button, Spin, Tag } from 'antd';
import { ReloadOutlined } from '@ant-design/icons';
import { api, type SystemUpdateStatus } from './api';
import { configurationDateTime } from './configurationName';
import { usePageCache } from './pageCache';
import { useVisiblePolling } from './useVisiblePolling';
const runningStates = new Set(['queued', 'downloading', 'backingUp', 'installing', 'restarting', 'rollingBack']);
function errorStatus(error: unknown): number | undefined {
return error && typeof error === 'object' && 'status' in error && typeof error.status === 'number' ? error.status : undefined;
}
function reconnectable(error: unknown): boolean {
const status = errorStatus(error);
return status === 0 || status === 408 || (status !== undefined && status >= 500 && status <= 599);
}
export function SystemUpdatePanel({ onUnauthorized }: { onUnauthorized: (error: unknown) => boolean }) {
const { active } = usePageCache();
const [status, setStatus] = useState<SystemUpdateStatus>();
const [busy, setBusy] = useState(false);
const [error, setError] = useState('');
const [reconnecting, setReconnecting] = useState(false);
const [permissionDenied, setPermissionDenied] = useState(false);
const inFlight = useRef(false);
const restarting = useRef(false);
const running = !!status && runningStates.has(status.state);
const load = useCallback(async () => {
try {
const next = await api.systemUpdateStatus();
setStatus(next);
setError('');
setReconnecting(false);
setPermissionDenied(false);
restarting.current = runningStates.has(next.state);
} catch (failure) {
const denied = [401, 403].includes(errorStatus(failure) ?? -1);
const handled = onUnauthorized(failure);
if (handled || denied) {
restarting.current = false;
setReconnecting(false);
setPermissionDenied(true);
if (!handled) setError(failure instanceof Error ? failure.message : '无权查看更新状态');
} else if (restarting.current && reconnectable(failure)) {
setReconnecting(true);
} else {
restarting.current = false;
setReconnecting(false);
setError(failure instanceof Error ? failure.message : '更新状态读取失败');
}
}
}, [onUnauthorized]);
useEffect(() => { if (active) void load(); }, [active, load]);
useVisiblePolling(load, running || reconnecting ? 4_000 : 30_000, active && !permissionDenied);
const run = async (apply: boolean) => {
if (inFlight.current) return;
inFlight.current = true;
setBusy(true);
setError('');
try {
if (apply) restarting.current = true;
const next = await (apply ? api.applySystemUpdate() : api.checkSystemUpdate());
setStatus(next);
restarting.current = runningStates.has(next.state);
} catch (failure) {
// A successful updater intentionally takes the server offline. A lost
// response is resolved by polling the persisted task after restart.
if (apply && reconnectable(failure)) {
setReconnecting(true);
} else {
restarting.current = false;
const denied = [401, 403].includes(errorStatus(failure) ?? -1);
setPermissionDenied(denied);
if (!onUnauthorized(failure)) setError(failure instanceof Error ? failure.message : '更新操作失败');
if (!denied) void load();
}
} finally {
setBusy(false);
inFlight.current = false;
}
};
return <section className="panel section-panel">
<div className="completion-panel-heading">
<strong>系统更新</strong>
<div className="completion-actions">
<Button icon={<ReloadOutlined />} loading={busy && !restarting.current} disabled={!status?.enabled || running || reconnecting || busy} onClick={() => void run(false)}>检查更新</Button>
{status?.available && !running && !reconnecting && <Button type="primary" loading={busy && restarting.current} disabled={busy} onClick={() => void run(true)}>更新并重启</Button>}
</div>
</div>
{!status && !error && <Spin />}
{error && <Alert showIcon type="error" message={error} action={<Button size="small" onClick={() => void load()}>重试</Button>} />}
{reconnecting ? <Alert showIcon type="info" icon={<Spin size="small" />} message="正在重新启动,完成后将自动连接。" />
: status && <>
{status.enabled && <div className="governance-monitor-grid">
<div><small>当前版本</small><strong>{status.currentVersion || '—'}</strong></div>
{status.version && <div><small>{status.available ? '可用版本' : '最近检查版本'}</small><strong>{status.version}</strong></div>}
{status.checkedAt && <div><small>检查时间</small><strong>{configurationDateTime(status.checkedAt)}</strong></div>}
</div>}
<Alert showIcon type={status.state === 'failed' ? 'error' : status.state === 'completed' ? 'success' : 'info'}
message={<>{running && <Tag color="processing">更新中</Tag>}{status.message || '点击检查更新获取最新版本。'}</>} />
</>}
</section>;
}
+2 -1
View File
@@ -12,6 +12,7 @@ import { configurationDateTime } from './configurationName';
import { backupSummary, isActive, isFailed, monitorLabel, orderedTasks, taskFailure, taskPercent, type MonitorRow } from './taskMonitorModel'; import { backupSummary, isActive, isFailed, monitorLabel, orderedTasks, taskFailure, taskPercent, type MonitorRow } from './taskMonitorModel';
import { useDelayedBusy } from './useDelayedBusy'; import { useDelayedBusy } from './useDelayedBusy';
import { useVisiblePolling } from './useVisiblePolling'; import { useVisiblePolling } from './useVisiblePolling';
import { SystemUpdatePanel } from './SystemUpdatePanel';
export function TasksMonitorPage({user,onUnauthorized}:{user:AuthUser;onUnauthorized:(error:unknown)=>boolean}) { export function TasksMonitorPage({user,onUnauthorized}:{user:AuthUser;onUnauthorized:(error:unknown)=>boolean}) {
const {message}=App.useApp(); const {message}=App.useApp();
@@ -72,7 +73,7 @@ export function TasksMonitorPage({user,onUnauthorized}:{user:AuthUser;onUnauthor
{loading&&!backupLoaded?<Spin/>:backupLoaded&&<div className="governance-monitor-grid"><div><small>最近备份</small><strong>{backupSummary(latest)}</strong></div><div><small>备份时间</small><strong>{configurationDateTime(latest.createdAt)||'—'}</strong></div><div><small>恢复验证时间</small><strong>{configurationDateTime(latest.restoreVerifiedAt)||'—'}</strong></div><div><small>自动备份</small><strong>{backup.planActive===true?'已启用':'未启用'}</strong></div></div>} {loading&&!backupLoaded?<Spin/>:backupLoaded&&<div className="governance-monitor-grid"><div><small>最近备份</small><strong>{backupSummary(latest)}</strong></div><div><small>备份时间</small><strong>{configurationDateTime(latest.createdAt)||'—'}</strong></div><div><small>恢复验证时间</small><strong>{configurationDateTime(latest.restoreVerifiedAt)||'—'}</strong></div><div><small>自动备份</small><strong>{backup.planActive===true?'已启用':'未启用'}</strong></div></div>}
</section> </section>
</> }, </> },
...(canManage ? [{ key:'monitor', label:'运行监控与告警', children:<OperationsMonitorPanel user={user} onUnauthorized={onUnauthorized}/> },{ key:'backups', label:'备份管理', children:<BackupOperationsPanel user={user} onUnauthorized={onUnauthorized}/> }] : []), ...(canManage ? [{ key:'monitor', label:'运行监控与告警', children:<OperationsMonitorPanel user={user} onUnauthorized={onUnauthorized}/> },{ key:'backups', label:'备份管理', children:<BackupOperationsPanel user={user} onUnauthorized={onUnauthorized}/> },{ key:'systemUpdate', label:'系统更新', children:<SystemUpdatePanel onUnauthorized={onUnauthorized}/> }] : []),
]} /> ]} />
<Drawer open={!!selected} title="任务详情" width={Math.min(620,window.innerWidth)} onClose={()=>setSelected(null)}>{selected&&<dl className="nc-form-grid"><dt>任务</dt><dd>{monitorLabel(selected.type,'后台任务')}</dd><dt>状态</dt><dd>{monitorLabel(selected.status)}</dd><dt>阶段</dt><dd>{monitorLabel(selected.stage,monitorLabel(selected.status))}</dd>{Number(selected.total)>0&&<><dt>已处理 / 总数</dt><dd>{String(selected.processed??0)} / {String(selected.total)}</dd></>}<dt>提交时间</dt><dd>{configurationDateTime(selected.createdAt)}</dd>{isFailed(selected)&&<><dt>失败原因</dt><dd style={{whiteSpace:'pre-wrap',overflowWrap:'anywhere'}}>{taskFailure(selected)}</dd></>}<dt>任务编号</dt><dd style={{overflowWrap:'anywhere'}}>{String(selected.id)}</dd></dl>}{selected?.sourceKind==='TASK'&&<div className="completion-actions">{selected.status==='FAILED'&&!!selected.importRunId&&user.roles.some(role=>['SYSTEM_ADMIN', 'DATA_ADMIN'].includes(role))&&<Button loading={busy} onClick={()=>void runAction(selected,'retry')}>重试导入</Button>}{selected.status==='QUEUED'&&user.roles.some(role=>['SYSTEM_ADMIN', 'DATA_ADMIN'].includes(role))&&<Button danger loading={busy} onClick={()=>void runAction(selected,'cancel')}>取消排队任务</Button>}</div>}{selected?.sourceKind==='ANALYSIS'&&!!selected.sessionId&&<Button href={`?page=analysis&sessionId=${encodeURIComponent(String(selected.sessionId))}&turnId=${encodeURIComponent(String(selected.id))}`}>查看分析记录</Button>}</Drawer> <Drawer open={!!selected} title="任务详情" width={Math.min(620,window.innerWidth)} onClose={()=>setSelected(null)}>{selected&&<dl className="nc-form-grid"><dt>任务</dt><dd>{monitorLabel(selected.type,'后台任务')}</dd><dt>状态</dt><dd>{monitorLabel(selected.status)}</dd><dt>阶段</dt><dd>{monitorLabel(selected.stage,monitorLabel(selected.status))}</dd>{Number(selected.total)>0&&<><dt>已处理 / 总数</dt><dd>{String(selected.processed??0)} / {String(selected.total)}</dd></>}<dt>提交时间</dt><dd>{configurationDateTime(selected.createdAt)}</dd>{isFailed(selected)&&<><dt>失败原因</dt><dd style={{whiteSpace:'pre-wrap',overflowWrap:'anywhere'}}>{taskFailure(selected)}</dd></>}<dt>任务编号</dt><dd style={{overflowWrap:'anywhere'}}>{String(selected.id)}</dd></dl>}{selected?.sourceKind==='TASK'&&<div className="completion-actions">{selected.status==='FAILED'&&!!selected.importRunId&&user.roles.some(role=>['SYSTEM_ADMIN', 'DATA_ADMIN'].includes(role))&&<Button loading={busy} onClick={()=>void runAction(selected,'retry')}>重试导入</Button>}{selected.status==='QUEUED'&&user.roles.some(role=>['SYSTEM_ADMIN', 'DATA_ADMIN'].includes(role))&&<Button danger loading={busy} onClick={()=>void runAction(selected,'cancel')}>取消排队任务</Button>}</div>}{selected?.sourceKind==='ANALYSIS'&&!!selected.sessionId&&<Button href={`?page=analysis&sessionId=${encodeURIComponent(String(selected.sessionId))}&turnId=${encodeURIComponent(String(selected.id))}`}>查看分析记录</Button>}</Drawer>
</>; </>;
+15
View File
@@ -29,6 +29,18 @@ import type { ConfigureSourceBody, DataStage, DataWorkflowOperation, DataWorkflo
export { configureRequestScope, getRequestDiagnostics, invalidateReadResources, resetRequestScope }; export { configureRequestScope, getRequestDiagnostics, invalidateReadResources, resetRequestScope };
export interface SystemUpdateStatus {
enabled: boolean;
currentVersion?: string;
state: string;
message?: string;
checkedAt?: string;
version?: string;
available?: boolean;
startedAt?: string;
finishedAt?: string;
}
export class ApiError extends Error { export class ApiError extends Error {
status: number; status: number;
details: unknown; details: unknown;
@@ -206,6 +218,9 @@ async function mutateForm<T>(path: string, body: FormData): Promise<T> {
} }
export const api = { export const api = {
systemUpdateStatus() { return request<SystemUpdateStatus>('/api/system-update', {}, 12_000); },
checkSystemUpdate() { return mutate<SystemUpdateStatus>('/api/system-update/check', undefined, 'POST', 55_000); },
applySystemUpdate() { return mutate<SystemUpdateStatus>('/api/system-update/apply', undefined, 'POST', 55_000); },
exportSalesOrderTypes(signal?:AbortSignal) { return requestBlob('/api/sales-order-types/export',signal); }, exportSalesOrderTypes(signal?:AbortSignal) { return requestBlob('/api/sales-order-types/export',signal); },
salesOrderTypeStatus() { return request<SalesOrderTypeStatus>('/api/sales-order-types'); }, salesOrderTypeStatus() { return request<SalesOrderTypeStatus>('/api/sales-order-types'); },
async salesOrderTypePreview(file:File,mode:string) { async salesOrderTypePreview(file:File,mode:string) {
+2
View File
@@ -16,6 +16,7 @@
.nc-workflow-modal { top: 24px; padding-bottom: 24px; } .nc-workflow-modal { top: 24px; padding-bottom: 24px; }
.nc-workflow-modal .ant-modal-container, .nc-workflow-modal .ant-modal-content, .nc-workflow-modal .ant-input, .nc-workflow-modal .ant-input-number, .nc-workflow-modal .ant-select-selector, .nc-workflow-modal .ant-btn, .nc-workflow-modal .ant-alert { border-radius: 0 !important; } .nc-workflow-modal .ant-modal-container, .nc-workflow-modal .ant-modal-content, .nc-workflow-modal .ant-input, .nc-workflow-modal .ant-input-number, .nc-workflow-modal .ant-select-selector, .nc-workflow-modal .ant-btn, .nc-workflow-modal .ant-alert { border-radius: 0 !important; }
.nc-workflow-modal .ant-modal-body { display: flex; flex-direction: column; gap: 18px; max-height: calc(var(--viewport-height) - 240px); overflow-y: auto; overflow-x: hidden; padding-right: 4px; color: var(--ink); } .nc-workflow-modal .ant-modal-body { display: flex; flex-direction: column; gap: 18px; max-height: calc(var(--viewport-height) - 240px); overflow-y: auto; overflow-x: hidden; padding-right: 4px; color: var(--ink); }
.nc-workflow-modal .ant-modal-body > .quality-table-frame { flex-shrink: 0; }
.nc-workflow-modal .ant-modal-title { font-size: var(--font-dialog-title); color: var(--ink); } .nc-workflow-modal .ant-modal-title { font-size: var(--font-dialog-title); color: var(--ink); }
.nc-workflow-modal h3 { font-size: var(--font-section); margin: 4px 0 12px; color: var(--ink); } .nc-workflow-modal h3 { font-size: var(--font-section); margin: 4px 0 12px; color: var(--ink); }
.nc-workflow-modal .ant-form-item { margin-bottom: 16px; } .nc-workflow-modal .ant-form-item { margin-bottom: 16px; }
@@ -156,3 +157,4 @@
@media (max-width: 1050px) { .nc-intake-start { grid-template-columns: 1fr; } .nc-primary-actions { margin-top: 20px; } .nc-intake-links { flex-wrap: wrap; gap: 0 16px; padding: 8px 0; } .nc-intake-links > span { margin-left: 0; width: 100%; padding: 5px 0; } .nc-column-matching { grid-template-columns: 1fr; } .nc-pending-strip { align-items: flex-start; flex-direction: column; } .nc-pending-strip .ant-select { min-width: 0; width: 100%; } } @media (max-width: 1050px) { .nc-intake-start { grid-template-columns: 1fr; } .nc-primary-actions { margin-top: 20px; } .nc-intake-links { flex-wrap: wrap; gap: 0 16px; padding: 8px 0; } .nc-intake-links > span { margin-left: 0; width: 100%; padding: 5px 0; } .nc-column-matching { grid-template-columns: 1fr; } .nc-pending-strip { align-items: flex-start; flex-direction: column; } .nc-pending-strip .ant-select { min-width: 0; width: 100%; } }
@media (max-width: 680px) { .nc-intake-start { padding: 20px 18px 0; } .nc-intake-intro h2 { font-size: var(--font-page-title); } .nc-primary-actions { flex-wrap: wrap; } .nc-form-grid { grid-template-columns: 1fr; } .nc-modal-footer { flex-direction: column; align-items: stretch; } .nc-modal-footer > div { justify-content: flex-end; flex-wrap: wrap; } .nc-file-summary { flex-wrap: wrap; } .nc-file-summary > span:last-child { margin-left: 0; } .nc-workbook-upload-intro { flex-direction: column; align-items: flex-start; } .nc-pending-strip > div:first-child { align-items: flex-start; flex-direction: column; gap: 6px; } .nc-review-facts { grid-template-columns: 1fr 1fr; } } @media (max-width: 680px) { .nc-intake-start { padding: 20px 18px 0; } .nc-intake-intro h2 { font-size: var(--font-page-title); } .nc-primary-actions { flex-wrap: wrap; } .nc-form-grid { grid-template-columns: 1fr; } .nc-modal-footer { flex-direction: column; align-items: stretch; } .nc-modal-footer > div { justify-content: flex-end; flex-wrap: wrap; } .nc-file-summary { flex-wrap: wrap; } .nc-file-summary > span:last-child { margin-left: 0; } .nc-workbook-upload-intro { flex-direction: column; align-items: flex-start; } .nc-pending-strip > div:first-child { align-items: flex-start; flex-direction: column; gap: 6px; } .nc-review-facts { grid-template-columns: 1fr 1fr; } }
@media (max-width: 680px) { .nc-source-delete-modal .nc-deletion-impact { grid-template-columns: 1fr 1fr; } } @media (max-width: 680px) { .nc-source-delete-modal .nc-deletion-impact { grid-template-columns: 1fr 1fr; } }
.nc-source-maintenance-actions { display: flex; flex-wrap: wrap; gap: 10px; margin: 0 16px 16px; }
+32
View File
@@ -0,0 +1,32 @@
export type SourceConfigureBody = { uploadId: string; fileId?: string };
export type ConfiguredSourceCandidate = { signature: string; sourceId: string };
type ConfigureItem<TBody extends SourceConfigureBody> = { id: string; body: TBody };
type ConfigureResult = { source: { id: string } };
/** Reuses confirmed candidate IDs and retries changed discovered candidates with the same upload/file identity. */
export async function configureSourceBatch<TBody extends SourceConfigureBody>(
items: ConfigureItem<TBody>[],
configured: Map<string, ConfiguredSourceCandidate>,
configure: (body: TBody) => Promise<ConfigureResult>,
onConfigured?: (candidate: ConfiguredSourceCandidate) => void,
): Promise<string[]> {
const sourceIds: string[] = [];
for (const item of items) {
const signature = JSON.stringify(item.body);
const prior = configured.get(item.id);
if (prior?.signature === signature) {
sourceIds.push(prior.sourceId);
continue;
}
// The server updates an existing DISCOVERED candidate in place when the same
// uploadId/fileId is sent again. Do not mint another upload or logical file ID.
const result = await configure(item.body);
const candidate = { signature, sourceId: result.source.id };
configured.set(item.id, candidate);
onConfigured?.(candidate);
sourceIds.push(candidate.sourceId);
}
return sourceIds;
}
+78
View File
@@ -0,0 +1,78 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { configureSourceBatch } from '../src/sourceBatchConfigure.ts';
type Body = { uploadId: string; fileId: string; year: number; fieldMap: Record<string, string> };
type Candidate = { id: string; status: 'DISCOVERED' | 'RUNNING'; body: Body };
function candidateService() {
const byIdentity = new Map<string, Candidate>();
const calls: Body[] = [];
const configure = async (body: Body) => {
calls.push(body);
const identity = `${body.uploadId}:${body.fileId}`;
const previous = byIdentity.get(identity);
if (previous && previous.status !== 'DISCOVERED') throw new Error('409 candidate is no longer editable');
const candidate = previous || { id: `candidate-${body.fileId}`, status: 'DISCOVERED' as const, body };
candidate.body = body;
byIdentity.set(identity, candidate);
return { source: { id: candidate.id } };
};
return { byIdentity, calls, configure };
}
const item = (id: string, fileId = id, year = 2026): { id: string; body: Body } => ({
id,
body: { uploadId: `upload-${id}`, fileId, year, fieldMap: { weight: '重量' } },
});
test('keeps the confirmed source ID when a later file fails, then retries only the failed file', async () => {
const service = candidateService();
const configured = new Map();
const files = [item('A'), item('B')];
let bAttempts = 0;
const configure = async (body: Body) => {
if (body.fileId === 'B' && ++bAttempts === 1) throw new Error('temporary configure failure');
return service.configure(body);
};
await assert.rejects(configureSourceBatch(files, configured, configure), /temporary configure failure/);
assert.equal(configured.get('A').sourceId, 'candidate-A');
assert.equal(configured.has('B'), false);
const sourceIds = await configureSourceBatch(files, configured, configure);
assert.deepEqual(sourceIds, ['candidate-A', 'candidate-B']);
assert.equal(service.calls.filter(body => body.fileId === 'A').length, 1);
assert.equal(service.calls.filter(body => body.fileId === 'B').length, 1);
});
test('reconfigures changed metadata with the original uploadId and fileId while preserving the candidate ID', async () => {
const service = candidateService();
const configured = new Map();
const first = item('A', 'logical-A', 2025);
const second = item('A', 'logical-A', 2026);
const [initialId] = await configureSourceBatch([first], configured, service.configure);
const [updatedId] = await configureSourceBatch([second], configured, service.configure);
assert.equal(initialId, updatedId);
assert.equal(service.calls.length, 2);
assert.equal(service.calls[0].uploadId, service.calls[1].uploadId);
assert.equal(service.calls[0].fileId, service.calls[1].fileId);
assert.equal(service.byIdentity.get('upload-A:logical-A')?.body.year, 2026);
});
test('does not replace the cached candidate when the server rejects edits after processing starts', async () => {
const service = candidateService();
const configured = new Map();
const first = item('A', 'logical-A', 2025);
const changed = item('A', 'logical-A', 2026);
await configureSourceBatch([first], configured, service.configure);
const candidate = service.byIdentity.get('upload-A:logical-A');
assert.ok(candidate);
candidate.status = 'RUNNING';
await assert.rejects(configureSourceBatch([changed], configured, service.configure), /409 candidate is no longer editable/);
assert.equal(configured.get('A').signature, JSON.stringify(first.body));
});
+1 -1
View File
@@ -8,7 +8,7 @@ function fixture() {
const end=source.includes('export function TrendTable(')?source.indexOf('export function TrendTable('):source.indexOf('function TrendTable('); const end=source.includes('export function TrendTable(')?source.indexOf('export function TrendTable('):source.indexOf('function TrendTable(');
const code=source.slice(source.indexOf('export function TrendChart('),end).replace('export function','function'); const code=source.slice(source.indexOf('export function TrendChart('),end).replace('export function','function');
const slots=new Map(),effects=new Map(),observers=[],charts=[];let cursor=0,dirty=false,queued=[],active=true,visible=true; const slots=new Map(),effects=new Map(),observers=[],charts=[];let cursor=0,dirty=false,queued=[],active=true,visible=true;
const node={clientWidth:900,clientHeight:380}; const node={clientWidth:900,clientHeight:380,closest(){return null;}};
const hooks={useRef(initial){const i=cursor++;if(!slots.has(i))slots.set(i,{current:initial});return slots.get(i);},useState(initial){const i=cursor++;if(!slots.has(i))slots.set(i,initial);return [slots.get(i),v=>{const next=typeof v==='function'?v(slots.get(i)):v;if(!Object.is(next,slots.get(i))){slots.set(i,next);dirty=true;}}];},useCallback(fn,deps){const i=cursor++;const old=slots.get(i);if(!old||deps.some((d,j)=>!Object.is(d,old.deps[j])))slots.set(i,{fn,deps});return slots.get(i).fn;},useEffect(fn,deps){const i=cursor++,old=effects.get(i);if(!old||deps.some((d,j)=>!Object.is(d,old.deps[j])))queued.push(()=>{old?.cleanup?.();effects.set(i,{deps,cleanup:fn()});});}}; const hooks={useRef(initial){const i=cursor++;if(!slots.has(i))slots.set(i,{current:initial});return slots.get(i);},useState(initial){const i=cursor++;if(!slots.has(i))slots.set(i,initial);return [slots.get(i),v=>{const next=typeof v==='function'?v(slots.get(i)):v;if(!Object.is(next,slots.get(i))){slots.set(i,next);dirty=true;}}];},useCallback(fn,deps){const i=cursor++;const old=slots.get(i);if(!old||deps.some((d,j)=>!Object.is(d,old.deps[j])))slots.set(i,{fn,deps});return slots.get(i).fn;},useEffect(fn,deps){const i=cursor++,old=effects.get(i);if(!old||deps.some((d,j)=>!Object.is(d,old.deps[j])))queued.push(()=>{old?.cleanup?.();effects.set(i,{deps,cleanup:fn()});});}};
hooks.useLayoutEffect=hooks.useEffect; hooks.useLayoutEffect=hooks.useEffect;
const ec={init(_node,_theme,opts){const c={opts,options:[],resizes:[],disposed:false,getWidth:()=>c.opts.width??100,getHeight:()=>c.opts.height??380,on(){},resize(o){c.resizes.push(o);},setOption(o){c.options.push(o);},convertToPixel:()=>0,dispose(){c.disposed=true;}};charts.push(c);return c;}}; const ec={init(_node,_theme,opts){const c={opts,options:[],resizes:[],disposed:false,getWidth:()=>c.opts.width??100,getHeight:()=>c.opts.height??380,on(){},resize(o){c.resizes.push(o);},setOption(o){c.options.push(o);},convertToPixel:()=>0,dispose(){c.disposed=true;}};charts.push(c);return c;}};
@@ -10,7 +10,7 @@ function fixture(){
const source=fs.readFileSync(process.env.QUERY_PARETO_SOURCE || new URL('../src/QueryPage.tsx',import.meta.url),'utf8'); const source=fs.readFileSync(process.env.QUERY_PARETO_SOURCE || new URL('../src/QueryPage.tsx',import.meta.url),'utf8');
const component=source.slice(source.indexOf('export function DefectParetoPanel('),source.indexOf('export function SectionHeading(')).replace('export function','function'); const component=source.slice(source.indexOf('export function DefectParetoPanel('),source.indexOf('export function SectionHeading(')).replace('export function','function');
const requests=[],options=[],effects=new Map(),states=new Map(),refs=new Map();let cursor=0,dirty=false,queued=[],tree,creates=0,clears=0; const requests=[],options=[],effects=new Map(),states=new Map(),refs=new Map();let cursor=0,dirty=false,queued=[],tree,creates=0,clears=0;
const node={clientWidth:900,clientHeight:470}; const node={clientWidth:900,clientHeight:470,closest(){return null;}};
const hooks={useState(initial){const i=cursor++;if(!states.has(i))states.set(i,initial);return [states.get(i),v=>{states.set(i,typeof v==='function'?v(states.get(i)):v);dirty=true;}];},useRef(initial){const i=cursor++;if(!refs.has(i))refs.set(i,{current:initial});return refs.get(i);},useMemo(fn){cursor++;return fn();},useEffect(fn,deps){const i=cursor++,old=effects.get(i);if(!old||deps.some((x,j)=>!Object.is(x,old.deps[j])))queued.push(()=>{old?.cleanup?.();effects.set(i,{deps,cleanup:fn()});});}}; const hooks={useState(initial){const i=cursor++;if(!states.has(i))states.set(i,initial);return [states.get(i),v=>{states.set(i,typeof v==='function'?v(states.get(i)):v);dirty=true;}];},useRef(initial){const i=cursor++;if(!refs.has(i))refs.set(i,{current:initial});return refs.get(i);},useMemo(fn){cursor++;return fn();},useEffect(fn,deps){const i=cursor++,old=effects.get(i);if(!old||deps.some((x,j)=>!Object.is(x,old.deps[j])))queued.push(()=>{old?.cleanup?.();effects.set(i,{deps,cleanup:fn()});});}};
const api={fineDefectTop(params,signal){const d=deferred();requests.push({...d,params,signal});return d.promise;}}; const api={fineDefectTop(params,signal){const d=deferred();requests.push({...d,params,signal});return d.promise;}};
const chart={setOption:(value,config)=>options.push({value,config}),clear(){clears++;},resize(){},dispose(){}}; const chart={setOption:(value,config)=>options.push({value,config}),clear(){clears++;},resize(){},dispose(){}};
@@ -0,0 +1,32 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
const deferred=()=>{let resolve,reject;const promise=new Promise((r,j)=>{resolve=r;reject=j;});return{promise,resolve,reject};};
function fixture(){
const source=fs.readFileSync(new URL('../src/useDeleteConfirmation.ts',import.meta.url),'utf8').replace(/^import .*;\n/gm,'').replace('export function','function');
const dialogs=[],cleanups=[];
const App={useApp:()=>({modal:{confirm(options){const dialog={options,updates:[],destroyed:false,update(value){this.updates.push(value);},destroy(){this.destroyed=true;options.afterClose();}};dialogs.push(dialog);return dialog;}}})};
const compiled=ts.transpileModule(source,{compilerOptions:{target:ts.ScriptTarget.ES2022,module:ts.ModuleKind.None}}).outputText;
const open=new Function('App','useCallback','useEffect','useRef',compiled+'\nreturn useDeleteConfirmation();')(App,fn=>fn,effect=>cleanups.push(effect()),value=>({current:value}));
return{open,dialogs,unmount:()=>cleanups.forEach(fn=>fn?.())};
}
test('opening or cancelling a deletion never changes data, including a late confirm callback',async()=>{
const f=fixture(),rows=['a','b'];f.open({title:'删除a?',description:'仅修改草稿',onConfirm:()=>rows.splice(0,1)});
assert.deepEqual(rows,['a','b']);const d=f.dialogs[0];assert.equal(d.options.autoFocusButton,'cancel');assert.equal(d.options.maskClosable,false);assert.equal(d.options.okButtonProps.danger,true);
d.options.onCancel();await d.options.onOk();assert.deepEqual(rows,['a','b']);d.destroy();
});
test('explicit confirmation performs an async action only once and disables cancellation while running',async()=>{
const f=fixture(),work=deferred();let writes=0;f.open({title:'删除a?',description:'立即生效',onConfirm:()=>{writes++;return work.promise;}});
const d=f.dialogs[0],first=d.options.onOk();await d.options.onOk();assert.equal(writes,1);assert.equal(d.updates.at(-1).cancelButtonProps.disabled,true);work.resolve();await first;d.destroy();
});
test('a failed confirmed request can be retried without losing the active confirmation',async()=>{
const f=fixture();let writes=0;f.open({title:'删除a?',description:'立即生效',onConfirm:()=>{if(++writes===1)throw Error('request failed');}});
const d=f.dialogs[0];await assert.rejects(d.options.onOk(),/request failed/);assert.equal(d.destroyed,false);assert.equal(d.updates.at(-1).cancelButtonProps.disabled,false);await d.options.onOk();assert.equal(writes,2);
});
test('leaving the component closes its confirmation and cannot trigger deletion afterwards',async()=>{
const f=fixture();let writes=0;f.open({title:'删除a?',description:'立即生效',onConfirm:()=>writes++});const d=f.dialogs[0];f.unmount();assert.equal(d.destroyed,true);await d.options.onOk();assert.equal(writes,0);
});
test('repeated delete clicks show one confirmation, and closing it allows a different target',async()=>{
const f=fixture();const writes=[];f.open({title:'a',description:'a',onConfirm:()=>writes.push('a')});f.open({title:'b',description:'b',onConfirm:()=>writes.push('b')});assert.equal(f.dialogs.length,1);f.dialogs[0].destroy();f.open({title:'b',description:'b',onConfirm:()=>writes.push('b')});assert.equal(f.dialogs.length,2);await f.dialogs[1].options.onOk();assert.deepEqual(writes,['b']);
});
@@ -0,0 +1,64 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import ts from 'typescript';
import React from 'react';
const tick = () => new Promise(resolve => setImmediate(resolve));
function fixture() {
const source = fs.readFileSync(new URL('../src/SystemUpdatePanel.tsx', import.meta.url), 'utf8')
.replace(/^import[\s\S]*?;\n/gm, '').replace(/export /g, '');
let cursor = 0, dirty = false, queued = [], polling, pollingEnabled, tree, failure;
const states = new Map(), refs = new Map(), memo = new Map(), effects = new Map(), authHandled = [];
let server = { enabled: true, state: 'available', available: true, version: '2', currentVersion: '1' };
const props = { onUnauthorized(error) { if ([401, 403].includes(error?.status)) authHandled.push(error.status); return error?.status === 401; } };
const hooks = {
useState(initial) { const i = cursor++; if (!states.has(i)) states.set(i, initial); return [states.get(i), next => { states.set(i, next); dirty = true; }]; },
useRef(initial) { const i = cursor++; if (!refs.has(i)) refs.set(i, { current: initial }); return refs.get(i); },
useCallback(fn, deps) { const i = cursor++, old = memo.get(i); if (!old || deps.some((value, j) => value !== old.deps[j])) memo.set(i, { fn, deps }); return memo.get(i).fn; },
useEffect(fn, deps) { const i = cursor++, old = effects.get(i); if (!old || deps.some((value, j) => value !== old.deps[j])) queued.push(() => { effects.set(i, { deps }); fn(); }); },
};
const api = {
async systemUpdateStatus() { if (failure !== undefined) { const status = failure; failure = undefined; throw Object.assign(Error('connection-or-session-error'), { status }); } return { ...server }; },
async applySystemUpdate() { server = { ...server, state: 'queued', available: false }; return { ...server }; },
async checkSystemUpdate() { return { ...server }; },
};
const prelude = 'const {useState,useRef,useCallback,useEffect}=hooks;';
const compiled = ts.transpileModule(prelude + source + '\nreturn SystemUpdatePanel;',
{ fileName: 'fixture.tsx', compilerOptions: { target: ts.ScriptTarget.ES2022, jsx: ts.JsxEmit.React } }).outputText;
const component = new Function('React', 'hooks', 'api', 'usePageCache', 'useVisiblePolling', 'configurationDateTime',
'Alert', 'Button', 'Spin', 'Tag', 'ReloadOutlined', compiled)(React, hooks, api, () => ({ active: true }),
(callback, interval, enabled) => { polling = callback; pollingEnabled = enabled; }, value => value, 'Alert', 'Button', 'Spin', 'Tag', 'ReloadOutlined');
const render = () => { let attempts = 0; do { cursor = 0; dirty = false; queued = []; tree = component(props); queued.forEach(fn => fn()); assert.ok(++attempts < 15); } while (dirty); return tree; };
const buttons = () => { const result = []; function walk(node) { if (!node || typeof node !== 'object') return; if (node.type === 'Button') result.push(node); React.Children.forEach(node.props?.children, walk); } walk(tree); return result; };
return {
authHandled, reconnecting: () => states.get(3), error: () => states.get(2), pollingEnabled: () => pollingEnabled,
async start() { render(); await tick(); render(); },
async apply() { const button = buttons().find(node => node.props.children === '更新并重启'); assert.ok(button); button.props.onClick(); await tick(); render(); },
async failPoll(status) { failure = status; await polling(); render(); },
async complete() { server = { ...server, state: 'completed', message: '更新完成。', currentVersion: '2' }; await polling(); render(); },
};
}
for (const status of [401, 403]) test(`restart polling forwards ${status} to authentication handling`, async () => {
const ui = fixture(); await ui.start(); await ui.apply(); await ui.failPoll(status);
assert.deepEqual(ui.authHandled, [status]);
assert.equal(ui.reconnecting(), false);
assert.equal(ui.pollingEnabled(), false);
if (status === 403) assert.equal(ui.error(), 'connection-or-session-error');
});
test('temporary restart connection loss reconnects and clears after completion', async () => {
const ui = fixture(); await ui.start(); await ui.apply(); await ui.failPoll(503);
assert.equal(ui.reconnecting(), true);
assert.deepEqual(ui.authHandled, []);
await ui.complete();
assert.equal(ui.reconnecting(), false);
});
test('a permanent route failure is displayed rather than treated as a restart connection loss', async () => {
const ui = fixture(); await ui.start(); await ui.apply(); await ui.failPoll(404);
assert.equal(ui.reconnecting(), false);
assert.equal(ui.error(), 'connection-or-session-error');
});
@@ -5,6 +5,7 @@ import ts from 'typescript';
import React from 'react'; import React from 'react';
import { readQueryLocation } from '../src/queryLoadingScope.ts'; import { readQueryLocation } from '../src/queryLoadingScope.ts';
import { pageUsesResources } from '../src/resourceInvalidation.ts'; import { pageUsesResources } from '../src/resourceInvalidation.ts';
import { canAccessWorkspacePage } from '../src/userRoles.ts';
const tick = () => new Promise(resolve => setImmediate(resolve)); const tick = () => new Promise(resolve => setImmediate(resolve));
const deferred = () => { let resolve, reject; const promise = new Promise((a, b) => { resolve = a; reject = b; }); return { promise, resolve, reject }; }; const deferred = () => { let resolve, reject; const promise = new Promise((a, b) => { resolve = a; reject = b; }); return { promise, resolve, reject }; };
@@ -15,8 +16,8 @@ function fixture() {
const values = new Map(), refs = new Map(), effects = new Map(), events = new Map(), navigations = [], scopes = []; const values = new Map(), refs = new Map(), effects = new Map(), events = new Map(), navigations = [], scopes = [];
let cursor = 0, dirty = false, queued = [], tree; let cursor = 0, dirty = false, queued = [], tree;
let state = { authRevision: 'a1', dataRevision: 'd1', currentReleaseId: 'r1' }, failState = false, failStatus, me, optionReads=0, optionDenied=false; let state = { authRevision: 'a1', dataRevision: 'd1', currentReleaseId: 'r1' }, failState = false, failStatus, me, optionReads=0, optionDenied=false;
const props = { page: 'overall', user: { id: 'A' }, onUnauthorized: () => false, const props = { page: 'overall', user: { id: 'A', roles: ['USER'] }, onUnauthorized: () => false,
onPageChange(page) { props.page = page; dirty = true; }, onUserChange(user) { props.user = user; dirty = true; } }; onPageChange(page) { props.page = page; dirty = true; }, onUserChange(user) { props.user = { roles: ['USER'], ...user }; dirty = true; } };
const ref = { current: null }; const ref = { current: null };
const hooks = { const hooks = {
useState(initial) { const i = cursor++; if (!values.has(i)) values.set(i, typeof initial === 'function' ? initial() : initial); return [values.get(i), next => { values.set(i, typeof next === 'function' ? next(values.get(i)) : next); dirty = true; }]; }, useState(initial) { const i = cursor++; if (!values.has(i)) values.set(i, typeof initial === 'function' ? initial() : initial); return [values.get(i), next => { values.set(i, typeof next === 'function' ? next(values.get(i)) : next); dirty = true; }]; },
@@ -26,6 +27,7 @@ function fixture() {
}; };
const api = { async options() { optionReads++; if(optionDenied) { events.get('quality-permission-recheck')?.(); throw Object.assign(Error('denied'),{status:403}); } return {}; }, async sourceColumnCatalog() { return {}; }, async workspaceState() { if (failState) { failState = false; throw Object.assign(Error('temporary-offline'), { status: failStatus }); } return { ...state }; }, async me() { return me ? me.promise : props.user; } }; const api = { async options() { optionReads++; if(optionDenied) { events.get('quality-permission-recheck')?.(); throw Object.assign(Error('denied'),{status:403}); } return {}; }, async sourceColumnCatalog() { return {}; }, async workspaceState() { if (failState) { failState = false; throw Object.assign(Error('temporary-offline'), { status: failStatus }); } return { ...state }; }, async me() { return me ? me.promise : props.user; } };
const env = { const env = {
canAccessWorkspacePage,
location: { search: '?page=overall', pathname: '/' }, location: { search: '?page=overall', pathname: '/' },
window: { scrollY: 0, scrollTo() {}, matchMedia: () => ({ matches: false }), addEventListener: (name, fn) => events.set(name, fn), removeEventListener: name => events.delete(name) }, window: { scrollY: 0, scrollTo() {}, matchMedia: () => ({ matches: false }), addEventListener: (name, fn) => events.set(name, fn), removeEventListener: name => events.delete(name) },
document: { visibilityState: 'visible', addEventListener: (name, fn) => events.set(name, fn), removeEventListener: name => events.delete(name), querySelector() {} }, document: { visibilityState: 'visible', addEventListener: (name, fn) => events.set(name, fn), removeEventListener: name => events.delete(name), querySelector() {} },
@@ -43,7 +45,7 @@ function fixture() {
const render = () => { let attempts = 0; do { cursor = 0; dirty = false; queued = []; tree = component(props, ref); queued.forEach(fn => fn()); assert.ok(++attempts < 15); } while (dirty); return tree; }; const render = () => { let attempts = 0; do { cursor = 0; dirty = false; queued = []; tree = component(props, ref); queued.forEach(fn => fn()); assert.ok(++attempts < 15); } while (dirty); return tree; };
const pages = () => { const result = []; function walk(node) { if (!node || typeof node !== 'object') return; if (node.props?.['data-page']) result.push(node.props['data-page']); React.Children.forEach(node.props?.children, walk); } walk(tree); return result; }; const pages = () => { const result = []; function walk(node) { if (!node || typeof node !== 'object') return; if (node.props?.['data-page']) result.push(node.props['data-page']); React.Children.forEach(node.props?.children, walk); } walk(tree); return result; };
return { render, pages, props, scopes, ref, async start() { render(); await tick(); render(); }, async settle() { await Promise.all(navigations); await tick(); render(); }, return { render, pages, props, scopes, ref, async start() { render(); await tick(); render(); }, async settle() { await Promise.all(navigations); await tick(); render(); },
getOptionReads:()=>optionReads,denyOptions(){optionDenied=true;},failNextState(status) { failState = true; failStatus = status; }, changeIdentity() { state = { ...state, authRevision: 'b2' }; me = deferred(); return me; }, locationSearch:()=>env.location.search,getOptionReads:()=>optionReads,denyOptions(){optionDenied=true;},failNextState(status) { failState = true; failStatus = status; }, changeIdentity() { state = { ...state, authRevision: 'b2' }; me = deferred(); return me; },
async focus() { events.get('focus')?.(); await tick(); render(); }, cleanup() { effects.forEach(effect => effect.cleanup?.()); } }; async focus() { events.get('focus')?.(); await tick(); render(); }, cleanup() { effects.forEach(effect => effect.cleanup?.()); } };
} }
@@ -86,3 +88,20 @@ test('a forbidden options resource does not trigger repeated warm reads during p
const ui=fixture();ui.denyOptions();await ui.start();await tick();ui.render();const count=ui.getOptionReads(); const ui=fixture();ui.denyOptions();await ui.start();await tick();ui.render();const count=ui.getOptionReads();
await ui.focus();await tick();ui.render();assert.equal(ui.getOptionReads(),count);assert.ok(count<=1);ui.cleanup(); await ui.focus();await tick();ui.render();assert.equal(ui.getOptionReads(),count);assert.ok(count<=1);ui.cleanup();
}); });
test('sales order type survives cross-stage navigation and clearing without copying stage source filters',async()=>{
const ui=fixture();await ui.start();
ui.ref.current.navigate('blackProduct',{year:2026,view:'product',extraFilters:{salesOrderType:['样品销售订单']},columnFilters:[{field:'blackOnly',stages:['black'],operator:'in',values:['A']}]});await ui.settle();
ui.ref.current.navigate('brightProduct');await ui.settle();
const bright=readQueryLocation(ui.locationSearch());assert.deepEqual(bright.extraFilters,{salesOrderType:['样品销售订单']});assert.equal(bright.columnFilters,undefined);
ui.ref.current.navigate('brightProduct',{year:2026,view:'product',extraFilters:{}});await ui.settle();
ui.ref.current.navigate('blackProduct');await ui.settle();
const black=readQueryLocation(ui.locationSearch());assert.deepEqual(black.extraFilters,{});assert.equal(black.columnFilters[0].field,'blackOnly');ui.cleanup();
});
test('sales order exclusions retain their mode on navigation',async()=>{
const ui=fixture();await ui.start();
ui.ref.current.navigate('overall',{year:2026,view:'overall',extraFilters:{salesOrderTypeNotIn:['样品销售订单']}});await ui.settle();
ui.ref.current.navigate('blackProduct');await ui.settle();
assert.deepEqual(readQueryLocation(ui.locationSearch()).extraFilters,{salesOrderTypeNotIn:['样品销售订单']});ui.cleanup();
});
+29
View File
@@ -0,0 +1,29 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import ts from 'typescript';
const source = ts.transpileModule(readFileSync(new URL('../src/userRoles.ts', import.meta.url), 'utf8'), {
compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext },
}).outputText.replace(/export /g, '');
const { USER_ROLE_OPTIONS, canAccessWorkspacePage } = new Function(`${source};return { USER_ROLE_OPTIONS, canAccessWorkspacePage };`)();
test('only three assigned roles are exposed with the agreed names', () => {
assert.deepEqual(USER_ROLE_OPTIONS, [{ value: 'USER', label: '用户' }, { value: 'DATA_ADMIN', label: '数据管理员' }, { value: 'SYSTEM_ADMIN', label: '系统管理员' }]);
});
test('higher roles inherit business pages while system settings stay system-only', () => {
for (const role of ['USER', 'DATA_ADMIN', 'SYSTEM_ADMIN']) {
const user = { roles: [role] };
for (const page of ['overall', 'department', 'product', 'blackProduct', 'brightProduct', 'analysis'])
assert.equal(canAccessWorkspacePage(user, page), true, `${role}: ${page}`);
for (const page of ['sources', 'imports', 'rules', 'mappings', 'masterData', 'tasks'])
assert.equal(canAccessWorkspacePage(user, page), role !== 'USER', `${role}: ${page}`);
assert.equal(canAccessWorkspacePage(user, 'access'), role === 'SYSTEM_ADMIN');
}
});
test('unexpected or multiple role assignments do not expose management pages', () => {
for (const roles of [[], ['QUALITY_VIEWER'], ['USER', 'SYSTEM_ADMIN']])
assert.equal(canAccessWorkspacePage({ roles }, 'access'), false);
});