diff --git a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisApiSettingsService.java b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisApiSettingsService.java index acbf9f5..64fa567 100644 --- a/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisApiSettingsService.java +++ b/backend/src/main/java/cn/yunsheng/quality/analysis/AnalysisApiSettingsService.java @@ -88,7 +88,9 @@ public class AnalysisApiSettingsService { Files.createDirectories(target.getParent()); if (Files.isSymbolicLink(target.getParent()) || Files.isSymbolicLink(target)) throw new IOException(); temp = Files.createTempFile(target.getParent(), ".nao-model-", ".tmp"); - Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------")); + if (Files.getFileStore(temp).supportsFileAttributeView("posix")) { + Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------")); + } List> storedProfiles = updated.stream().map(p -> Map.of( "id",p.id(),"name",p.name(),"baseUrl",p.config().baseUrl(),"apiKey",p.config().apiKey(),"model",p.config().model(),"thinkingEnabled",p.config().thinkingEnabled(),"thinkingEffort",p.config().thinkingEffort())).toList(); Files.writeString(temp, json.writeValueAsString(Map.of("schemaVersion", 3, "baseUrl", next.baseUrl(), diff --git a/backend/src/main/java/cn/yunsheng/quality/governance/SourceCoverageGuard.java b/backend/src/main/java/cn/yunsheng/quality/governance/SourceCoverageGuard.java index 38bb017..984a0c3 100644 --- a/backend/src/main/java/cn/yunsheng/quality/governance/SourceCoverageGuard.java +++ b/backend/src/main/java/cn/yunsheng/quality/governance/SourceCoverageGuard.java @@ -19,8 +19,18 @@ import org.springframework.stereotype.Service; } static boolean overlaps(Coverage a,Coverage b) { if(!Objects.equals(a.stage(),b.stage())||a.year()!=b.year()||a.month()!=b.month()||!SourceIdentity.intersects(a.scope(),b.scope()))return false; - return Objects.equals(SourceIdentity.canonicalSlot(a.slot()),SourceIdentity.canonicalSlot(b.slot())) - ||!"SHARD_SNAPSHOT".equals(a.mode())||!"SHARD_SNAPSHOT".equals(b.mode())||Objects.equals(a.sha(),b.sha()); + if(Objects.equals(SourceIdentity.canonicalSlot(a.slot()),SourceIdentity.canonicalSlot(b.slot()))||Objects.equals(a.sha(),b.sha()))return true; + if("SHARD_SNAPSHOT".equals(a.mode())&&"SHARD_SNAPSHOT".equals(b.mode()))return false; + // APPEND_SNAPSHOT is created only by the explicit upload workflow. Its scope must match + // exactly before it can augment a complete monthly file; intersecting declarations still conflict. + if(isUploadAppend(a.mode())||isUploadAppend(b.mode())){ + boolean supportedPair=(isUploadAppend(a.mode())&&isUploadAppend(b.mode())) + ||("MONTHLY_SNAPSHOT".equals(a.mode())&&isUploadAppend(b.mode())) + ||("MONTHLY_SNAPSHOT".equals(b.mode())&&isUploadAppend(a.mode())); + return !supportedPair||!Objects.equals(SourceIdentity.scope(a.scope()),SourceIdentity.scope(b.scope())); + } + return true; } + private static boolean isUploadAppend(String mode){return "APPEND_SNAPSHOT".equals(mode);} record Coverage(String stage,int year,int month,String scope,String mode,String sha,String slot) {} } diff --git a/backend/src/main/java/cn/yunsheng/quality/ingestion/ImportWorker.java b/backend/src/main/java/cn/yunsheng/quality/ingestion/ImportWorker.java index dce818b..db1eddc 100644 --- a/backend/src/main/java/cn/yunsheng/quality/ingestion/ImportWorker.java +++ b/backend/src/main/java/cn/yunsheng/quality/ingestion/ImportWorker.java @@ -255,8 +255,7 @@ public class ImportWorker { issues.add(new Issue(candidate, row, "INVALID_NUMBER", field, "数值字段不能转换为重量数值,原值已保留在raw中", true)); } if (value.unmappedCategory()) { - blocking[0]++; - issues.add(new Issue(candidate, row, "CATEGORY_MAPPING_MISSING", "PDT代码/销售片区", "非空来源键未命中已验证的产业部-地区映射", true)); + issues.add(new Issue(candidate, row, "CATEGORY_MAPPING_MISSING", "PDT代码/销售片区", "未命中业务分组对应关系,已保留为未分组,可在对应关系中补充归类", false)); } else if ("source_key_blank".equals(value.components().get("categoryMappingStatus"))) { blankCategory[0]++; issues.add(new Issue(candidate, row, "SOURCE_CATEGORY_KEY_BLANK", "PDT代码/销售片区", "原始匹配键为空,按源模型趋势上下文默认排除;原始行保留", false)); diff --git a/backend/src/main/java/cn/yunsheng/quality/ingestion/TransformBuildKey.java b/backend/src/main/java/cn/yunsheng/quality/ingestion/TransformBuildKey.java index 90614ad..aa0aa41 100644 --- a/backend/src/main/java/cn/yunsheng/quality/ingestion/TransformBuildKey.java +++ b/backend/src/main/java/cn/yunsheng/quality/ingestion/TransformBuildKey.java @@ -19,6 +19,8 @@ final class TransformBuildKey { Integer.toString(source.year()), Integer.toString(source.month()), parserVersion, rulesVersion, mappingVersion); if(parserConfigHash!=null&&!"legacy".equals(parserConfigHash))identity+="\n"+parserConfigHash; + // Validation policy is part of cache identity; old mapping rejections must not survive a retry. + identity += "\nunassigned-mapping-nonblocking-v1"; try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(identity.getBytes(StandardCharsets.UTF_8))); } catch (NoSuchAlgorithmException e) { diff --git a/backend/src/main/java/cn/yunsheng/quality/source/SourceUploadService.java b/backend/src/main/java/cn/yunsheng/quality/source/SourceUploadService.java index 1755387..4493a14 100644 --- a/backend/src/main/java/cn/yunsheng/quality/source/SourceUploadService.java +++ b/backend/src/main/java/cn/yunsheng/quality/source/SourceUploadService.java @@ -32,8 +32,8 @@ public class SourceUploadService { } public Map options(AppUserPrincipal original){ access.requireUserActivePolicy(original);AppUserPrincipal actor=access.resolvePrincipal(original);String current=current(false);List> targets=new ArrayList<>(); - if(current!=null)for(Map row:g.rows("SELECT d.id AS document_id,r.id AS revision_id,d.source_slot,d.stage,d.source_year AS year,d.source_month AS month,d.scope_code,r.source_file,r.sha256,r.parser_config_hash FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id=? ORDER BY d.source_year,d.source_month,d.stage",current)){ - try{sourceAccess.requireSource((String)row.get("documentId"),(String)row.get("revisionId"),actor);targets.add(row);}catch(ResponseStatusException denied){if(denied.getStatusCode()!=HttpStatus.FORBIDDEN)throw denied;} + if(current!=null)for(Map row:g.rows("SELECT d.id AS document_id,r.id AS revision_id,d.source_slot,d.source_mode,d.stage,d.source_year AS year,d.source_month AS month,d.scope_code,r.source_file,r.sha256,r.parser_config_hash FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id=? ORDER BY d.source_year,d.source_month,d.stage,d.source_slot",current)){ + try{sourceAccess.requireSource((String)row.get("documentId"),(String)row.get("revisionId"),actor);row.put("fileId",fileId(row.get("sourceSlot")));targets.add(row);}catch(ResponseStatusException denied){if(denied.getStatusCode()!=HttpStatus.FORBIDDEN)throw denied;} } List currentSourceIds=new ArrayList<>(); for(var row:targets){var ids=jdbc.query("SELECT id FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND sha256=? AND parser_config_hash=? AND deleted_at IS NULL AND status IN('DISCOVERED','IMPORTED') AND NOT "+SourceDeletionService.pendingDeletionSql("source_candidate")+" ORDER BY discovered_at DESC,id DESC LIMIT 1",(rs,n)->rs.getString(1),SourceIdentity.canonicalSlot((String)row.get("sourceSlot")),row.get("sha256"),row.get("parserConfigHash"));if(!ids.isEmpty())currentSourceIds.add(ids.getFirst());} @@ -41,39 +41,90 @@ public class SourceUploadService { } @Transactional public Map configure(ConfigureRequest r,AppUserPrincipal original)throws IOException { requireActor(original);AppUserPrincipal actor=access.resolvePrincipal(original); - if(r.year()<1900||r.year()>2100||r.month()<1||r.month()>12||!Set.of("ADD","REPLACE").contains(r.mode()==null?"":r.mode()))throw new IllegalArgumentException("请选择有效年月和新增/替换模式"); + if(r.year()<1900||r.year()>2100||r.month()<1||r.month()>12||!Set.of("ADD","REPLACE","APPEND").contains(r.mode()==null?"":r.mode()))throw new IllegalArgumentException("请选择有效年月和新增、追加或替换模式"); + boolean append="APPEND".equals(r.mode()),appendReplacement="REPLACE".equals(r.mode())&&r.fileId()!=null&&!r.fileId().isBlank(); + if((append||appendReplacement)&&!validFileId(r.fileId()))throw new IllegalArgumentException("追加或替换追加文件时必须提供有效fileId"); + if(!append&&!appendReplacement&&r.fileId()!=null&&!r.fileId().isBlank())throw new IllegalArgumentException("仅追加文件替换可提供fileId"); if(r.templateId()!=null&&!r.templateId().isBlank())throw new IllegalArgumentException("表头模板功能已停用,请直接提交工作表、表头行和字段映射"); String reason=g.required(r.reason(),"操作理由");String scope=SourceIdentity.scope(r.scopeCode()==null?"ALL":r.scopeCode()); if(scope==null||scope.isBlank()||scope.length()>160||scope.contains("|")||!SourceIdentity.fullOrganizationScope(actor)&&(!actor.getDepartments().stream().map(SourceIdentity::scope).toList().contains(scope)||!actor.getGroups().isEmpty()&&!actor.getGroups().contains("*")))throw new ResponseStatusException(HttpStatus.FORBIDDEN,"来源范围须完整属于当前组织及分组授权"); - Map draft=draft(r.uploadId(),actor,true);if(draft.get("candidateId")!=null)throw new ResponseStatusException(HttpStatus.CONFLICT,"此上传已确认,请刷新来源列表或重新上传"); + Map draft=draft(r.uploadId(),actor,true);String existingCandidateId=(String)draft.get("candidateId");Map priorCandidate=existingCandidateId==null?null:reconfigurableCandidate(existingCandidateId,actor,draft); Path file=verified(draft);SourceParserConfig config=new SourceParserConfig(r.stage(),r.sheetName(),r.headerRow(),r.fieldMap(),null); Map actual=reader.preview(file,r.sheetName(),r.headerRow());Map preview=(Map)actual.get("preview");config.validate((List)preview.get("headers")); String current=current(true);if(!Objects.equals(current,r.expectedCurrentReleaseId()))throw new ResponseStatusException(HttpStatus.CONFLICT,"当前正式版本已变化,请刷新后重新确认"); - String slot=SourceIdentity.slot(r.stage(),r.year(),r.month(),scope,"MAIN"); + String shard=(append||appendReplacement)?"UPLOAD_"+r.fileId():"MAIN"; + String slot=SourceIdentity.slot(r.stage(),r.year(),r.month(),scope,shard); + if(slot.length()>190)throw new IllegalArgumentException("来源身份过长"); var documents=g.rows("SELECT id FROM source_document WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? FOR UPDATE",slot); String replacementRevision=null,replacementDocument=null; if("REPLACE".equals(r.mode())){ if(r.targetDocumentId()==null||r.targetRevisionId()==null||current==null)throw new IllegalArgumentException("请选择当前正式版中的替换来源"); - var old=g.rows("SELECT d.id,r.source_revision_id FROM release_member r JOIN source_document d ON d.id=r.source_document_id WHERE r.release_id=? AND d.id=? AND r.source_revision_id=? AND "+SourceIdentity.canonicalSlotSql("d.source_slot")+"=? FOR UPDATE",current,r.targetDocumentId(),r.targetRevisionId(),slot); - if(old.size()!=1)throw new ResponseStatusException(HttpStatus.CONFLICT,"替换目标的月份、阶段、范围或旧修订已变化");sourceAccess.requireSource(r.targetDocumentId(),r.targetRevisionId(),actor);replacementDocument=r.targetDocumentId();replacementRevision=r.targetRevisionId(); - }else if(!documents.isEmpty())throw new ResponseStatusException(HttpStatus.CONFLICT,"该月份、阶段及范围已有来源,请选择替换旧数据"); + var old=g.rows("SELECT d.id,r.source_revision_id,d.source_mode FROM release_member r JOIN source_document d ON d.id=r.source_document_id WHERE r.release_id=? AND d.id=? AND r.source_revision_id=? AND "+SourceIdentity.canonicalSlotSql("d.source_slot")+"=? FOR UPDATE",current,r.targetDocumentId(),r.targetRevisionId(),slot); + if(old.size()!=1)throw new ResponseStatusException(HttpStatus.CONFLICT,"替换目标的月份、阶段、范围、追加文件ID或旧修订已变化"); + if(appendReplacement&&!("APPEND_SNAPSHOT".equals(old.getFirst().get("sourceMode"))))throw new ResponseStatusException(HttpStatus.CONFLICT,"替换目标不是由追加上传创建的文件批次"); + sourceAccess.requireSource(r.targetDocumentId(),r.targetRevisionId(),actor);replacementDocument=r.targetDocumentId();replacementRevision=r.targetRevisionId(); + }else if(!documents.isEmpty())throw new ResponseStatusException(HttpStatus.CONFLICT,append?"该fileId已有来源,请选择这份追加文件并替换其当前修订":"该月份、阶段及范围已有来源,请选择替换旧数据"); + if(append){ + var prior=g.rows("SELECT id FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND deleted_at IS NULL AND status IN('DISCOVERED','QUEUED','RUNNING','IMPORTED') FOR UPDATE",slot); + if(prior.stream().anyMatch(row->!Objects.equals(existingCandidateId,row.get("id"))))throw new ResponseStatusException(HttpStatus.CONFLICT,"该fileId已登记来源,请选择已有来源替换,或为新的独立文件批次生成fileId"); + } Integer pending=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND status IN('QUEUED','RUNNING')",Integer.class,slot);if(pending!=null&&pending>0)throw new ResponseStatusException(HttpStatus.CONFLICT,"该来源正在构建,请稍后重试"); - String protocol=protocol(slot,r,scope,actor,reason);String key=GovernanceSupport.sha(slot+"\n"+draft.get("sha256")+"\n"+config.hash()+"\n"+r.uploadId());String id=UUID.randomUUID().toString(); - Map frozenConfiguration=new LinkedHashMap<>(config.snapshot());frozenConfiguration.put("sourceSlot",slot); - jdbc.update("INSERT INTO source_candidate(id,candidate_key,source_slot,stage,source_year,source_month,scope_code,source_file,source_path,sha256,file_size,status,protocol_id,resolution,resolution_reason,parser_config,parser_config_hash,expected_current_release_id,replacement_document_id,replacement_revision_id,configured_by) VALUES(?,?,?,?,?,?,?,?,?,?,?,'DISCOVERED',?,?,?,CAST(? AS JSON),?,?,?,?,?)",id,key,slot,r.stage(),r.year(),r.month(),scope,draft.get("sourceFile"),file.toString(),draft.get("sha256"),draft.get("fileSize"),protocol,"REPLACE".equals(r.mode())?"REPLACE":"CONFIRMED",reason,json.writeValueAsString(frozenConfiguration),config.hash(),current,replacementDocument,replacementRevision,actor.getId().toString()); - jdbc.update("UPDATE source_upload_draft SET candidate_id=? WHERE id=?",id,r.uploadId());g.audit(actor.getId().toString(),"SOURCE_UPLOAD_CONFIGURED","source_candidate",id,Map.of("mode",r.mode(),"slot",slot,"reason",reason,"configuration",frozenConfiguration)); - Map result=new LinkedHashMap<>();result.put("source",new SourceCatalogService.SourceView(id,(String)draft.get("sourceFile"),r.stage(),r.year(),r.month(),((Number)draft.get("fileSize")).longValue(),(String)draft.get("sha256"),"DISCOVERED"));result.put("configuration",frozenConfiguration);result.put("currentReleaseId",current);return result; + ensureUniqueContent((String)draft.get("sha256"),r.stage(),r.year(),r.month(),scope,current,replacementDocument,existingCandidateId); + String protocol=protocol(slot,r,scope,actor,reason,shard);String key=GovernanceSupport.sha(slot+"\n"+draft.get("sha256")+"\n"+config.hash()+"\n"+r.uploadId());String id=existingCandidateId==null?UUID.randomUUID().toString():existingCandidateId; + Map frozenConfiguration=new LinkedHashMap<>(config.snapshot());frozenConfiguration.put("sourceSlot",slot);if(append||appendReplacement)frozenConfiguration.put("fileId",r.fileId()); + String resolution=append?"APPEND_CONFIRMED":"REPLACE".equals(r.mode())?"REPLACE":"CONFIRMED"; + if(priorCandidate==null){ + jdbc.update("INSERT INTO source_candidate(id,candidate_key,source_slot,stage,source_year,source_month,scope_code,source_file,source_path,sha256,file_size,status,protocol_id,resolution,resolution_reason,parser_config,parser_config_hash,expected_current_release_id,replacement_document_id,replacement_revision_id,configured_by) VALUES(?,?,?,?,?,?,?,?,?,?,?,'DISCOVERED',?,?,?,CAST(? AS JSON),?,?,?,?,?)",id,key,slot,r.stage(),r.year(),r.month(),scope,draft.get("sourceFile"),file.toString(),draft.get("sha256"),draft.get("fileSize"),protocol,resolution,reason,json.writeValueAsString(frozenConfiguration),config.hash(),current,replacementDocument,replacementRevision,actor.getId().toString()); + }else{ + String oldSlot=SourceIdentity.canonicalSlot((String)priorCandidate.get("sourceSlot"));String oldProtocol=Objects.toString(priorCandidate.get("protocolId"),null); + if(!Objects.equals(oldSlot,SourceIdentity.canonicalSlot(slot))&&oldProtocol!=null)deactivateUnusedUploadProtocol(oldProtocol,id); + int changed=jdbc.update("UPDATE source_candidate SET candidate_key=?,source_slot=?,stage=?,source_year=?,source_month=?,scope_code=?,source_file=?,source_path=?,sha256=?,file_size=?,protocol_id=?,resolution=?,resolution_reason=?,parser_config=CAST(? AS JSON),parser_config_hash=?,expected_current_release_id=?,replacement_document_id=?,replacement_revision_id=?,configured_by=? WHERE id=? AND status='DISCOVERED' AND deleted_at IS NULL",key,slot,r.stage(),r.year(),r.month(),scope,draft.get("sourceFile"),file.toString(),draft.get("sha256"),draft.get("fileSize"),protocol,resolution,reason,json.writeValueAsString(frozenConfiguration),config.hash(),current,replacementDocument,replacementRevision,actor.getId().toString(),id); + if(changed!=1)throw new ResponseStatusException(HttpStatus.CONFLICT,"来源已排队或状态变化,请刷新来源列表后重试"); + } + jdbc.update("UPDATE source_upload_draft SET candidate_id=? WHERE id=?",id,r.uploadId());g.audit(actor.getId().toString(),priorCandidate==null?"SOURCE_UPLOAD_CONFIGURED":"SOURCE_UPLOAD_RECONFIGURED","source_candidate",id,Map.of("mode",r.mode(),"slot",slot,"reason",reason,"configuration",frozenConfiguration)); + Map result=new LinkedHashMap<>();result.put("source",new SourceCatalogService.SourceView(id,(String)draft.get("sourceFile"),r.stage(),r.year(),r.month(),((Number)draft.get("fileSize")).longValue(),(String)draft.get("sha256"),"DISCOVERED"));result.put("configuration",frozenConfiguration);result.put("currentReleaseId",current);result.put("reconfigured",priorCandidate!=null);return result; } - private String protocol(String slot,ConfigureRequest r,String scope,AppUserPrincipal actor,String reason){ + private String protocol(String slot,ConfigureRequest r,String scope,AppUserPrincipal actor,String reason,String shard){ + String mode="MAIN".equals(shard)?"MONTHLY_SNAPSHOT":"APPEND_SNAPSHOT"; var existing=g.rows("SELECT id,source_mode,applicable FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? AND active=TRUE FOR UPDATE",slot); if(existing.size()>1)throw new IllegalArgumentException("来源协议存在重复,请先完成来源治理"); - if(!existing.isEmpty()){if(!"MONTHLY_SNAPSHOT".equals(existing.getFirst().get("sourceMode"))||!Boolean.TRUE.equals(existing.getFirst().get("applicable")))throw new IllegalArgumentException("此向导仅接收适用阶段的完整月度快照");return (String)existing.getFirst().get("id");} - String id=UUID.randomUUID().toString();Integer version=jdbc.queryForObject("SELECT COALESCE(MAX(version_no),0)+1 FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=?",Integer.class,slot);jdbc.update("INSERT INTO source_protocol(id,source_slot,version_no,stage,source_year,source_month,scope_code,source_mode,required_sheet,applicable,expected,reason,created_by) VALUES(?,?,?,?,?,?,?,'MONTHLY_SNAPSHOT',?,TRUE,TRUE,?,?)",id,slot,version,r.stage(),r.year(),r.month(),scope,r.sheetName(),reason,actor.getId().toString());return id; + if(!existing.isEmpty()){if(!mode.equals(existing.getFirst().get("sourceMode"))||!Boolean.TRUE.equals(existing.getFirst().get("applicable")))throw new IllegalArgumentException("来源协议与完整快照/追加批次身份不匹配,须先完成来源治理");return (String)existing.getFirst().get("id");} + var versions=g.rows("SELECT version_no FROM source_protocol WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=? ORDER BY version_no DESC LIMIT 1 FOR UPDATE",slot);int version=versions.isEmpty()?1:((Number)versions.getFirst().get("versionNo")).intValue()+1; + String id=UUID.randomUUID().toString();jdbc.update("INSERT INTO source_protocol(id,source_slot,version_no,stage,source_year,source_month,scope_code,shard_code,source_mode,required_sheet,applicable,expected,reason,created_by) VALUES(?,?,?,?,?,?,?,?,?,?,TRUE,TRUE,?,?)",id,slot,version,r.stage(),r.year(),r.month(),scope,shard,mode,r.sheetName(),reason,actor.getId().toString());return id; } + private Map reconfigurableCandidate(String id,AppUserPrincipal actor,Map draft){ + var rows=g.rows("SELECT * FROM source_candidate WHERE id=? AND configured_by=? FOR UPDATE",id,actor.getId().toString()); + if(rows.isEmpty())throw new ResponseStatusException(HttpStatus.CONFLICT,"已确认来源不属于当前上传人,请重新上传");Map candidate=rows.getFirst(); + if(!"DISCOVERED".equals(candidate.get("status"))||!Objects.equals(candidate.get("sha256"),draft.get("sha256"))||!Objects.equals(candidate.get("sourcePath"),draft.get("sourcePath")))throw new ResponseStatusException(HttpStatus.CONFLICT,"此来源已排队、处理或内容变化,不能重新确认"); + Integer runs=jdbc.queryForObject("SELECT COUNT(*) FROM import_run WHERE JSON_CONTAINS(requested_sources,JSON_QUOTE(?))",Integer.class,id); + String sameSlot=SourceIdentity.canonicalSlotSql("d.source_slot")+"="+SourceIdentity.canonicalSlotSql("c.source_slot"); + Integer documents=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate c JOIN source_document d ON "+sameSlot+" JOIN source_revision r ON r.source_document_id=d.id AND r.sha256=c.sha256 AND r.parser_config_hash=c.parser_config_hash JOIN release_member m ON m.source_document_id=d.id AND m.source_revision_id=r.id WHERE c.id=?",Integer.class,id); + if((runs!=null&&runs>0)||(documents!=null&&documents>0))throw new ResponseStatusException(HttpStatus.CONFLICT,"来源已被导入任务或版本引用,不能重新确认"); + return candidate; + } + private void deactivateUnusedUploadProtocol(String protocolId,String candidateId){ + var protocol=g.rows("SELECT source_slot,active FROM source_protocol WHERE id=? AND active=TRUE FOR UPDATE",protocolId);if(protocol.isEmpty())return; + Integer others=jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate WHERE protocol_id=? AND id<>? AND deleted_at IS NULL AND status NOT IN('RETURNED','SUPERSEDED')",Integer.class,protocolId,candidateId); + Integer documents=jdbc.queryForObject("SELECT COUNT(*) FROM source_document WHERE "+SourceIdentity.canonicalSlotSql("source_slot")+"=?",Integer.class,SourceIdentity.canonicalSlot((String)protocol.getFirst().get("sourceSlot"))); + if((others==null||others==0)&&(documents==null||documents==0))jdbc.update("UPDATE source_protocol SET active=FALSE WHERE id=? AND active=TRUE",protocolId); + } + private void ensureUniqueContent(String sha,String stage,int year,int month,String scope,String releaseId,String excludedDocument,String excludedCandidate){ + if(releaseId!=null){ + List> currentRows=g.rows("SELECT d.id,d.scope_code FROM release_member m JOIN source_document d ON d.id=m.source_document_id JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id=? AND r.sha256=? AND d.stage=? AND d.source_year=? AND d.source_month=? FOR UPDATE",releaseId,sha,stage,year,month); + if(currentRows.stream().anyMatch(row->!Objects.equals(excludedDocument,row.get("id"))&&SourceIdentity.intersects(scope,(String)row.get("scopeCode"))))throw duplicateContent(); + } + List> pending=g.rows("SELECT id,source_slot,scope_code FROM source_candidate WHERE sha256=? AND stage=? AND source_year=? AND source_month=? AND deleted_at IS NULL AND status IN('DISCOVERED','QUEUED','RUNNING') AND NOT "+SourceDeletionService.pendingDeletionSql("source_candidate")+" FOR UPDATE",sha,stage,year,month); + if(pending.stream().anyMatch(row->!Objects.equals(excludedCandidate,row.get("id"))&&SourceIdentity.intersects(scope,(String)row.get("scopeCode"))))throw duplicateContent(); + } + private ResponseStatusException duplicateContent(){return new ResponseStatusException(HttpStatus.CONFLICT,"该工作簿内容已用于同一年月、阶段及重叠范围的另一份来源,请勿重复计入;更正版请精确选择原来源替换");} + private static boolean validFileId(String value){return value!=null&&value.length()<=80&&value.matches("[A-Za-z0-9_-]{1,80}");} + private static String fileId(Object slotValue){if(!(slotValue instanceof String slot))return null;String[] parts=SourceIdentity.canonicalSlot(slot).split("\\|",-1);if(parts.length!=5||!parts[4].startsWith("UPLOAD_"))return null;String value=parts[4].substring("UPLOAD_".length());return validFileId(value)?value:null;} private Map draft(String id,AppUserPrincipal actor,boolean lock){var rows=g.rows("SELECT * FROM source_upload_draft WHERE id=? AND created_by=? AND expires_at>CURRENT_TIMESTAMP(6)"+(lock?" FOR UPDATE":""),id,actor.getId().toString());if(rows.isEmpty())throw new ResponseStatusException(HttpStatus.NOT_FOUND,"上传预览已过期或不属于当前用户,请重新上传");return rows.getFirst();} private Path verified(Map draft)throws IOException{Path path=Path.of((String)draft.get("sourcePath"));if(Files.isSymbolicLink(path)||!Files.isRegularFile(path)||!path.toRealPath().startsWith(scanner.archiveRoot())||Files.size(path)!=((Number)draft.get("fileSize")).longValue()||!SourceScanner.sha256(path).equals(draft.get("sha256")))throw new IllegalArgumentException("上传文件已变化,请重新上传");return path.toRealPath();} private String current(boolean lock){return jdbc.queryForObject("SELECT release_id FROM current_release WHERE dataset_id='quality'"+(lock?" FOR UPDATE":""),String.class);} private void requireActor(AppUserPrincipal actor){access.requireUserActivePolicy(actor);access.requireBoundAction("RECEIVER","source.receive");} /** templateId remains an explicitly disabled compatibility field for older clients. */ - public record ConfigureRequest(String uploadId,String stage,int year,int month,String sheetName,int headerRow,Map fieldMap,String templateId,String mode,String expectedCurrentReleaseId,String targetDocumentId,String targetRevisionId,String reason,String scopeCode){} + public record ConfigureRequest(String uploadId,String stage,int year,int month,String sheetName,int headerRow,Map fieldMap,String templateId,String mode,String expectedCurrentReleaseId,String targetDocumentId,String targetRevisionId,String reason,String scopeCode,String fileId){ + public ConfigureRequest(String uploadId,String stage,int year,int month,String sheetName,int headerRow,Map fieldMap,String templateId,String mode,String expectedCurrentReleaseId,String targetDocumentId,String targetRevisionId,String reason,String scopeCode){this(uploadId,stage,year,month,sheetName,headerRow,fieldMap,templateId,mode,expectedCurrentReleaseId,targetDocumentId,targetRevisionId,reason,scopeCode,null);} + } } diff --git a/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateController.java b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateController.java new file mode 100644 index 0000000..66d1787 --- /dev/null +++ b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateController.java @@ -0,0 +1,27 @@ +package cn.yunsheng.quality.update; + +import java.util.Map; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/system-update") +@PreAuthorize("hasRole('SYSTEM_ADMIN')") +public class SystemUpdateController { + private final SystemUpdateService updates; + + public SystemUpdateController(SystemUpdateService updates) { this.updates = updates; } + + @GetMapping + public Map status() { return updates.status(); } + + // Both mutations use the application's existing session + CSRF filter. + @PostMapping("/check") + public Map check() { return updates.check(); } + + @PostMapping("/apply") + public Map apply() { return updates.apply(); } +} diff --git a/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateMaintenanceFilter.java b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateMaintenanceFilter.java new file mode 100644 index 0000000..e9e35d8 --- /dev/null +++ b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateMaintenanceFilter.java @@ -0,0 +1,39 @@ +package cn.yunsheng.quality.update; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Set; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +/** Keeps the startup-to-update-commit interval free of business writes. */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE + 20) +public class SystemUpdateMaintenanceFilter extends OncePerRequestFilter { + private static final Set READ_METHODS = Set.of("GET", "HEAD", "OPTIONS", "TRACE"); + private static final Set AUTH_ROUTES = Set.of("/api/auth/login", "/api/auth/logout", "/api/auth/csrf", "/api/auth/me"); + private final SystemUpdateService updates; + + public SystemUpdateMaintenanceFilter(SystemUpdateService updates) { this.updates = updates; } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + String path = request.getRequestURI().substring(request.getContextPath().length()); + if (!READ_METHODS.contains(request.getMethod()) && path.startsWith("/api/") + && !path.equals("/api/system-update") && !path.startsWith("/api/system-update/") + && !AUTH_ROUTES.contains(path) && updates.maintenanceActive()) { + response.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + response.setContentType("application/problem+json;charset=UTF-8"); + response.setHeader("Retry-After", "10"); + response.getWriter().write("{\"title\":\"Service Unavailable\",\"status\":503,\"detail\":\"系统正在更新,请稍后重试。\"}"); + return; + } + chain.doFilter(request, response); + } +} diff --git a/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateService.java b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateService.java new file mode 100644 index 0000000..05f1f57 --- /dev/null +++ b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateService.java @@ -0,0 +1,138 @@ +package cn.yunsheng.quality.update; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; +import tools.jackson.databind.json.JsonMapper; + +/** Restricted bridge to the packaged updater; neither URLs nor paths are input. */ +@Service +public class SystemUpdateService { + private static final Set RESPONSE_FIELDS = Set.of("enabled", "currentVersion", "state", "message", + "checkedAt", "version", "available", "startedAt", "finishedAt"); + private final JsonMapper json; + private final Path root; + private final boolean enabled; + + public SystemUpdateService(JsonMapper json, + @Value("${QUALITY_PROJECT_ROOT:}") String root, + @Value("${QUALITY_UPDATE_ENABLED:false}") boolean requested) { + this.json = json; + this.root = root.isBlank() ? null : Path.of(root).toAbsolutePath().normalize(); + this.enabled = requested && System.getProperty("os.name", "").startsWith("Windows") + && this.root != null && Files.isRegularFile(this.root.resolve("manifest.json")) + && Files.isRegularFile(this.root.resolve("runtimes/python/python.exe")) + && Files.isRegularFile(this.root.resolve("deploy/offline/update_manager.py")) + && Files.isRegularFile(this.root.resolve("deploy/offline/update-public-key.pem")); + } + + public synchronized Map status() { + if (!enabled) return Map.of("enabled", false, "state", "disabled", "message", "此部署不支持软件内更新。"); + return invoke("status"); + } + + public synchronized Map check() { + requireEnabled(); + return invoke("check"); + } + + public synchronized Map apply() { + requireEnabled(); + return invoke("launch"); + } + + /** Fail closed during installation/recovery, including a process crash. */ + public boolean maintenanceActive() { + if (!enabled) return false; + try { + Map state = readMetadata("status.json"); + if (state.get("state") instanceof String phase + && Set.of("backingUp", "installing", "restarting", "rollingBack").contains(phase)) return true; + Map journal = readMetadata("journal.json"); + return journal.get("phase") instanceof String phase && Set.of("stopping", "installing").contains(phase); + } catch (Exception failure) { + return true; + } + } + + private Map readMetadata(String name) throws IOException { + Path path = root.resolve(".runtime/update/" + name); + if (!Files.exists(path)) return Map.of(); + try (var stream = Files.newInputStream(path)) { + byte[] raw = stream.readNBytes(8193); + if (raw.length > 8192) throw new IOException("Oversized update state"); + Object decoded = json.readValue(raw, Object.class); + if (!(decoded instanceof Map value)) throw new IOException("Invalid update state"); + return value; + } + } + + private void requireEnabled() { + if (!enabled) throw new ResponseStatusException(HttpStatus.CONFLICT, "此部署不支持软件内更新。"); + } + + private Map invoke(String action) { + Process process = null; + try { + ProcessBuilder builder = new ProcessBuilder(root.resolve("runtimes/python/python.exe").toString(), + root.resolve("deploy/offline/update_manager.py").toString(), action); + builder.directory(root.toFile()); + builder.redirectError(ProcessBuilder.Redirect.DISCARD); + builder.environment().put("PYTHONUTF8", "1"); + builder.environment().put("PYTHONIOENCODING", "utf-8"); + builder.environment().put("PYTHONDONTWRITEBYTECODE", "1"); + process = builder.start(); + process.getOutputStream().close(); + // The CLI prints one small, allowlisted JSON receipt. Waiting first + // also bounds hangs caused by a failed DNS/TLS/network operation. + if (!process.waitFor(45, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new ResponseStatusException(HttpStatus.GATEWAY_TIMEOUT, "更新服务器暂未响应,请稍后重试。"); + } + byte[] raw = process.getInputStream().readNBytes(8193); + if (raw.length > 8192) throw new IOException("Oversized updater receipt"); + Object decoded = json.readValue(new String(raw, StandardCharsets.UTF_8), Object.class); + if (!(decoded instanceof Map value)) throw new IOException("Invalid updater receipt"); + if (process.exitValue() != 0) { + Object detail = value.get("error"); + String message = detail instanceof String s && s.length() <= 300 ? s : "更新操作未完成,请稍后重试。"; + throw new ResponseStatusException(HttpStatus.CONFLICT, message); + } + return safeResponse(value); + } catch (ResponseStatusException failure) { + throw failure; + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "更新操作已中断。"); + } catch (Exception failure) { + throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "更新服务暂不可用,请稍后重试。"); + } finally { + if (process != null && process.isAlive()) process.destroyForcibly(); + } + } + + static Map safeResponse(Map value) throws IOException { + Map result = new LinkedHashMap<>(); + for (String field : RESPONSE_FIELDS) { + Object item = value.get(field); + if (item == null) continue; + if ((field.equals("enabled") || field.equals("available")) && !(item instanceof Boolean)) + throw new IOException("Invalid updater boolean"); + if (!(item instanceof Boolean) && !(item instanceof String s && s.length() <= 300)) + throw new IOException("Invalid updater field"); + result.put(field, item); + } + if (!(result.get("state") instanceof String) || !(result.get("enabled") instanceof Boolean)) + throw new IOException("Incomplete updater receipt"); + return result; + } +} diff --git a/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateStartupBarrier.java b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateStartupBarrier.java new file mode 100644 index 0000000..0d53e89 --- /dev/null +++ b/backend/src/main/java/cn/yunsheng/quality/update/SystemUpdateStartupBarrier.java @@ -0,0 +1,97 @@ +package cn.yunsheng.quality.update; + +import jakarta.annotation.PreDestroy; +import java.lang.reflect.Method; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.aopalliance.intercept.MethodInterceptor; +import org.springframework.aop.framework.AopProxyUtils; +import org.springframework.aop.framework.ProxyFactory; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.AnnotatedElementUtils; +import org.springframework.core.env.Environment; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.ReflectionUtils; + +/** Defers business recovery and suppresses schedules until the update commits. */ +@Component +public class SystemUpdateStartupBarrier implements BeanPostProcessor, Ordered { + private static final Set RECOVERY_METHODS = Set.of( + "cn.yunsheng.quality.ingestion.ImportRecoveryRunner#run", + "cn.yunsheng.quality.analysis.AnalysisTurnService#recoverQueuedTurns", + "cn.yunsheng.quality.analysis.AnalysisReportService#recoverPendingReports", + "cn.yunsheng.quality.analytics.InspectionService#recoverPendingDrilldowns"); + private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(SystemUpdateStartupBarrier.class); + private final ObjectProvider updates; + private final Environment environment; + private final ConcurrentLinkedQueue pending = new ConcurrentLinkedQueue<>(); + private final ScheduledExecutorService release = Executors.newSingleThreadScheduledExecutor(task -> { + Thread thread = new Thread(task, "quality-update-recovery"); + thread.setDaemon(true); + return thread; + }); + + public SystemUpdateStartupBarrier(ObjectProvider updates, Environment environment) { + this.updates = updates; + this.environment = environment; + release.scheduleWithFixedDelay(this::drain, 500, 500, TimeUnit.MILLISECONDS); + } + + @Override public int getOrder() { return Ordered.HIGHEST_PRECEDENCE + 50; } + + @Override + public Object postProcessAfterInitialization(Object bean, String name) { + if (!System.getProperty("os.name", "").startsWith("Windows") + || !environment.getProperty("QUALITY_UPDATE_ENABLED", Boolean.class, false)) return bean; + Class type = AopProxyUtils.ultimateTargetClass(bean); + if (!type.getName().startsWith("cn.yunsheng.quality.") || type.getName().contains(".update.")) return bean; + boolean candidate = false; + for (Method method : ReflectionUtils.getAllDeclaredMethods(type)) + if (isRecovery(type, method) || AnnotatedElementUtils.hasAnnotation(method, Scheduled.class)) candidate = true; + return candidate ? wrap(bean, type) : bean; + } + + Object wrap(Object bean, Class type) { + ProxyFactory proxy = new ProxyFactory(bean); + proxy.setProxyTargetClass(true); + proxy.addAdvice((MethodInterceptor) invocation -> { + Method method = ReflectionUtils.findMethod(type, invocation.getMethod().getName(), invocation.getMethod().getParameterTypes()); + if (method == null || !updates.getObject().maintenanceActive()) return invocation.proceed(); + if (isRecovery(type, method)) { + pending.add(() -> { + try { invocation.proceed(); } + catch (Throwable failure) { LOG.error("Deferred application recovery failed", failure); } + }); + return null; + } + if (AnnotatedElementUtils.hasAnnotation(method, Scheduled.class)) return null; + return invocation.proceed(); + }); + return proxy.getProxy(type.getClassLoader()); + } + + private static boolean isRecovery(Class type, Method method) { + return RECOVERY_METHODS.contains(type.getName() + "#" + method.getName()); + } + + void drain() { + if (pending.isEmpty()) return; + try { + while (!updates.getObject().maintenanceActive()) { + Runnable task = pending.poll(); + if (task == null) return; + task.run(); + } + } catch (Exception failure) { + LOG.warn("Waiting for application update to complete"); + } + } + + @PreDestroy public void close() { release.shutdownNow(); pending.clear(); } +} diff --git a/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java b/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java index cbbd47e..6121127 100644 --- a/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java +++ b/backend/src/main/java/cn/yunsheng/quality/workflow/DataWorkflowService.java @@ -506,10 +506,40 @@ public class DataWorkflowService { throw failure("CANDIDATE_VERSION_MISMATCH", "候选版本未使用本次工作流冻结的映射和规则,候选不会发布"); Integer invalidBuilds = jdbc.queryForObject(""" SELECT COUNT(*) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id - WHERE rm.release_id=? AND (b.mapping_version<>? OR b.rule_version<>? OR b.status<>'COMPLETED' OR b.rejected_count>0) + WHERE rm.release_id=? AND (b.mapping_version<>? OR b.rule_version<>?) """, Integer.class, releaseId, mappingId, ruleId); if (invalidBuilds != null && invalidBuilds > 0) throw failure("CANDIDATE_BUILD_VERSION_MISMATCH", "候选中存在未使用本次冻结版本的转换产物,候选不会发布"); + Integer unfinishedBuilds = jdbc.queryForObject(""" + SELECT COUNT(*) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id + WHERE rm.release_id=? AND b.status<>'COMPLETED' + """, Integer.class, releaseId); + if (unfinishedBuilds != null && unfinishedBuilds > 0) + throw failure("CANDIDATE_BUILD_INCOMPLETE", "有 " + unfinishedBuilds + " 个来源尚未完成转换,候选不会发布"); + Integer rejected = jdbc.queryForObject(""" + SELECT COALESCE(SUM(b.rejected_count),0) FROM release_member rm JOIN transform_build b ON b.id=rm.transform_build_id + WHERE rm.release_id=? + """, Integer.class, releaseId); + if (rejected != null && rejected > 0) { + List> issues = jdbc.queryForList(""" + SELECT issue_code AS issueCode,field_name AS fieldName,COUNT(*) AS issueCount FROM import_issue + WHERE import_run_id=(SELECT id FROM import_run WHERE result_release_id=? ORDER BY created_at DESC LIMIT 1) + AND JSON_EXTRACT(details,'$.blocking')=TRUE + GROUP BY issue_code,field_name ORDER BY issueCount DESC,issue_code,field_name LIMIT 3 + """, releaseId); + String detail = issues.stream().map(issue -> { + String reason = switch (Objects.toString(issue.get("issueCode"), "")) { + case "INVALID_NUMBER" -> "数值格式无效"; + case "REQUIRED_HEADER_MISSING" -> "缺少必需列"; + case "NO_DETAIL_ROWS" -> "没有可导入明细"; + default -> "来源校验未通过"; + }; + String field = Objects.toString(issue.get("fieldName"), ""); + return (field.isBlank() ? "" : field + ":") + reason + "(" + issue.get("issueCount") + "项)"; + }).collect(java.util.stream.Collectors.joining(";")); + throw failure("CANDIDATE_SOURCE_VALIDATION_FAILED", "来源数据有 " + rejected + " 项阻断问题,候选不会发布。" + + (detail.isBlank() ? "请查看导入记录中的问题明细" : detail)); + } } private void publishExisting(WorkflowOperation operation, AppUserPrincipal actor, String token) { diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisAgentLoopTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisAgentLoopTest.java index 5bbb257..92cfaeb 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisAgentLoopTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisAgentLoopTest.java @@ -132,7 +132,7 @@ class AnalysisAgentLoopTest { private static final class Fixture { final AppUserPrincipal actor = new AppUserPrincipal(UUID.fromString("29000000-0000-0000-0000-000000000001"), - "agent-test", "unused", "Agent Test", true, false, List.of("BUSINESS_ANALYST"), + "agent-test", "unused", "Agent Test", true, false, List.of("USER"), List.of("质量中心"), List.of("锻造"), List.of()); final JdbcTemplate jdbc = mock(JdbcTemplate.class); final JsonMapper json = JsonMapper.builder().build(); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisApiSettingsTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisApiSettingsTest.java index 3648d3e..8fa90a4 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisApiSettingsTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisApiSettingsTest.java @@ -184,7 +184,7 @@ class AnalysisApiSettingsTest { context.registerBean(AnalysisApiSettingsService.class, this::service); context.register(AnalysisApiSettingsController.class); context.refresh(); var c = context.getBean(AnalysisApiSettingsController.class); - SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken("viewer", "", List.of(new SimpleGrantedAuthority("ROLE_QUALITY_VIEWER")))); + SecurityContextHolder.getContext().setAuthentication(new UsernamePasswordAuthenticationToken("viewer", "", List.of(new SimpleGrantedAuthority("ROLE_USER")))); assertThrows(AccessDeniedException.class, c::get); assertThrows(AccessDeniedException.class, () -> c.test(null)); assertThrows(AccessDeniedException.class, () -> c.save(null)); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisChangesServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisChangesServiceTest.java index abee8be..899bb01 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisChangesServiceTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisChangesServiceTest.java @@ -27,7 +27,7 @@ class AnalysisChangesServiceTest { private final GovernanceAccessService governance = mock(GovernanceAccessService.class); private final OptionsController options = mock(OptionsController.class); private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "analyst", "unused", "Analyst", true, - false, List.of("BUSINESS_ANALYST"), List.of("消费电子产业部"), List.of("消费电子-3C"), List.of()); + false, List.of("USER"), List.of("消费电子产业部"), List.of("消费电子-3C"), List.of()); private final AnalysisChangesService service = new AnalysisChangesService(metrics, responseAccess, governance, options); @Test diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisControllerHistoryAccessTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisControllerHistoryAccessTest.java index 34be744..defe908 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisControllerHistoryAccessTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisControllerHistoryAccessTest.java @@ -56,7 +56,7 @@ class AnalysisControllerHistoryAccessTest { } private static final class Fixture { final String session=UUID.randomUUID().toString(),turn=UUID.randomUUID().toString(),release=UUID.randomUUID().toString(); - final AppUserPrincipal actor=new AppUserPrincipal(UUID.randomUUID(),"history-viewer","unused","测试",true,false,List.of("QUALITY_VIEWER"),List.of(),List.of(),List.of()); + final AppUserPrincipal actor=new AppUserPrincipal(UUID.randomUUID(),"history-viewer","unused","测试",true,false,List.of("USER"),List.of(),List.of(),List.of()); final AnalysisProtocolService protocol=mock(AnalysisProtocolService.class); final MetricsService metrics=mock(MetricsService.class); final AnalysisController controller; diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisExplorationToolsTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisExplorationToolsTest.java index 8bb8529..4a6c393 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisExplorationToolsTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisExplorationToolsTest.java @@ -43,7 +43,7 @@ class AnalysisExplorationToolsTest { private final JdbcTemplate jdbc = mock(JdbcTemplate.class); private final AnalysisExplorationTools tools = new AnalysisExplorationTools(metrics, dictionary, governance, responseAccess, jdbc); private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "analyst", "unused", "Analyst", true, - false, List.of("BUSINESS_ANALYST"), List.of("质量中心"), List.of("锻造"), List.of()); + false, List.of("USER"), List.of("质量中心"), List.of("锻造"), List.of()); @Test void detailsReturnsOnlyAuthorizedFieldsFromFilteredPublishedReleaseSample() throws Exception { diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisNativeAgentContextTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisNativeAgentContextTest.java index b7fda54..08c2efa 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisNativeAgentContextTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisNativeAgentContextTest.java @@ -157,7 +157,7 @@ class AnalysisNativeAgentContextTest { JdbcTemplate jdbc = mock(JdbcTemplate.class); GovernanceAccessService governance = mock(GovernanceAccessService.class); AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "agent-context-test", "unused", "测试", true, false, - List.of("QUALITY_VIEWER"), List.of(), List.of(), List.of()); + List.of("USER"), List.of(), List.of(), List.of()); when(governance.allowedFields(actor)).thenReturn(Set.of("black_good_weight", "black_total_weight")); doReturn(List.of()).when(jdbc).query(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any(RowMapper.class)); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisNativeRuntimeRecoveryTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisNativeRuntimeRecoveryTest.java index 1ad15db..91e414b 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisNativeRuntimeRecoveryTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisNativeRuntimeRecoveryTest.java @@ -112,7 +112,7 @@ class AnalysisNativeRuntimeRecoveryTest { final GovernanceAccessService governance = mock(GovernanceAccessService.class); final NaoAnalysisRunner runner = mock(NaoAnalysisRunner.class); final AppUserPrincipal actor = new AppUserPrincipal(UUID.fromString("32cd8848-cfa1-4b10-8d2f-91c7bf433103"), - "runtime-review", "unused", "隔离测试", true, false, List.of("QUALITY_VIEWER"), List.of("D"), List.of("G"), List.of()); + "runtime-review", "unused", "隔离测试", true, false, List.of("USER"), List.of("D"), List.of("G"), List.of()); final JsonMapper json = JsonMapper.builder().build(); final AnalysisTurnService service; final Map firstEvents = new LinkedHashMap<>(); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolServiceMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolServiceMySqlTest.java index a5fb898..cf2a95f 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolServiceMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolServiceMySqlTest.java @@ -25,7 +25,7 @@ class AnalysisProtocolServiceMySqlTest { private String database; private boolean created; private final String session=UUID.randomUUID().toString(),release=UUID.randomUUID().toString(); - private final AppUserPrincipal actor=new AppUserPrincipal(UUID.randomUUID(),"protocol-test","unused","测试",true,false,List.of("QUALITY_VIEWER"),List.of(),List.of(),List.of()); + private final AppUserPrincipal actor=new AppUserPrincipal(UUID.randomUUID(),"protocol-test","unused","测试",true,false,List.of("USER"),List.of(),List.of(),List.of()); private final GovernanceAccessService governance=mock(GovernanceAccessService.class); private final OptionsController options=mock(OptionsController.class); private final AnalysisToolsController tools=mock(AnalysisToolsController.class); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisSemanticsTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisSemanticsTest.java index 45f0811..58eadfc 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisSemanticsTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisSemanticsTest.java @@ -67,7 +67,7 @@ class AnalysisSemanticsTest { @Test void modelIntentResolvesAutomotiveAliasToAllAuthorizedGroupsAndDefaultsGenericRateToComposite() throws Exception { AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "semantic-test", "unused", "Semantic Test", true, - false, List.of("BUSINESS_ANALYST"), List.of("工业电机产业部", "新能源汽车磁钢", "欧洲磁钢", "消费电子产业部", "美洲产业部"), + false, List.of("USER"), List.of("工业电机产业部", "新能源汽车磁钢", "欧洲磁钢", "消费电子产业部", "美洲产业部"), List.of("汽车交通-国内", "汽车交通-海外"), List.of()); List> catalog = List.of(Map.of("code", "COMPOSITE_RATE", "name", "综合合格率", "enabled", true), Map.of("code", "BLACK_RATE", "name", "黑片合格率", "enabled", true), @@ -110,7 +110,7 @@ class AnalysisSemanticsTest { @Test void modelCannotSilentlyDropAnUnmappedDimension() throws Exception { AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "semantic-test", "unused", "Semantic Test", true, - false, List.of("BUSINESS_ANALYST"), List.of("工业电机产业部"), List.of("汽车交通-国内"), List.of()); + false, List.of("USER"), List.of("工业电机产业部"), List.of("汽车交通-国内"), List.of()); List> catalog = List.of(Map.of("code", "COMPOSITE_RATE", "name", "综合合格率", "enabled", true)); JsonNode parsed = mapper.readTree("{\"intent\":\"metric\",\"metricCode\":\"COMPOSITE_RATE\",\"group\":\"其他业务组\"}"); assertThrows(IllegalArgumentException.class, () -> AnalysisModelGateway.decodeIntent(parsed, diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryControllerTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryControllerTest.java index 6e7c6bc..5f50219 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryControllerTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisSourceQueryControllerTest.java @@ -12,7 +12,7 @@ import org.springframework.web.server.ResponseStatusException; class AnalysisSourceQueryControllerTest { @Test void aPolicyChangeWhileScanningPreventsReturningTheProjectedData() { - var user=new AppUserPrincipal(UUID.randomUUID(),"source-reader","unused","测试",true,false,List.of("QUALITY_VIEWER"),List.of(),List.of(),List.of()); + var user=new AppUserPrincipal(UUID.randomUUID(),"source-reader","unused","测试",true,false,List.of("USER"),List.of(),List.of(),List.of()); var governance=mock(cn.yunsheng.quality.governance.GovernanceAccessService.class); var users=mock(cn.yunsheng.quality.identity.JdbcAppUserDetailsService.class); when(governance.allowedFields(user)).thenReturn(Set.of("*")); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisTurnSnapshotTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisTurnSnapshotTest.java index ac21dd1..40e176d 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisTurnSnapshotTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisTurnSnapshotTest.java @@ -222,7 +222,7 @@ class AnalysisTurnSnapshotTest { private final MetricsService metrics = mock(MetricsService.class); private final JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); private final GovernanceAccessService governance = mock(GovernanceAccessService.class); - private final AppUserPrincipal submittedActor = actor("submitted-user", "QUALITY_VIEWER"); + private final AppUserPrincipal submittedActor = actor("submitted-user", "USER"); private final AppUserPrincipal freshActor = actor("fresh-user", "SYSTEM_ADMIN"); private final String fingerprint; private final AnalysisTurnService service; diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisVisualSnapshotAccessTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisVisualSnapshotAccessTest.java index 255032d..101506f 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisVisualSnapshotAccessTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisVisualSnapshotAccessTest.java @@ -643,7 +643,7 @@ class AnalysisVisualSnapshotAccessTest { JdbcTemplate jdbc = mock(JdbcTemplate.class); GovernanceAccessService governance = mock(GovernanceAccessService.class); AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "visual-test", "unused", "测试", true, false, - List.of("QUALITY_VIEWER"), List.of(), List.of(), List.of()); + List.of("USER"), List.of(), List.of(), List.of()); when(governance.allowedFields(actor)).thenReturn(RATE_FIELDS); doReturn(List.of()).when(jdbc).query(anyString(), any(RowMapper.class)); AnalysisTurnService service = new AnalysisTurnService(jdbc, JsonMapper.builder().build(), null, null, diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java index 973b0a7..80d081f 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java @@ -43,7 +43,7 @@ class FineDefectRollupsMySqlTest { jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(100),status VARCHAR(32),published_at TIMESTAMP NULL,rule_version VARCHAR(100),mapping_version VARCHAR(100))"); jdbc.execute("CREATE TABLE release_member(release_id CHAR(36),source_revision_id CHAR(36),transform_build_id CHAR(36))"); for (String stage : List.of("black", "bright")) jdbc.execute("CREATE TABLE fact_" + stage + "(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),source_revision_id CHAR(36),month_key CHAR(7),department VARCHAR(190),group_name VARCHAR(190),project_group VARCHAR(190),product VARCHAR(190),qualified_weight DECIMAL(38,8),scrap_weight DECIMAL(38,8),total_weight DECIMAL(38,8),raw_values JSON,components JSON,KEY ix_fact_" + stage + "_month_dims(build_id,month_key))"); - for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql")) { + for (String migration : List.of("V16__metric_rollups.sql", "V36__fine_defect_rollups.sql", "V39__sales_order_types.sql", "V40__sales_order_fact_indexes.sql", "V41__sales_order_metric_rollups.sql")) { try (var stream = getClass().getResourceAsStream("/db/migration/" + migration)) { for (String sql : new String(stream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split(";")) if (!sql.isBlank()) jdbc.execute(sql); } @@ -108,7 +108,7 @@ class FineDefectRollupsMySqlTest { @Test void fieldAndOrganizationPermissionsRemainEffective() { var viewer = new AppUserPrincipal(UUID.randomUUID(), "viewer", "", "Viewer", true, false, - List.of("QUALITY_VIEWER"), List.of("A"), List.of("G"), List.of()); + List.of("USER"), List.of("A"), List.of("G"), List.of()); when(governance.allowedFields(viewer)).thenReturn(Set.of("责废尺寸重量")); equivalent(request("black", Map.of()), viewer); when(governance.allowedFields(actor)).thenReturn(Set.of("镀废缺角重量", "bright_plating_scrap")); @@ -139,6 +139,27 @@ class FineDefectRollupsMySqlTest { equivalent(request("bright", Map.of()), actor); } + @Test void excludingASmallTypeSubtractsExactCountsAndWeightsAndLargeExclusionsKeepSourceFallback() { + String version=UUID.randomUUID().toString(); + jdbc.update("INSERT INTO sales_order_type_version(id,source_file,source_sha256,import_mode,row_count,created_by) VALUES(?,'fixture',?,'REPLACE',2,?)",version,"0".repeat(64),actor.getId().toString()); + jdbc.update("INSERT INTO sales_order_type_entry VALUES(?,'sample','样品销售订单'),(?,'normal','标准销售订单')",version,version); + jdbc.update("UPDATE sales_order_type_current SET version_id=?",version); + jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64),label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)"); + for(String stage:List.of("black","bright")) jdbc.update("UPDATE fact_"+stage+" SET raw_values=JSON_SET(raw_values,?,CASE WHEN id=1 THEN 'sample' WHEN id=2 THEN NULL ELSE 'normal' END)","$.\""+cn.yunsheng.quality.filters.SalesOrderTypes.column(stage)+"\""); + var typed=new cn.yunsheng.quality.metrics.SalesOrderMetricRollups(jdbc);typed.prepareBuild(version,build); + for(String stage:List.of("black","bright")) for(var selection:List.of(List.of("样品销售订单"),List.of("标准销售订单"),List.of("__EMPTY__"),List.of("样品销售订单","__EMPTY__"))) { + for(Map scope:List.>of(Map.of(),Map.of("includeUnassigned",false),Map.of("department","A"),Map.of("productSearch","P0"),Map.of("month",1))) { + var input=new LinkedHashMap<>(scope);input.put("extraFilters",Map.of("salesOrderTypeNotIn",selection)); + var query=request(stage,input); + assertEquals(canonical(source.query(query,actor)),canonical(fast.query(query,actor)),stage+selection+scope); + } + } + // An unavailable fine-defect snapshot must use the original complete query. + jdbc.update("DELETE FROM fine_defect_rollup_build"); + var query=request("black",Map.of("extraFilters",Map.of("salesOrderTypeNotIn",List.of("样品销售订单")))); + assertEquals(canonical(source.query(query,actor)),canonical(fast.query(query,actor))); + } + @AfterEach void cleanup() { if (admin != null && database != null && database.matches("quality_fine_rollup_[a-f0-9]{32}")) admin.execute("DROP DATABASE " + database); } diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoAnalysisRunnerRevocationTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoAnalysisRunnerRevocationTest.java index c9c84c4..d449def 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoAnalysisRunnerRevocationTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoAnalysisRunnerRevocationTest.java @@ -40,7 +40,7 @@ class NaoAnalysisRunnerRevocationTest { return null; }).when(access).authorizeModel("offline-delegation"); AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "offline", "unused", "测试", true, false, - List.of("QUALITY_VIEWER"), List.of("D"), List.of(), List.of()); + List.of("USER"), List.of("D"), List.of(), List.of()); NaoAnalysisRunner runner = new NaoAnalysisRunner(JsonMapper.builder().build(), access); try { System.setProperty("quality.project.root", root.toString()); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoProtocolRunnerTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoProtocolRunnerTest.java index de22817..573947a 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoProtocolRunnerTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoProtocolRunnerTest.java @@ -23,7 +23,7 @@ class NaoProtocolRunnerTest { print(json.dumps({'type':'result','status':'COMPLETED','answer':'verified','stopReason':'stop'}),flush=True) """); var access=mock(NaoRuntimeAccessService.class);when(access.issueProtocol(any(),anyString(),anyString(),anyString())).thenReturn("synthetic-token"); - var actor=new AppUserPrincipal(UUID.randomUUID(),"test","unused","测试",true,false,List.of("QUALITY_VIEWER"),List.of(),List.of(),List.of()); + var actor=new AppUserPrincipal(UUID.randomUUID(),"test","unused","测试",true,false,List.of("USER"),List.of(),List.of(),List.of()); var runner=new NaoAnalysisRunner(JsonMapper.builder().build(),access); String prior=System.getProperty("quality.project.root"),command=System.getProperty("quality.analysis.nao.command"); try { diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoRuntimeAccessServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoRuntimeAccessServiceTest.java index e63d563..f89a21b 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoRuntimeAccessServiceTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoRuntimeAccessServiceTest.java @@ -100,8 +100,8 @@ class NaoRuntimeAccessServiceTest { private final AtomicInteger policyVersion = new AtomicInteger(1); private final MutableClock clock = new MutableClock(Instant.parse("2026-10-22T04:00:00Z")); private final AppUserPrincipal actor = new AppUserPrincipal(USER_ID, "quality-user", "hash", "质量用户", true, - false, List.of("QUALITY_VIEWER"), List.of("D-1"), List.of("G-1"), - List.of(new SimpleGrantedAuthority("ROLE_QUALITY_VIEWER"))); + false, List.of("USER"), List.of("D-1"), List.of("G-1"), + List.of(new SimpleGrantedAuthority("ROLE_USER"))); private final JdbcTemplate jdbc = mock(JdbcTemplate.class); private final JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); private final GovernanceAccessService governance = mock(GovernanceAccessService.class); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoRuntimeQueryControllerTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoRuntimeQueryControllerTest.java index cea09ef..36b4c6e 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoRuntimeQueryControllerTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoRuntimeQueryControllerTest.java @@ -55,8 +55,8 @@ class NaoRuntimeQueryControllerTest { org.mockito.Mockito.verify(protocol,org.mockito.Mockito.never()).recordQuery(org.mockito.ArgumentMatchers.anyString(),org.mockito.ArgumentMatchers.anyString(),org.mockito.ArgumentMatchers.anyString(),org.mockito.ArgumentMatchers.anyMap(),org.mockito.ArgumentMatchers.any(),org.mockito.ArgumentMatchers.any(),org.mockito.ArgumentMatchers.any(),org.mockito.ArgumentMatchers.any()); } private static final AppUserPrincipal ACTOR = new AppUserPrincipal(UUID.randomUUID(), "viewer", "hash", "查看者", - true, false, List.of("QUALITY_VIEWER"), List.of(), List.of(), - List.of(new SimpleGrantedAuthority("ROLE_QUALITY_VIEWER"))); + true, false, List.of("USER"), List.of(), List.of(), + List.of(new SimpleGrantedAuthority("ROLE_USER"))); @Test void delegatedModelAuthorizationRechecksTheBoundModelService() { Fixture fixture = new Fixture(); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoSemanticCompletionTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoSemanticCompletionTest.java index 5bdfa84..7f01f73 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/NaoSemanticCompletionTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/NaoSemanticCompletionTest.java @@ -9,8 +9,10 @@ import org.junit.jupiter.api.Test; class NaoSemanticCompletionTest { @Test void historyCompletionNeedsCurrentImportAndCompleteCoverage() { - var message=new LinkedHashMap(Map.of("type","result","status","COMPLETED","answer","","stopReason","history_report_complete","finishReason","stop","completionMethod","confirmed_history_report","agentTask",Map.of("kind","report"),"historyEvidence",Map.of("importId","current"),"historyCoverage",Map.of("complete",true))); + var message=new LinkedHashMap(Map.of("type","result","status","COMPLETED","answer","历史报告证据已完整核对。","stopReason","history_report_complete","finishReason","stop","completionMethod","confirmed_history_report","agentTask",Map.of("kind","report"),"historyEvidence",Map.of("importId","current"),"historyCoverage",Map.of("complete",true))); assertEquals("COMPLETED",NaoAnalysisRunner.Result.from(message,"",false).status()); + var blank = new LinkedHashMap<>(message); blank.put("answer", ""); + assertEquals("PARTIAL", NaoAnalysisRunner.Result.from(blank,"",false).status()); message.put("historyCoverage",Map.of("complete",false));assertEquals("PARTIAL",NaoAnalysisRunner.Result.from(message,"",false).status()); message.put("historyCoverage",Map.of("complete",true));message.remove("historyEvidence");assertEquals("PARTIAL",NaoAnalysisRunner.Result.from(message,"",false).status()); } diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/RoleMigrationFingerprintTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/RoleMigrationFingerprintTest.java new file mode 100644 index 0000000..5d0b759 --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/RoleMigrationFingerprintTest.java @@ -0,0 +1,45 @@ +package cn.yunsheng.quality.analysis; + +import cn.yunsheng.quality.governance.GovernanceAccessService; +import cn.yunsheng.quality.identity.AppUserPrincipal; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import tools.jackson.databind.json.JsonMapper; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.*; + +class RoleMigrationFingerprintTest { + @Test @SuppressWarnings("unchecked") void roleRenamePreservesHistoryWhileRealGrantChangesStillInvalidateIt() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + List oldRoles = List.of("SYSTEM_ADMIN","QUALITY_VIEWER","BUSINESS_ANALYST","DATA_MAINTAINER","DATA_PUBLISHER","RULE_MAINTAINER","SYSTEM_OPERATOR","AUDIT_VIEWER"); + when(jdbc.query(contains("user_role_migration_anchor"),any(RowMapper.class),any(Object[].class))) + .thenReturn(List.of(Map.of("role","SYSTEM_ADMIN","previous",oldRoles))); + when(jdbc.query(contains("FROM user_access_policy"),any(RowMapper.class),any(Object[].class))).thenReturn(List.of()); + GovernanceAccessService access = new GovernanceAccessService(jdbc,JsonMapper.builder().build()); + UUID id=UUID.randomUUID(); + String original=AuthorizationFingerprint.of(user(id,oldRoles,List.of("A"),List.of("G")),access); + assertEquals(original,AuthorizationFingerprint.of(user(id,List.of("SYSTEM_ADMIN"),List.of("A"),List.of("G")),access)); + assertNotEquals(original,AuthorizationFingerprint.of(user(id,List.of("USER"),List.of("A"),List.of("G")),access)); + assertNotEquals(original,AuthorizationFingerprint.of(user(id,List.of("SYSTEM_ADMIN"),List.of("B"),List.of("G")),access)); + assertNotEquals(original,AuthorizationFingerprint.of(user(id,List.of("SYSTEM_ADMIN"),List.of("A"),List.of("H")),access)); + when(jdbc.query(contains("FROM user_access_policy"),any(RowMapper.class),any(Object[].class))) + .thenReturn(List.of(Map.of("detailAllowed",false,"evidenceAllowed",false,"originalDownloadAllowed",false,"allowedFields",List.of("product"),"rowVersion",1L))); + assertNotEquals(original,AuthorizationFingerprint.of(user(id,List.of("SYSTEM_ADMIN"),List.of("A"),List.of("G")),access)); + } + + @Test @SuppressWarnings("unchecked") void inconsistentAnchorCannotKeepAnOldAdministratorFingerprintForAUser() { + JdbcTemplate jdbc=mock(JdbcTemplate.class); + when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenReturn(List.of(Map.of("role","USER","previous",List.of("SYSTEM_ADMIN")))); + GovernanceAccessService access=new GovernanceAccessService(jdbc,JsonMapper.builder().build()); + assertEquals(List.of("USER"),access.fingerprintRoles(user(UUID.randomUUID(),List.of("USER"),List.of("A"),List.of()))); + } + + private AppUserPrincipal user(UUID id,List roles,List departments,List groups) { + return new AppUserPrincipal(id,"test","unused","test",true,false,roles,departments,groups,List.of()); + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/analytics/ComparisonViewSharingServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/analytics/ComparisonViewSharingServiceTest.java index aab5ae5..b7a297a 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analytics/ComparisonViewSharingServiceTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analytics/ComparisonViewSharingServiceTest.java @@ -16,15 +16,15 @@ class ComparisonViewSharingServiceTest { void legacyDepartmentSpellingUsesTheSameSharingScope() { var policy = Map.of("detailAllowed", true, "evidenceAllowed", false, "originalDownloadAllowed", false, "inheritDepartmentChildren", false); - var oldName = principal(List.of("QUALITY_VIEWER"), List.of("质量管理部"), List.of()); - var currentName = principal(List.of("QUALITY_VIEWER"), List.of("质量中心"), List.of()); + var oldName = principal(List.of("USER"), List.of("质量管理部"), List.of()); + var currentName = principal(List.of("USER"), List.of("质量中心"), List.of()); assertEquals(ComparisonViewSharingService.semanticFingerprint(oldName, Set.of("*"), policy), ComparisonViewSharingService.semanticFingerprint(currentName, Set.of("*"), policy)); } @Test void equivalentAuthorizationSemanticsIgnorePolicyEditCountAndCollectionOrder() { - AppUserPrincipal first = principal(List.of("QUALITY_VIEWER", "BUSINESS_ANALYST"), List.of("部二", "部一"), List.of("组B", "组A")); - AppUserPrincipal second = principal(List.of("BUSINESS_ANALYST", "QUALITY_VIEWER"), List.of("部一", "部二"), List.of("组A", "组B")); + AppUserPrincipal first = principal(List.of("USER"), List.of("部二", "部一"), List.of("组B", "组A")); + AppUserPrincipal second = principal(List.of("USER"), List.of("部一", "部二"), List.of("组A", "组B")); String firstFingerprint = ComparisonViewSharingService.semanticFingerprint(first, Set.of("序号", "批次"), Map.of("detailAllowed", true, "evidenceAllowed", false, "originalDownloadAllowed", false, @@ -38,8 +38,8 @@ class ComparisonViewSharingServiceTest { @Test void changedOrganizationFieldOrPolicySemanticsCannotShare() { - AppUserPrincipal owner = principal(List.of("QUALITY_VIEWER"), List.of("部一"), List.of("组A")); - AppUserPrincipal narrowed = principal(List.of("QUALITY_VIEWER"), List.of("部二"), List.of("组A")); + AppUserPrincipal owner = principal(List.of("USER"), List.of("部一"), List.of("组A")); + AppUserPrincipal narrowed = principal(List.of("USER"), List.of("部二"), List.of("组A")); Map policy = Map.of("detailAllowed", true, "evidenceAllowed", false, "originalDownloadAllowed", false, "inheritDepartmentChildren", false, "validUntil", "2026-12-31T00:00:00Z"); String original = ComparisonViewSharingService.semanticFingerprint(owner, Set.of("序号"), policy); diff --git a/backend/src/test/java/cn/yunsheng/quality/common/WorkspaceStateControllerTest.java b/backend/src/test/java/cn/yunsheng/quality/common/WorkspaceStateControllerTest.java index ad4e53c..84ed073 100644 --- a/backend/src/test/java/cn/yunsheng/quality/common/WorkspaceStateControllerTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/common/WorkspaceStateControllerTest.java @@ -14,7 +14,7 @@ class WorkspaceStateControllerTest { final WorkspaceStateController controller = new WorkspaceStateController(jdbc, access); final UUID id = UUID.randomUUID(); AppUserPrincipal actor(List departments) { - return new AppUserPrincipal(id,"test","","test",true,false,List.of("QUALITY_VIEWER"),departments,List.of("*"),List.of()); + return new AppUserPrincipal(id,"test","","test",true,false,List.of("USER"),departments,List.of("*"),List.of()); } @Test void stableDigestIgnoresScopeOrderAndDoesNotExposePolicy() { when(jdbc.queryForList(contains("user_access_policy"), any(Object[].class))).thenReturn(List.of(Map.of("allowed_fields","private-field"))); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index b7dbac3..060af22 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -33,7 +33,7 @@ class SourceColumnQueriesMySqlTest { private boolean created; private final GovernanceAccessService governance = mock(GovernanceAccessService.class); private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "source-column-test", "", "测试", - true, false, List.of("QUALITY_VIEWER"), List.of("部门A"), List.of("组A"), List.of()); + true, false, List.of("USER"), List.of("部门A"), List.of("组A"), List.of()); private SourceColumnQueries queries; @BeforeEach void setup() { diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java index 7e9c9b0..899cb9b 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesTest.java @@ -19,7 +19,7 @@ class SourceColumnQueriesTest { private static final String SOURCE = "src_0123456789abcdef"; private final GovernanceAccessService governance = mock(GovernanceAccessService.class); private final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "source-test", "", "test", - true, false, List.of("QUALITY_VIEWER"), List.of(), List.of(), List.of()); + true, false, List.of("USER"), List.of(), List.of(), List.of()); @Test void parsesJsonStringAndKeepsLeadingZeroAndInjectionTextAsExactStrings() { List> parsed = SourceColumnQueries.parse( diff --git a/backend/src/test/java/cn/yunsheng/quality/governance/GovernanceAccessServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/governance/GovernanceAccessServiceTest.java index e487267..5adbfd1 100644 --- a/backend/src/test/java/cn/yunsheng/quality/governance/GovernanceAccessServiceTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/governance/GovernanceAccessServiceTest.java @@ -37,7 +37,7 @@ class GovernanceAccessServiceTest { when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenAnswer(call->call.getArgument(0,String.class).startsWith("SELECT detail_allowed")?List.of(Map.of("inheritDepartmentChildren",true)):List.of(definition)); when(jdbc.query(anyString(),any(RowMapper.class))).thenReturn(List.of(definition)); GovernanceAccessService service=new GovernanceAccessService(jdbc,JsonMapper.builder().build()); - AppUserPrincipal original=new AppUserPrincipal(UUID.randomUUID(),"reader","unchanged","Reader",true,false,List.of("QUALITY_VIEWER"),List.of("parent"),List.of("G1"),List.of()); + AppUserPrincipal original=new AppUserPrincipal(UUID.randomUUID(),"reader","unchanged","Reader",true,false,List.of("USER"),List.of("parent"),List.of("G1"),List.of()); AppUserPrincipal expanded=service.resolvePrincipal(original); assertEquals(List.of("parent"),original.getDepartments());assertEquals(Set.of("parent","child"),new HashSet<>(expanded.getDepartments())); assertEquals(original.getId(),expanded.getId());assertEquals(original.getRoleCodes(),expanded.getRoleCodes());assertEquals(original.getGroups(),expanded.getGroups()); @@ -47,7 +47,7 @@ class GovernanceAccessServiceTest { when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenReturn(List.of(Map.of("inheritDepartmentChildren",true))); when(jdbc.query(anyString(),any(RowMapper.class))).thenReturn(List.of(definition)); GovernanceAccessService service=new GovernanceAccessService(jdbc,JsonMapper.builder().build()); - AppUserPrincipal reader=new AppUserPrincipal(UUID.randomUUID(),"reader","unused","Reader",true,false,List.of("QUALITY_VIEWER"),List.of("p"),List.of(),List.of()); + AppUserPrincipal reader=new AppUserPrincipal(UUID.randomUUID(),"reader","unused","Reader",true,false,List.of("USER"),List.of("p"),List.of(),List.of()); assertThrows(ResponseStatusException.class,()->service.resolvePrincipal(reader)); } } diff --git a/backend/src/test/java/cn/yunsheng/quality/governance/SourceChangeAuthorizationBoundaryTest.java b/backend/src/test/java/cn/yunsheng/quality/governance/SourceChangeAuthorizationBoundaryTest.java index 8234520..2a74b55 100644 --- a/backend/src/test/java/cn/yunsheng/quality/governance/SourceChangeAuthorizationBoundaryTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/governance/SourceChangeAuthorizationBoundaryTest.java @@ -18,7 +18,7 @@ import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; class SourceChangeAuthorizationBoundaryTest { - private AppUserPrincipal actor(){return new AppUserPrincipal(UUID.randomUUID(),"a","unused","A",true,false,List.of("DATA_MAINTAINER","DATA_PUBLISHER"),List.of("A"),List.of(),List.of());} + private AppUserPrincipal actor(){return new AppUserPrincipal(UUID.randomUUID(),"a","unused","A",true,false,List.of("DATA_ADMIN"),List.of("A"),List.of(),List.of());} private SourceAuthorizationService denyingAccess(){SourceAuthorizationService access=mock(SourceAuthorizationService.class);doThrow(new ResponseStatusException(HttpStatus.FORBIDDEN,"B is outside scope")).when(access).requireSource(eq("doc-b"),eq("rev-b"),any(AppUserPrincipal.class));return access;} private GovernanceSupport support(JdbcTemplate jdbc){return new GovernanceSupport(jdbc,JsonMapper.builder().build());} diff --git a/backend/src/test/java/cn/yunsheng/quality/governance/SourceSecurityRegressionTest.java b/backend/src/test/java/cn/yunsheng/quality/governance/SourceSecurityRegressionTest.java index eb5da99..e80fbd3 100644 --- a/backend/src/test/java/cn/yunsheng/quality/governance/SourceSecurityRegressionTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/governance/SourceSecurityRegressionTest.java @@ -13,7 +13,7 @@ import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; class SourceSecurityRegressionTest { - private AppUserPrincipal reader(String department){return new AppUserPrincipal(UUID.randomUUID(),"reader","unused","Reader",true,false,List.of("DATA_MAINTAINER","DATA_PUBLISHER"),List.of(department),List.of(),List.of());} + private AppUserPrincipal reader(String department){return new AppUserPrincipal(UUID.randomUUID(),"reader","unused","Reader",true,false,List.of("DATA_ADMIN"),List.of(department),List.of(),List.of());} @Test @SuppressWarnings("unchecked") void parsedAuthorizedRowsCannotGrantWholeWorkbookDownload(){ JdbcTemplate jdbc=mock(JdbcTemplate.class); @@ -35,6 +35,17 @@ class SourceSecurityRegressionTest { assertFalse(SourceCoverageGuard.overlaps(old,nextMonth)); } + @Test void uploadAppendOnlyCoexistsWithExactScopeMonthlySnapshotAndOtherAppendBatches(){ + var monthly=new SourceCoverageGuard.Coverage("black",2026,2,"质量管理部","MONTHLY_SNAPSHOT","sha-month","black|2026|02|质量管理部"); + var appendA=new SourceCoverageGuard.Coverage("black",2026,2,"质量中心","APPEND_SNAPSHOT","sha-a","black|2026|02|质量管理部|UPLOAD_A"); + var appendB=new SourceCoverageGuard.Coverage("black",2026,2,"质量管理部","APPEND_SNAPSHOT","sha-b","black|2026|02|质量管理部|UPLOAD_B"); + assertFalse(SourceCoverageGuard.overlaps(monthly,appendA)); + assertFalse(SourceCoverageGuard.overlaps(appendA,appendB)); + assertTrue(SourceCoverageGuard.overlaps(monthly,new SourceCoverageGuard.Coverage("black",2026,2,"ALL","APPEND_SNAPSHOT","sha-c","black|2026|02|ALL|UPLOAD_C"))); + assertTrue(SourceCoverageGuard.overlaps(monthly,new SourceCoverageGuard.Coverage("black",2026,2,"质量管理部","APPEND_SNAPSHOT","sha-month","black|2026|02|质量管理部|UPLOAD_D"))); + assertTrue(SourceCoverageGuard.overlaps(monthly,new SourceCoverageGuard.Coverage("black",2026,2,"质量管理部","SHARD_SNAPSHOT","sha-shard","black|2026|02|质量管理部|P1"))); + } + @Test @SuppressWarnings("unchecked") void localMaintainerCannotRemoveAnotherDepartmentsSource(){ JdbcTemplate jdbc=mock(JdbcTemplate.class); when(jdbc.query(anyString(),any(RowMapper.class),any(Object[].class))).thenReturn(List.of("B")); diff --git a/backend/src/test/java/cn/yunsheng/quality/identity/AppRolesTest.java b/backend/src/test/java/cn/yunsheng/quality/identity/AppRolesTest.java new file mode 100644 index 0000000..7be53d6 --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/identity/AppRolesTest.java @@ -0,0 +1,34 @@ +package cn.yunsheng.quality.identity; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.springframework.web.server.ResponseStatusException; +import static org.junit.jupiter.api.Assertions.*; + +class AppRolesTest { + @Test void exactlyOneOfThreeRolesCanBeAssigned() { + for (String role : AppRoles.CODES) assertEquals(List.of(role), AppRoles.validate(List.of(role))); + for (List roles : List.of(List.of(), List.of("USER", "DATA_ADMIN"), + List.of("USER", "USER"), List.of("QUALITY_VIEWER"), Arrays.asList((String) null))) + assertEquals(400, assertThrows(ResponseStatusException.class, () -> AppRoles.validate(roles)).getStatusCode().value()); + assertThrows(ResponseStatusException.class, () -> AppRoles.validate(null)); + } + + @Test void operationAuthoritiesAreInheritedWhileAssignedRoleStaysSingle() { + assertEquals(List.of("ROLE_USER"), codes("USER")); + assertEquals(List.of("ROLE_DATA_ADMIN", "ROLE_USER"), codes("DATA_ADMIN")); + assertEquals(List.of("ROLE_SYSTEM_ADMIN", "ROLE_DATA_ADMIN", "ROLE_USER"), codes("SYSTEM_ADMIN")); + assertEquals(List.of("DATA_ADMIN"), AppRoles.validate(List.of("DATA_ADMIN"))); + } + + @Test void unknownOrMultipleAssignmentsCannotGainAuthorities() { + assertTrue(AppRoles.authorities(List.of("QUALITY_VIEWER")).isEmpty()); + assertTrue(AppRoles.authorities(List.of("USER", "SYSTEM_ADMIN")).isEmpty()); + assertTrue(AppRoles.authorities(Arrays.asList((String) null)).isEmpty()); + } + + private List codes(String role) { + return AppRoles.authorities(List.of(role)).stream().map(a -> a.getAuthority()).toList(); + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/identity/CurrentUserRefreshFilterTest.java b/backend/src/test/java/cn/yunsheng/quality/identity/CurrentUserRefreshFilterTest.java index 08b697a..2141939 100644 --- a/backend/src/test/java/cn/yunsheng/quality/identity/CurrentUserRefreshFilterTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/identity/CurrentUserRefreshFilterTest.java @@ -45,7 +45,7 @@ class CurrentUserRefreshFilterTest { @Test void refreshesRolesAndScopesFromTheCurrentUserRecord() throws Exception { UUID id = UUID.randomUUID(); AppUserPrincipal old = principal(id, List.of("SYSTEM_ADMIN"), List.of("*"), List.of("*"), true); - AppUserPrincipal current = principal(id, List.of("QUALITY_VIEWER"), List.of("质量管理部"), List.of("VCM"), true); + AppUserPrincipal current = principal(id, List.of("USER"), List.of("质量管理部"), List.of("VCM"), true); JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); when(users.loadUserByUsername("alice")).thenReturn(current); SecurityContextRepository repository = mock(SecurityContextRepository.class); @@ -60,14 +60,14 @@ class CurrentUserRefreshFilterTest { assertTrue(continued.get()); AppUserPrincipal refreshed = (AppUserPrincipal) SecurityContextHolder.getContext().getAuthentication().getPrincipal(); - assertEquals(List.of("QUALITY_VIEWER"), refreshed.getRoleCodes()); + assertEquals(List.of("USER"), refreshed.getRoleCodes()); assertEquals(List.of("质量管理部"), refreshed.getDepartments()); assertEquals(List.of("VCM"), refreshed.getGroups()); verify(repository).saveContext(any(), eq(request), eq(response)); } @Test void rejectsAndInvalidatesARevokedSessionBeforeTheRequestRuns() throws Exception { - AppUserPrincipal old = principal(UUID.randomUUID(), List.of("QUALITY_VIEWER"), List.of("*"), List.of("*"), true); + AppUserPrincipal old = principal(UUID.randomUUID(), List.of("USER"), List.of("*"), List.of("*"), true); JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); when(users.loadUserByUsername("alice")).thenThrow(new UsernameNotFoundException("inactive")); SecurityContextHolder.getContext().setAuthentication(UsernamePasswordAuthenticationToken.authenticated( @@ -87,7 +87,7 @@ class CurrentUserRefreshFilterTest { } @Test void expiresAnExistingSessionBeforeAnyBusinessRequestRuns() throws Exception { - AppUserPrincipal current = principal(UUID.randomUUID(), List.of("QUALITY_VIEWER"), List.of("*"), List.of("*"), true); + AppUserPrincipal current = principal(UUID.randomUUID(), List.of("USER"), List.of("*"), List.of("*"), true); JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); when(users.loadUserByUsername("alice")).thenReturn(current); GovernanceAccessService access = mock(GovernanceAccessService.class); @@ -111,8 +111,8 @@ class CurrentUserRefreshFilterTest { @Test void usesTheServerResolvedHierarchyForTheRefreshedIdentity() throws Exception { UUID id = UUID.randomUUID(); - AppUserPrincipal current = principal(id, List.of("QUALITY_VIEWER"), List.of("parent"), List.of(), true); - AppUserPrincipal expanded = principal(id, List.of("QUALITY_VIEWER"), List.of("parent", "child"), List.of(), true); + AppUserPrincipal current = principal(id, List.of("USER"), List.of("parent"), List.of(), true); + AppUserPrincipal expanded = principal(id, List.of("USER"), List.of("parent", "child"), List.of(), true); JdbcAppUserDetailsService users = mock(JdbcAppUserDetailsService.class); when(users.loadUserByUsername("alice")).thenReturn(current); GovernanceAccessService access = mock(GovernanceAccessService.class); diff --git a/backend/src/test/java/cn/yunsheng/quality/ingestion/ImportBeforeImageAuthorizationTest.java b/backend/src/test/java/cn/yunsheng/quality/ingestion/ImportBeforeImageAuthorizationTest.java index 8853fd6..55dbe1b 100644 --- a/backend/src/test/java/cn/yunsheng/quality/ingestion/ImportBeforeImageAuthorizationTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/ingestion/ImportBeforeImageAuthorizationTest.java @@ -26,12 +26,12 @@ class ImportBeforeImageAuthorizationTest { } private GovernanceAccessService access(){GovernanceAccessService access=mock(GovernanceAccessService.class);when(access.resolvePrincipal(any())).thenAnswer(call->call.getArgument(0));return access;} @Test void historicalCandidateCannotReplaceCurrentBeforeImageOutsideMaintainerScope(){ - JdbcTemplate jdbc=jdbc("B","DATA_MAINTAINER"); + JdbcTemplate jdbc=jdbc("B","DATA_ADMIN"); var denied=assertThrows(ResponseStatusException.class,()->ImportRunService.requireBeforeImages(jdbc,access(),List.of("historical-candidate-with-only-A-rows"),actor,"current-release")); assertEquals(HttpStatus.FORBIDDEN,denied.getStatusCode());verify(jdbc,never()).update(anyString(),any(Object[].class)); } @Test void currentRevisionIsCheckedInsteadOfHistoricalCandidateRevision(){ - JdbcTemplate jdbc=jdbc("A","DATA_MAINTAINER");SourceAuthorizationService sourceAccess=mock(SourceAuthorizationService.class); + JdbcTemplate jdbc=jdbc("A","DATA_ADMIN");SourceAuthorizationService sourceAccess=mock(SourceAuthorizationService.class); ImportRunService.requireBeforeImages(jdbc,access(),sourceAccess,List.of("historical-candidate"),actor,"current-release"); verify(sourceAccess).requireSource(eq("current-doc"),eq("current-revision"),argThat(user->user.getDepartments().equals(List.of("A")))); } diff --git a/backend/src/test/java/cn/yunsheng/quality/ingestion/ImportUnassignedRowsTest.java b/backend/src/test/java/cn/yunsheng/quality/ingestion/ImportUnassignedRowsTest.java new file mode 100644 index 0000000..fdcf128 --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/ingestion/ImportUnassignedRowsTest.java @@ -0,0 +1,126 @@ +package cn.yunsheng.quality.ingestion; + +import cn.yunsheng.quality.analytics.TargetVersionService; +import cn.yunsheng.quality.governance.*; +import cn.yunsheng.quality.metrics.MetricsRollupService; +import cn.yunsheng.quality.source.*; +import java.nio.file.Path; +import java.util.*; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.*; +import org.springframework.test.util.ReflectionTestUtils; +import tools.jackson.databind.json.JsonMapper; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.*; + +class ImportUnassignedRowsTest { + @Test + @org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable(named="QUALITY_UNASSIGNED_REPLAY_INPUT", matches=".+") + @SuppressWarnings("unchecked") + void failedDecemberWorkbookKeepsAllRowsAndItsTwelveUnassignedMappings() throws Exception { + JsonMapper json = JsonMapper.builder().build(); + Map input = json.readValue(java.nio.file.Files.readString(Path.of(System.getenv("QUALITY_UNASSIGNED_REPLAY_INPUT"))), Map.class); + Map mapping = (Map) input.get("mapping"); + Map config = (Map) input.get("parserConfig"); + JdbcTemplate jdbc = mock(JdbcTemplate.class); + SourceScanner scanner = mock(SourceScanner.class); + Path file = Path.of((String) input.get("archivePath")); + when(scanner.archiveRevision(any())).thenReturn(file); + when(jdbc.queryForObject(anyString(), eq(String.class), any(Object[].class))).thenAnswer(call -> { + String sql = call.getArgument(0); + return sql.contains("CAST(parser_config") ? json.writeValueAsString(config) : sql.contains("parser_config_hash") ? input.get("parserConfigHash") : sql.contains("scope_code") ? "ALL" : "actor"; + }); + when(jdbc.query(anyString(), any(RowMapper.class), any(Object[].class))).thenAnswer(call -> { + String sql = call.getArgument(0); + return sql.startsWith("SELECT id FROM source_document") ? List.of("doc") : sql.startsWith("SELECT id FROM source_revision") ? List.of("revision") : List.of(); + }); + int[] counters = {0, 0, -1}; + Set unmappedRows = new TreeSet<>(); + doAnswer(call -> { + String sql = call.getArgument(0); + if (sql.contains("INSERT INTO fact_bright")) { + counters[0]++; + if (call.getArgument(7) == null) counters[1]++; + Map components = json.readValue(call.getArgument(14, String.class), Map.class); + if ("unmapped".equals(components.get("categoryMappingStatus"))) unmappedRows.add(call.getArgument(4, Integer.class)); + } + if (sql.contains("UPDATE transform_build SET status='COMPLETED'")) counters[2] = call.getArgument(2, Integer.class); + return 1; + }).when(jdbc).update(anyString(), any(Object[].class)); + var rules = ConfigurableRuleEngine.compile((Map) input.get("rules")); + var mapper = new BusinessRowMapper(lookup(mapping,"categoryLookup"), lookup(mapping,"projectGroupLookup"), (String)input.get("mappingVersion"), lookup(mapping,"departmentLookup"), lookup(mapping,"productLookup"), rules, mapping); + var worker = new ImportWorker(jdbc, mock(SourceCatalogService.class), scanner, new XlsxStreamingReader(), json, mock(GovernanceAccessService.class), mock(TargetVersionService.class), mock(SourceCoverageGuard.class), mock(TaskProgressService.class), mock(CandidateAcceptanceService.class), mock(MappingIssueService.class), mock(MetricsRollupService.class)); + var candidate = new SourceFileCandidate(UUID.randomUUID(), "replay", "2025:bright:12", "bright", 2025, 12, "12月亮片.xlsx", file, (String)input.get("sha256"), java.nio.file.Files.size(file), "DISCOVERED"); + ReflectionTestUtils.invokeMethod(worker, "build", candidate, mapper, input.get("ruleVersion"), "replay-run"); + assertArrayEquals(new int[]{18627,13,0}, counters, "all original rows survive, including unassigned rows; no blocking data errors"); + assertEquals(Set.of(9104,11838,12576,12682,13205,13682,13773,13774,13945,14347,15419,17290), unmappedRows); + } + + @SuppressWarnings("unchecked") + private Map lookup(Map definition, String name) { + Map result = new HashMap<>(); + ((Map)definition.getOrDefault(name, Map.of())).forEach((key,value) -> result.put(key, value == null ? null : MappingValueCodec.fromStorage(String.valueOf(value)))); + return result; + } + + @Test void missingBusinessGroupIsStoredWithoutBlockingEitherStage() throws Exception { + for (String stage : List.of("black", "bright")) checkBuild(stage, false, false); + } + + @Test void unassignedRowsStillRejectInvalidNumbersAndMissingHeaders() throws Exception { + checkBuild("black", true, false); + checkBuild("bright", false, true); + } + + @SuppressWarnings("unchecked") + private void checkBuild(String stage, boolean invalidNumber, boolean missingHeader) throws Exception { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + SourceScanner scanner = mock(SourceScanner.class); + XlsxStreamingReader reader = mock(XlsxStreamingReader.class); + when(jdbc.queryForObject(anyString(), eq(String.class), any(Object[].class))).thenAnswer(call -> { + String sql = call.getArgument(0); + return sql.contains("CAST(parser_config") ? "{}" : sql.contains("parser_config_hash") ? "legacy" : sql.contains("scope_code") ? "ALL" : "actor"; + }); + when(jdbc.query(anyString(), any(RowMapper.class), any(Object[].class))).thenAnswer(call -> { + String sql = call.getArgument(0); + return sql.startsWith("SELECT id FROM source_document") ? List.of("doc") : sql.startsWith("SELECT id FROM source_revision") ? List.of("revision") : List.of(); + }); + when(scanner.archiveRevision(any())).thenReturn(Path.of("/unused")); + BusinessRowMapper mapper = new BusinessRowMapper(Map.of(), Map.of(), "mapping"); + Map cells = new LinkedHashMap<>(); + for (String header : mapper.requiredHeaders(stage)) cells.put(header, cell("0")); + cells.put("PDT代码", cell("P001")); cells.put("销售片区", cell("欧洲事业部")); + cells.put("事业部", cell("新能源汽车磁钢")); cells.put("成品规格", cell("32.05X18X3.7+T")); + if (invalidNumber) cells.put("送电镀重量", cell("bad")); + if (missingHeader) cells.remove("晶界扩散报废合计重量"); + String sheet = stage.equals("black") ? "报废明细" : "明细"; + doAnswer(call -> { + Consumer consumer = call.getArgument(3); + consumer.accept(new XlsxStreamingReader.RawRow(sheet, 9104, cells, Map.of("PDT代码", "P001", "销售片区", "欧洲事业部"))); + return new XlsxStreamingReader.ReadSummary(sheet, 1); + }).when(reader).read(any(Path.class), anyString(), isNull(), any(Consumer.class)); + List> facts = new ArrayList<>(), issues = new ArrayList<>(); + int[] rejected = {-1}; + doAnswer(call -> { + String sql = call.getArgument(0); + List args = Arrays.asList(call.getArguments()).subList(1, call.getArguments().length); + if (sql.contains("INSERT INTO fact_")) facts.add(new ArrayList<>(args)); + if (sql.contains("INSERT INTO import_issue")) issues.add(new ArrayList<>(args)); + if (sql.contains("UPDATE transform_build SET status='COMPLETED'")) rejected[0] = ((Number) args.get(1)).intValue(); + return 1; + }).when(jdbc).update(anyString(), any(Object[].class)); + ImportWorker worker = new ImportWorker(jdbc, mock(SourceCatalogService.class), scanner, reader, JsonMapper.builder().build(), mock(GovernanceAccessService.class), mock(TargetVersionService.class), mock(SourceCoverageGuard.class), mock(TaskProgressService.class), mock(CandidateAcceptanceService.class), mock(MappingIssueService.class), mock(MetricsRollupService.class)); + var candidate = new SourceFileCandidate(UUID.randomUUID(), "key", "2025:" + stage + ":12", stage, 2025, 12, "synthetic.xlsx", Path.of("/unused"), "a".repeat(64), 1, "DISCOVERED"); + ReflectionTestUtils.invokeMethod(worker, "build", candidate, mapper, "pq-dax-v1", "run"); + assertEquals(1, facts.size()); + assertNull(facts.getFirst().get(6), "unmatched business group must remain unassigned"); + assertTrue(String.valueOf(facts.getFirst().get(12)).contains("欧洲事业部"), "original values are retained"); + var unassigned = issues.stream().filter(issue -> "CATEGORY_MAPPING_MISSING".equals(issue.get(5))).findFirst().orElseThrow(); + assertTrue(String.valueOf(unassigned.get(8)).contains("\"blocking\":false")); + assertEquals(invalidNumber || missingHeader ? 1 : 0, rejected[0]); + } + + private XlsxStreamingReader.CellValue cell(String value) { return new XlsxStreamingReader.CellValue(value, value, "s"); } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/ingestion/TransformBuildKeyTest.java b/backend/src/test/java/cn/yunsheng/quality/ingestion/TransformBuildKeyTest.java index bb82ba0..0d3d759 100644 --- a/backend/src/test/java/cn/yunsheng/quality/ingestion/TransformBuildKeyTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/ingestion/TransformBuildKeyTest.java @@ -11,6 +11,14 @@ import org.junit.jupiter.api.Test; class TransformBuildKeyTest { private static final String SAME_FILE_HASH = "a".repeat(64); + @Test + void previousBlockingMappingCacheIsNotReusedAfterUnassignedPolicyChange() throws Exception { + var source = source("2026:black:01", "black", 1); + String oldIdentity = String.join("\n", source.sha256(), source.sourceSlot(), "revision", source.stage(), "2026", "1", "parser", "rule", "mapping"); + String oldKey = java.util.HexFormat.of().formatHex(java.security.MessageDigest.getInstance("SHA-256").digest(oldIdentity.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + assertNotEquals(oldKey, TransformBuildKey.create(source, "revision", "parser", "rule", "mapping")); + } + @Test void identicalWorkbookBytesAreSeparatedBySourcePeriodAndRevision() { SourceFileCandidate january = source("2026:black:01", "black", 1); diff --git a/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java index 02dd9b7..a10c511 100644 --- a/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java @@ -30,6 +30,9 @@ class UnclassifiedMappingRebuildMySqlTest { jdbc.execute("CREATE TABLE defect_component(id BIGINT AUTO_INCREMENT PRIMARY KEY,build_id CHAR(36),stage VARCHAR(16),sheet_name VARCHAR(100),physical_row_no INT,defect_name VARCHAR(100),weight DECIMAL(24,8),source_field VARCHAR(100))"); try(var stream=getClass().getResourceAsStream("/db/migration/V16__metric_rollups.sql")){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);} try(var stream=getClass().getResourceAsStream("/db/migration/V36__fine_defect_rollups.sql")){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);} + for(String migration:List.of("V39__sales_order_types.sql","V40__sales_order_fact_indexes.sql","V41__sales_order_metric_rollups.sql")) { + try(var stream=getClass().getResourceAsStream("/db/migration/"+migration)){for(String sql:new String(stream.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8).split(";"))if(!sql.isBlank())jdbc.execute(sql);} + } jdbc.execute("CREATE TABLE import_run(id CHAR(36) PRIMARY KEY,idempotency_key VARCHAR(190),status VARCHAR(32),stage VARCHAR(80),requested_sources JSON,expected_current_release_id CHAR(36),created_by CHAR(36),processed BIGINT,total BIGINT,result_release_id CHAR(36),expected_mapping_version VARCHAR(100),expected_rule_version VARCHAR(100),workflow_operation_id CHAR(36),validation JSON,started_at TIMESTAMP(6),completed_at TIMESTAMP(6))"); jdbc.execute("CREATE TABLE release_protocol_dependency(release_id CHAR(36),protocol_id VARCHAR(100),PRIMARY KEY(release_id,protocol_id))");jdbc.execute("CREATE TABLE release_source_exclusion(release_id CHAR(36),source_document_id VARCHAR(100),request_id VARCHAR(100))"); jdbc.update("INSERT INTO data_release VALUES(?,'parent','PUBLISHED',NULL,NULL,'rule','old-map',NULL,TRUE,JSON_OBJECT(),?)",parent,UUID.randomUUID().toString()); diff --git a/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsResponseAccessTest.java b/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsResponseAccessTest.java index 4951fa7..5a8d503 100644 --- a/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsResponseAccessTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsResponseAccessTest.java @@ -15,7 +15,7 @@ import org.junit.jupiter.api.Test; class MetricsResponseAccessTest { final GovernanceAccessService governance = mock(GovernanceAccessService.class); final AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "scope", "unused", "测试", true, false, - List.of("QUALITY_VIEWER"), List.of("D"), List.of(), List.of()); + List.of("USER"), List.of("D"), List.of(), List.of()); final MetricStage stage = new MetricStage(new BigDecimal("90"), new BigDecimal("10"), new BigDecimal("100"), new BigDecimal("0.9"), List.of(new DefectRow("责废", BigDecimal.TEN, BigDecimal.ONE)), BigDecimal.ONE, new BigDecimal("0.91")); final MetricRow row = new MetricRow("x", "x", "D", "G", "P", null, new BigDecimal("0.81"), stage, stage, Map.of(), new BigDecimal("0.82")); diff --git a/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsTrendControllerTest.java b/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsTrendControllerTest.java index 4ff9b06..f4df541 100644 --- a/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsTrendControllerTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/metrics/MetricsTrendControllerTest.java @@ -19,7 +19,7 @@ class MetricsTrendControllerTest { MetricsService metrics = mock(MetricsService.class); MetricsResponseAccess access = mock(MetricsResponseAccess.class); AppUserPrincipal actor = new AppUserPrincipal(UUID.randomUUID(), "trend", "unused", "趋势", true, false, - List.of("QUALITY_VIEWER"), List.of("D"), List.of("G"), List.of()); + List.of("USER"), List.of("D"), List.of("G"), List.of()); MetricsService.MetricTrendSnapshot response = new MetricsService.MetricTrendSnapshot( Map.of(), Map.of(), List.of(), Map.of()); when(metrics.calculateTrend(any(), eq(actor))).thenReturn(response); diff --git a/backend/src/test/java/cn/yunsheng/quality/source/SourceDeletionServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/source/SourceDeletionServiceTest.java index b92fcb3..cdbc095 100644 --- a/backend/src/test/java/cn/yunsheng/quality/source/SourceDeletionServiceTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/source/SourceDeletionServiceTest.java @@ -62,7 +62,7 @@ class SourceDeletionServiceTest { verify(jdbc,never()).update(anyString(),any(Object[].class));verify(governance,never()).request(any(),any()); } @Test void restrictedMaintainerCannotHideARevisionOutsideTheirOrganization(){ - currentUsed=true;var service=service();var restricted=new AppUserPrincipal(actor.getId(),"A","","",true,false,List.of("DATA_MAINTAINER"),List.of("A"),List.of(),List.of()); + currentUsed=true;var service=service();var restricted=new AppUserPrincipal(actor.getId(),"A","","",true,false,List.of("DATA_ADMIN"),List.of("A"),List.of(),List.of()); doThrow(new ResponseStatusException(HttpStatus.FORBIDDEN,"B outside scope")).when(sourceAccess).requireSource(eq("document"),eq("current-revision"),any()); assertEquals(HttpStatus.FORBIDDEN,assertThrows(ResponseStatusException.class,()->service.delete("candidate",new SourceDeletionService.DeleteRequest(pointer,0L,"current-revision","理由","key"),restricted)).getStatusCode()); verify(jdbc,never()).update(anyString(),any(Object[].class)); diff --git a/backend/src/test/java/cn/yunsheng/quality/source/SourceUploadAppendMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/source/SourceUploadAppendMySqlTest.java new file mode 100644 index 0000000..d6a5e5d --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/source/SourceUploadAppendMySqlTest.java @@ -0,0 +1,284 @@ +package cn.yunsheng.quality.source; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +import cn.yunsheng.quality.analytics.TargetVersionService; +import cn.yunsheng.quality.config.QualityProperties; +import cn.yunsheng.quality.governance.*; +import cn.yunsheng.quality.identity.AppRoles; +import cn.yunsheng.quality.identity.AppUserPrincipal; +import cn.yunsheng.quality.ingestion.*; +import cn.yunsheng.quality.metrics.MetricsRollupService; +import cn.yunsheng.quality.metrics.MetricsService; +import cn.yunsheng.quality.release.ReleaseService; +import java.io.ByteArrayOutputStream; +import java.math.BigDecimal; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.http.HttpStatus; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.web.server.ResponseStatusException; +import tools.jackson.databind.json.JsonMapper; + +/** Exercises the upload, import, release, and metric paths against a real isolated MySQL schema. */ +@EnabledIfEnvironmentVariable(named="QUALITY_MULTIFILE_TEST_CREATE_DATABASE",matches="true") +class SourceUploadAppendMySqlTest { + @TempDir Path tempDir; + private JdbcTemplate admin,jdbc; + private DriverManagerDataSource dataSource; + private TransactionTemplate tx; + private String database; + private Path archiveRoot; + private UUID adminId; + private AppUserPrincipal adminActor; + private JsonMapper json; + private GovernanceAccessService access; + private SourceAuthorizationService sourceAccess; + private SourceScanner scanner; + private XlsxStreamingReader reader; + private SourceUploadService uploads; + private SourceCatalogService sources; + private ImportRunService imports; + private ImportWorker worker; + private SourceCoverageGuard coverage; + private ReleaseService releases; + private SourceGovernanceService governance; + private MetricsService metrics; + + @BeforeEach void setup() throws Exception { + String serverUrl=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_SERVER_URL"); + assertNotNull(serverUrl,"set the isolated local MySQL server URL"); + assertTrue(serverUrl.startsWith("jdbc:mysql://127.0.0.1:")&&serverUrl.contains("/?")); + java.util.regex.Matcher port=java.util.regex.Pattern.compile("jdbc:mysql://127\\.0\\.0\\.1:(\\d+)/\\?").matcher(serverUrl); + assertTrue(port.find());assertNotEquals("53306",port.group(1),"tests must never connect to the formal MySQL port"); + String user=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_USER"),password=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_PASSWORD"); + admin=new JdbcTemplate(new DriverManagerDataSource(serverUrl,user,password)); + database="quality_verify_multifile_"+UUID.randomUUID().toString().replace("-",""); + admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + dataSource=new DriverManagerDataSource(serverUrl.replaceFirst("/\\?","/"+database+"?"),user,password); + Flyway.configure().dataSource(dataSource).locations("classpath:db/migration") + .placeholders(Map.of("legacyTimestampOffsetSeconds","0")).load().migrate(); + jdbc=new JdbcTemplate(dataSource);tx=new TransactionTemplate(new DataSourceTransactionManager(dataSource)); + archiveRoot=tempDir.resolve("archive");Files.createDirectories(archiveRoot); + adminId=UUID.randomUUID(); + jdbc.update("INSERT INTO app_user(id,username,display_name,password_hash,active) VALUES(?,?,?,?,TRUE)",adminId.toString(),"multifile-test-admin","Multifile test admin","fixture-hash"); + jdbc.update("INSERT INTO app_user_role(user_id,role_code) VALUES(?,'SYSTEM_ADMIN')",adminId.toString()); + adminActor=new AppUserPrincipal(adminId,"multifile-test-admin","fixture-hash","Multifile test admin",true,false,List.of("SYSTEM_ADMIN"),List.of("*"),List.of("*"),AppRoles.authorities(List.of("SYSTEM_ADMIN"))); + + QualityProperties properties=new QualityProperties();properties.setArchiveRoot(archiveRoot);scanner=new SourceScanner(properties);reader=new XlsxStreamingReader(); + json=JsonMapper.builder().build();access=new GovernanceAccessService(jdbc,json);sourceAccess=new SourceAuthorizationService(jdbc); + GovernanceSupport support=new GovernanceSupport(jdbc,json);sources=new SourceCatalogService(jdbc,scanner);coverage=new SourceCoverageGuard(jdbc); + uploads=new SourceUploadService(jdbc,scanner,reader,sourceAccess,access,json,support); + imports=new ImportRunService(jdbc,sources,json,access); + MetricsRollupService rollups=new MetricsRollupService(jdbc); + worker=new ImportWorker(jdbc,sources,scanner,reader,json,access,new TargetVersionService(jdbc),coverage,new TaskProgressService(jdbc),mock(CandidateAcceptanceService.class),mock(MappingIssueService.class),rollups); + ReflectionTestUtils.setField(worker,"testWorkflowMode",true); + releases=new ReleaseService(jdbc,json,mock(ApplicationEventPublisher.class),scanner,coverage,sourceAccess,mock(CandidateAcceptanceService.class)); + ReflectionTestUtils.setField(releases,"testWorkflowMode",true); + governance=new SourceGovernanceService(support,imports,worker,coverage,sourceAccess,mock(CandidateAcceptanceService.class)); + ReflectionTestUtils.setField(governance,"testWorkflowMode",true); + metrics=new MetricsService(jdbc,access); + } + + @AfterEach void cleanup(){ + if(admin!=null&&database!=null&&database.matches("quality_verify_multifile_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database); + } + + @Test void mainAndTwoAppendFilesImportPublishReplaceWithdrawAndKeepHistory() throws Exception { + ConfiguredSource mainBlack=configure("main-black.xlsx","black","ADD",null,11,"main black",null); + ConfiguredSource mainBright=configure("main-bright.xlsx","bright","ADD",null,5,"main bright",null); + String initialCandidate=importSources(List.of(mainBlack.sourceId(),mainBright.sourceId()),null); + publish(initialCandidate,null,"publish-main"); + String mainRelease=currentRelease(); + Map originalMembers=members(mainRelease); + assertEquals(2,originalMembers.size()); + assertWeight(mainRelease,11); + + ConfiguredSource appendA=configure("append-a.xlsx","black","APPEND","batch-A",4,"append batch A",mainRelease); + Map retry=configureAgain(appendA,"black","APPEND","batch-A",4,"retry same file identity",mainRelease); + assertEquals(true,retry.get("reconfigured")); + assertEquals(appendA.sourceId(),sourceId(retry)); + ConfiguredSource appendB=configure("append-b.xlsx","black","APPEND","batch-B",6,"append batch B",mainRelease); + String appendCandidate=importSources(List.of(appendA.sourceId(),appendB.sourceId()),mainRelease); + assertTrue(Boolean.TRUE.equals(jdbc.queryForObject("SELECT complete FROM data_release WHERE id=?",Boolean.class,appendCandidate))); + assertEquals(0,coverage.overlapCount(appendCandidate)); + publish(appendCandidate,mainRelease,"publish-appends"); + String appendRelease=currentRelease(); + Map appendedMembers=members(appendRelease); + assertEquals(4,appendedMembers.size()); + assertEquals(originalMembers.get("black|2026|02|ALL"),appendedMembers.get("black|2026|02|ALL")); + assertEquals(originalMembers.get("bright|2026|02|ALL"),appendedMembers.get("bright|2026|02|ALL")); + assertEquals("APPEND_SNAPSHOT",sourceMode("black|2026|02|ALL|UPLOAD_batch-A")); + assertEquals("APPEND_SNAPSHOT",sourceMode("black|2026|02|ALL|UPLOAD_batch-B")); + assertWeight(appendRelease,21); + + assertThrows(ResponseStatusException.class,()->configure("same-content.xlsx","black","APPEND","batch-C",0,"duplicate bytes",appendRelease,appendB.bytes())); + AppUserPrincipal limitedActor=limitedActor(); + var unauthorized=new SourceUploadService.ConfigureRequest("unused","black",2026,2,"报废明细",1,fieldMap("black"),null,"APPEND",appendRelease,null,null,"out of range","ALL","limited-batch"); + ResponseStatusException forbidden=assertThrows(ResponseStatusException.class,()->uploads.configure(unauthorized,limitedActor)); + assertEquals(HttpStatus.FORBIDDEN,forbidden.getStatusCode()); + + Map target=targets().stream().filter(t->Objects.equals("batch-A",t.get("fileId"))).findFirst().orElseThrow(); + assertEquals("APPEND_SNAPSHOT",target.get("sourceMode")); + String aDocument=String.valueOf(target.get("documentId")),aRevision=String.valueOf(target.get("revisionId")); + ConfiguredSource correctedA=configureReplace("append-a-corrected.xlsx","black","batch-A",8,"replace batch A only",appendRelease,aDocument,aRevision,blackWorkbook(8)); + String replacementCandidate=importSources(List.of(correctedA.sourceId()),appendRelease); + publish(replacementCandidate,appendRelease,"publish-replace-a"); + String replacementRelease=currentRelease(); + Map replacedMembers=members(replacementRelease); + assertEquals(4,replacedMembers.size()); + assertEquals(appendedMembers.get("black|2026|02|ALL"),replacedMembers.get("black|2026|02|ALL")); + assertEquals(appendedMembers.get("bright|2026|02|ALL"),replacedMembers.get("bright|2026|02|ALL")); + assertEquals(appendedMembers.get("black|2026|02|ALL|UPLOAD_batch-B"),replacedMembers.get("black|2026|02|ALL|UPLOAD_batch-B")); + assertEquals(aDocument,replacedMembers.get("black|2026|02|ALL|UPLOAD_batch-A").documentId()); + assertNotEquals(aRevision,replacedMembers.get("black|2026|02|ALL|UPLOAD_batch-A").revisionId()); + assertWeight(replacementRelease,25); + assertWeight(appendRelease,21); + assertEquals(originalMembers, members(mainRelease)); + + Map requested=tx.execute(status->governance.request(new SourceGovernanceService.ChangeRequest("WITHDRAW",aDocument,null,"withdraw appended batch A","withdraw-batch-a"),adminActor)); + String requestId=String.valueOf(((Map)requested.get("request")).get("id")); + tx.executeWithoutResult(status->governance.review(requestId,new SourceGovernanceService.ReviewRequest(true,"approve batch A withdrawal",0L),adminActor)); + Map built=tx.execute(status->governance.build(requestId,new SourceGovernanceService.BuildRequest(replacementRelease,"build-withdraw-a",null),adminActor)); + String withdrawalCandidate=String.valueOf(built.get("releaseId")); + assertTrue(Boolean.TRUE.equals(jdbc.queryForObject("SELECT complete FROM data_release WHERE id=?",Boolean.class,withdrawalCandidate))); + publish(withdrawalCandidate,replacementRelease,"publish-withdraw-a"); + String withdrawalRelease=currentRelease(); + Map withdrawnMembers=members(withdrawalRelease); + assertEquals(3,withdrawnMembers.size()); + assertFalse(withdrawnMembers.containsKey("black|2026|02|ALL|UPLOAD_batch-A")); + assertEquals(replacedMembers.get("black|2026|02|ALL"),withdrawnMembers.get("black|2026|02|ALL")); + assertEquals(replacedMembers.get("bright|2026|02|ALL"),withdrawnMembers.get("bright|2026|02|ALL")); + assertEquals(replacedMembers.get("black|2026|02|ALL|UPLOAD_batch-B"),withdrawnMembers.get("black|2026|02|ALL|UPLOAD_batch-B")); + assertWeight(withdrawalRelease,17); + assertWeight(replacementRelease,25); + assertWeight(appendRelease,21); + assertEquals(originalMembers,members(mainRelease)); + + ConfiguredSource departmentAppend=configureScoped("department-append.xlsx","black","APPEND","dept-batch",7,"scope overlap regression",withdrawalRelease,"质量管理部",blackWorkbook(7)); + String scopeOverlapCandidate=importSources(List.of(departmentAppend.sourceId()),withdrawalRelease); + assertFalse(Boolean.TRUE.equals(jdbc.queryForObject("SELECT complete FROM data_release WHERE id=?",Boolean.class,scopeOverlapCandidate))); + assertTrue(coverage.overlapCount(scopeOverlapCandidate)>0); + } + + @Test void concurrentSameShaUploadsUseCurrentLockingReadsAndOnlyOneIsConfirmed() throws Exception { + byte[] bytes=blackWorkbook(11); + Map firstPreview=uploads.previewUpload(multipart("same-one.xlsx",bytes),adminActor); + Map secondPreview=uploads.previewUpload(multipart("same-two.xlsx",bytes),adminActor); + var first=request(firstPreview,"black","APPEND","concurrent-A",null,"same content A"); + var second=request(secondPreview,"black","APPEND","concurrent-B",null,"same content B"); + CountDownLatch ready=new CountDownLatch(2),start=new CountDownLatch(1); + try(var pool=Executors.newFixedThreadPool(2)){ + var a=pool.submit(()->concurrentConfigure(first,ready,start)); + var b=pool.submit(()->concurrentConfigure(second,ready,start)); + assertTrue(ready.await(10,TimeUnit.SECONDS));start.countDown(); + List outcomes=List.of(a.get(20,TimeUnit.SECONDS),b.get(20,TimeUnit.SECONDS)); + assertEquals(1,outcomes.stream().filter("CONFIRMED"::equals).count()); + assertEquals(1,outcomes.stream().filter("DUPLICATE_SHA"::equals).count()); + } + assertEquals(1,jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate WHERE status='DISCOVERED' AND resolution='APPEND_CONFIRMED'",Integer.class)); + } + + private String concurrentConfigure(SourceUploadService.ConfigureRequest request,CountDownLatch ready,CountDownLatch start){ + try{return tx.execute(status->{ + jdbc.queryForObject("SELECT COUNT(*) FROM source_candidate",Integer.class); // Establish a repeatable-read snapshot before the shared pointer lock. + ready.countDown();try{if(!start.await(10,TimeUnit.SECONDS))throw new IllegalStateException("barrier timeout");}catch(InterruptedException e){Thread.currentThread().interrupt();throw new IllegalStateException(e);} + try{uploads.configure(request,adminActor);return "CONFIRMED";}catch(ResponseStatusException duplicate){if(duplicate.getReason()!=null&&duplicate.getReason().contains("请勿重复计入"))return "DUPLICATE_SHA";throw duplicate;}catch(java.io.IOException e){throw new IllegalStateException(e);} + });}catch(RuntimeException failure){throw failure;} + } + + private ConfiguredSource configure(String filename,String stage,String mode,String fileId,int weight,String reason,String expected)throws Exception{ + return configure(filename,stage,mode,fileId,weight,reason,expected,"black".equals(stage)?blackWorkbook(weight):brightWorkbook(weight)); + } + private ConfiguredSource configure(String filename,String stage,String mode,String fileId,int weight,String reason,String expected,byte[] bytes)throws Exception{ + return configureScoped(filename,stage,mode,fileId,weight,reason,expected,"ALL",bytes); + } + private ConfiguredSource configureScoped(String filename,String stage,String mode,String fileId,int weight,String reason,String expected,String scope,byte[] bytes)throws Exception{ + Map preview=uploads.previewUpload(multipart(filename,bytes),adminActor); + var request=request(preview,stage,mode,fileId,expected,reason,scope); + Map configured=tx.execute(status->{try{return uploads.configure(request,adminActor);}catch(java.io.IOException e){throw new IllegalStateException(e);}}); + return new ConfiguredSource(((SourceCatalogService.SourceView)configured.get("source")).id(),String.valueOf(preview.get("uploadId")),bytes); + } + private ConfiguredSource configureReplace(String filename,String stage,String fileId,int weight,String reason,String expected,String targetDocument,String targetRevision,byte[] bytes)throws Exception{ + Map preview=uploads.previewUpload(multipart(filename,bytes),adminActor); + var request=new SourceUploadService.ConfigureRequest(String.valueOf(preview.get("uploadId")),stage,2026,2,"black".equals(stage)?"报废明细":"明细",1,fieldMap(stage),null,"REPLACE",expected,targetDocument,targetRevision,reason,"ALL",fileId); + Map configured=tx.execute(status->{try{return uploads.configure(request,adminActor);}catch(java.io.IOException e){throw new IllegalStateException(e);}}); + return new ConfiguredSource(((SourceCatalogService.SourceView)configured.get("source")).id(),String.valueOf(preview.get("uploadId")),bytes); + } + private Map configureAgain(ConfiguredSource source,String stage,String mode,String fileId,int weight,String reason,String expected)throws Exception{ + var request=new SourceUploadService.ConfigureRequest(source.uploadId(),stage,2026,2,"black".equals(stage)?"报废明细":"明细",1,fieldMap(stage),null,mode,expected,null,null,reason,"ALL",fileId); + return tx.execute(status->{try{return uploads.configure(request,adminActor);}catch(java.io.IOException e){throw new IllegalStateException(e);}}); + } + private SourceUploadService.ConfigureRequest request(Map preview,String stage,String mode,String fileId,String expected,String reason){ + return request(preview,stage,mode,fileId,expected,reason,"ALL"); + } + private SourceUploadService.ConfigureRequest request(Map preview,String stage,String mode,String fileId,String expected,String reason,String scope){ + return new SourceUploadService.ConfigureRequest(String.valueOf(preview.get("uploadId")),stage,2026,2,"black".equals(stage)?"报废明细":"明细",1,fieldMap(stage),null,mode,expected,null,null,reason,scope,fileId); + } + private Map fieldMap(String stage){Map result=new LinkedHashMap<>();SourceParserConfig.required(stage).forEach(header->result.put(header,header));return result;} + private String importSources(List ids,String expected){ + var run=tx.execute(status->imports.create(new ImportRunService.CreateImportRequest(ids,UUID.randomUUID().toString(),expected),adminId)); + Map result=tx.execute(status->worker.execute(run.id())); + assertEquals("COMPLETED",result.get("status"),String.valueOf(result.get("errorMessage"))); + return String.valueOf(result.get("resultReleaseId")); + } + private void publish(String releaseId,String expected,String key){ + tx.execute(status->releases.transitionInTestWorkflow(releaseId,"PUBLISH",new ReleaseService.ReleaseActionRequest(expected,key,"multifile integration acceptance"),adminActor,false)); + } + private String currentRelease(){return jdbc.query("SELECT release_id FROM current_release WHERE dataset_id='quality'",(rs,n)->rs.getString(1)).getFirst();} + private Map members(String releaseId){ + return jdbc.query("SELECT d.source_slot,d.id,m.source_revision_id,m.transform_build_id FROM release_member m JOIN source_document d ON d.id=m.source_document_id WHERE m.release_id=? ORDER BY d.source_slot",(rs,n)->Map.entry(rs.getString(1),new Member(rs.getString(2),rs.getString(3),rs.getString(4))),releaseId).stream().collect(java.util.stream.Collectors.toMap(Map.Entry::getKey,Map.Entry::getValue, (a,b)->a,LinkedHashMap::new)); + } + private String sourceMode(String slot){return jdbc.queryForObject("SELECT source_mode FROM source_document WHERE source_slot=?",String.class,slot);} + private void assertWeight(String releaseId,int expected){ + var query=new MetricsService.MetricsQuery(2026,2,null,null,null,null,null,"overall",false,releaseId,"month",Map.of()); + BigDecimal total=metrics.calculate(query,adminActor).summary().black().totalWeight(); + assertNotNull(total);assertEquals(0,new BigDecimal(expected).compareTo(total),"black-stage accumulated total for "+releaseId); + } + private List> targets(){return (List>)uploads.options(adminActor).get("targets");} + private String sourceId(Map response){return ((SourceCatalogService.SourceView)response.get("source")).id();} + private AppUserPrincipal limitedActor(){ + UUID id=UUID.randomUUID();jdbc.update("INSERT INTO app_user(id,username,display_name,password_hash,active) VALUES(?,?,?,?,TRUE)",id.toString(),"limited-upload-user","Limited upload user","fixture-hash"); + jdbc.update("INSERT INTO app_user_role(user_id,role_code) VALUES(?,'DATA_ADMIN')",id.toString()); + jdbc.update("INSERT INTO app_user_scope(user_id,scope_type,scope_value) VALUES(?,'DEPARTMENT','DEPT_A')",id.toString()); + return new AppUserPrincipal(id,"limited-upload-user","fixture-hash","Limited upload user",true,false,List.of("DATA_ADMIN"),List.of("DEPT_A"),List.of(),AppRoles.authorities(List.of("DATA_ADMIN"))); + } + private MockMultipartFile multipart(String filename,byte[] bytes){return new MockMultipartFile("file",filename,"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",bytes);} + private byte[] blackWorkbook(int weight)throws Exception{return workbook("报废明细","black",weight);} + private byte[] brightWorkbook(int weight)throws Exception{return workbook("明细","bright",weight);} + private byte[] workbook(String sheetName,String stage,int weight)throws Exception{ + List headers=SourceParserConfig.required(stage);Map values=new HashMap<>(); + if("black".equals(stage)){ + values.put("序号",Integer.toString(weight));values.put("PDT代码","P002");values.put("销售片区","中国区");values.put("事业部","质量管理部");values.put("成品规格","P");values.put("送电镀重量",Integer.toString(weight));values.put("黑片出货重量","0");values.put("责废总重量","0");values.put("料废重量","0");values.put("外废重量","0");values.put("发料地点","常规");values.put("检验备注",""); + }else{ + values.put("序号",Integer.toString(weight));values.put("PDT代码","P002");values.put("销售片区","中国区");values.put("事业部","质量管理部");values.put("成品规格","P");values.put("是否电镀来料/二等品库来料","是");values.put("合格重量",Integer.toString(weight));values.put("检包入废品库重量","1");values.put("本部报废重量","0");values.put("镀废重量合计","1");values.put("材料废重量合计","0");values.put("加工废重量合计","0");values.put("晶界扩散报废合计重量","0"); + } + try(XSSFWorkbook workbook=new XSSFWorkbook();ByteArrayOutputStream out=new ByteArrayOutputStream()){ + Sheet sheet=workbook.createSheet(sheetName);Row header=sheet.createRow(0);for(int i=0;i candidate=ArgumentCaptor.forClass(Object[].class); + verify(f.jdbc).update(contains("INSERT INTO source_candidate"),candidate.capture()); + assertEquals("black|2026|02|ALL|UPLOAD_file-1",candidate.getValue()[2]); + assertEquals("APPEND_CONFIRMED",candidate.getValue()[12]); + ArgumentCaptor protocol=ArgumentCaptor.forClass(Object[].class); + verify(f.jdbc).update(contains("INSERT INTO source_protocol"),protocol.capture()); + assertEquals("UPLOAD_file-1",protocol.getValue()[7]); + assertEquals("APPEND_SNAPSHOT",protocol.getValue()[8]); + } + + @Test void sameWorkbookHashInCurrentOverlappingScopeIsRejected() throws Exception { + Fixture f=fixture("APPEND"); + when(f.jdbc.queryForList(startsWith("SELECT d.id,d.scope_code FROM release_member"),any(Object[].class))) + .thenReturn(List.of(Map.of("id","monthly-document","scope_code","ALL"))); + + var error=assertThrows(org.springframework.web.server.ResponseStatusException.class,()->f.service.configure(f.request(null),f.actor)); + + assertEquals(org.springframework.http.HttpStatus.CONFLICT,error.getStatusCode()); + assertTrue(error.getReason().contains("请勿重复计入")); + verify(f.jdbc,never()).update(contains("INSERT INTO source_candidate"),any(Object[].class)); + } + + @Test void replacementTargetsOneAppendFileByItsStableFileIdAndRevision() throws Exception { + Fixture f=fixture("REPLACE"); + when(f.jdbc.queryForList(startsWith("SELECT d.id,r.source_revision_id,d.source_mode"),any(Object[].class))) + .thenReturn(List.of(Map.of("id","document-current","source_revision_id","revision-current","source_mode","APPEND_SNAPSHOT"))); + SourceUploadService.ConfigureRequest base=f.request(null); + var request=new SourceUploadService.ConfigureRequest(base.uploadId(),base.stage(),base.year(),base.month(),base.sheetName(),base.headerRow(),base.fieldMap(),base.templateId(),base.mode(),base.expectedCurrentReleaseId(),base.targetDocumentId(),base.targetRevisionId(),base.reason(),base.scopeCode(),"file-1"); + + f.service.configure(request,f.actor); + + ArgumentCaptor candidate=ArgumentCaptor.forClass(Object[].class); + verify(f.jdbc).update(contains("INSERT INTO source_candidate"),candidate.capture()); + assertEquals("black|2026|02|ALL|UPLOAD_file-1",candidate.getValue()[2]); + assertEquals("document-current",candidate.getValue()[17]); + assertEquals("revision-current",candidate.getValue()[18]); + } + + @Test void sameAppendFileIdCannotBeRegisteredAsAnotherNewBatch() throws Exception { + Fixture f=fixture("APPEND"); + when(f.jdbc.queryForList(startsWith("SELECT id FROM source_candidate WHERE"),any(Object[].class))) + .thenReturn(List.of(Map.of("id","prior-candidate"))); + + var error=assertThrows(org.springframework.web.server.ResponseStatusException.class,()->f.service.configure(f.request(null),f.actor)); + + assertTrue(error.getReason().contains("该fileId已登记")); + verify(f.jdbc,never()).update(contains("INSERT INTO source_candidate"),any(Object[].class)); + } + @Test void uploadOptionsDoNotExposeOrReadTemplateCatalog() throws Exception { Fixture f = fixture("ADD"); when(f.jdbc.queryForList(anyString(), any(Object[].class))).thenReturn(List.of()); @@ -134,7 +187,7 @@ class SourceUploadServiceTest { boolean replace = "REPLACE".equals(mode); return new SourceUploadService.ConfigureRequest("upload-draft", "black", 2026, 2, "工作表", 2, fieldMap, templateId, mode, "release-current", replace ? "document-current" : null, - replace ? "revision-current" : null, "测试直接字段映射", "ALL"); + replace ? "revision-current" : null, "测试直接字段映射", "ALL", "APPEND".equals(mode)?"file-1":null); } } } diff --git a/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateMaintenanceFilterTest.java b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateMaintenanceFilterTest.java new file mode 100644 index 0000000..3ba6089 --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateMaintenanceFilterTest.java @@ -0,0 +1,49 @@ +package cn.yunsheng.quality.update; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +class SystemUpdateMaintenanceFilterTest { + @Test + void blocksBusinessWritesDuringUpdateButKeepsReadsAuthAndUpdateStatusAvailable() throws Exception { + var updates = mock(SystemUpdateService.class); + when(updates.maintenanceActive()).thenReturn(true); + var filter = new SystemUpdateMaintenanceFilter(updates); + for (String method : List.of("POST", "PUT", "PATCH", "DELETE")) { + var request = new MockHttpServletRequest(method, "/api/sources/test"); + var response = new MockHttpServletResponse(); + var continued = new AtomicBoolean(); + filter.doFilter(request, response, (req, res) -> continued.set(true)); + assertFalse(continued.get()); + assertEquals(503, response.getStatus()); + assertEquals("10", response.getHeader("Retry-After")); + } + var password = new MockHttpServletResponse(); + filter.doFilter(new MockHttpServletRequest("POST", "/api/auth/change-password"), password, + (req, res) -> fail("Password changes must remain blocked during update")); + assertEquals(503, password.getStatus()); + for (var route : List.of(new String[]{"GET", "/api/sources"}, new String[]{"GET", "/api/system-update"}, + new String[]{"POST", "/api/system-update/check"}, new String[]{"POST", "/api/auth/login"}, + new String[]{"GET", "/actuator/health"})) { + var continued = new AtomicBoolean(); + filter.doFilter(new MockHttpServletRequest(route[0], route[1]), new MockHttpServletResponse(), + (req, res) -> continued.set(true)); + assertTrue(continued.get()); + } + } + + @Test + void normalAndSourceDeploymentsContinueToAllowWrites() throws Exception { + var updates = mock(SystemUpdateService.class); + when(updates.maintenanceActive()).thenReturn(false); + var continued = new AtomicBoolean(); + new SystemUpdateMaintenanceFilter(updates).doFilter(new MockHttpServletRequest("POST", "/api/sources"), + new MockHttpServletResponse(), (req, res) -> continued.set(true)); + assertTrue(continued.get()); + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateSecurityTest.java b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateSecurityTest.java new file mode 100644 index 0000000..512f7fa --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateSecurityTest.java @@ -0,0 +1,88 @@ +package cn.yunsheng.quality.update; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import cn.yunsheng.quality.config.SecurityConfig; +import cn.yunsheng.quality.governance.GovernanceAccessService; +import cn.yunsheng.quality.identity.JdbcAppUserDetailsService; +import jakarta.servlet.Filter; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.mock.web.MockHttpSession; +import org.springframework.mock.web.MockServletContext; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.web.csrf.CsrfToken; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.support.AnnotationConfigWebApplicationContext; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; + +/** Exercises the real application security chain, including CSRF and method roles. */ +class SystemUpdateSecurityTest { + @Configuration + @EnableWebMvc + @Import(SecurityConfig.class) + static class TestConfig { + @Bean JdbcAppUserDetailsService users() { return mock(JdbcAppUserDetailsService.class); } + @Bean GovernanceAccessService access() { return mock(GovernanceAccessService.class); } + @Bean SystemUpdateService updates() { + var service = mock(SystemUpdateService.class); + var result = Map.of("enabled", true, "state", "idle"); + when(service.status()).thenReturn(result); + when(service.check()).thenReturn(result); + when(service.apply()).thenReturn(result); + return service; + } + @Bean SystemUpdateController controller(SystemUpdateService updates) { return new SystemUpdateController(updates); } + } + + private MockHttpSession session(String role) { + var context = SecurityContextHolder.createEmptyContext(); + context.setAuthentication(UsernamePasswordAuthenticationToken.authenticated("test-user", null, + List.of(new SimpleGrantedAuthority("ROLE_" + role)))); + var session = new MockHttpSession(); + session.setAttribute("SPRING_SECURITY_CONTEXT", context); + return session; + } + + @Test + void requiresAdminAndCsrfForBothUpdateMutations() throws Exception { + try (var context = new AnnotationConfigWebApplicationContext()) { + context.setServletContext(new MockServletContext()); + context.register(TestConfig.class); + context.refresh(); + var mvc = MockMvcBuilders.webAppContextSetup(context) + .addFilters(context.getBean("springSecurityFilterChain", Filter.class)).build(); + mvc.perform(get("/api/system-update")).andExpect(status().isUnauthorized()); + mvc.perform(get("/api/system-update").session(session("USER"))).andExpect(status().isForbidden()); + var admin = session("SYSTEM_ADMIN"); + var initial = mvc.perform(get("/api/system-update").session(admin)).andExpect(status().isOk()).andReturn(); + var csrf = (CsrfToken) initial.getRequest().getAttribute(CsrfToken.class.getName()); + assertNotNull(csrf); + String token = csrf.getToken(); + var cookie = initial.getResponse().getCookie("XSRF-TOKEN"); + assertNotNull(cookie); + for (String path : List.of("/api/system-update/check", "/api/system-update/apply")) { + mvc.perform(post(path).session(admin)).andExpect(status().isForbidden()); + mvc.perform(post(path).session(admin).cookie(cookie).header(csrf.getHeaderName(), "bad-token")) + .andExpect(status().isForbidden()); + mvc.perform(post(path).session(session("DATA_ADMIN")).cookie(cookie).header(csrf.getHeaderName(), token)) + .andExpect(status().isForbidden()); + mvc.perform(post(path).session(admin).cookie(cookie).header(csrf.getHeaderName(), token)) + .andExpect(status().isOk()); + } + verify(context.getBean(SystemUpdateService.class), times(1)).check(); + verify(context.getBean(SystemUpdateService.class), times(1)).apply(); + } finally { + SecurityContextHolder.clearContext(); + } + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateServiceTest.java new file mode 100644 index 0000000..a7ad4c1 --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateServiceTest.java @@ -0,0 +1,63 @@ +package cn.yunsheng.quality.update; + +import static org.junit.jupiter.api.Assertions.*; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.server.ResponseStatusException; +import tools.jackson.databind.json.JsonMapper; + +class SystemUpdateServiceTest { + @Test + void sourceDeploymentIsDisabledEvenIfFlagIsAccidentallyEnabled() { + var service = new SystemUpdateService(JsonMapper.builder().build(), "", true); + assertEquals(false, service.status().get("enabled")); + assertThrows(ResponseStatusException.class, service::check); + assertThrows(ResponseStatusException.class, service::apply); + } + + @Test + void updaterReceiptsNeverExposePrivateConfigurationOrPaths() throws Exception { + var result = SystemUpdateService.safeResponse(Map.of("enabled", true, "state", "available", + "version", "2026.11.04", "rootPassword", "secret", "backupPath", "C:/private")); + assertEquals(Map.of("enabled", true, "state", "available", "version", "2026.11.04"), result); + assertThrows(IOException.class, () -> SystemUpdateService.safeResponse(Map.of("enabled", "true", "state", "idle"))); + assertThrows(IOException.class, () -> SystemUpdateService.safeResponse(Map.of("enabled", true, "state", Map.of()))); + } + + @Test + void allUpdateRoutesRequireSystemAdministrator() { + assertEquals("hasRole('SYSTEM_ADMIN')", SystemUpdateController.class.getAnnotation(PreAuthorize.class).value()); + } + + @Test + void maintenanceStartsBeforeBackupAndPersistsAcrossAnInterruptedInstall(@TempDir Path root) throws Exception { + var service = new SystemUpdateService(JsonMapper.builder().build(), root.toString(), false); + // Exercise host-independent metadata handling without changing the + // production Windows/runtime enablement guard. + var enabled = SystemUpdateService.class.getDeclaredField("enabled"); + enabled.setAccessible(true); + enabled.setBoolean(service, true); + assertFalse(service.maintenanceActive()); + Path directory = Files.createDirectories(root.resolve(".runtime/update")); + for (String phase : new String[]{"queued", "downloading", "completed", "failed"}) { + Files.writeString(directory.resolve("status.json"), "{\"state\":\"" + phase + "\"}"); + assertFalse(service.maintenanceActive(), phase); + } + for (String phase : new String[]{"backingUp", "installing", "restarting", "rollingBack"}) { + Files.writeString(directory.resolve("status.json"), "{\"state\":\"" + phase + "\"}"); + assertTrue(service.maintenanceActive(), phase); + } + Files.writeString(directory.resolve("status.json"), "{\"state\":\"failed\"}"); + Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"installing\"}"); + assertTrue(service.maintenanceActive()); + Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"rolledBack\"}"); + assertFalse(service.maintenanceActive()); + Files.writeString(directory.resolve("status.json"), "broken JSON"); + assertTrue(service.maintenanceActive()); + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateSpringLifecycleTest.java b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateSpringLifecycleTest.java new file mode 100644 index 0000000..334cebd --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateSpringLifecycleTest.java @@ -0,0 +1,79 @@ +package cn.yunsheng.quality.update; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import cn.yunsheng.quality.analysis.AnalysisReportService; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.core.env.MapPropertySource; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.ScheduledAnnotationBeanPostProcessor; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +/** Actual Spring event/scheduling registration, including transaction proxies. */ +class SystemUpdateSpringLifecycleTest { + private static final AtomicBoolean MAINTENANCE = new AtomicBoolean(); + private static AnalysisReportService originalReports; + + @Configuration + @EnableScheduling + @EnableTransactionManagement + static class Config { + @Bean static SystemUpdateStartupBarrier barrier(ObjectProvider updates, Environment environment) { + return new SystemUpdateStartupBarrier(updates, environment); + } + @Bean SystemUpdateService updates() { + var updates = mock(SystemUpdateService.class); + when(updates.maintenanceActive()).thenAnswer(call -> MAINTENANCE.get()); + return updates; + } + @Bean AnalysisReportService reports() { originalReports = mock(AnalysisReportService.class); return originalReports; } + @Bean PlatformTransactionManager transactionManager() { return mock(PlatformTransactionManager.class); } + } + + @Test + void actualSpringReadyEventAndScheduledTaskRemainPausedUntilCommit() { + // Initialize Mockito before temporarily simulating the target OS flag; + // no external process or filesystem/runtime guard is changed. + mock(SystemUpdateService.class); + String originalOs = System.getProperty("os.name"); + MAINTENANCE.set(true); + try (var context = new AnnotationConfigApplicationContext()) { + context.getEnvironment().getPropertySources().addFirst(new MapPropertySource("update-test", + Map.of("QUALITY_UPDATE_ENABLED", "true"))); + context.register(Config.class); + try { + System.setProperty("os.name", "Windows 11"); + context.refresh(); + } finally { + System.setProperty("os.name", originalOs); + } + var event = new ApplicationReadyEvent(new SpringApplication(Config.class), new String[0], context, Duration.ZERO); + context.publishEvent(event); + verify(originalReports, never()).recoverPendingReports(any(ApplicationReadyEvent.class)); + var schedules = context.getBean(ScheduledAnnotationBeanPostProcessor.class).getScheduledTasks(); + assertEquals(1, schedules.size()); + var task = schedules.iterator().next().getTask().getRunnable(); + task.run(); + verify(originalReports, never()).scheduledReports(); + MAINTENANCE.set(false); + context.getBean(SystemUpdateStartupBarrier.class).drain(); + verify(originalReports, times(1)).recoverPendingReports(event); + task.run(); + verify(originalReports, times(1)).scheduledReports(); + } finally { + System.setProperty("os.name", originalOs); + MAINTENANCE.set(false); + } + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateStartupBarrierTest.java b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateStartupBarrierTest.java new file mode 100644 index 0000000..6a172df --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/update/SystemUpdateStartupBarrierTest.java @@ -0,0 +1,58 @@ +package cn.yunsheng.quality.update; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; +import cn.yunsheng.quality.analysis.AnalysisTurnService; +import cn.yunsheng.quality.ingestion.ImportRecoveryRunner; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.core.env.Environment; +import org.springframework.scheduling.annotation.Scheduled; + +class SystemUpdateStartupBarrierTest { + public static class ScheduledProbe { + int ticks; + @Scheduled(fixedDelay = 1000) public void tick() { ticks++; } + public void ordinaryRead() { ticks += 10; } + } + + @Test + @SuppressWarnings("unchecked") + void defersBusinessStartupExactlyOnceUntilCommitAndSkipsScheduledWrites() { + var maintenance = new AtomicBoolean(true); + var updates = mock(SystemUpdateService.class); + when(updates.maintenanceActive()).thenAnswer(call -> maintenance.get()); + var provider = (ObjectProvider) mock(ObjectProvider.class); + when(provider.getObject()).thenReturn(updates); + try (var scope = new BarrierScope(new SystemUpdateStartupBarrier(provider, mock(Environment.class)))) { + var barrier = scope.barrier; + var imports = mock(ImportRecoveryRunner.class); + var importProxy = (ImportRecoveryRunner) barrier.wrap(imports, ImportRecoveryRunner.class); + var turns = mock(AnalysisTurnService.class); + var turnProxy = (AnalysisTurnService) barrier.wrap(turns, AnalysisTurnService.class); + var raw = new ScheduledProbe(); + var scheduled = (ScheduledProbe) barrier.wrap(raw, ScheduledProbe.class); + importProxy.run(null); + turnProxy.recoverQueuedTurns(); + scheduled.tick(); + scheduled.ordinaryRead(); + assertEquals(10, raw.ticks); + verify(imports, never()).run(null); + verify(turns, never()).recoverQueuedTurns(); + barrier.drain(); + verify(imports, never()).run(null); + maintenance.set(false); + barrier.drain(); + barrier.drain(); + verify(imports, times(1)).run(null); + verify(turns, times(1)).recoverQueuedTurns(); + scheduled.tick(); + assertEquals(11, raw.ticks); + } + } + + private record BarrierScope(SystemUpdateStartupBarrier barrier) implements AutoCloseable { + @Override public void close() { barrier.close(); } + } +} diff --git a/backend/src/test/java/cn/yunsheng/quality/workflow/DataWorkflowServiceTest.java b/backend/src/test/java/cn/yunsheng/quality/workflow/DataWorkflowServiceTest.java index 25fae3f..512a9de 100644 --- a/backend/src/test/java/cn/yunsheng/quality/workflow/DataWorkflowServiceTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/workflow/DataWorkflowServiceTest.java @@ -62,6 +62,41 @@ class DataWorkflowServiceTest { verifyNoMoreInteractions(jdbc); } + @Test + void candidateDataErrorsAreNotReportedAsVersionMismatches() { + when(jdbc.queryForMap(contains("SELECT mapping_version"), eq("candidate"))) + .thenReturn(Map.of("mappingVersion", "mapping", "ruleVersion", "rule")); + when(jdbc.queryForObject(contains("rejected_count"), eq(Integer.class), any(Object[].class))).thenReturn(2); + when(jdbc.queryForList(contains("FROM import_issue"), any(Object[].class))).thenReturn(List.of( + Map.of("issueCode", "INVALID_NUMBER", "fieldName", "合格重量", "issueCount", 2L))); + RuntimeException error = assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(service, + "assertWorkflowCandidate", "candidate", "mapping", "rule")); + assertTrue(error.getMessage().contains("合格重量"), error.getMessage()); + assertTrue(error.getMessage().contains("数值"), error.getMessage()); + assertFalse(error.getMessage().contains("未使用本次冻结版本")); + } + + @Test + void incompleteCandidateBuildHasItsOwnFailureReason() { + when(jdbc.queryForMap(contains("SELECT mapping_version"), eq("candidate"))) + .thenReturn(Map.of("mappingVersion", "mapping", "ruleVersion", "rule")); + when(jdbc.queryForObject(contains("b.status<>'COMPLETED'"), eq(Integer.class), any(Object[].class))).thenReturn(1); + RuntimeException error = assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(service, + "assertWorkflowCandidate", "candidate", "mapping", "rule")); + assertTrue(error.getMessage().contains("尚未完成"), error.getMessage()); + assertFalse(error.getMessage().contains("未使用本次冻结版本")); + } + + @Test + void actualCandidateVersionMismatchStillBlocksPublication() { + when(jdbc.queryForMap(contains("SELECT mapping_version"), eq("candidate"))) + .thenReturn(Map.of("mappingVersion", "mapping", "ruleVersion", "rule")); + when(jdbc.queryForObject(contains("b.mapping_version<>? OR b.rule_version<>?"), eq(Integer.class), any(Object[].class))).thenReturn(1); + RuntimeException error = assertThrows(RuntimeException.class, () -> ReflectionTestUtils.invokeMethod(service, + "assertWorkflowCandidate", "candidate", "mapping", "rule")); + assertTrue(error.getMessage().contains("未使用本次冻结版本")); + } + @Test void disabledModeKeepsTheLegacyPathAndDoesNotQueueMalformedRequests() { ReflectionTestUtils.setField(service, "testMode", false); @@ -72,55 +107,35 @@ class DataWorkflowServiceTest { } @Test - void writeActionsStillRequirePublisherEvenWhenApprovalsAreSkipped() { + void usersCannotQueueDataWrites() { ResponseStatusException error = assertThrows(ResponseStatusException.class, - () -> service.recalculate(user("DATA_MAINTAINER"))); + () -> service.recalculate(user("USER"))); assertEquals(HttpStatus.FORBIDDEN, error.getStatusCode()); verify(access).requireUserActivePolicy(any()); verifyNoInteractions(jdbc); } @Test - void sourceCapabilitiesReflectManagerAndPublisherIntersection() { - Map maintainer = DataWorkflowService.capabilities(true, user("DATA_MAINTAINER")); - assertEquals(true, maintainer.get("canDeleteSource")); - assertEquals(false, maintainer.get("canDeleteCurrentSource")); - assertEquals(false, maintainer.get("canRestoreSource")); - assertEquals(false, maintainer.get("canApplyMapping")); - assertEquals(false, maintainer.get("canImport")); - - Map publisher = DataWorkflowService.capabilities(true, user("DATA_PUBLISHER")); - assertEquals(false, publisher.get("canDeleteSource")); - assertEquals(false, publisher.get("canRestoreSource")); - assertEquals(false, publisher.get("canApplyMapping")); - assertEquals(false, publisher.get("canApplyRule")); - assertEquals(false, publisher.get("canImport")); - assertEquals(true, publisher.get("canRecalculate")); - - Map publisherAndDataMaintainer = DataWorkflowService.capabilities(true, - user("DATA_PUBLISHER", "DATA_MAINTAINER")); - assertEquals(true, publisherAndDataMaintainer.get("canApplyMapping")); - assertEquals(false, publisherAndDataMaintainer.get("canApplyRule")); - assertEquals(true, publisherAndDataMaintainer.get("canImport")); - - Map publisherAndRuleMaintainer = DataWorkflowService.capabilities(true, - user("DATA_PUBLISHER", "RULE_MAINTAINER")); - assertEquals(true, publisherAndRuleMaintainer.get("canApplyMapping")); - assertEquals(true, publisherAndRuleMaintainer.get("canApplyRule")); - assertEquals(false, publisherAndRuleMaintainer.get("canImport")); - - Map rulesMaintainer = DataWorkflowService.capabilities(true, user("RULE_MAINTAINER")); - assertEquals(false, rulesMaintainer.get("canApply")); - assertEquals(false, rulesMaintainer.get("canApplyRule")); - assertEquals(false, rulesMaintainer.get("canDeleteSource")); - assertEquals(false, rulesMaintainer.get("canRestoreSource")); - + void threeRolesInheritDataCapabilitiesButKeepRuleEditingRestricted() { + Map reader = DataWorkflowService.capabilities(true, user("USER")); + for (String key : List.of("canApply", "canImport", "canPublish", "canDeleteSource", "canRestoreSource")) + assertEquals(false, reader.get(key), key); + Map dataAdmin = DataWorkflowService.capabilities(true, user("DATA_ADMIN")); + for (String key : List.of("canApplyMapping", "canImport", "canPublish", "canRecalculate", "canDeleteSource", "canDeleteCurrentSource", "canRestoreSource")) + assertEquals(true, dataAdmin.get(key), key); + assertEquals(false, dataAdmin.get("canApplyRule")); + assertEquals(true, DataWorkflowService.capabilities(true, admin()).get("canApplyRule")); Map disabled = DataWorkflowService.capabilities(false, admin()); - assertEquals(false, disabled.get("canApply")); - assertEquals(false, disabled.get("canApplyMapping")); - assertEquals(false, disabled.get("canApplyRule")); - assertEquals(false, disabled.get("canImport")); - assertEquals(false, disabled.get("canPublish")); + for (String key : List.of("canApply", "canApplyMapping", "canApplyRule", "canImport", "canPublish")) + assertEquals(false, disabled.get(key), key); + } + + @Test + void dataAdministratorCannotQueueGlobalRuleChanges() { + var error = assertThrows(ResponseStatusException.class, () -> service.apply( + new DataWorkflowController.ApplyRequest("rule", "rules", Map.of(), null), user("DATA_ADMIN"))); + assertEquals(HttpStatus.FORBIDDEN, error.getStatusCode()); + verifyNoInteractions(jdbc); } @Test @@ -192,13 +207,13 @@ class DataWorkflowServiceTest { } @Test - void sourceMutationEndpointsRequireTheUnderlyingManagerAndPublisherRoles() { + void usersCannotDeleteOrRestoreDataSources() { ResponseStatusException deleteError = assertThrows(ResponseStatusException.class, - () -> service.deleteSource("candidate", user("DATA_PUBLISHER"))); + () -> service.deleteSource("candidate", user("USER"))); assertEquals(HttpStatus.FORBIDDEN, deleteError.getStatusCode()); ResponseStatusException restoreError = assertThrows(ResponseStatusException.class, - () -> service.restoreSource("candidate", user("DATA_MAINTAINER"))); + () -> service.restoreSource("candidate", user("USER"))); assertEquals(HttpStatus.FORBIDDEN, restoreError.getStatusCode()); verifyNoInteractions(jdbc); } @@ -333,18 +348,18 @@ class DataWorkflowServiceTest { } @Test - void batchDeletePreflightsAllSourcesAndEnforcesCurrentSourcePublishingPermission() { + void dataAdministratorPreflightsAllSourcesBeforeQueueingBatchDeletion() { when(jdbc.query(eq("SELECT release_id FROM current_release WHERE dataset_id=?"), any(RowMapper.class), eq("quality"))) .thenReturn(List.of("current")); when(deletion.preview(eq("a"), any())).thenReturn(Map.of("hidden", false, "currentUsed", false)); when(deletion.preview(eq("b"), any())).thenReturn(Map.of("hidden", false, "currentUsed", true)); - var error = assertThrows(ResponseStatusException.class, () -> service.deleteSources( - new DataWorkflowController.DeleteSourcesRequest(List.of("a", "b"), "current"), user("DATA_MAINTAINER"))); - assertEquals(HttpStatus.FORBIDDEN, error.getStatusCode()); + var queued = service.deleteSources( + new DataWorkflowController.DeleteSourcesRequest(List.of("a", "b"), "current"), user("DATA_ADMIN")); + assertEquals("DELETE_SOURCES", queued.get("operationType")); verify(deletion).preview(eq("a"), any()); verify(deletion).preview(eq("b"), any()); verify(deletion, never()).delete(anyString(), any(), any()); - verify(jdbc, never()).update(anyString(), any(Object[].class)); + verify(jdbc).update(anyString(), any(Object[].class)); } @Test @@ -357,13 +372,13 @@ class DataWorkflowServiceTest { @Test void workflowWriteEndpointsKeepRoleGuards() throws Exception { - assertPreAuthorize("apply", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); - assertPreAuthorize("recalculate", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); - assertPreAuthorize("importSources", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); - assertPreAuthorize("publish", "SYSTEM_ADMIN", "DATA_PUBLISHER"); - assertPreAuthorize("deleteSource", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); - assertPreAuthorize("deleteSources", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); - assertPreAuthorize("restoreSource", "SYSTEM_ADMIN", "DATA_MAINTAINER", "DATA_PUBLISHER"); + assertPreAuthorize("apply", "SYSTEM_ADMIN", "DATA_ADMIN"); + assertPreAuthorize("recalculate", "SYSTEM_ADMIN", "DATA_ADMIN"); + assertPreAuthorize("importSources", "SYSTEM_ADMIN", "DATA_ADMIN"); + assertPreAuthorize("publish", "SYSTEM_ADMIN", "DATA_ADMIN"); + assertPreAuthorize("deleteSource", "SYSTEM_ADMIN", "DATA_ADMIN"); + assertPreAuthorize("deleteSources", "SYSTEM_ADMIN", "DATA_ADMIN"); + assertPreAuthorize("restoreSource", "SYSTEM_ADMIN", "DATA_ADMIN"); } private void assertPreAuthorize(String method, String... roles) throws Exception { diff --git a/deploy/offline/README.md b/deploy/offline/README.md new file mode 100644 index 0000000..88492e3 --- /dev/null +++ b/deploy/offline/README.md @@ -0,0 +1,36 @@ +# 质量中心离线部署 + +此目录是 Windows / Linux 原生离线包的打包与启动工具。编译后的前端、Spring Boot fat JAR、Java 21、MySQL 8.4、Python 3.11.15 和固定版本的 AI 依赖全部随包交付;目标机不执行 pip、pnpm、Maven 或联网下载。不需要 Docker。 + +Windows 推荐将离线包解压到 `%LOCALAPPDATA%\QualityCenter`,或本机固定 NTFS 磁盘上更短的当前用户可写目录。启动器会读取包内 `manifest.json`,要求部署根路径长度加最长相对文件路径不超过 248 个字符;若用户名较长导致 `%LOCALAPPDATA%\QualityCenter` 超限,请选择更短的用户可写目录。请用当前用户运行 `windows-start.ps1`;服务进程在当前用户会话中启动,运行环境随包提供,不安装系统组件、不注册全局服务,也不修改 Windows 防火墙。网页服务会监听内网网卡并显示本机及内网访问链接;MySQL 和后端接口只监听 `127.0.0.1`。其他设备能否访问网页取决于系统现有防火墙和网络策略,需要放行时由用户或管理员按组织策略处理。 + +Linux 默认只允许本机访问,使用 `./start.sh --lan` 开启内网监听;已有防火墙策略由部署者按实际网段配置。使用普通用户启动,不能以 root 运行 MySQL。 + +`build_bundle.py --work output/offline-20261104` 使用隔离构建目录中的 JAR / 前端产物及已下载的官方发行文件组装包。`downloads/assets.json` 记录来源及摘要,`requirements-*.lock` 固定各平台依赖,`wheels-*` 是已通过 pip 哈希验证的平台 wheel。该命令不会读取本机数据库、用户配置、API 密钥或 AI 会话。 + +为保留本机运行环境,跨平台适配只应用在包内的 Python 脚本副本上;原有 macOS 脚本不被改写。适配包括 Windows 的文件锁、进程创建时间校验和 NTFS 继承权限,以及随包 Python / nao-core 的相对位置。 + +包内 `manifest.json` 记录每个只读文件的 SHA-256 和符号链接目标。首次及后续启动都会检查文件完整性和平台原生程序;运行数据单独保存在 `.runtime/` 和 `runtimedata/`。初始管理员密码在目标机随机生成,不使用公共默认密码。 + +包内不包含业务数据库、工作簿、现有用户、模型密钥或本机私有配置。全离线时可使用常规查询、导入和分析功能;AI 调用在线模型时需要网络,或自行配置可达的内网模型服务。代码执行沙箱和开发测试工具不在业务部署范围内。 + +不能在服务运行时移动安装目录。迁移业务数据应使用逻辑备份与对应版本的恢复流程,不得直接将 macOS 的 MySQL 数据目录复制到其他系统。 + +## Windows 增量更新包(维护人员) + +离线签名工具读取 Windows x64 部署包的 `manifest.json`,只打包 `update_manager.program_path` 认可的完整程序清单,并使用包内运行环境和 wheel 信息计算运行时指纹。私有配置、业务数据、迁移文件、启动引导和密钥不会进入更新包。普通发布的序号必须大于部署包当前的 `updateSequence`。首次通道种子可显式加 `--allow-baseline`,但版本和序号必须分别与 `updateVersion`、`updateSequence` 完全相同;低序号始终拒绝。私钥必须是与包内更新公钥匹配的 Ed25519 PEM 文件,并保存在部署包及产物目录之外。 + +```sh +python3 deploy/offline/publish_update.py \ + --root /path/to/QualityCenter-Windows-x64 \ + --version 2026.11.04.2 \ + --sequence 2 \ + --private-key /secure/path/update-private-key.pem \ + --output-dir /path/to/new-update-output +``` + +若需要为 `updateVersion=2026.11.04.1`、`updateSequence=1` 的原始包签署首个通道基线,使用与该包完全相同的版本和序号,并显式传入 `--allow-baseline`。不要修改包内 manifest 来绕过检查。 + +工具只在本地产生 `quality-update-.zip` 和签名 `latest.json`,不会上传或发布文件。每个版本请使用新的输出目录;发布到更新服务器由维护人员另行完成。 + +参考:[Java 21 官方发行](https://adoptium.net/temurin/releases?version=21)、[MySQL Windows 安装要求](https://dev.mysql.com/doc/refman/8.4/en/windows-installation.html)、[MySQL Linux 二进制安装](https://dev.mysql.com/doc/refman/8.4/en/binary-installation.html)、[Python standalone](https://github.com/astral-sh/python-build-standalone)、[nao-core 0.3.26](https://pypi.org/project/nao-core/0.3.26/)、[Microsoft 运行库](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist/)。 diff --git a/deploy/offline/build_bundle.py b/deploy/offline/build_bundle.py new file mode 100644 index 0000000..63a8a12 --- /dev/null +++ b/deploy/offline/build_bundle.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Assemble native offline packages from reviewed builds and verified downloads.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import io +import json +from pathlib import Path +import re +import shutil +import stat +import subprocess +import tarfile +import tempfile +import zipfile + +ROOT = Path(__file__).resolve().parents[2] +TEMPLATES = Path(__file__).resolve().parent + + +def digest(path, algorithm='sha256'): + with path.open('rb') as stream: + return hashlib.file_digest(stream, algorithm).hexdigest() + + +def extract(archive, target): + target.mkdir(parents=True) + with tempfile.TemporaryDirectory(dir=target.parent) as directory: + stage = Path(directory) + if archive.suffix == '.zip': + with zipfile.ZipFile(archive) as source: + source.extractall(stage) + else: + with tarfile.open(archive) as source: + # Terminfo contains case-distinct names. They alias on a Mac's + # default filesystem and are unused by this noninteractive service. + members = [member for member in source.getmembers() + if '/share/terminfo/' not in member.name and not member.name.endswith('/share/terminfo')] + source.extractall(stage, members=members, filter='data') + roots = list(stage.iterdir()) + if len(roots) != 1 or not roots[0].is_dir(): + raise RuntimeError('Expected one archive root: ' + archive.name) + shutil.copytree(roots[0], target, symlinks=True, dirs_exist_ok=True) + + +def extract_deb(archive, target): + raw = archive.read_bytes() + if raw[:8] != b'!\n': + raise RuntimeError('Invalid Debian archive') + position = 8 + while position + 60 <= len(raw): + header = raw[position:position + 60] + name = header[:16].decode().strip().rstrip('/') + length = int(header[48:58].decode().strip()) + position += 60 + data = raw[position:position + length] + position += length + length % 2 + if name.startswith('data.tar'): + with tempfile.TemporaryDirectory(dir=target.parent) as directory: + if name.endswith('.zst'): + import zstandard + with zstandard.ZstdDecompressor().stream_reader(io.BytesIO(data)) as reader: + with tarfile.open(fileobj=reader, mode='r|') as source: + source.extractall(directory, filter='data') + else: + with tarfile.open(fileobj=io.BytesIO(data)) as source: + source.extractall(directory, filter='data') + for path in Path(directory).rglob('*'): + if '.so' in path.name and (path.is_file() or path.is_symlink()): + dest = target / path.name + if path.is_symlink(): + dest.symlink_to(Path(path.readlink()).name) + else: + shutil.copy2(path, dest) + docs = Path(directory) / 'usr/share/doc' + if docs.exists(): + shutil.copytree(docs, target / 'licenses', dirs_exist_ok=True, symlinks=True) + return + raise RuntimeError('No data member in ' + archive.name) + + +def install_wheel(wheel, python_root, windows): + site = python_root / ('Lib/site-packages' if windows else 'lib/python3.11/site-packages') + site.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(wheel) as source: + for member in source.infolist(): + if member.is_dir(): + continue + relative = Path(member.filename) + if relative.is_absolute() or '..' in relative.parts: + raise RuntimeError('Wheel path escape') + target = site / relative + if relative.parts[0].endswith('.data'): + scheme = relative.parts[1] + prefix = {'purelib': site, 'platlib': site, 'data': python_root, + 'scripts': python_root / ('Scripts' if windows else 'bin'), + 'headers': python_root / 'include'}[scheme] + target = prefix.joinpath(*relative.parts[2:]) + target.parent.mkdir(parents=True, exist_ok=True) + with source.open(member) as reader, target.open('wb') as writer: + shutil.copyfileobj(reader, writer) + mode = member.external_attr >> 16 + target.chmod((mode & 0o777) or 0o644) + + +def adapt_runtime(source): + def replace(old, new, count=None): + nonlocal source + actual = source.count(old) + if not actual or (count is not None and actual != count): + raise RuntimeError('Runtime source changed; inspect the portability patch: ' + old) + source = source.replace(old, new) + + replace('import fcntl\n', 'import runtime_platform as _host\nfrom runtime_platform import fcntl\n', 1) + replace('PYTHON = VENV / "bin" / "python"', + 'PYTHON = Path(os.environ.get("QUALITY_ASSISTANT_PYTHON", str(VENV / "bin" / "python")))', 1) + old = 'VENV / "lib" / "python3.11" / "site-packages" / "nao_core"' + replace(old, 'Path(os.environ.get("QUALITY_ASSISTANT_PACKAGE", str(' + old + ')))', 2) + replace('/ "nao-chat-server"', '/ ("nao-chat-server.exe" if os.name == "nt" else "nao-chat-server")', 1) + source, count = re.subn(r'\b(info|directory_info)\.st_mode & 0o077', + lambda match: '_host.private_bits(' + match[1] + '.st_mode)', source) + if count != 3: + raise RuntimeError('Runtime private-permission checks changed; inspect before packaging') + replace('os.fchmod(', '_host.fchmod(') + replace('os.getpgid(pid) != pid', 'not _host.group_leader(pid)', 1) + replace('os.killpg(', '_host.killpg(') + replace('signal.SIGKILL', 'getattr(signal, "SIGKILL", 9)') + replace('def _ps_value(pid: int, field: str) -> str | None:\n', + 'def _ps_value(pid: int, field: str) -> str | None:\n' + ' if os.name == "nt":\n return _host.ps_value(pid, field)\n', 1) + compile(source, 'nao_quality_runtime.py', 'exec') + return source + + +def adapt_script(name, source): + if name == 'nao_quality_runtime.py': + return adapt_runtime(source) + if name == 'analysis_visual_evidence.py': + source = source.replace('import stat\n', 'import stat\nimport runtime_platform as _host\n', 1) + source = source.replace('os.O_NOFOLLOW', 'getattr(os, "O_NOFOLLOW", 0)') + source = source.replace('os.O_NONBLOCK', 'getattr(os, "O_NONBLOCK", 0)') + source = source.replace('before.st_mode & 0o077', '_host.private_bits(before.st_mode)') + source = source.replace('before.st_uid != os.getuid()', '_host.owner_mismatch(before)') + source = source.replace(' fd = os.open(path,', + ' if path.is_symlink():\n raise ValueError("invalid evidence link")\n fd = os.open(path,', 1) + elif name == 'nao_evidence_finalizer.py': + source = source.replace('import copy\n', 'import copy\nimport runtime_platform as _host\n', 1) + source = source.replace('path.stat().st_mode & 0o077', '_host.private_bits(path.stat().st_mode)') + compile(source, name, 'exec') + return source + + +def windows_local_runtime(root): + """Use the signed vendor JRE's redistributable runtime; never install globally.""" + libraries = ('msvcp140.dll', 'vcruntime140.dll', 'vcruntime140_1.dll') + for name in libraries: + source = root / 'runtimes/java/bin' / name + if not source.is_file(): + raise RuntimeError('The official JRE is missing an app-local Microsoft runtime: ' + name) + for folder in ('runtimes/mysql/bin', 'runtimes/mysql/lib'): + shutil.copy2(source, root / folder / name) + mysql = root / 'runtimes/mysql' + # Debug-only tools depend on non-redistributable Microsoft debug runtimes. + for path in sorted(mysql.rglob('*'), reverse=True): + if path.is_file() and (any(part.lower() == 'debug' for part in path.relative_to(mysql).parts) + or '-debug.' in path.name.lower()): + path.unlink() + elif path.is_dir() and path.name.lower() == 'debug': + shutil.rmtree(path) + return {'name': 'app-local-msvc', 'os': 'windows', 'source': 'bundled-official-temurin-jre', + 'libraries': {name: digest(root / 'runtimes/java/bin' / name) for name in libraries}} + + +def payload(root, work, osname, assets): + windows = osname == 'windows' + app = root / 'app' + app.mkdir(parents=True) + jars = list((work / 'source/backend/target').glob('*.jar')) + if len(jars) != 1: + raise RuntimeError('Expected one freshly built backend JAR') + with zipfile.ZipFile(jars[0]) as archive: + if 'BOOT-INF/classes/verification/original-workbook-acceptance.json' in archive.namelist(): + raise RuntimeError('Exclude the private workbook acceptance input before building a distributable JAR') + shutil.copy2(jars[0], app / 'backend.jar') + shutil.copytree(work / 'source/web/dist', app / 'web') + for name in ('java', 'mysql', 'python'): + kind = 'jre' if name == 'java' else name + item = next(a for a in assets if a['name'] == kind and a['os'] == osname) + archive = work / 'downloads' / item['file'] + if digest(archive) != item['sha256']: + raise RuntimeError('Download checksum mismatch: ' + item['file']) + extract(archive, root / 'runtimes' / name) + wheels = sorted((work / ('wheels-' + osname)).glob('*.whl')) + if len(wheels) < 60: + raise RuntimeError('The offline Python dependency set is incomplete') + lock_text = (work / f'requirements-{osname}.lock').read_text() + for wheel in wheels: + if '--hash=sha256:' + digest(wheel) not in lock_text: + raise RuntimeError('Wheel absent from hash lock: ' + wheel.name) + install_wheel(wheel, root / 'runtimes/python', windows) + scripts = root / 'scripts' + scripts.mkdir() + for pattern in ('nao_*.py', 'analysis_visual*.py'): + for path in (ROOT / 'scripts').glob(pattern): + (scripts / path.name).write_text(adapt_script(path.name, path.read_text()), encoding='utf-8') + for name in ('dbgpt_trial_data.py', 'local_api.py', 'project.py'): + shutil.copy2(ROOT / 'scripts' / name, scripts / name) + shutil.copy2(TEMPLATES / 'runtime_platform.py', scripts / 'runtime_platform.py') + policy = root / 'backend/src/main/resources/analysis-runtime-policy.json' + policy.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / 'backend/src/main/resources/analysis-runtime-policy.json', policy) + shutil.copytree(ROOT / 'ai/nao-quality', root / 'ai/nao-quality', ignore=shutil.ignore_patterns('.DS_Store')) + deploy = root / 'deploy/offline' + deploy.mkdir(parents=True) + for name in ('control.py', 'gateway.py', 'runtime_platform.py', 'migration_common.py', + 'restore_migration.py', 'update_manager.py', 'update-public-key.pem', + 'windows-start.ps1', 'windows-prepare.ps1', 'README.md'): + if not windows and name.endswith('.ps1'): + continue + source = TEMPLATES / name + if name.endswith('.ps1'): + (deploy / name).write_text(source.read_text(), encoding='utf-8-sig') + else: + shutil.copy2(source, deploy / name) + shutil.copy2(work / f'requirements-{osname}.lock', root / 'requirements.lock') + if windows: + local_runtime = windows_local_runtime(root) + shutil.copy2(root / 'runtimes/python/python.exe', root / 'runtimes/python/python3.exe') + bootstrap = root / 'deploy/bootstrap' + bootstrap.mkdir() + shutil.copy2(TEMPLATES / 'update_bootstrap.py', bootstrap / 'update-recovery.py') + for name in ('windows-start.ps1', 'windows-prepare.ps1'): + (bootstrap / name).write_text((TEMPLATES / name).read_text(), encoding='utf-8-sig') + (root / '一键部署启动.cmd').write_bytes( + b'@echo off\r\nchcp 65001 >nul\r\npowershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0deploy\\bootstrap\\windows-start.ps1"\r\n' + b'if errorlevel 1 (echo Startup failed. See the message above.)\r\npause\r\n') + for name, action in (('停止服务.cmd', 'stop'), ('查看状态.cmd', 'status'), ('检查环境.cmd', 'verify')): + (root / name).write_bytes(('@echo off\r\nchcp 65001 >nul\r\nset PYTHONUTF8=1\r\nset PYTHONIOENCODING=utf-8\r\n' + '"%~dp0runtimes\\python\\python.exe" "%~dp0deploy\\offline\\control.py" ' + action + '\r\npause\r\n').encode()) + else: + libs = root / 'runtimes/linux-libs' + libs.mkdir() + for item in assets: + if item.get('kind') == 'deb': + archive = work / 'downloads' / item['file'] + if digest(archive) != item['sha256']: + raise RuntimeError('Debian library checksum mismatch') + extract_deb(archive, libs) + for name, action in (('start.sh', 'start'), ('stop.sh', 'stop'), ('status.sh', 'status'), ('verify.sh', 'verify')): + (root / name).write_text('#!/bin/sh\nset -eu\nROOT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)\n' + 'if [ "$(id -u)" -eq 0 ]; then echo "Run as a normal user, not root." >&2; exit 1; fi\n' + 'export PYTHONUTF8=1 PYTHONIOENCODING=utf-8 PYTHONDONTWRITEBYTECODE=1\n' + 'export LD_LIBRARY_PATH="$ROOT/runtimes/linux-libs:$ROOT/runtimes/mysql/lib:$ROOT/runtimes/mysql/lib/private"\n' + 'exec "$ROOT/runtimes/python/bin/python3" "$ROOT/deploy/offline/control.py" ' + action + ' "$@"\n') + (root / name).chmod(0o755) + system = ('Windows 10/11 或 Windows Server 2019/2022/2025,Intel/AMD x64。仅当前用户运行,无需管理员权限。' + if windows else 'Ubuntu 22.04/24.04、Debian 12/13 的 Intel/AMD x86_64。需要 glibc 2.35 及以上;不支持 Alpine/musl、ARM 或旧版 CentOS。') + usage = ('解压到当前用户的 %LOCALAPPDATA%\\QualityCenter 等较短的本地 NTFS 路径,双击“一键部署启动.cmd”。不要在压缩包中直接打开。\n' + '运行库随包提供;不安装系统服务,不修改系统环境变量、注册表或防火墙,不申请管理员权限。\n' + '启动后自动打开浏览器,命令窗口显示本机和内网地址。关闭此窗口不会停止服务。\n' + '其他电脑连接同一内网后,用浏览器打开显示的 http://内网IP:5280 即可。\n' + '停止服务请双击“停止服务.cmd”;重新启动仍双击原入口。\n' if windows else + '解压后以普通用户运行 ./start.sh;内网共享运行 ./start.sh --lan。\n' + '同事用浏览器打开显示的 http://内网IP:5280;如系统有防火墙,仅对可信内网放行该端口。\n' + '停止运行 ./stop.sh,查看状态运行 ./status.sh,检查完整性运行 ./verify.sh。\n') + (root / '使用说明.txt').write_text('质量中心离线部署包\n\n系统要求\n' + system + '\n建议内存 8GB 以上,至少预留 5GB 磁盘及后续业务数据空间。\n\n' + + usage + '\n登录信息保存在 runtimedata/登录信息.txt,初始管理员密码在本机随机生成。登录后请修改密码。\n' + '首次数据库为空,不包含原电脑的用户、业务数据、来源文件和模型密钥。通过网页上传授权的数据文件。\n' + '普通功能和安装不需要外网;AI 在线模型调用需要网络,或自行配置内网模型。\n' + '不要在服务运行时移动目录,不要覆盖 .runtime 和 runtimedata。数据迁移使用逻辑备份,不能复制 macOS MySQL 原始数据目录。\n' + '内网访问需要主机保持开机、服务运行、网络互通;本机 MySQL 和内部 API 端口不对内网开放。\n' + '默认网页端口 5280。更改 .runtime/deployment.json 中的 webPort 后重新运行入口。内网连接受现有防火墙规则限制,程序不会自动更改这些规则。\n' + '详细技术说明见 deploy/offline/README.md。\n\n' + '验证范围:前后端隔离构建及本机自动检查已完成;Windows/Linux 原生首次启动和跨电脑内网访问需要在目标机实测。\n', encoding='utf-8-sig') + metadata = {'platform': osname, 'architecture': 'x86_64', 'builtAtUtc': datetime.now(timezone.utc).isoformat(), + 'updateVersion': '2026.11.04.1', 'updateSequence': 1, 'installationScope': 'current-user', 'requiresAdministrator': False, + 'databaseIncluded': False, 'credentialsIncluded': False, 'requiresDownloadAtInstall': False, + 'nativeTargetSmokeTest': 'not-run-on-macos-host', + 'runtimes': [{k: v for k, v in a.items() if k not in ('kind',)} for a in assets if a['os'] == osname and a['name'] != 'vcredist'] + ([local_runtime] if windows else []), + 'pythonWheels': [{'file': p.name, 'sha256': digest(p)} for p in wheels], 'files': []} + for path in sorted(root.rglob('*')): + if path.is_symlink(): + metadata['files'].append({'path': path.relative_to(root).as_posix(), 'symlink': str(path.readlink())}) + elif path.is_file(): + metadata['files'].append({'path': path.relative_to(root).as_posix(), 'sha256': digest(path)}) + (root / 'manifest.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + return metadata + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--work', type=Path, required=True) + parser.add_argument('--platform', choices=('windows', 'linux', 'both'), default='both') + parser.add_argument('--assemble-only', action='store_true') + args = parser.parse_args() + work = args.work.resolve() + assets = json.loads((work / 'downloads/assets.json').read_text()) + platforms = ('windows', 'linux') if args.platform == 'both' else (args.platform,) + summaries = [] + for osname in platforms: + name = 'QualityCenter-' + ('Windows' if osname == 'windows' else 'Linux') + '-x64' + directory = work / name + if directory.exists(): + raise RuntimeError('Preserving an existing bundle; choose an empty work directory: ' + str(directory)) + print('Assembling ' + name, flush=True) + metadata = payload(directory, work, osname, assets) + print('Files verified: ' + str(len(metadata['files'])), flush=True) + if args.assemble_only: + continue + if osname == 'windows': + archive = work / (name + '.zip') + with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED, compresslevel=6) as target: + for path in sorted(directory.rglob('*')): + if path.is_file(): + target.write(path, path.relative_to(work).as_posix()) + else: + archive = work / (name + '.tar.gz') + with tarfile.open(archive, 'w:gz', compresslevel=6) as target: + target.add(directory, arcname=name) + summaries.append({'file': archive.name, 'bytes': archive.stat().st_size, 'sha256': digest(archive)}) + print(json.dumps(summaries[-1]), flush=True) + if summaries: + (work / 'SHA256SUMS.txt').write_text(''.join(a['sha256'] + ' ' + a['file'] + '\n' for a in summaries)) + (work / 'packages.json').write_text(json.dumps(summaries, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/deploy/offline/control.py b/deploy/offline/control.py new file mode 100644 index 0000000..d47033e --- /dev/null +++ b/deploy/offline/control.py @@ -0,0 +1,486 @@ +#!/usr/bin/env python3 +"""Offline, unprivileged service supervisor for the standalone distribution.""" +from __future__ import annotations + +import argparse +import contextlib +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import secrets +import signal +import socket +import subprocess +import sys +import threading +import time +from urllib.request import ProxyHandler, build_opener + +import psutil + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts')) +import runtime_platform as host +from gateway import make_server + +PRIVATE = ROOT / '.runtime' +DATA = ROOT / 'runtimedata' +CONFIG = PRIVATE / 'deployment.json' +STATE = PRIVATE / 'service.json' +STOP = PRIVATE / 'stop-request.json' +IS_WINDOWS = os.name == 'nt' +EXE = '.exe' if IS_WINDOWS else '' +PYTHON = ROOT / 'runtimes/python' / ('python.exe' if IS_WINDOWS else 'bin/python3') +JAVA = ROOT / 'runtimes/java/bin' / ('java' + EXE) +MYSQL = ROOT / 'runtimes/mysql/bin' / ('mysql' + EXE) +MYSQLD = ROOT / 'runtimes/mysql/bin' / ('mysqld' + EXE) +ADMIN = ROOT / 'runtimes/mysql/bin' / ('mysqladmin' + EXE) +OPENER = build_opener(ProxyHandler({})) + + +def atomic(path, value): + temporary = path.with_name(path.name + '.' + secrets.token_hex(5) + '.tmp') + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, 'w', encoding='utf-8') as stream: + json.dump(value, stream, ensure_ascii=False, indent=2) + temporary.replace(path) + + +def private_directory(path): + if path.is_symlink(): + raise RuntimeError('运行目录不能是符号链接。') + path.mkdir(parents=True, exist_ok=True, mode=0o700) + if IS_WINDOWS: + result = subprocess.run(['whoami', '/user', '/fo', 'csv', '/nh'], capture_output=True, check=True) + sid = re.search(rb'S-1-\d+(?:-\d+)+', result.stdout) + if not sid: + raise RuntimeError('无法确定当前 Windows 用户。') + subprocess.run(['icacls', str(path), '/inheritance:r', '/grant:r', + '*' + sid.group().decode() + ':(OI)(CI)F', '*S-1-5-18:(OI)(CI)F', '/Q'], + stdout=subprocess.DEVNULL, check=True) + else: + path.chmod(0o700) + + +def configuration(): + if CONFIG.exists(): + value = json.loads(CONFIG.read_text(encoding='utf-8')) + else: + value = {'mysqlPort': 53306, 'backendPort': 5281, 'webPort': 5280, + 'webBind': '0.0.0.0' if IS_WINDOWS else '127.0.0.1', + 'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32), + 'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)} + atomic(CONFIG, value) + value.setdefault('mysqlDatabase', 'quality_platform') + if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', value['mysqlDatabase']): + raise RuntimeError('部署数据库名称无效。') + relative_archive = Path(value.get('archiveDirectory', 'runtimedata/archive')) + if relative_archive.is_absolute() or '..' in relative_archive.parts or not (ROOT / relative_archive).resolve().is_relative_to(DATA.resolve()): + raise RuntimeError('来源归档目录无效。') + for key in ('mysqlPort', 'backendPort', 'webPort'): + if type(value.get(key)) is not int or not 1024 <= value[key] <= 65535: + raise RuntimeError('部署端口配置无效。') + if value['backendPort'] not in (5281, 5283, 5284, 5285): + raise RuntimeError('接口端口必须为 5281、5283、5284 或 5285。') + if len({value[k] for k in ('mysqlPort', 'backendPort', 'webPort')}) != 3: + raise RuntimeError('数据库、接口和网页端口不能重复。') + if value.get('webBind') not in ('127.0.0.1', '0.0.0.0'): + raise RuntimeError('网页监听配置无效。') + for key in ('mysqlPassword', 'rootPassword', 'adminPassword'): + if not re.fullmatch(r'[A-Za-z0-9_-]{20,100}', value.get(key, '')): + raise RuntimeError('本机随机凭据配置无效,原配置已保留。') + return value + + +def environment(value): + env = os.environ.copy() + package = ROOT / 'runtimes/python' / ('Lib/site-packages' if IS_WINDOWS else 'lib/python3.11/site-packages') / 'nao_core' + env.update({ + 'JAVA_HOME': str(JAVA.parent.parent), + 'PATH': os.pathsep.join([str(PYTHON.parent), str(JAVA.parent), str(MYSQL.parent), env.get('PATH', '')]), + 'PYTHONUTF8': '1', 'PYTHONIOENCODING': 'utf-8', 'PYTHONDONTWRITEBYTECODE': '1', + 'QUALITY_PROJECT_ROOT': str(ROOT), 'QUALITY_ASSISTANT_PYTHON': str(PYTHON), + 'QUALITY_ASSISTANT_PACKAGE': str(package), + 'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{value["mysqlPort"]}/{value.get("mysqlDatabase", "quality_platform")}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true', + 'MYSQL_USER': 'quality_app', 'MYSQL_PASSWORD': value['mysqlPassword'], + 'QUALITY_BOOTSTRAP_ADMIN_USERNAME': '' if value.get('restoredFromMigration') else value['adminUsername'], + 'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': '' if value.get('restoredFromMigration') else value['adminPassword'], + 'QUALITY_ARCHIVE_ROOT': str(ROOT / value.get('archiveDirectory', 'runtimedata/archive')), 'QUALITY_ALLOWED_ROOTS': str(DATA / 'sources'), + 'QUALITY_ANALYSIS_RUNTIME_QUERY_PORT': str(value['backendPort']), + 'PORT': str(value['backendPort']), 'SERVER_ADDRESS': '127.0.0.1', + 'QUALITY_DATA_WORKFLOW_TEST_MODE': 'false', + 'MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS': 'UNCONFIRMED', + }) + env['QUALITY_DATA_WORKFLOW_TEST_MODE'] = str(value.get('dataWorkflowTestMode', False)).lower() + env['QUALITY_ANALYSIS_AUTONOMOUS_ENABLED'] = str(value.get('analysisAutonomousEnabled', False)).lower() + env['QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED'] = str(value.get('analysisProtocolV1Enabled', False)).lower() + env['QUALITY_UPDATE_ENABLED'] = 'true' if IS_WINDOWS else 'false' + # Do not accidentally inherit a development or source-machine AI command. + env.pop('QUALITY_ANALYSIS_NAO_COMMAND', None) + env.pop('QUALITY_ANALYSIS_NAO_SCRIPT', None) + env.pop('PYTHONHOME', None) + env.pop('PYTHONPATH', None) + if not IS_WINDOWS: + env['LD_LIBRARY_PATH'] = os.pathsep.join([str(ROOT / 'runtimes/linux-libs'), + str(ROOT / 'runtimes/mysql/lib'), + str(ROOT / 'runtimes/mysql/lib/private')]) + return env + + +def record(process, command): + observed = psutil.Process(process.pid) + return {'pid': process.pid, 'created': observed.create_time(), + 'executable': str(Path(observed.exe()).resolve()), 'args': [str(a) for a in command[1:]]} + + +def matches(info): + try: + process = psutil.Process(int(info['pid'])) + return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE + and abs(process.create_time() - info['created']) < .01 + and os.path.normcase(str(Path(process.exe()).resolve())) == os.path.normcase(info['executable']) + and process.cmdline()[1:] == info['args']) + except (psutil.Error, KeyError, ValueError, TypeError): + return False + + +def read_state(): + try: + return json.loads(STATE.read_text(encoding='utf-8')) + except FileNotFoundError: + return {} + + +def free_port(port, bind='127.0.0.1'): + # A recently closed TCP connection may remain in TIME_WAIT. Reject a live + # listener, but allow a clean restart without waiting for that state to expire. + try: + connection = socket.create_connection(('127.0.0.1', port), timeout=.3) + except OSError: + pass + else: + connection.close() + raise RuntimeError(f'端口 {port} 已被其他服务使用。') + with socket.socket() as test: + test.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + test.bind((bind, port)) + + +def mysql_command(value, binary=MYSQL): + return [str(binary), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1', + '--port=' + str(value['mysqlPort']), '--user=root', '--connect-timeout=3'] + + +def sql(value, query): + env = environment(value) + env['MYSQL_PWD'] = value['rootPassword'] + return subprocess.run(mysql_command(value) + ['--batch', '--skip-column-names'], input=query, + text=True, encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env, timeout=10) + + +def verify(check_hashes=True): + manifest = json.loads((ROOT / 'manifest.json').read_text(encoding='utf-8')) + expected_os = 'windows' if IS_WINDOWS else platform.system().lower() + if manifest['platform'] != expected_os or platform.machine().lower() not in ('amd64', 'x86_64'): + raise RuntimeError('此包仅用于 ' + manifest['platform'] + ' 的 Intel/AMD 64 位电脑。') + if check_hashes: + for entry in manifest['files']: + path = ROOT / entry['path'] + if entry.get('symlink') is not None: + valid = path.is_symlink() and os.readlink(path) == entry['symlink'] + else: + valid = path.is_file() and hashlib.file_digest(path.open('rb'), 'sha256').hexdigest() == entry['sha256'] + if not valid: + raise RuntimeError('文件缺失或校验失败:' + entry['path']) + value = configuration() + env = environment(value) + for command in ([JAVA, '-version'], [MYSQLD, '--no-defaults', '--version']): + result = subprocess.run([str(a) for a in command], capture_output=True, env=env, timeout=30) + if result.returncode: + raise RuntimeError('运行环境检查失败:' + Path(command[0]).name + '。请查看安装说明中的系统要求。') + probe = subprocess.run([str(PYTHON), '-c', 'import duckdb,yaml,fastapi,psutil,nao_core;import sys;assert sys.version_info[:3]==(3,11,15)'], + capture_output=True, env=env, timeout=45) + if probe.returncode: + raise RuntimeError('AI Python 运行环境检查失败。') + application_probe = subprocess.run([str(PYTHON), '-c', + 'import sys;sys.path.insert(0,sys.argv[1]);import nao_quality_runtime as r;' + 'import nao_native_run,nao_quality_runtime_mcp,nao_visual_recovery;' + 'assert len(r._config_fingerprint("https://offline.invalid/v1","offline-check","offline-check"))==64', + str(ROOT / 'scripts')], capture_output=True, env=env, timeout=45) + if application_probe.returncode: + raise RuntimeError('AI 应用脚本或策略资源检查失败。') + package = Path(env['QUALITY_ASSISTANT_PACKAGE']) + native = subprocess.run([str(package / 'bin' / ('nao-chat-server' + EXE)), '--help'], + capture_output=True, env=env, timeout=30) + if native.returncode: + raise RuntimeError('AI 原生服务无法运行,请检查系统版本和 CPU 支持。') + return value + + +def addresses(value): + ips = set() + if value['webBind'] == '0.0.0.0': + for entries in psutil.net_if_addrs().values(): + for entry in entries: + if entry.family == socket.AF_INET and not entry.address.startswith(('127.', '169.254.')): + ips.add(entry.address) + return ['http://127.0.0.1:' + str(value['webPort'])] + [f'http://{ip}:{value["webPort"]}' for ip in sorted(ips)] + + +def write_login(value): + path = DATA / '登录信息.txt' + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, 'w', encoding='utf-8') as stream: + if value.get('restoredFromMigration'): + stream.write('质量中心\n' + '\n'.join(addresses(value)) + '\n业务数据已迁移,请使用原有账号和密码登录。\n') + else: + stream.write('质量中心\n' + '\n'.join(addresses(value)) + '\n用户名:' + value['adminUsername'] + + '\n初始密码:' + value['adminPassword'] + '\n登录后请在个人账户修改密码。此文件仅记录初始密码,请勿分享。\n') + + +def serve(database_only=False): + lock = os.open(PRIVATE / 'supervisor.lock', os.O_RDWR | os.O_CREAT, 0o600) + try: + host.fcntl.flock(lock, host.fcntl.LOCK_EX | host.fcntl.LOCK_NB) + except BlockingIOError: + raise RuntimeError('此目录的服务已经启动。') from None + children = [] + value = configuration() + state = {'supervisor': record(psutil.Process(), [sys.executable, *sys.argv]), + 'token': secrets.token_hex(24), 'ready': False, 'databaseOnly': database_only, 'children': children} + stopped = threading.Event() + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: stopped.set()) + + def stopping(): + if STOP.exists(): + try: + if json.loads(STOP.read_text(encoding='utf-8')).get('token') == state['token']: + stopped.set() + except (OSError, ValueError): + pass + return stopped.is_set() + + def spawn(name, command): + with (DATA / 'logs' / (name + '.log')).open('ab') as log: + child = subprocess.Popen([str(a) for a in command], cwd=ROOT, env=environment(value), + stdin=subprocess.DEVNULL, stdout=log, stderr=log) + info = record(child, command) + info['name'] = name + children.append(info) + atomic(STATE, state) + return child + + def wait_ready(child, check, seconds): + end = time.monotonic() + seconds + while time.monotonic() < end: + if stopping(): + raise RuntimeError('启动已停止。') + if child.poll() is not None: + raise RuntimeError('服务启动失败,请查看 runtimedata/logs。') + try: + if check(): + return + except (OSError, ValueError, subprocess.TimeoutExpired): + pass + time.sleep(.5) + raise RuntimeError('服务启动超时,请查看 runtimedata/logs。') + + try: + for key in (('mysqlPort',) if database_only else ('mysqlPort', 'backendPort', 'webPort')): + free_port(value[key], value['webBind'] if key == 'webPort' else '127.0.0.1') + atomic(STATE, state) + db = DATA / 'mysql' + if not (db / 'auto.cnf').exists(): + if db.exists() and any(db.iterdir()): + raise RuntimeError('数据库目录含有未完成的初始化文件,已保留原目录;请查看初始化日志。') + db.mkdir(exist_ok=True) + with (DATA / 'logs/mysql-initialize.log').open('ab') as log: + result = subprocess.run([str(MYSQLD), '--no-defaults', '--initialize-insecure', + '--basedir=' + str(MYSQLD.parent.parent), '--datadir=' + str(db), + '--lower-case-table-names=1'], env=environment(value), + stdout=log, stderr=log, timeout=180) + if result.returncode: + raise RuntimeError('数据库初始化失败,请查看 runtimedata/logs/mysql-initialize.log。') + bootstrap = PRIVATE / 'bootstrap.sql' + fd = os.open(bootstrap, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, 'w', encoding='utf-8') as stream: + stream.write(f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{value['rootPassword']}';\n" + f"CREATE USER IF NOT EXISTS 'root'@'127.0.0.1' IDENTIFIED BY '{value['rootPassword']}';\n" + "GRANT ALL PRIVILEGES ON *.* TO 'root'@'127.0.0.1' WITH GRANT OPTION;\n" + f"CREATE DATABASE IF NOT EXISTS `{value['mysqlDatabase']}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;\n" + f"CREATE USER IF NOT EXISTS 'quality_app'@'127.0.0.1' IDENTIFIED BY '{value['mysqlPassword']}';\n" + f"GRANT ALL PRIVILEGES ON `{value['mysqlDatabase']}`.* TO 'quality_app'@'127.0.0.1';\n") + mysql_args = [MYSQLD, '--no-defaults', '--basedir=' + str(MYSQLD.parent.parent), '--datadir=' + str(db), + '--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1', '--mysqlx=OFF', + '--lower-case-table-names=1', '--skip-name-resolve', '--skip-log-bin', + '--max-connections=60', '--innodb-buffer-pool-size=' + ('1024M' if database_only else '256M'), '--init-file=' + bootstrap.as_posix()] + if database_only: + # The restored database is unpublished until validation and a clean + # MySQL shutdown finish. Normal startup restores durable defaults. + mysql_args += ['--innodb-redo-log-capacity=1G', '--innodb-flush-log-at-trx-commit=2'] + if not IS_WINDOWS: + mysql_args.append('--socket=' + str(PRIVATE / 'mysql.sock')) + database = spawn('mysql', mysql_args) + wait_ready(database, lambda: sql(value, 'SELECT 1;').returncode == 0, 120) + # The server does not reread this file after startup. + bootstrap.unlink(missing_ok=True) + if database_only: + state['ready'] = True + atomic(STATE, state) + while not stopping(): + if database.poll() is not None: + raise RuntimeError('恢复数据库服务已退出,请查看日志。') + time.sleep(.25) + return + backend = spawn('backend', [JAVA, '-Djava.awt.headless=true', '-Xms128m', '-Xmx2048m', '-jar', ROOT / 'app/backend.jar']) + + def backend_ready(): + with OPENER.open(f'http://127.0.0.1:{value["backendPort"]}/actuator/health', timeout=3) as response: + return json.load(response).get('status') == 'UP' + + wait_ready(backend, backend_ready, 900) + server = make_server(ROOT / 'app/web', value['webBind'], value['webPort'], value['backendPort']) + server.timeout = .5 + state['ready'] = True + write_login(value) + atomic(STATE, state) + print('质量中心已启动:' + ' '.join(addresses(value)), flush=True) + try: + while not stopping(): + if database.poll() is not None or backend.poll() is not None: + raise RuntimeError('后台服务已退出,请查看 runtimedata/logs。') + server.handle_request() + finally: + server.server_close() + finally: + state['ready'] = False + atomic(STATE, state) + # Stop backend first, then its isolated AI processes, then the database. + for info in reversed(children): + if info['name'] != 'backend' or not matches(info): + continue + process = psutil.Process(info['pid']) + descendants = process.children(recursive=True) + process.terminate() + try: + process.wait(35) + except psutil.TimeoutExpired: + if matches(info): + process.kill() + for child in descendants: + with contextlib.suppress(psutil.NoSuchProcess): + child.terminate() + with contextlib.suppress(OSError, subprocess.SubprocessError): + subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'], + env=environment(value), cwd=ROOT, capture_output=True, timeout=25) + if any(info['name'] == 'mysql' and matches(info) for info in children): + env = environment(value) + env['MYSQL_PWD'] = value['rootPassword'] + with contextlib.suppress(OSError, subprocess.SubprocessError): + subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env, + capture_output=True, timeout=60) + # mysqladmin can return before InnoDB has finished closing files. + for info in children: + if info['name'] == 'mysql' and matches(info): + with contextlib.suppress(psutil.NoSuchProcess, psutil.TimeoutExpired): + psutil.Process(info['pid']).wait(60) + survivors = [info for info in children if matches(info)] + if survivors: + state['children'] = survivors + atomic(STATE, state) + else: + STATE.unlink(missing_ok=True) + STOP.unlink(missing_ok=True) + (PRIVATE / 'bootstrap.sql').unlink(missing_ok=True) + os.close(lock) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'serve')) + parser.add_argument('--lan', action='store_true') + parser.add_argument('--database-only', action='store_true', help=argparse.SUPPRESS) + args = parser.parse_args() + if args.action in ('start', 'serve', 'verify'): + private_directory(PRIVATE) + private_directory(DATA) + (DATA / 'logs').mkdir(exist_ok=True) + (DATA / 'sources').mkdir(exist_ok=True) + (DATA / 'archive').mkdir(exist_ok=True) + if args.action == 'serve': + return serve(args.database_only) + updater = ROOT / 'deploy/offline/update_manager.py' + if args.action == 'start' and IS_WINDOWS and updater.is_file(): + recovered = subprocess.run([str(PYTHON), str(updater), 'recover'], cwd=ROOT, + env=environment(configuration()), capture_output=True, text=True, + encoding='utf-8') + if recovered.returncode: + try: + message = json.loads(recovered.stdout).get('error') + except (ValueError, AttributeError): + message = None + raise RuntimeError(message or '程序更新正在进行或尚未安全恢复,请查看更新记录。') + state = read_state() + running = matches(state.get('supervisor', {})) + if args.action == 'status': + print('已启动' if running and state.get('ready') else '正在启动或停止' if running else '已停止') + return + if args.action == 'stop': + if not running: + if any(matches(info) for info in state.get('children', [])): + raise RuntimeError('发现上次退出后遗留的进程,请根据日志检查;未结束任何不明进程。') + print('服务已停止。') + return + atomic(STOP, {'token': state['token']}) + for _ in range(240): + if not matches(state['supervisor']): + print('服务已停止。') + return + time.sleep(.5) + raise RuntimeError('服务仍在安全关闭,请稍后检查状态。') + if running: + if args.lan and configuration()['webBind'] != '0.0.0.0': + raise RuntimeError('请先停止服务,再使用 --lan 启动。') + print('服务已启动。' if state.get('ready') else '服务正在启动,请稍候。') + for url in addresses(configuration()): + print(url) + return + if any(matches(info) for info in state.get('children', [])): + raise RuntimeError('发现上次退出后的服务进程,原数据已保留;请检查日志。') + value = verify() + if args.action == 'verify': + print('文件完整性、Java、MySQL、AI Python 与 AI 原生程序检查通过。') + return + if args.lan and value['webBind'] != '0.0.0.0': + value['webBind'] = '0.0.0.0' + atomic(CONFIG, value) + with (DATA / 'logs/supervisor.log').open('ab') as log: + options = {'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP} if IS_WINDOWS else {'start_new_session': True} + process = subprocess.Popen([str(PYTHON), str(Path(__file__).resolve()), 'serve'], cwd=ROOT, + env=environment(value), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **options) + for _ in range(2000): + state = read_state() + if state.get('ready') and state.get('supervisor', {}).get('pid') == process.pid and matches(state['supervisor']): + for url in addresses(value): + print(url) + print('管理员初始登录信息:' + str(DATA / '登录信息.txt')) + if IS_WINDOWS: + os.startfile(addresses(value)[0]) + return + if process.poll() is not None: + raise RuntimeError('启动未完成,请查看 runtimedata/logs/supervisor.log。') + time.sleep(.5) + raise RuntimeError('服务仍在启动,请检查状态与日志。') + + +if __name__ == '__main__': + try: + main() + except (RuntimeError, OSError, subprocess.SubprocessError) as error: + print(str(error), file=sys.stderr) + raise SystemExit(1) diff --git a/deploy/offline/gateway.py b/deploy/offline/gateway.py new file mode 100644 index 0000000..967c54c --- /dev/null +++ b/deploy/offline/gateway.py @@ -0,0 +1,102 @@ +"""Serve built assets and stream the authenticated API on one LAN port.""" +from __future__ import annotations + +import functools +import http.client +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import urlsplit + +HOP_HEADERS = {'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', + 'te', 'trailer', 'transfer-encoding', 'upgrade'} + + +class Gateway(SimpleHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + backend_port = 5281 + extensions_map = {**SimpleHTTPRequestHandler.extensions_map, '.js': 'text/javascript', + '.mjs': 'text/javascript', '.css': 'text/css', '.svg': 'image/svg+xml', + '.wasm': 'application/wasm'} + + def log_message(self, *_): + # Do not record query strings, authentication data or business values. + pass + + def _proxy(self): + upstream = http.client.HTTPConnection('127.0.0.1', self.backend_port, timeout=1800) + try: + if self.headers.get('Transfer-Encoding'): + self.send_error(400, 'Content-Length is required') + return + length = int(self.headers.get('Content-Length', '0')) + if length < 0 or length > 273 * 1024 * 1024: + self.send_error(413) + return + upstream.putrequest(self.command, self.path, skip_host=True, skip_accept_encoding=True) + connection_headers = {s.strip().lower() for s in self.headers.get('Connection', '').split(',')} + for name, value in self.headers.items(): + if name.lower() not in HOP_HEADERS | connection_headers: + upstream.putheader(name, value) + upstream.putheader('Connection', 'close') + upstream.endheaders() + remaining = length + while remaining: + chunk = self.rfile.read(min(remaining, 1024 * 1024)) + if not chunk: + raise ConnectionError('Incomplete upload') + upstream.send(chunk) + remaining -= len(chunk) + response = upstream.getresponse() + self.send_response(response.status) + response_connection = {s.strip().lower() for s in response.getheader('Connection', '').split(',')} + for name, value in response.getheaders(): + if name.lower() not in HOP_HEADERS | response_connection: + self.send_header(name, value) + self.send_header('Connection', 'close') + self.send_header('X-Content-Type-Options', 'nosniff') + self.end_headers() + self.close_connection = True + if self.command != 'HEAD': + while chunk := response.read1(64 * 1024): + self.wfile.write(chunk) + self.wfile.flush() # SSE events must arrive without buffering. + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + except (OSError, http.client.HTTPException, ValueError): + self.close_connection = True + # Never expose upstream exception text or an internal file path. + if not self.wfile.closed: + try: + self.send_error(502, 'Backend unavailable') + except OSError: + pass + finally: + upstream.close() + + def _dispatch(self): + path = urlsplit(self.path).path + if path == '/api' or path.startswith('/api/') or path == '/actuator/health': + return self._proxy() + if self.command not in ('GET', 'HEAD'): + self.send_error(405) + return + candidate = Path(self.translate_path(self.path)).resolve() + root = Path(self.directory).resolve() + if not candidate.is_relative_to(root): + self.send_error(403) + return + if candidate.is_dir() and not (candidate / 'index.html').is_file(): + self.send_error(404) + return + if not candidate.exists() and not Path(path).suffix: + self.path = '/index.html' + return super().do_HEAD() if self.command == 'HEAD' else super().do_GET() + + do_GET = do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = _dispatch + + +def make_server(root, address, port, backend_port=5281): + handler = type('ConfiguredGateway', (Gateway,), {'backend_port': backend_port}) + server = ThreadingHTTPServer((address, port), functools.partial(handler, directory=str(root))) + server.daemon_threads = True + return server diff --git a/deploy/offline/migration_common.py b/deploy/offline/migration_common.py new file mode 100644 index 0000000..76c7e20 --- /dev/null +++ b/deploy/offline/migration_common.py @@ -0,0 +1,154 @@ +"""Portable migration validation; no source-machine paths or credentials are logged.""" +from __future__ import annotations + +import gzip +import hashlib +import json +from pathlib import Path, PurePosixPath, PureWindowsPath +import re +import tarfile +import uuid + + +def sha256(path): + with Path(path).open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def sql_text(value): + return 'CONVERT(0x' + value.encode('utf-8').hex() + ' USING utf8mb4)' + + +def relative_name(name): + if not isinstance(name, str) or not name or '\\' in name or ':' in name: + raise ValueError('迁移文件路径无效。') + path = PurePosixPath(name) + if not path.parts or path.is_absolute() or path.as_posix() != name or any(part in ('.', '..', '') for part in path.parts): + raise ValueError('迁移文件路径无效。') + for part in path.parts: + if part.endswith((' ', '.')) or re.fullmatch(r'(?i)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(?:\..*)?', part): + raise ValueError('迁移文件名不兼容 Windows。') + return path + + +def path_under_source(stored, source_root): + # A Windows restore must interpret the old Mac path as POSIX, not Path(). + path, root = PurePosixPath(stored), PurePosixPath(source_root) + if not path.is_absolute() or '..' in path.parts or not path.is_relative_to(root): + raise ValueError('来源文件引用不在已验证的归档目录内。') + return relative_name(path.relative_to(root).as_posix()).as_posix() + + +def extract_checked(archive, destination, expected): + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=False, mode=0o700) + observed = set() + for name in expected: + relative_name(name) + folded = [name.casefold() for name in expected] + if len(folded) != len(set(folded)): + raise ValueError('迁移文件在 Windows 上存在重名。') + with tarfile.open(archive, 'r:gz') as source: + for member in source: + rel = relative_name(member.name) + if not member.isfile() or member.name in observed or member.name not in expected: + raise ValueError('迁移归档包含未登记或无效的文件。') + entry = expected[member.name] + target = destination.joinpath(*rel.parts) + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if not target.resolve().is_relative_to(destination.resolve()): + raise ValueError('迁移文件路径越界。') + content = source.extractfile(member) + if content is None: + raise ValueError('迁移文件无法读取。') + digest = hashlib.sha256() + with content, target.open('xb') as output: + while chunk := content.read(1024 * 1024): + digest.update(chunk) + output.write(chunk) + target.chmod(0o600) + if target.stat().st_size != entry['bytes'] or digest.hexdigest() != entry['sha256']: + raise ValueError('迁移文件完整性检查失败。') + observed.add(member.name) + if observed != set(expected): + raise ValueError('迁移归档缺少文件。') + + +def table_counts(query): + result = {} + for name in query('SHOW TABLES;').splitlines(): + if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', name): + raise ValueError('数据库表名无效。') + result[name] = int(query('SELECT COUNT(*) FROM `' + name + '`;').strip()) + return result + + +def query_evidence(query): + current = query("SELECT release_id FROM current_release WHERE dataset_id='quality';").strip() + if not current: + return {'releaseId': None, 'months': []} + if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current): + raise ValueError('正式数据版本无效。') + result = {'releaseId': current, 'months': []} + components = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap', + 'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap', + 'bright_processing_scrap', 'bright_diffusion_scrap'] + for stage in ('black', 'bright'): + pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*),'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)" + for key in components: + pairs += ", '" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))" + result['months'].extend(json.loads(line) for line in query("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;").splitlines()) + result['members'] = [json.loads(line) for line in query("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;").splitlines()] + return result + + +_VALUES_TOKENS = re.compile(rb"'(?:[^'\\]+|\\.)*'|(\),\()") + + +def inserted_rows(line): + if not line.startswith(b'INSERT INTO `') or b' VALUES (' not in line: + raise ValueError('不支持的数据库导出行。') + values = line.split(b' VALUES ', 1)[1] + if not values.rstrip().endswith(b';'): + raise ValueError('数据库导出行不完整。') + return 1 + sum(match[1] is not None for match in _VALUES_TOKENS.finditer(values)) + + +def dump_counts(path): + counts, uncompressed = {}, 0 + with gzip.open(path, 'rb') as source: + for line in source: + uncompressed += len(line) + create = re.match(rb'^CREATE TABLE `([A-Za-z0-9_]+)`', line) + if create: + counts[create[1].decode()] = 0 + insert = re.match(rb'^INSERT INTO `([A-Za-z0-9_]+)`', line) + if insert: + name = insert[1].decode() + if name not in counts: + raise ValueError('导出数据缺少表结构。') + counts[name] += inserted_rows(line) + return counts, uncompressed + + +def filter_models(value): + profiles = value.get('profiles') + if not isinstance(profiles, list): + profiles = [{**value, 'id': 'migrated-profile', 'name': '迁移模型'}] + def excluded(profile): + return any('mimo' in str(profile.get(key, '')).lower() for key in ('model', 'name', 'baseUrl')) + kept = [profile.copy() for profile in profiles if not excluded(profile)] + removed = [profile for profile in profiles if excluded(profile)] + if not kept: + return None, removed + selected = next((profile for profile in kept if profile.get('id') == value.get('activeProfileId')), kept[0]) + required = ('baseUrl', 'apiKey', 'model') + if any(not isinstance(selected.get(key), str) or not selected[key] for key in required): + raise ValueError('需要迁移的模型配置不完整。') + filtered = {key: value[key] for key in ('schemaVersion',) if key in value} + filtered.update(selected) + filtered.pop('id', None) + filtered.pop('name', None) + filtered.update({'schemaVersion': 3, 'profiles': kept, 'activeProfileId': selected['id']}) + filtered['revision'] = value.get('revision') or str(uuid.uuid4()) + return filtered, removed diff --git a/deploy/offline/prepare_migration.py b/deploy/offline/prepare_migration.py new file mode 100644 index 0000000..a21d70d --- /dev/null +++ b/deploy/offline/prepare_migration.py @@ -0,0 +1,200 @@ +"""Capture verified source files and private model settings for a Windows migration.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import sys +import tarfile +import uuid + +from migration_common import filter_models, relative_name, sha256 + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts')) +import backup + + +def private_json(path, data): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, 'w', encoding='utf-8') as output: + json.dump(data, output, ensure_ascii=False, indent=2) + output.write('\n') + + +def native_copy(source, target): + source_db = sqlite3.connect(source.resolve().as_uri() + '?mode=ro', uri=True) + destination = sqlite3.connect(target) + try: + source_db.backup(destination) + destination.execute('PRAGMA foreign_keys=OFF') + destination.execute('PRAGMA secure_delete=ON') + tables = {row[0] for row in destination.execute("SELECT name FROM sqlite_master WHERE type='table'")} + # Access sessions, external integrations and cached provider keys are + # recreated on Windows. Historical chats and their ownership stay intact. + for name in ('session', 'verification', 'api_key', 'jwks', 'oauth_access_token', + 'oauth_refresh_token', 'oauth_consent', 'oauth_client', 'mcp_user_token', + 'mcp_oauth_client', 'sandbox_secret', 'log', 'mcp_call_log'): + if name in tables: + destination.execute('DELETE FROM "' + name + '"') + if 'project_llm_config' in tables: + destination.execute('DELETE FROM project_llm_config') + for name in ('user', 'account', 'organization'): + if name not in tables: + continue + columns = list(destination.execute('PRAGMA table_info("' + name + '")')) + for column in columns: + key = column[1] + if key in ('github_access_token', 'gitlab_access_token', 'access_token', 'refresh_token', + 'id_token', 'google_client_secret'): + replacement = "''" if column[3] else 'NULL' + destination.execute('UPDATE "' + name + '" SET "' + key + '"=' + replacement) + destination.commit() + destination.execute('VACUUM') + assert destination.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' + finally: + destination.close() + source_db.close() + target.chmod(0o600) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--work', type=Path, required=True) + args = parser.parse_args() + work = args.work.resolve() + payload = work / 'migration' + manifest = json.loads((payload / 'source-snapshot.json').read_text()) + database = manifest['sourceDatabase'] + query = lambda sql: backup.client(sql, database) + archive_root = ROOT / 'runtimedata/archive' + files, sources, digest_index = {}, {}, {} + for path in sorted(archive_root.rglob('*')): + if path.is_symlink(): + raise RuntimeError('来源归档包含符号链接。') + if not path.is_file(): + continue + rel = relative_name(path.relative_to(archive_root).as_posix()).as_posix() + digest = sha256(path) + files[rel] = {'sha256': digest, 'bytes': path.stat().st_size} + sources[rel] = path + digest_index.setdefault(digest, rel) + bindings = [] + missing_optional = 0 + for table, column, state in ( + ('source_revision', 'archive_path', "'REQUIRED'"), + ('source_candidate', 'source_path', 'status'), + ('source_candidate', 'archive_path', 'status'), + ('source_upload_draft', 'source_path', "IF(expires_at dict: + try: + value = update_manager.read(root / 'manifest.json') + except Exception as error: + raise PublishError('无法读取部署包 manifest.json。') from error + if not isinstance(value, dict) or value.get('platform') != 'windows' or value.get('architecture') != 'x86_64': + raise PublishError('输入目录必须是 Windows x64 离线部署包。') + if not isinstance(value.get('files'), list) or len(value['files']) > 50000: + raise PublishError('部署包文件清单无效。') + for key in ('runtimes', 'pythonWheels'): + if not isinstance(value.get(key), list): + raise PublishError('部署包缺少运行环境指纹信息。') + current_sequence = value.get('updateSequence', 0) + if type(current_sequence) is not int or current_sequence < 0: + raise PublishError('部署包更新序号无效。') + return value + + +def _sensitive_path(name: str) -> bool: + path = update_manager.safe_path(name) + parts = [part.casefold() for part in path.parts] + return (name == update_manager.KEY + or name.startswith('deploy/bootstrap/') + or any(part in SENSITIVE_PARTS for part in parts) + or path.suffix.casefold() in SENSITIVE_SUFFIXES) + + +def _program_inventory(root: Path, manifest: dict) -> list[dict]: + entries: dict[str, dict] = {} + folded: set[str] = set() + for entry in manifest['files']: + if not isinstance(entry, dict): + raise PublishError('部署包文件清单无效。') + name = entry.get('path') + try: + update_manager.safe_path(name) + except update_manager.UpdateError as error: + raise PublishError('部署包文件清单含有无效路径。') from error + if name.casefold() in folded: + raise PublishError('部署包文件清单含有重复或大小写冲突路径。') + folded.add(name.casefold()) + if not update_manager.program_path(name): + continue + if _sensitive_path(name): + raise PublishError('程序清单中包含受保护的私有、迁移、引导或密钥文件路径。') + if entry.get('symlink') is not None or not update_manager.SHA.fullmatch(str(entry.get('sha256', ''))): + raise PublishError('程序文件清单缺少有效摘要或包含符号链接。') + path = update_manager.destination(root, name) + if not path.is_file() or path.is_symlink(): + raise PublishError('程序清单文件缺失或不是普通文件。') + try: + digest = update_manager.digest(path) + except OSError as error: + raise PublishError('读取程序清单文件失败。') from error + if digest != entry['sha256']: + raise PublishError('部署包程序文件与 manifest.json 摘要不一致。') + entries[name] = {'path': name, 'sha256': digest, 'size': path.stat().st_size} + + if not REQUIRED_PROGRAM_FILES.issubset(entries) or not any(name.startswith('app/web/') for name in entries): + raise PublishError('程序清单缺少更新所需的后端、网页、策略或更新管理程序。') + + # The updater replaces program files as a complete snapshot. Refuse to + # sign an incomplete manifest if a program file exists outside its list. + present: set[str] = set() + for path in root.rglob('*'): + name = path.relative_to(root).as_posix() + if not update_manager.program_path(name): + continue + if path.is_symlink(): + raise PublishError('程序目录含有符号链接或特殊文件。') + if path.is_dir(): + continue + if not path.is_file(): + raise PublishError('程序目录含有符号链接或特殊文件。') + present.add(name) + if present != set(entries): + raise PublishError('manifest.json 与磁盘上的完整程序文件清单不一致。') + return [entries[name] for name in sorted(entries)] + + +def _load_private_key(path: Path, root: Path, output_dir: Path) -> Ed25519PrivateKey: + try: + resolved = path.expanduser().resolve(strict=True) + if resolved.is_relative_to(root) or resolved.is_relative_to(output_dir): + raise PublishError('签名私钥必须保存在部署包和更新产物目录之外。') + key = serialization.load_pem_private_key(resolved.read_bytes(), password=None) + except PublishError: + raise + except Exception as error: + raise PublishError('无法读取未加密 PEM 格式的 Ed25519 签名私钥。') from error + if not isinstance(key, Ed25519PrivateKey): + raise PublishError('签名私钥必须是 Ed25519。') + return key + + +def _write_archive(root: Path, archive_temp: Path, files: list[dict]) -> None: + with zipfile.ZipFile(archive_temp, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=9, allowZip64=True) as output: + for entry in files: + name = entry['path'] + source = update_manager.destination(root, name) + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + digest = hashlib.sha256() + size = 0 + with source.open('rb') as reader, output.open(info, 'w') as writer: + while block := reader.read(1024 * 1024): + size += len(block) + if size > entry['size'] or size > update_manager.MAX_EXPANDED: + raise PublishError('程序文件在打包期间发生变化。') + digest.update(block) + writer.write(block) + if size != entry['size'] or digest.hexdigest() != entry['sha256']: + raise PublishError('程序文件在打包期间发生变化,已停止发布。') + + +def publish(root: Path, version: str, sequence: int, private_key_path: Path, output_dir: Path, + allow_baseline: bool = False) -> PublishedArtifacts: + """Create quality-update-.zip and signed latest.json locally.""" + root = root.expanduser().resolve(strict=True) + if not root.is_dir(): + raise PublishError('部署包根路径必须是目录。') + if not isinstance(version, str) or not VERSION.fullmatch(version): + raise PublishError('版本号只能包含字母、数字、点、下划线和连字符。') + if not ARCHIVE_VERSION.fullmatch(version): + raise PublishError('版本号不能用于更新文件名。') + if type(sequence) is not int or sequence <= 0: + raise PublishError('更新序号必须是正整数。') + + manifest = _manifest(root) + current_sequence = manifest['updateSequence'] + if sequence < current_sequence: + raise PublishError('更新序号不能低于部署包当前序号。') + if sequence == current_sequence and not allow_baseline: + raise PublishError('更新序号必须大于部署包当前序号。') + if sequence == current_sequence and version != manifest.get('updateVersion'): + raise PublishError('基线版本必须与部署包当前 updateVersion 完全一致。') + files = _program_inventory(root, manifest) + if sum(entry['size'] for entry in files) > update_manager.MAX_EXPANDED: + raise PublishError('更新程序文件总大小超过限制。') + + output_dir = output_dir.expanduser().resolve() + if output_dir.is_relative_to(root): + raise PublishError('更新产物目录必须位于部署包目录之外。') + private_key = _load_private_key(Path(private_key_path), root, output_dir) + public_key_path = update_manager.destination(root, update_manager.KEY) + try: + public_key = serialization.load_pem_public_key(public_key_path.read_bytes()) + except Exception as error: + raise PublishError('无法读取部署包中的更新公钥。') from error + if not isinstance(public_key, Ed25519PublicKey) or public_key.public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw) != private_key.public_key().public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw): + raise PublishError('签名私钥与部署包中的更新公钥不匹配。') + + output_dir.mkdir(parents=True, exist_ok=True) + archive_name = f'quality-update-{version}.zip' + archive_path = output_dir / archive_name + latest_path = output_dir / 'latest.json' + if archive_path.exists() or latest_path.exists(): + raise PublishError('输出目录已包含同名发布文件;请使用新的空输出目录。') + + archive_temp: Path | None = None + latest_temp: Path | None = None + published_archive = False + try: + with tempfile.NamedTemporaryFile(prefix='.quality-update-', suffix='.tmp', dir=output_dir, delete=False) as stream: + archive_temp = Path(stream.name) + _write_archive(root, archive_temp, files) + archive_size = archive_temp.stat().st_size + if not 0 < archive_size <= update_manager.MAX_ARCHIVE: + raise PublishError('更新压缩包大小超过限制。') + archive_hash = update_manager.digest(archive_temp) + release = { + 'schema': 1, + 'version': version, + 'sequence': sequence, + 'platform': 'windows-x64', + 'runtimeFingerprint': update_manager.runtime_fingerprint(manifest), + 'archive': {'file': archive_name, 'size': archive_size, 'sha256': archive_hash}, + 'files': files, + } + raw = update_manager.canonical(release) + if len(raw) > update_manager.MAX_MANIFEST: + raise PublishError('更新文件清单超过大小限制。') + envelope = { + 'payload': base64.b64encode(raw).decode('ascii'), + 'signature': base64.b64encode(private_key.sign(raw)).decode('ascii'), + } + try: + update_manager.validate_release(envelope, root) + except update_manager.UpdateError as error: + raise PublishError('生成的更新包未通过客户端契约校验:' + str(error)) from error + latest_bytes = update_manager.canonical(envelope) + with tempfile.NamedTemporaryFile(prefix='.latest-', suffix='.tmp', dir=output_dir, delete=False) as stream: + latest_temp = Path(stream.name) + stream.write(latest_bytes) + stream.flush() + os.fsync(stream.fileno()) + os.link(archive_temp, archive_path) + archive_temp.unlink() + archive_temp = None + published_archive = True + os.link(latest_temp, latest_path) + latest_temp.unlink() + latest_temp = None + return PublishedArtifacts(archive_path, latest_path, len(files), archive_hash) + except FileExistsError as error: + raise PublishError('输出目录出现同名发布文件;请使用新的空输出目录。') from error + except OSError as error: + raise PublishError('写入更新产物失败。') from error + finally: + if archive_temp is not None: + archive_temp.unlink(missing_ok=True) + if latest_temp is not None: + latest_temp.unlink(missing_ok=True) + if published_archive and not latest_path.exists(): + archive_path.unlink(missing_ok=True) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, required=True, help='Windows x64 offline bundle directory') + parser.add_argument('--version', required=True, help='Release version, for example 2026.11.04.2') + parser.add_argument('--sequence', type=int, required=True, help='Monotonically increasing update sequence') + parser.add_argument('--private-key', type=Path, required=True, help='Ed25519 PEM private key kept outside the bundle') + parser.add_argument('--output-dir', type=Path, required=True, help='New output directory for the ZIP and latest.json') + parser.add_argument('--allow-baseline', action='store_true', + help='Allow a seed release only when version and sequence exactly match the bundle manifest') + args = parser.parse_args(argv) + try: + result = publish(args.root, args.version, args.sequence, args.private_key, args.output_dir, args.allow_baseline) + except (PublishError, update_manager.UpdateError) as error: + parser.error(str(error)) + print(f'已生成更新包:{result.archive}') + print(f'已生成签名清单:{result.latest}') + print(f'版本 {args.version},序号 {args.sequence},程序文件 {result.file_count} 个,SHA-256 {result.archive_sha256}') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/deploy/offline/restore_migration.py b/deploy/offline/restore_migration.py new file mode 100644 index 0000000..5bcbdc8 --- /dev/null +++ b/deploy/offline/restore_migration.py @@ -0,0 +1,327 @@ +"""Restore into a new database, validate, then switch the Windows deployment.""" +from __future__ import annotations + +from datetime import datetime, timezone +import gzip +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import sqlite3 +import subprocess +import sys +import time +import uuid + +import control +from migration_common import extract_checked, query_evidence, relative_name, sha256, sql_text, table_counts + +ROOT = control.ROOT +PAYLOAD = ROOT / 'migration' +JOURNAL = control.PRIVATE / 'migration-progress.json' +RECEIPT = control.PRIVATE / 'migration-receipt.json' + + +def checked_local(relative): + rel = relative_name(relative) + path = ROOT.joinpath(*rel.parts) + if not path.resolve().is_relative_to(ROOT.resolve()): + raise RuntimeError('恢复暂存路径越界。') + return path + + +def run_query(value, statement, database=None): + command = control.mysql_command(value) + command += ['--batch', '--skip-column-names', '--default-character-set=utf8mb4', '--max-allowed-packet=1G'] + if database: + if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', database): + raise RuntimeError('恢复数据库名称无效。') + command += ['--database=' + database] + env = control.environment(value) + env['MYSQL_PWD'] = value['rootPassword'] + result = subprocess.run(command, input=statement, text=True, encoding='utf-8', + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=1800) + if result.returncode: + raise RuntimeError('恢复数据库校验未完成;已有数据库保留。') + return result.stdout + + +def stop_owned_service(): + state = control.read_state() + if control.matches(state.get('supervisor', {})): + control.atomic(control.STOP, {'token': state['token']}) + for _ in range(300): + if not control.matches(state['supervisor']): + break + time.sleep(.5) + else: + raise RuntimeError('原服务仍在关闭,请稍后重新运行。') + if any(control.matches(info) for info in state.get('children', [])): + raise RuntimeError('发现本目录尚未关闭的后台进程,请检查服务日志。') + + +def start_database(value): + with (control.DATA / 'logs/migration-database-supervisor.log').open('ab') as log: + options = ({'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP} + if control.IS_WINDOWS else {'start_new_session': True}) + process = subprocess.Popen([str(control.PYTHON), str(Path(control.__file__).resolve()), + 'serve', '--database-only'], cwd=ROOT, env=control.environment(value), + stdin=subprocess.DEVNULL, stdout=log, stderr=log, **options) + for _ in range(360): + state = control.read_state() + if (state.get('ready') and state.get('databaseOnly') and + state.get('supervisor', {}).get('pid') == process.pid and control.matches(state['supervisor'])): + return + if process.poll() is not None: + raise RuntimeError('恢复数据库无法启动,请查看 runtimedata/logs。') + time.sleep(.5) + raise RuntimeError('恢复数据库仍未就绪,请查看日志后重试。') + + +def restore_dump(value, database, dump): + env = control.environment(value) + env['MYSQL_PWD'] = value['rootPassword'] + command = control.mysql_command(value) + ['--database=' + database, + '--default-character-set=utf8mb4', '--max-allowed-packet=1G', + '--init-command=SET SESSION autocommit=0'] + with (control.DATA / 'logs/migration-import.log').open('ab') as errors: + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors, env=env) + try: + total, next_progress = dump.stat().st_size, 0 + with dump.open('rb') as raw, gzip.GzipFile(fileobj=raw) as source: + while chunk := source.read(1024 * 1024): + process.stdin.write(chunk) + percent = min(99, raw.tell() * 100 // total) + if percent >= next_progress: + print(f'数据库导入进度约 {percent}%……', flush=True) + next_progress = percent + 10 + process.stdin.write(b'\nCOMMIT;\n') + process.stdin.close() + if process.wait(timeout=1800): + raise RuntimeError('业务数据导入失败,请查看私有恢复日志;原数据库未覆盖。') + except BaseException: + if process.poll() is None: + process.kill() + process.wait() + raise + + +def rebind_sources(query, manifest, archive): + counts = {'source_revision': 0, 'source_candidate': 0, 'source_upload_draft': 0} + for entry in manifest['pathBindings']: + pair = (entry['table'], entry['column']) + if pair not in {('source_revision', 'archive_path'), ('source_candidate', 'source_path'), + ('source_candidate', 'archive_path'), ('source_upload_draft', 'source_path')}: + raise RuntimeError('来源路径绑定类型无效。') + rel = relative_name(entry['relative']) + target = archive.joinpath(*rel.parts) + if not entry['expiredMissing']: + if not target.is_file() or target.stat().st_size != entry['bytes'] or sha256(target) != entry['sha256']: + raise RuntimeError('来源文件校验失败,恢复未切换。') + query('UPDATE `' + entry['table'] + '` SET `' + entry['column'] + '`=' + sql_text(str(target)) + + ' WHERE id=' + sql_text(entry['id']) + ' AND `' + entry['column'] + '`=' + sql_text(entry['oldPath']) + ';') + actual = query('SELECT `' + entry['column'] + '` FROM `' + entry['table'] + '` WHERE id=' + sql_text(entry['id']) + ';').strip() + if actual != str(target): + raise RuntimeError('来源文件路径重绑定失败。') + counts[entry['table']] += 1 + return counts + + +def relocate_native(stage, source_root): + source_root = PurePosixPath(source_root) + def relocated(value): + if isinstance(value, str) and value.startswith(str(source_root) + '/'): + relative = PurePosixPath(value).relative_to(source_root) + if '..' in relative.parts: + raise RuntimeError('原生历史路径无效。') + return str(ROOT.joinpath(*relative.parts)) + if isinstance(value, dict): + return {key: relocated(child) for key, child in value.items()} + if isinstance(value, list): + return [relocated(child) for child in value] + return value + for path in stage.rglob('*.json'): + control.atomic(path, relocated(json.loads(path.read_text(encoding='utf-8')))) + databases = 0 + for path in stage.rglob('nao.sqlite'): + connection = sqlite3.connect(path) + try: + connection.execute('PRAGMA secure_delete=ON') + for table, column in (('project', 'path'), ('message_part', 'storage_path')): + exists = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone() + if not exists: + continue + for rowid, original in connection.execute('SELECT rowid,"' + column + '" FROM "' + table + '" WHERE "' + column + '" IS NOT NULL').fetchall(): + replacement = relocated(original) + if replacement != original: + connection.execute('UPDATE "' + table + '" SET "' + column + '"=? WHERE rowid=?', (replacement, rowid)) + connection.commit() + connection.execute('VACUUM') + if connection.execute('PRAGMA integrity_check').fetchone()[0] != 'ok': + raise RuntimeError('原生分析历史数据库校验失败。') + databases += 1 + finally: + connection.close() + return databases + + +def rollback_plan(plan): + # Only paths generated by this migration are eligible. Databases and source + # archive trees are retained even after an interrupted restore. + attempt = plan['attempt'] + if not re.fullmatch(r'[a-f0-9]{16}', attempt): + raise RuntimeError('恢复日志无效。') + for item in plan['privateSwitches']: + target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged')) + failed = checked_local(item['failed']) + if previous.exists(): + if target.exists(): + target.rename(failed) + previous.rename(target) + elif not item['existed'] and not staged.exists() and target.exists(): + target.rename(failed) + control.atomic(control.CONFIG, plan['previousConfig']) + JOURNAL.rename(control.PRIVATE / ('migration-rolled-back-' + attempt + '.json')) + + +def commit_plan(plan, receipt): + control.atomic(JOURNAL, plan) + try: + for item in plan['privateSwitches']: + target, previous, staged = (checked_local(item[key]) for key in ('target', 'previous', 'staged')) + if target.is_symlink() or staged.is_symlink() or previous.exists(): + raise RuntimeError('私有配置切换路径无效。') + if target.exists(): + target.rename(previous) + staged.rename(target) + control.atomic(control.CONFIG, plan['newConfig']) + control.atomic(RECEIPT, receipt) + except BaseException: + rollback_plan(plan) + raise + JOURNAL.unlink(missing_ok=True) + + +def restore(check_runtime=True): + control.private_directory(control.PRIVATE) + control.private_directory(control.DATA) + (control.DATA / 'logs').mkdir(exist_ok=True) + (control.DATA / 'sources').mkdir(exist_ok=True) + (control.DATA / 'archive').mkdir(exist_ok=True) + lock = os.open(control.PRIVATE / 'migration.lock', os.O_CREAT | os.O_RDWR, 0o600) + try: + control.host.fcntl.flock(lock, control.host.fcntl.LOCK_EX | control.host.fcntl.LOCK_NB) + except BlockingIOError: + os.close(lock) + raise RuntimeError('另一项迁移正在进行,请等待原窗口完成。') from None + try: + return restore_locked(check_runtime) + finally: + os.close(lock) + + +def restore_locked(check_runtime): + manifest_path = PAYLOAD / 'manifest.json' + manifest = json.loads(manifest_path.read_text(encoding='utf-8')) + if manifest.get('formatVersion') != 1 or not re.fullmatch(r'[a-f0-9]{32}', manifest.get('migrationId', '')): + raise RuntimeError('迁移包格式无效。') + migration_id = manifest['migrationId'] + if RECEIPT.exists(): + receipt = json.loads(RECEIPT.read_text(encoding='utf-8')) + current = control.configuration() + if receipt.get('migrationId') == migration_id and current.get('restoredFromMigration') == migration_id and current.get('mysqlDatabase') == receipt.get('database'): + print('此迁移已完成,保留 Windows 上的后续数据,直接启动。', flush=True) + JOURNAL.unlink(missing_ok=True) + return receipt + raise RuntimeError('此目录已有其他迁移记录,请使用新的部署目录。') + stop_owned_service() + if JOURNAL.exists(): + rollback_plan(json.loads(JOURNAL.read_text(encoding='utf-8'))) + print('上次未完成的切换已撤回,原配置已保留。', flush=True) + value = control.verify() if check_runtime else control.configuration() + print('正在校验迁移包……', flush=True) + for name, key in (('database.sql.gz', 'databaseDumpSha256'), ('archive.tar.gz', 'archiveSha256'), + ('native-history.tar.gz', 'nativeHistorySha256')): + if sha256(PAYLOAD / name) != manifest[key]: + raise RuntimeError('迁移包文件不完整,已停止恢复。') + if manifest.get('modelConfigSha256') and sha256(PAYLOAD / 'model-config.json') != manifest['modelConfigSha256']: + raise RuntimeError('模型配置校验失败。') + if shutil.disk_usage(ROOT).free < manifest['minimumFreeBytes']: + raise RuntimeError('可用磁盘空间不足,请至少预留 ' + str((manifest['minimumFreeBytes'] + 1024**3 - 1) // 1024**3) + ' GB。') + attempt = uuid.uuid4().hex[:16] + stage = control.PRIVATE / ('migration-stage-' + attempt) + stage.mkdir(mode=0o700) + archive = control.DATA / 'migrations' / attempt / 'archive' + archive.parent.mkdir(parents=True, mode=0o700) + native = stage / 'nao-users' + print('正在恢复来源文件和分析历史……', flush=True) + extract_checked(PAYLOAD / 'archive.tar.gz', archive, manifest['archiveFiles']) + extract_checked(PAYLOAD / 'native-history.tar.gz', native, manifest['nativeHistoryFiles']) + native_count = relocate_native(native, manifest['sourceProjectRoot']) + if native_count != manifest['nativeHistoryDatabases']: + raise RuntimeError('分析历史文件数量不一致。') + model = stage / 'model-config.json' + if manifest.get('modelConfigSha256'): + shutil.copy2(PAYLOAD / 'model-config.json', model) + model.chmod(0o600) + settings = json.loads(model.read_text(encoding='utf-8')) + if any('mimo' in str(profile.get(key, '')).lower() for profile in settings['profiles'] for key in ('model', 'name', 'baseUrl')): + raise RuntimeError('迁移模型配置中仍含 MiMo。') + database = 'quality_restore_' + uuid.uuid4().hex + try: + start_database(value) + run_query(value, 'CREATE DATABASE `' + database + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;') + restore_dump(value, database, PAYLOAD / 'database.sql.gz') + query = lambda statement: run_query(value, statement, database) + print('正在核对全部数据表、原账号和正式统计结果……', flush=True) + actual_counts = table_counts(query) + if actual_counts != manifest['tableCounts']: + raise RuntimeError('恢复后的数据表行数不一致,未切换正式使用库。') + actual_users = [json.loads(line) for line in query("SELECT JSON_OBJECT('id',id,'username',username,'passwordHashSha256',SHA2(password_hash,256),'active',active) FROM app_user ORDER BY id;").splitlines()] + if sorted(actual_users, key=lambda row: row['id']) != sorted(manifest['sourceUsers'], key=lambda row: row['id']): + raise RuntimeError('原用户账号校验失败,未切换使用库。') + if query_evidence(query) != manifest['queryEvidence']: + raise RuntimeError('正式版本和月度统计结果不一致,未切换使用库。') + rebound = rebind_sources(query, manifest, archive) + run_query(value, 'GRANT ALL PRIVILEGES ON `' + database + "`.* TO 'quality_app'@'127.0.0.1';") + run_query(value, 'FLUSH TABLES;') + finally: + stop_owned_service() + new_config = {**value, 'mysqlDatabase': database, 'archiveDirectory': archive.relative_to(ROOT).as_posix(), + 'restoredFromMigration': migration_id, 'webBind': '0.0.0.0'} + for name in ('dataWorkflowTestMode', 'analysisAutonomousEnabled', 'analysisProtocolV1Enabled'): + if name in manifest.get('applicationSettings', {}): + new_config[name] = bool(manifest['applicationSettings'][name]) + switches = [] + for staged, target in [(native, control.PRIVATE / 'nao-users')] + ([(model, control.PRIVATE / 'nao-model-config.json')] if model.exists() else []): + switches.append({'staged': staged.relative_to(ROOT).as_posix(), 'target': target.relative_to(ROOT).as_posix(), + 'previous': (control.PRIVATE / ('before-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(), + 'failed': (control.PRIVATE / ('failed-migration-' + attempt + '-' + target.name)).relative_to(ROOT).as_posix(), + 'existed': target.exists()}) + receipt = {'migrationId': migration_id, 'migrationManifestSha256': sha256(manifest_path), 'database': database, + 'restoredAtUtc': datetime.now(timezone.utc).isoformat(), 'tables': len(actual_counts), + 'totalRows': sum(actual_counts.values()), 'sourceReferencesRebound': rebound, + 'modelProfilesMigrated': manifest['modelProfilesMigrated'], 'modelProfilesExcluded': manifest['modelProfilesExcluded'], + 'nativeHistoryDatabases': native_count, 'originalDatabasesPreserved': True, + 'archiveDirectory': new_config['archiveDirectory']} + plan = {'attempt': attempt, 'previousConfig': value, 'newConfig': new_config, 'privateSwitches': switches} + commit_plan(plan, receipt) + print('迁移恢复完成。请使用原来的账号和密码登录;MiMo 配置未迁移。', flush=True) + return receipt + + +if __name__ == '__main__': + try: + restore() + except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error: + import traceback + log = control.DATA / 'logs/migration-validation.log' + log.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(log, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + with os.fdopen(fd, 'w', encoding='utf-8') as output: + traceback.print_exc(file=output) + if isinstance(error, RuntimeError): + print(str(error), file=sys.stderr) + print('迁移未完成。已有数据库保留;原配置在切换失败时自动恢复。请查看私有恢复日志后重试。', file=sys.stderr) + raise SystemExit(1) diff --git a/deploy/offline/runtime_platform.py b/deploy/offline/runtime_platform.py new file mode 100644 index 0000000..60444e2 --- /dev/null +++ b/deploy/offline/runtime_platform.py @@ -0,0 +1,87 @@ +"""Platform helpers for the standalone distribution; POSIX behavior is preserved.""" +from __future__ import annotations + +import errno +import os +import signal +import time +IS_WINDOWS = os.name == 'nt' + +if os.name != 'nt': + import fcntl +else: + import msvcrt + + class _FileLocks: + LOCK_EX, LOCK_NB, LOCK_UN = 2, 4, 8 + + @staticmethod + def flock(fd, operation): + os.lseek(fd, 0, os.SEEK_SET) + if operation & _FileLocks.LOCK_UN: + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + return + while True: + try: + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + return + except OSError as error: + if error.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK): + raise + if operation & _FileLocks.LOCK_NB: + raise BlockingIOError(errno.EAGAIN, 'File is locked') from None + time.sleep(.05) + + fcntl = _FileLocks() + + +def private_bits(mode): + # NTFS access is protected by the inherited user-only ACL on .runtime. + # Windows' st_mode has no information about that ACL. + return not IS_WINDOWS and bool(mode & 0o077) + + +def owner_mismatch(info): + return not IS_WINDOWS and info.st_uid != os.getuid() + + +def fchmod(fd, mode): + if os.name != 'nt': + os.fchmod(fd, mode) + + +def ps_value(pid, field): + import psutil + try: + process = psutil.Process(int(pid)) + if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE: + return None + if field == 'lstart': + return repr(process.create_time()) + if field == 'command': + return ' '.join(process.cmdline()) + return None + except (psutil.Error, ValueError): + return None + + +def group_leader(pid): + return True if os.name == 'nt' else os.getpgid(pid) == pid + + +def killpg(pid, sig): + if os.name != 'nt': + return os.killpg(pid, sig) + import psutil + try: + process = psutil.Process(pid) + # Caller verifies creation time and executable before reaching this point. + children = process.children(recursive=True) + for child in reversed(children): + try: + child.kill() if sig == getattr(signal, 'SIGKILL', 9) else child.terminate() + except psutil.NoSuchProcess: + pass + process.kill() if sig == getattr(signal, 'SIGKILL', 9) else process.terminate() + except psutil.NoSuchProcess: + raise ProcessLookupError(pid) from None diff --git a/deploy/offline/smoke_local.py b/deploy/offline/smoke_local.py new file mode 100644 index 0000000..7f69b33 --- /dev/null +++ b/deploy/offline/smoke_local.py @@ -0,0 +1,117 @@ +"""Exercise an isolated empty MySQL database with the host's native test runtimes.""" +import argparse +import http.cookiejar +import json +import os +from pathlib import Path +import secrets +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import urllib.request + +from build_bundle import ROOT, TEMPLATES, adapt_runtime + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--work', type=Path, required=True) + args = parser.parse_args() + work = args.work.resolve() + stage = Path(tempfile.mkdtemp(prefix='qc-offline-', dir='/tmp')).resolve() + (work / 'smoke-root.txt').write_text(str(stage)) + for folder in ('deploy/offline', 'scripts', 'runtimes', 'app', '.runtime'): + (stage / folder).mkdir(parents=True, exist_ok=True, mode=0o700) + for name in ('control.py', 'gateway.py'): + shutil.copy2(TEMPLATES / name, stage / 'deploy/offline' / name) + shutil.copy2(TEMPLATES / 'runtime_platform.py', stage / 'scripts/runtime_platform.py') + for pattern in ('nao_*.py', 'analysis_visual*.py'): + for path in (ROOT / 'scripts').glob(pattern): + (stage / 'scripts' / path.name).write_text(adapt_runtime(path.read_text()) if path.name == 'nao_quality_runtime.py' else path.read_text()) + shutil.copytree(ROOT / 'ai/nao-quality', stage / 'ai/nao-quality') + shutil.copy2(next((work / 'source/backend/target').glob('*.jar')), stage / 'app/backend.jar') + shutil.copytree(work / 'source/web/dist', stage / 'app/web') + for name, target in (('java', ROOT / '.runtime/jdk/Contents/Home'), ('mysql', ROOT / '.runtime/mysql'), + ('python', ROOT / '.runtime/nao-trial/venv')): + (stage / 'runtimes' / name).symlink_to(target, target_is_directory=True) + def available(port): + try: + with socket.socket() as test: + test.bind(('127.0.0.1', port)) + return True + except OSError: + return False + port = next(p for p in (5284, 5283, 5285) if available(p)) + def free(): + with socket.socket() as test: + test.bind(('127.0.0.1', 0)) + return test.getsockname()[1] + value = {'mysqlPort': free(), 'backendPort': port, 'webPort': free(), 'webBind': '127.0.0.1', + 'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32), + 'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)} + config = stage / '.runtime/deployment.json' + config.write_text(json.dumps(value)) + config.chmod(0o600) + report = {'environment': 'macOS isolated empty database using host-native runtimes', + 'targetWindowsLinuxNativeValidated': False, 'checks': []} + process = None + try: + for cycle in range(2): + with (stage / f'cycle-{cycle}.log').open('w') as log: + process = subprocess.Popen([sys.executable, str(stage / 'deploy/offline/control.py'), 'serve'], stdout=log, stderr=subprocess.STDOUT) + for _ in range(180): + if process.poll() is not None: + raise RuntimeError('Supervisor exited; inspect ' + str(stage / f'cycle-{cycle}.log')) + try: + state = json.loads((stage / '.runtime/service.json').read_text()) + if state.get('ready'): + break + except (FileNotFoundError, ValueError): + pass + time.sleep(1) + else: + raise RuntimeError('Isolated startup timed out') + cookies = http.cookiejar.CookieJar() + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(cookies)) + base = f'http://127.0.0.1:{value["webPort"]}' + with opener.open(base + '/', timeout=5) as response: + assert response.status == 200 and b'
' in response.read() + with opener.open(base + '/actuator/health', timeout=5) as response: + assert json.load(response)['status'] == 'UP' + with opener.open(base + '/api/auth/csrf', timeout=5) as response: + token = json.load(response)['token'] + request = urllib.request.Request(base + '/api/auth/login', + data=json.dumps({'username': value['adminUsername'], 'password': value['adminPassword']}).encode(), + headers={'Content-Type': 'application/json', 'X-XSRF-TOKEN': token}) + with opener.open(request, timeout=10) as response: + assert response.status == 200 + for path in ('/api/auth/me', '/api/options'): + with opener.open(base + path, timeout=10) as response: + assert response.status == 200 + report['checks'].append({'cycle': cycle + 1, 'web': True, 'databaseMigration': True, + 'csrfAndLogin': True, 'authenticatedOptions': True}) + result = subprocess.run([sys.executable, str(stage / 'deploy/offline/control.py'), 'stop'], capture_output=True, text=True, timeout=150) + if result.returncode: + raise RuntimeError('Stop failed: ' + result.stderr) + process.wait(timeout=10) + process = None + assert not (stage / '.runtime/service.json').exists() + print('Isolated startup/login/stop cycle', cycle + 1, 'passed', flush=True) + report['passed'] = True + except Exception as error: + report['passed'] = False + report['error'] = str(error) + print(str(error), flush=True) + finally: + if process is not None and process.poll() is None: + subprocess.run([sys.executable, str(stage / 'deploy/offline/control.py'), 'stop'], capture_output=True, timeout=150) + (work / 'smoke-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') + print(json.dumps(report, ensure_ascii=False), flush=True) + return 0 if report['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/deploy/offline/test_migration.py b/deploy/offline/test_migration.py new file mode 100644 index 0000000..fcf4075 --- /dev/null +++ b/deploy/offline/test_migration.py @@ -0,0 +1,148 @@ +import gzip +import hashlib +import io +import json +import os +from pathlib import Path +import sqlite3 +import sys +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import control +import restore_migration as restore +from migration_common import dump_counts, extract_checked, filter_models, inserted_rows, path_under_source, relative_name +from prepare_migration import native_copy + + +class MigrationValidationTest(unittest.TestCase): + def test_extended_insert_counts_ignore_strings_and_escaped_quotes(self): + cases = [ + (b"INSERT INTO `x` VALUES (1,'a'),(2,'b');\n", 2), + (b"INSERT INTO `x` VALUES (1,'fake ),( delimiter'),(2,'quote \\' ),( escaped'),(3,0x001122);\n", 3), + (b"INSERT INTO `x` VALUES ('a\\\\',NULL,'(nested)'),(NULL,'\\n',4);\n", 2), + (b"INSERT INTO `x` VALUES ('\xe4\xb8\xad\xe6\x96\x87', '{\\\"text\\\":\\\"),(\\\"}');\n", 1), + ] + for statement, count in cases: + with self.subTest(statement=statement): + self.assertEqual(inserted_rows(statement), count) + + def test_dump_counts_include_empty_tables(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / 'dump.gz' + with gzip.open(path, 'wb') as stream: + stream.write(b"CREATE TABLE `empty` (id INT);\nCREATE TABLE `x` (id INT);\nINSERT INTO `x` VALUES (1),(2);\n") + counts, size = dump_counts(path) + self.assertEqual(counts, {'empty': 0, 'x': 2}) + self.assertGreater(size, 0) + + def test_mimo_profiles_and_root_selection_are_excluded(self): + config = {'revision': 'original', 'activeProfileId': 'mimo', 'profiles': [ + {'id': 'mimo', 'name': 'MiMo', 'model': 'mimo-v2.6', 'apiKey': 'exclude-this', 'baseUrl': 'https://api.example.test'}, + {'id': 'other', 'name': 'Qwen', 'model': 'qwen-test', 'apiKey': 'retain-this', 'baseUrl': 'https://other.example.test'}]} + filtered, removed = filter_models(config) + self.assertEqual(len(removed), 1) + self.assertEqual(filtered['activeProfileId'], 'other') + self.assertEqual(filtered['apiKey'], 'retain-this') + self.assertEqual(filtered['revision'], 'original') + self.assertNotIn('exclude-this', json.dumps(filtered)) + self.assertEqual(len(filtered['profiles']), 1) + + def test_mac_path_is_parsed_portably(self): + self.assertEqual(path_under_source('/Users/a/archive/source/文件.xlsx', '/Users/a/archive'), 'source/文件.xlsx') + for name in ('../outside', '/absolute', 'C:/data', 'a\\b', '.', 'a/./b', 'CON.txt', 'x/name.'): + with self.subTest(name=name), self.assertRaises(ValueError): + relative_name(name) + with self.assertRaises(ValueError): + path_under_source('/Users/a/archive-other/secret', '/Users/a/archive') + + def test_archive_rejects_traversal_links_duplicates_and_bad_hash(self): + for kind in ('traversal', 'link', 'duplicate', 'checksum'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + archive = folder / 'archive.gz' + body = b'contents' + with tarfile.open(archive, 'w:gz') as output: + info = tarfile.TarInfo('../outside' if kind == 'traversal' else 'file') + if kind == 'link': + info.type, info.linkname = tarfile.SYMTYPE, '/outside' + output.addfile(info) + else: + info.size = len(body) + output.addfile(info, io.BytesIO(body)) + if kind == 'duplicate': + output.addfile(info, io.BytesIO(body)) + expected = {'file': {'bytes': len(body), 'sha256': '0' * 64 if kind == 'checksum' else hashlib.sha256(body).hexdigest()}} + with self.assertRaises(ValueError): + extract_checked(archive, folder / 'target', expected) + self.assertFalse((folder / 'outside').exists()) + + def test_native_history_preserves_chats_and_scrubs_credentials(self): + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + source = folder / 'source.sqlite' + db = sqlite3.connect(source) + db.executescript("CREATE TABLE chat(id TEXT,body TEXT);INSERT INTO chat VALUES('a','history');" + "CREATE TABLE project_llm_config(api_key TEXT);INSERT INTO project_llm_config VALUES('mimo-exclusive-secret');" + "CREATE TABLE session(token TEXT);INSERT INTO session VALUES('transient-token');") + db.commit() + db.close() + target = folder / 'target.sqlite' + native_copy(source, target) + migrated = sqlite3.connect(target) + self.assertEqual(migrated.execute('SELECT body FROM chat').fetchone()[0], 'history') + self.assertEqual(migrated.execute('SELECT COUNT(*) FROM project_llm_config').fetchone()[0], 0) + self.assertEqual(migrated.execute('SELECT COUNT(*) FROM session').fetchone()[0], 0) + migrated.close() + self.assertNotIn(b'mimo-exclusive-secret', target.read_bytes()) + + +class MigrationCommitTest(unittest.TestCase): + def test_partial_file_switch_rolls_back_existing_files_and_configuration(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + private = root / '.runtime' + private.mkdir() + target = private / 'nao-users' + target.mkdir() + (target / 'old').write_text('original') + staged = private / 'stage-native' + staged.mkdir() + (staged / 'new').write_text('new') + model = private / 'nao-model-config.json' + model.write_text('original model') + attempt = 'a' * 16 + plan = {'attempt': attempt, 'previousConfig': {'previous': True}, 'newConfig': {'new': True}, 'privateSwitches': [ + {'target': '.runtime/nao-users', 'previous': '.runtime/previous-native', 'staged': '.runtime/stage-native', 'failed': '.runtime/failed-native', 'existed': True}, + {'target': '.runtime/nao-model-config.json', 'previous': '.runtime/previous-model', 'staged': '.runtime/missing-stage-model', 'failed': '.runtime/failed-model', 'existed': True}]} + with patch.multiple(restore, ROOT=root, JOURNAL=private / 'journal', RECEIPT=private / 'receipt'), patch.object(control, 'CONFIG', private / 'deployment.json'): + with self.assertRaises(FileNotFoundError): + restore.commit_plan(plan, {}) + self.assertEqual((target / 'old').read_text(), 'original') + self.assertEqual(model.read_text(), 'original model') + self.assertEqual(json.loads((private / 'deployment.json').read_text()), {'previous': True}) + self.assertTrue((private / 'failed-native/new').exists()) + + def test_repeated_restore_does_not_reimport_old_snapshot(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + payload = root / 'migration' + payload.mkdir() + migration_id = 'a' * 32 + (payload / 'manifest.json').write_text(json.dumps({'formatVersion': 1, 'migrationId': migration_id})) + receipt = root / 'receipt' + value = {'migrationId': migration_id, 'database': 'quality_restore_' + 'b' * 32} + receipt.write_text(json.dumps(value)) + config = {'restoredFromMigration': migration_id, 'mysqlDatabase': value['database']} + with patch.multiple(restore, PAYLOAD=payload, RECEIPT=receipt, JOURNAL=root / 'journal'), patch.object(control, 'configuration', return_value=config), patch.object(restore, 'stop_owned_service') as stop, patch.object(restore, 'run_query') as query: + self.assertEqual(restore.restore_locked(False), value) + stop.assert_not_called() + query.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/offline/test_offline.py b/deploy/offline/test_offline.py new file mode 100644 index 0000000..c50190d --- /dev/null +++ b/deploy/offline/test_offline.py @@ -0,0 +1,179 @@ +import functools +import http.client +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import importlib.util +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import tempfile +import threading +import time +import unittest + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from gateway import make_server +from build_bundle import adapt_runtime +import runtime_platform + + +class Upstream(BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + if self.path == '/api/events': + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.end_headers() + self.wfile.write(b'data: first\n\n') + self.wfile.flush() + time.sleep(.8) + self.wfile.write(b'data: second\n\n') + self.wfile.flush() + else: + body = json.dumps({'cookie': self.headers.get('Cookie'), 'host': self.headers.get('Host')}).encode() + self.send_response(401) + self.send_header('Set-Cookie', 'one=1; HttpOnly') + self.send_header('Set-Cookie', 'two=2; SameSite=Strict') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + body = self.rfile.read(int(self.headers['Content-Length'])) + self.send_response(201) + self.send_header('Content-Length', str(len(body))) + self.send_header('X-CSRF-Echo', self.headers.get('X-XSRF-TOKEN', '')) + self.end_headers() + self.wfile.write(body) + + +class GatewayTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp = tempfile.TemporaryDirectory() + root = Path(cls.temp.name) + (root / 'index.html').write_text('QUALITY APP', encoding='utf-8') + (root / 'assets').mkdir() + (root / 'assets/file.js').write_text('const x=1;') + cls.backend = ThreadingHTTPServer(('127.0.0.1', 0), Upstream) + cls.frontend = make_server(root, '127.0.0.1', 0, cls.backend.server_port) + for server in (cls.backend, cls.frontend): + threading.Thread(target=server.serve_forever, daemon=True).start() + + @classmethod + def tearDownClass(cls): + for server in (cls.frontend, cls.backend): + server.shutdown() + server.server_close() + cls.temp.cleanup() + + def connection(self): + return http.client.HTTPConnection('127.0.0.1', self.frontend.server_port, timeout=3) + + def test_static_and_spa(self): + for url, body in [('/', b'QUALITY APP'), ('/?page=home', b'QUALITY APP'), ('/analysis', b'QUALITY APP'), ('/assets/file.js', b'const x=1;')]: + connection = self.connection() + connection.request('GET', url) + response = connection.getresponse() + self.assertEqual(response.status, 200) + self.assertEqual(response.read(), body) + connection.close() + + def test_no_directory_listing_or_private_files(self): + for url in ('/assets/', '/.runtime/deployment.json', '/../.runtime/deployment.json', '/missing.js'): + connection = self.connection() + connection.request('GET', url) + response = connection.getresponse() + self.assertEqual(response.status, 404) + response.read() + connection.close() + + def test_auth_cookies_and_host_are_preserved(self): + connection = self.connection() + connection.request('GET', '/api/auth/me', headers={'Cookie': 'SESSION=abc', 'Host': '192.168.1.22:5280'}) + response = connection.getresponse() + self.assertEqual(response.status, 401) + self.assertEqual(len(response.headers.get_all('Set-Cookie')), 2) + self.assertEqual(json.loads(response.read()), {'cookie': 'SESSION=abc', 'host': '192.168.1.22:5280'}) + connection.close() + + def test_upload_and_csrf(self): + connection = self.connection() + data = os.urandom(2 * 1024 * 1024 + 19) + connection.request('POST', '/api/sources/upload', data, {'X-XSRF-TOKEN': 'csrf-value'}) + response = connection.getresponse() + self.assertEqual(response.status, 201) + self.assertEqual(response.getheader('X-CSRF-Echo'), 'csrf-value') + self.assertEqual(response.read(), data) + connection.close() + + def test_sse_is_streamed_before_upstream_finishes(self): + connection = self.connection() + started = time.monotonic() + connection.request('GET', '/api/events') + response = connection.getresponse() + self.assertEqual(response.readline(), b'data: first\n') + self.assertLess(time.monotonic() - started, .65) + self.assertIn(b'data: second', response.read()) + connection.close() + + +class PortabilityTest(unittest.TestCase): + def test_port_probe_rejects_live_listener_and_allows_restart(self): + import control + with socket.socket() as listener: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen() + with self.assertRaises(RuntimeError): + control.free_port(port) + accepted, _ = listener.accept() + accepted.close() + control.free_port(port) + + def test_process_identity_requires_start_time_executable_and_arguments(self): + import psutil + import control + process = psutil.Process() + info = control.record(process, process.cmdline()) + self.assertTrue(control.matches(info)) + self.assertFalse(control.matches({**info, 'created': info['created'] - 10})) + self.assertFalse(control.matches({**info, 'executable': '/unrelated/program'})) + self.assertFalse(control.matches({**info, 'args': ['unrelated-command']})) + + def test_posix_permissions_are_not_relaxed(self): + if os.name != 'nt': + self.assertFalse(runtime_platform.private_bits(0o100600)) + self.assertTrue(runtime_platform.private_bits(0o100640)) + + def test_lock_excludes_another_process(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'lock' + fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600) + runtime_platform.fcntl.flock(fd, runtime_platform.fcntl.LOCK_EX | runtime_platform.fcntl.LOCK_NB) + code = ('import sys,os;sys.path.insert(0,sys.argv[1]);import runtime_platform as h;' + 'fd=os.open(sys.argv[2],os.O_RDWR);h.fcntl.flock(fd,h.fcntl.LOCK_EX|h.fcntl.LOCK_NB)') + result = subprocess.run([sys.executable, '-c', code, str(HERE), str(path)], capture_output=True) + self.assertNotEqual(result.returncode, 0) + os.close(fd) + + def test_source_adapter_preserves_mac_source(self): + path = HERE.parents[1] / 'scripts/nao_quality_runtime.py' + source = path.read_text() + adapted = adapt_runtime(source) + self.assertIn('import fcntl', source) + self.assertIn('from runtime_platform import fcntl', adapted) + self.assertIn('_host.private_bits(directory_info.st_mode)', adapted) + self.assertIn('_host.killpg', adapted) + compile(adapted, str(path), 'exec') + self.assertEqual(path.read_text(), source) + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/offline/test_publish_update.py b/deploy/offline/test_publish_update.py new file mode 100644 index 0000000..e49b463 --- /dev/null +++ b/deploy/offline/test_publish_update.py @@ -0,0 +1,175 @@ +"""Offline tests for the signed Windows update publisher.""" +import base64 +import hashlib +import json +from pathlib import Path +import tempfile +import sys +import unittest +import zipfile + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import publish_update +import update_manager + + +class PublishUpdateTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.base = Path(self.temp.name) + self.root = self.base / 'windows-bundle' + self.root.mkdir() + self.output = self.base / 'published' + self.key_path = self.base / 'release-private-key.pem' + self.private = Ed25519PrivateKey.generate() + self.key_path.write_bytes(self.private.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + )) + public_key_path = self.root / update_manager.KEY + public_key_path.parent.mkdir(parents=True, exist_ok=True) + public_key_path.write_bytes(self.private.public_key().public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + )) + self.data = { + 'app/backend.jar': b'backend program', + 'app/web/index.html': b'program', + 'deploy/offline/control.py': b'control program', + 'deploy/offline/update_manager.py': b'update manager program', + 'deploy/offline/publish_update.py': b'publisher program', + 'deploy/offline/windows-start.ps1': b'start script', + 'deploy/offline/README.md': b'package instructions', + 'backend/src/main/resources/analysis-runtime-policy.json': b'{"enabled":false}', + 'scripts/runtime.py': b'helper program', + 'ai/nao-quality/app.py': b'ai program', + 'runtimes/python/python.exe': b'local runtime must not ship in an update', + 'migration/manifest.json': b'private migration data', + 'deploy/bootstrap/update-recovery.py': b'immutable bootstrap', + '.runtime/deployment.json': b'{"mysqlPassword":"private"}', + 'runtimedata/archive/source.xlsx': b'user business data', + update_manager.KEY: public_key_path.read_bytes(), + } + for name, value in self.data.items(): + target = self.root.joinpath(*name.split('/')) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(value) + self.manifest = { + 'platform': 'windows', + 'architecture': 'x86_64', + 'updateVersion': '2026.11.04.1', + 'updateSequence': 1, + 'runtimes': [{'name': 'python', 'version': '3.11.15'}], + 'pythonWheels': [{'file': 'cryptography.whl', 'sha256': 'a' * 64}], + 'files': [ + {'path': name, 'sha256': hashlib.sha256(value).hexdigest()} + for name, value in self.data.items() + ], + } + (self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8') + + def build(self, **changes): + args = { + 'root': self.root, + 'version': '2026.11.04.2', + 'sequence': 2, + 'private_key_path': self.key_path, + 'output_dir': self.output, + } + args.update(changes) + return publish_update.publish(**args) + + def test_release_is_signed_program_snapshot_with_runtime_fingerprint(self): + artifacts = self.build() + self.assertTrue(artifacts.archive.is_file()) + self.assertTrue(artifacts.latest.is_file()) + envelope = update_manager.read(artifacts.latest) + release = update_manager.validate_release(envelope, self.root) + raw = base64.b64decode(envelope['payload'], validate=True) + self.private.public_key().verify(base64.b64decode(envelope['signature'], validate=True), raw) + self.assertEqual('2026.11.04.2', release['version']) + self.assertEqual(2, release['sequence']) + self.assertEqual(update_manager.runtime_fingerprint(self.manifest), release['runtimeFingerprint']) + self.assertNotIn('bundleManifest', release) + included = {entry['path'] for entry in release['files']} + self.assertEqual({name for name in self.data if update_manager.program_path(name)}, included) + self.assertFalse(included & { + 'runtimes/python/python.exe', + 'migration/manifest.json', + 'deploy/bootstrap/update-recovery.py', + '.runtime/deployment.json', + 'runtimedata/archive/source.xlsx', + update_manager.KEY, + }) + with zipfile.ZipFile(artifacts.archive) as archive: + self.assertEqual(included, set(archive.namelist())) + self.assertEqual(self.data['app/backend.jar'], archive.read('app/backend.jar')) + self.assertEqual(hashlib.sha256(artifacts.archive.read_bytes()).hexdigest(), release['archive']['sha256']) + + def test_rejects_wrong_key_stale_sequence_and_package_mutation(self): + with self.assertRaisesRegex(publish_update.PublishError, '不匹配'): + wrong_path = self.base / 'wrong-private-key.pem' + wrong_path.write_bytes(Ed25519PrivateKey.generate().private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + )) + self.build(private_key_path=wrong_path) + with self.assertRaisesRegex(publish_update.PublishError, '大于部署包当前序号'): + self.build(sequence=1) + (self.root / 'app/backend.jar').write_bytes(b'changed after manifest') + with self.assertRaisesRegex(publish_update.PublishError, '摘要不一致'): + self.build() + self.assertFalse(self.output.exists()) + + def test_allows_only_explicit_exact_baseline_and_still_rejects_lower_sequence(self): + with self.assertRaisesRegex(publish_update.PublishError, '大于部署包当前序号'): + self.build(version='2026.11.04.1', sequence=1) + with self.assertRaisesRegex(publish_update.PublishError, '完全一致'): + self.build(version='2026.11.04.2', sequence=1, allow_baseline=True) + baseline = self.build(version='2026.11.04.1', sequence=1, allow_baseline=True) + release = update_manager.validate_release(update_manager.read(baseline.latest), self.root) + self.assertEqual('2026.11.04.1', release['version']) + self.assertEqual(1, release['sequence']) + + self.manifest['updateVersion'] = '2026.11.04.1' + self.manifest['updateSequence'] = 2 + (self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8') + self.output = self.base / 'lower-sequence-output' + with self.assertRaisesRegex(publish_update.PublishError, '不能低于'): + self.build(version='2026.11.04.1', sequence=1, allow_baseline=True) + + def test_rejects_sensitive_program_path_and_missing_program_inventory(self): + sensitive_name = 'ai/nao-quality/private/model.json' + sensitive_path = self.root.joinpath(*sensitive_name.split('/')) + sensitive_path.parent.mkdir(parents=True, exist_ok=True) + sensitive_path.write_bytes(b'private model config') + self.manifest['files'].append({'path': sensitive_name, 'sha256': hashlib.sha256(sensitive_path.read_bytes()).hexdigest()}) + (self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8') + with self.assertRaisesRegex(publish_update.PublishError, '受保护'): + self.build() + self.manifest['files'].pop() + self.manifest['files'] = [entry for entry in self.manifest['files'] if entry['path'] != 'scripts/runtime.py'] + (self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8') + with self.assertRaisesRegex(publish_update.PublishError, '完整程序文件清单'): + self.build() + + def test_refuses_key_inside_source_or_output_and_refuses_overwrite(self): + with self.assertRaisesRegex(publish_update.PublishError, '部署包和更新产物目录之外'): + self.build(private_key_path=self.root / 'manifest.json') + with self.assertRaisesRegex(publish_update.PublishError, '部署包目录之外'): + self.build(output_dir=self.root / 'updates') + self.build() + original_latest = (self.output / 'latest.json').read_bytes() + with self.assertRaisesRegex(publish_update.PublishError, '同名发布文件'): + self.build() + self.assertEqual(original_latest, (self.output / 'latest.json').read_bytes()) + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/offline/test_update_bootstrap.py b/deploy/offline/test_update_bootstrap.py new file mode 100644 index 0000000..6127c73 --- /dev/null +++ b/deploy/offline/test_update_bootstrap.py @@ -0,0 +1,339 @@ +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +import time +import unittest +from unittest.mock import patch + +import psutil + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import update_bootstrap as bootstrap + + +class ImmutableRecoveryTest(unittest.TestCase): + def fixture(self, folder): + root = Path(folder) + directory = root / '.runtime/update' + backup = directory / 'backup-20261104T120000Z-abcdef12' + (backup / 'program/deploy/offline').mkdir(parents=True) + (root / 'deploy/offline').mkdir(parents=True) + old = { + 'app/backend.jar': b'verified old backend jar', + 'deploy/offline/control.py': b'VALID_OLD_VERSION = True\n', + 'deploy/offline/update_manager.py': b'def recover_locked(root):\n (root / ".runtime/restored").write_text("done")\n return {"recovered": True}\n', + } + entries = [] + for name, content in old.items(): + path = backup / 'program' / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_bytes(b'def broken syntax\n') + entries.append({'path': name, 'sha256': hashlib.sha256(content).hexdigest()}) + (backup / 'manifest.json').write_text(json.dumps({'files': entries})) + (backup / 'program.json').write_text(json.dumps({'all': list(old), 'present': list(old)})) + journal = {'phase': 'installing', 'databaseBackedUp': True, 'backup': backup.name, + 'backupManifestSha256': bootstrap.digest(backup / 'manifest.json'), + 'programInventorySha256': bootstrap.digest(backup / 'program.json')} + (directory / 'journal.json').write_text(json.dumps(journal)) + return root, directory, backup, journal, old + + def runtime_aliases(self, root): + for role in ('supervisor', 'backend'): + executable = bootstrap.runtime_executable(root, role) + executable.parent.mkdir(parents=True, exist_ok=True) + executable.symlink_to(psutil.Process().exe()) + + def record(self, process, name=None): + observed = psutil.Process(process.pid) + value = {'pid': process.pid, 'created': observed.create_time(), + 'executable': str(Path(observed.exe()).resolve()), 'args': observed.cmdline()[1:]} + if name: + value['name'] = name + return value + + def stop_test_child(self, process): + if process.poll() is not None: + return + descendants = [] + try: + descendants = psutil.Process(process.pid).children(recursive=True) + except psutil.Error: + pass + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + for child in descendants: + try: + if child.is_running() and child.status() != psutil.STATUS_ZOMBIE: + child.kill() + except psutil.Error: + pass + + def wait_file(self, path, process): + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if path.exists(): + return + self.assertIsNone(process.poll(), 'Owned test child exited before readiness') + time.sleep(.02) + self.fail('Owned test child did not become ready') + + def backend_script(self, root): + path = root / 'backend-test.py' + path.write_text(textwrap.dedent(''' + import hashlib, os, signal, sys, time + from pathlib import Path + root = Path(__file__).parent + jar = root / 'app/backend.jar' + held = jar.open('rb') + def stop(*ignored): + (root / '.runtime/backend-stopped-before-copy').write_text(hashlib.sha256(jar.read_bytes()).hexdigest()) + held.close() + raise SystemExit(0) + signal.signal(signal.SIGTERM, stop) + (root / '.runtime/backend-ready').write_text(str(os.getpid())) + while True: + time.sleep(.02) + ''')) + return path + + def start_backend(self, root): + self.runtime_aliases(root) + script = self.backend_script(root) + process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'backend')), str(script), '-jar', str(root / 'app/backend.jar')], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.addCleanup(self.stop_test_child, process) + self.wait_file(root / '.runtime/backend-ready', process) + return process + + def start_supervisor(self, root): + self.runtime_aliases(root) + self.backend_script(root) + script = root / 'deploy/offline/control.py' + script.write_text(textwrap.dedent(''' + import json, os, signal, subprocess, sys, time + from pathlib import Path + import psutil + root = Path(__file__).resolve().parents[2] + token = 'a' * 48 + java = root / 'runtimes/java/bin' / ('java.exe' if os.name == 'nt' else 'java') + child = subprocess.Popen([str(java), str(root / 'backend-test.py'), '-jar', str(root / 'app/backend.jar')]) + def record(pid): + process = psutil.Process(pid) + return {'pid': pid, 'created': process.create_time(), 'executable': str(Path(process.exe()).resolve()), 'args': process.cmdline()[1:]} + state = {'supervisor': record(os.getpid()), 'token': token, 'children': [{**record(child.pid), 'name': 'backend'}], 'ready': True} + def stop(*ignored): + child.terminate() + child.wait(timeout=5) + (root / '.runtime/supervisor-stopped').write_text('done') + (root / '.runtime/service.json').unlink(missing_ok=True) + raise SystemExit(0) + signal.signal(signal.SIGTERM, stop) + (root / '.runtime/service.json').write_text(json.dumps(state)) + (root / '.runtime/supervisor-ready').write_text('ready') + while True: + try: + request = json.loads((root / '.runtime/stop-request.json').read_text()) + if request.get('token') == token: + (root / '.runtime/token-observed').write_text(token) + stop() + except (FileNotFoundError, ValueError): + pass + time.sleep(.02) + ''')) + process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'supervisor')), str(script), 'serve'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.addCleanup(self.stop_test_child, process) + self.wait_file(root / '.runtime/supervisor-ready', process) + self.wait_file(root / '.runtime/backend-ready', process) + return process + + def test_recovers_before_importing_broken_control_and_updater(self): + with tempfile.TemporaryDirectory() as folder: + root, directory, backup, journal, old = self.fixture(folder) + result = bootstrap.recover(root) + self.assertTrue(result['recovered']) + self.assertEqual((root / '.runtime/restored').read_text(), 'done') + for name, content in old.items(): + self.assertEqual((root / name).read_bytes(), content) + self.assertFalse((directory / 'worker.json').exists()) + + def test_corrupt_backup_is_rejected_before_any_replacement(self): + with tempfile.TemporaryDirectory() as folder: + root, directory, backup, journal, old = self.fixture(folder) + (backup / 'program/deploy/offline/control.py').write_text('corruption') + with self.assertRaises(RuntimeError): + bootstrap.recover(root) + self.assertEqual((root / 'deploy/offline/control.py').read_text(), 'def broken syntax\n') + + def test_active_worker_cannot_be_overridden(self): + with tempfile.TemporaryDirectory() as folder: + root, directory, backup, journal, old = self.fixture(folder) + (directory / 'worker.json').write_text(json.dumps({'pid': os.getpid(), 'created': psutil.Process().create_time()})) + with self.assertRaises(RuntimeError): + bootstrap.recover(root) + self.assertFalse((root / '.runtime/restored').exists()) + + def test_program_protection_and_no_update_are_read_only(self): + for name in ('deploy/bootstrap/windows-start.ps1', 'deploy/offline/update-public-key.pem', '.runtime/deployment.json', 'runtimedata/mysql/file'): + self.assertFalse(bootstrap.program(name)) + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + self.assertEqual(bootstrap.recover(root), {'recovered': False}) + self.assertFalse((root / '.runtime').exists()) + + def test_orphaned_backend_stops_before_jar_copy_after_worker_crash(self): + with tempfile.TemporaryDirectory() as folder: + root, directory, _, _, old = self.fixture(folder) + child = self.start_backend(root) + state = {'supervisor': {'pid': 999999999, 'created': 1, 'executable': str(sys.executable), + 'args': [str(root / 'deploy/offline/control.py'), 'serve']}, 'token': 'b' * 48, + 'children': [self.record(child, 'backend')]} + (root / '.runtime/service.json').write_text(json.dumps(state)) + (directory / 'worker.json').write_text(json.dumps({'pid': 999999998, 'created': 1})) + before = bootstrap.digest(root / 'app/backend.jar') + real_copy = bootstrap.atomic_copy + def checked_copy(source, target): + self.assertIsNotNone(child.poll(), 'A mutable program was copied while the old backend still held its JAR') + return real_copy(source, target) + with patch.object(bootstrap, 'atomic_copy', side_effect=checked_copy): + self.assertTrue(bootstrap.recover(root)['recovered']) + self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text()) + self.assertEqual(old['app/backend.jar'], (root / 'app/backend.jar').read_bytes()) + + def test_live_supervisor_receives_matching_stop_token_and_children_exit_before_copy(self): + with tempfile.TemporaryDirectory() as folder: + root, directory, _, _, _ = self.fixture(folder) + supervisor = self.start_supervisor(root) + (root / 'deploy/offline/control.py').write_text('def broken syntax\n') + before = bootstrap.digest(root / 'app/backend.jar') + result = bootstrap.recover(root) + self.assertTrue(result['recovered']) + self.assertIsNotNone(supervisor.poll()) + self.assertEqual('a' * 48, (root / '.runtime/token-observed').read_text()) + self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text()) + self.assertFalse((root / '.runtime/stop-request.json').exists()) + + def test_stale_record_pointing_at_an_unrelated_live_pid_never_kills_or_replaces(self): + with tempfile.TemporaryDirectory() as folder: + root, _, _, _, _ = self.fixture(folder) + self.runtime_aliases(root) + child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.addCleanup(self.stop_test_child, child) + info = self.record(child, 'backend') + info['created'] -= 60 + (root / '.runtime/service.json').write_text(json.dumps({'children': [info]})) + with self.assertRaisesRegex(RuntimeError, '归属'): + bootstrap.recover(root) + self.assertIsNone(child.poll()) + self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text()) + self.assertFalse((root / '.runtime/restored').exists()) + + def test_unregistered_root_backend_is_refused_without_signalling(self): + with tempfile.TemporaryDirectory() as folder: + root, _, _, _, _ = self.fixture(folder) + child = self.start_backend(root) + with self.assertRaisesRegex(RuntimeError, '未登记'): + bootstrap.recover(root) + self.assertIsNone(child.poll()) + self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text()) + + def test_other_deployment_root_cannot_be_claimed_by_a_valid_live_pid_record(self): + with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second: + root, _, _, _, _ = self.fixture(first) + other, _, _, _, _ = self.fixture(second) + self.runtime_aliases(root) + child = self.start_backend(other) + (root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]})) + with self.assertRaisesRegex(RuntimeError, '归属'): + bootstrap.recover(root) + self.assertIsNone(child.poll()) + self.assertFalse((root / '.runtime/restored').exists()) + + def test_invalid_supervisor_token_does_not_signal_any_owned_process(self): + with tempfile.TemporaryDirectory() as folder: + root, _, _, _, _ = self.fixture(folder) + supervisor = self.start_supervisor(root) + state = json.loads((root / '.runtime/service.json').read_text()) + state['token'] = 'invalid-token' + (root / '.runtime/service.json').write_text(json.dumps(state)) + with self.assertRaisesRegex(RuntimeError, '令牌'): + bootstrap.recover(root) + self.assertIsNone(supervisor.poll()) + self.assertFalse((root / '.runtime/stop-request.json').exists()) + + def test_dead_pid_records_do_not_block_safe_recovery(self): + with tempfile.TemporaryDirectory() as folder: + root, _, _, _, _ = self.fixture(folder) + state = {'children': [{'pid': 999999999, 'created': 1, 'executable': str(sys.executable), + 'args': ['missing'], 'name': 'backend'}]} + (root / '.runtime/service.json').write_text(json.dumps(state)) + self.assertTrue(bootstrap.recover(root)['recovered']) + + def test_a_process_owned_by_another_user_is_never_stopped(self): + with tempfile.TemporaryDirectory() as folder: + root, _, _, _, _ = self.fixture(folder) + child = self.start_backend(root) + (root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]})) + with patch.object(bootstrap, 'current_user', return_value=False), self.assertRaisesRegex(RuntimeError, '归属'): + bootstrap.recover(root) + self.assertIsNone(child.poll()) + self.assertFalse((root / '.runtime/restored').exists()) + + @unittest.skipIf(os.name == 'nt', 'Host-side immutable-client fixture uses a POSIX shebang; real Windows binaries need native acceptance') + def test_orphaned_mysql_uses_immutable_shutdown_client_and_is_not_force_killed(self): + with tempfile.TemporaryDirectory() as folder: + root, _, _, _, _ = self.fixture(folder) + executable = bootstrap.runtime_executable(root, 'mysql') + executable.parent.mkdir(parents=True) + executable.symlink_to(psutil.Process().exe()) + server = root / 'mysql-test.py' + server.write_text(textwrap.dedent(''' + import os, signal, time + from pathlib import Path + root = Path(__file__).parent + def stop(*ignored): + (root / '.runtime/mysql-stopped-cleanly').write_text('done') + raise SystemExit(0) + signal.signal(signal.SIGTERM, stop) + (root / '.runtime/mysql-test-pid').write_text(str(os.getpid())) + while True: + time.sleep(.02) + ''')) + child = subprocess.Popen([str(executable), str(server), '--datadir=' + str(root / 'runtimedata/mysql'), '--port=54489'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.addCleanup(self.stop_test_child, child) + self.wait_file(root / '.runtime/mysql-test-pid', child) + admin = executable.parent / 'mysqladmin' + admin.write_text('#!' + psutil.Process().exe() + '\n' + textwrap.dedent(''' + import os, signal, sys + from pathlib import Path + root = Path(__file__).resolve().parents[3] + assert os.environ['MYSQL_PWD'] == 'isolated_password_value_1234' + assert '--no-defaults' in sys.argv and '--port=54489' in sys.argv and sys.argv[-1] == 'shutdown' + assert not any('password' in arg.lower() for arg in sys.argv) + (root / '.runtime/immutable-mysqladmin-called').write_text('done') + os.kill(int((root / '.runtime/mysql-test-pid').read_text()), signal.SIGTERM) + ''')) + admin.chmod(0o700) + (root / '.runtime/deployment.json').write_text(json.dumps({'mysqlPort': 54489, 'rootPassword': 'isolated_password_value_1234'})) + (root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'mysql')]})) + with patch.object(psutil.Process, 'kill', side_effect=AssertionError('MySQL must not be force killed')): + self.assertTrue(bootstrap.recover(root)['recovered']) + self.assertIsNotNone(child.poll()) + self.assertTrue((root / '.runtime/immutable-mysqladmin-called').exists()) + self.assertTrue((root / '.runtime/mysql-stopped-cleanly').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/offline/test_update_manager.py b/deploy/offline/test_update_manager.py new file mode 100644 index 0000000..d3392df --- /dev/null +++ b/deploy/offline/test_update_manager.py @@ -0,0 +1,303 @@ +"""Offline security and rollback tests. No network, production DB or services.""" +import base64 +import copy +import hashlib +import importlib.util +import json +from pathlib import Path +import shutil +import stat +import sys +import tempfile +import types +import unittest +from unittest.mock import patch +import zipfile + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives import serialization + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import update_manager as update + + +class UpdateTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) / 'bundle' + self.root.mkdir() + self.private = Ed25519PrivateKey.generate() + self.files = { + 'app/backend.jar': b'old backend', + 'app/web/index.html': b'old web', + 'app/web/obsolete.js': b'old asset', + 'deploy/offline/control.py': b'old supervisor', + 'deploy/offline/update_manager.py': b'old updater', + 'backend/src/main/resources/analysis-runtime-policy.json': b'{}', + update.KEY: self.private.public_key().public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo), + 'runtimes/python/python.exe': b'unchanged runtime', + 'migration/snapshot.sql': b'private migration data', + } + for name, data in self.files.items(): + self.write(name, data) + self.manifest = { + 'platform': 'windows', 'architecture': 'x86_64', + 'runtimes': [{'name': 'python', 'version': '3.11.15'}], + 'pythonWheels': [{'file': 'cryptography.whl', 'sha256': 'a' * 64}], + 'files': [{'path': k, 'sha256': hashlib.sha256(v).hexdigest()} for k, v in self.files.items()], + 'updateSequence': 5, 'updateVersion': '2026.11.03', + } + update.atomic(self.root / 'manifest.json', self.manifest) + self.config = {'mysqlDatabase': 'quality_restore_' + 'a' * 24, 'archiveDirectory': 'runtimedata/archive/restore-old', + 'mysqlPort': 53306, 'rootPassword': 'privateRootPasswordValue', 'mysqlPassword': 'privateAppPasswordValue'} + update.atomic(self.root / '.runtime/deployment.json', self.config) + self.write('.runtime/analysis-api-settings.json', b'{"apiKey":"keep-private"}') + self.program = {k: b'new ' + v for k, v in self.files.items() if update.program_path(k) and k != 'app/web/obsolete.js'} + self.release = {'schema': 1, 'version': '2026.11.04', 'sequence': 6, 'platform': 'windows-x64', + 'runtimeFingerprint': update.runtime_fingerprint(self.manifest), + 'archive': {'file': 'quality-update-2026.11.04.zip', 'size': 123, 'sha256': 'a' * 64}, + 'files': [{'path': k, 'size': len(v), 'sha256': hashlib.sha256(v).hexdigest()} for k, v in self.program.items()]} + + def write(self, name, data): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + def envelope(self, release=None, private=None): + raw = update.canonical(release or self.release) + return {'payload': base64.b64encode(raw).decode(), 'signature': base64.b64encode((private or self.private).sign(raw)).decode()} + + def validate(self, release=None): + return update.validate_release(self.envelope(release), self.root) + + def archive(self, entries=None): + archive = Path(self.temp.name) / 'update.zip' + with zipfile.ZipFile(archive, 'w') as out: + for name, data in (entries or self.program).items(): + out.writestr(name, data) + return archive + + def test_valid_signature_and_runtime_identity(self): + self.assertEqual('2026.11.04', self.validate()['version']) + self.assertEqual(update.runtime_fingerprint(self.manifest), update.runtime_fingerprint(copy.deepcopy(self.manifest))) + + def test_wrong_signature_key_and_modified_payload_are_rejected(self): + with self.assertRaises(update.UpdateError): + update.validate_release(self.envelope(private=Ed25519PrivateKey.generate()), self.root) + envelope = self.envelope() + envelope['payload'] = base64.b64encode(update.canonical({**self.release, 'sequence': 999})).decode() + with self.assertRaises(update.UpdateError): + update.validate_release(envelope, self.root) + + def test_immutable_trust_anchor_checked_against_local_manifest(self): + self.write(update.KEY, Ed25519PrivateKey.generate().public_key().public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo)) + with self.assertRaises(update.UpdateError): + self.validate() + + def test_runtime_mismatch_requires_full_bundle(self): + release = {**self.release, 'runtimeFingerprint': 'b' * 64} + with self.assertRaisesRegex(update.UpdateError, '完整安装包'): + self.validate(release) + + def test_forbidden_destinations_and_trust_key_are_never_update_files(self): + for name in ['.runtime/deployment.json', 'runtimedata/mysql/table.ibd', 'migration/snapshot.sql', + 'runtimes/python/python.exe', update.KEY, 'manifest.json', 'backend/src/private.key']: + with self.subTest(name=name): + release = copy.deepcopy(self.release) + release['files'].append({'path': name, 'size': 0, 'sha256': 'a' * 64}) + with self.assertRaises(update.UpdateError): + self.validate(release) + + def test_windows_aliases_traversal_and_case_collisions_rejected(self): + for name in ['../evil', 'app/web/../evil', 'app/web/C:evil', 'app\\web\\evil', '/absolute', + 'app/web/NUL.txt', 'app/web/trailing.', 'app/web/double//slash', 'app/web/control\x01']: + with self.subTest(name=name), self.assertRaises(update.UpdateError): + update.safe_path(name) + release = copy.deepcopy(self.release) + release['files'].append({**release['files'][0], 'path': release['files'][0]['path'].upper()}) + with self.assertRaises(update.UpdateError): + self.validate(release) + + def test_destination_symlink_rejected(self): + target = self.root / 'app/web/link' + target.symlink_to(Path(self.temp.name), target_is_directory=True) + with self.assertRaises(update.UpdateError): + update.destination(self.root, 'app/web/link/escape.html') + + def test_missing_required_program_or_deployment_manifest_rejected(self): + release = copy.deepcopy(self.release) + release['files'] = [e for e in release['files'] if e['path'] != 'backend/src/main/resources/analysis-runtime-policy.json'] + with self.assertRaises(update.UpdateError): + self.validate(release) + with self.assertRaisesRegex(update.UpdateError, '部署数据'): + self.validate({**self.release, 'bundleManifest': self.manifest}) + + def test_zip_inventory_and_hash_enforced(self): + stage = Path(self.temp.name) / 'good' + update.unpack(self.archive(), stage, self.release) + self.assertEqual(self.program['app/backend.jar'], (stage / 'app/backend.jar').read_bytes()) + bad = dict(self.program) + bad['../evil'] = b'escape' + with self.assertRaises(update.UpdateError): + update.unpack(self.archive(bad), Path(self.temp.name) / 'escape', self.release) + bad = dict(self.program) + bad['app/backend.jar'] = b'x' * len(bad['app/backend.jar']) + with self.assertRaisesRegex(update.UpdateError, '校验失败'): + update.unpack(self.archive(bad), Path(self.temp.name) / 'hash', self.release) + + def test_zip_symlink_rejected(self): + archive = Path(self.temp.name) / 'symlink.zip' + info = zipfile.ZipInfo('app/backend.jar') + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + with zipfile.ZipFile(archive, 'w') as out: + out.writestr(info, self.program['app/backend.jar']) + with self.assertRaises(update.UpdateError): + update.unpack(archive, Path(self.temp.name) / 'links', self.release) + + def test_snapshot_replace_preserves_local_data_and_manifest_entries(self): + backup = self.root / '.runtime/update/backup-test' + backup.mkdir(parents=True) + stage = Path(self.temp.name) / 'stage' + update.unpack(self.archive(), stage, self.release) + update.backup_program(self.root, backup, self.release) + update.replace_program(self.root, stage, backup, self.release) + result = update.read(self.root / 'manifest.json') + self.assertEqual(6, result['updateSequence']) + self.assertEqual(self.config, update.read(self.root / '.runtime/deployment.json')) + self.assertFalse((self.root / 'app/web/obsolete.js').exists()) + for name in (update.KEY, 'runtimes/python/python.exe', 'migration/snapshot.sql'): + self.assertEqual(self.files[name], (self.root / name).read_bytes()) + self.assertIn(next(e for e in self.manifest['files'] if e['path'] == name), result['files']) + update.restore_program(self.root, backup) + self.assertEqual(self.manifest, update.read(self.root / 'manifest.json')) + self.assertEqual(self.files['app/web/obsolete.js'], (self.root / 'app/web/obsolete.js').read_bytes()) + + def worker_harness(self, fail_verify=False): + directory = self.root / '.runtime/update' + update.atomic(directory / 'request.json', {'token': 'a' * 32}) + archive = self.archive() + envelope = self.envelope() + original_download = update.download + calls, restores, queries = [], [], [] + + def fake_download(url, target=None, *args): + if target: + shutil.copyfile(archive, target) + return b'' + return json.dumps(envelope).encode() + + def fake_control(root, action, database_only=False, timeout=0): + calls.append((action, database_only)) + if action == 'verify' and fail_verify: + raise update.UpdateError('模拟新版启动校验失败。') + return types.SimpleNamespace(pid=1234) + + def fake_mysql(root, config, binary='mysql', database=None, stdin=None, stdout=None): + if binary == 'mysqldump': + stdout.write(b'-- database snapshot\n' + b'SELECT 1;\n' * 30) + else: + restores.append((database, stdin.read())) + + process = types.SimpleNamespace(parents=lambda: [], create_time=lambda: 1, + is_running=lambda: True, status=lambda: 'running') + with patch.object(update.psutil, 'pid_exists', return_value=False), \ + patch.object(update.psutil, 'Process', return_value=process), \ + patch.object(update, 'download', side_effect=fake_download), \ + patch.object(update, 'controller', side_effect=fake_control), \ + patch.object(update, 'wait_ready'), \ + patch.object(update, 'mysql', side_effect=fake_mysql), \ + patch.object(update, 'sql', side_effect=lambda root, config, query: queries.append(query)): + update.worker(987654, 'a' * 32, self.root) + return calls, restores, queries + + def test_complete_update_orchestration_with_isolated_database_backup(self): + calls, restores, queries = self.worker_harness() + self.assertEqual([('stop', False), ('serve', True), ('stop', False), ('verify', False), ('serve', False)], calls) + self.assertFalse(restores) + self.assertEqual('completed', update.read(self.root / '.runtime/update/status.json')['state']) + self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase']) + self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence']) + + def test_failed_update_restores_old_program_and_new_rollback_database(self): + self.config['restoredFromMigration'] = 'isolated-test-migration' + update.atomic(self.root / '.runtime/deployment.json', self.config) + update.atomic(self.root / '.runtime/migration-receipt.json', {'migrationId': 'isolated-test-migration', + 'database': self.config['mysqlDatabase'], 'migrationManifestSha256': 'b' * 64}) + calls, restores, queries = self.worker_harness(fail_verify=True) + self.assertEqual(self.manifest, update.read(self.root / 'manifest.json')) + config = update.read(self.root / '.runtime/deployment.json') + self.assertRegex(config['mysqlDatabase'], r'^quality_restore_[a-f0-9]{24}$') + self.assertNotEqual(self.config['mysqlDatabase'], config['mysqlDatabase']) + self.assertEqual(self.config['archiveDirectory'], config['archiveDirectory']) + self.assertEqual(config['mysqlDatabase'], restores[0][0]) + self.assertEqual(config['mysqlDatabase'], update.read(self.root / '.runtime/migration-receipt.json')['database']) + self.assertTrue(any('CREATE DATABASE' in q for q in queries)) + self.assertFalse(any('DROP' in q for q in queries)) + self.assertEqual('failed', update.read(self.root / '.runtime/update/status.json')['state']) + self.assertIn('已恢复原版本', update.read(self.root / '.runtime/update/status.json')['message']) + self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase']) + + def test_recovery_blocks_live_worker_and_rejects_untrusted_backup_path(self): + directory = self.root / '.runtime/update' + update.atomic(directory / 'journal.json', {'phase': 'installing', 'backup': '../../escape'}) + with patch.object(update, 'live', return_value=False), self.assertRaises(update.UpdateError): + update.recover(self.root) + with patch.object(update, 'live', return_value=True), self.assertRaisesRegex(update.UpdateError, '正在进行'): + update.recover(self.root) + + def test_arbitrary_download_url_is_rejected_before_network(self): + for url in ['http://quality.lyunlab.me/quality-updates/windows-x64/latest.json', 'https://evil.example/update.zip', + update.BASE + '../private.zip', update.BASE + 'quality-update-x.zip?token=x']: + with self.subTest(url=url), self.assertRaises(update.UpdateError): + update.download(url) + + def test_source_is_disabled_and_status_never_exposes_private_fields(self): + update.atomic(self.root / '.runtime/update/status.json', {'state': 'idle', 'rootPassword': 'secret', 'backupPath': '/private'}) + with patch.object(update.os, 'name', 'posix'): + self.assertFalse(update.enabled(self.root)) + result = update.status(self.root) + self.assertNotIn('rootPassword', result) + self.assertNotIn('backupPath', result) + + def test_recovery_and_worker_share_lock_and_live_identity(self): + with update.exclusive_update(self.root): + self.assertTrue(update.live(update.read(self.root / '.runtime/update/worker.json'))) + with self.assertRaises(update.BusyUpdate): + update.recover(self.root) + # The lock remains authoritative if the identity file disappears. + (self.root / '.runtime/update/worker.json').unlink() + with self.assertRaises(update.BusyUpdate): + with update.exclusive_update(self.root): + self.fail('Concurrent update lock was acquired') + + def test_atomic_copy_failure_preserves_existing_control_and_configuration(self): + source = Path(self.temp.name) / 'new-content' + source.write_bytes(b'new file content') + for name in ('deploy/offline/control.py', '.runtime/deployment.json', 'manifest.json'): + target = self.root / name + previous = target.read_bytes() + with self.subTest(name=name), patch.object(update.os, 'replace', side_effect=OSError('simulated interruption')): + with self.assertRaises(OSError): + update.copy_atomic(source, target) + self.assertEqual(previous, target.read_bytes()) + self.assertFalse(list(target.parent.glob(target.name + '.update-*'))) + + def test_mysqldump_does_not_receive_mysql_only_connect_timeout_option(self): + with patch.object(update.subprocess, 'run', return_value=types.SimpleNamespace(returncode=0)) as run: + update.mysql(self.root, self.config, 'mysqldump', stdout=update.subprocess.DEVNULL) + self.assertFalse(any(arg.startswith('--connect-timeout') for arg in run.call_args.args[0])) + update.mysql(self.root, self.config, 'mysql') + self.assertIn('--connect-timeout=10', run.call_args.args[0]) + + def test_actual_dump_is_followed_by_a_second_recovery_space_check(self): + with patch.object(update.shutil, 'disk_usage', side_effect=[types.SimpleNamespace(free=10**15), types.SimpleNamespace(free=0)]): + calls, _, _ = self.worker_harness() + self.assertNotIn(('verify', False), calls) + self.assertEqual(self.manifest, update.read(self.root / 'manifest.json')) + self.assertIn('剩余磁盘空间不足', update.read(self.root / '.runtime/update/status.json')['message']) + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/offline/update-public-key.pem b/deploy/offline/update-public-key.pem new file mode 100644 index 0000000..6ee26f9 --- /dev/null +++ b/deploy/offline/update-public-key.pem @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAMPNTCeYpWygXdSxtaJzijXu0mKZF64lP7NcUuYZ4NcU= +-----END PUBLIC KEY----- diff --git a/deploy/offline/update_bootstrap.py b/deploy/offline/update_bootstrap.py new file mode 100644 index 0000000..98221e6 --- /dev/null +++ b/deploy/offline/update_bootstrap.py @@ -0,0 +1,357 @@ +"""Immutable stage-zero recovery. Installed under deploy/bootstrap, never updated.""" +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import secrets +import shutil +import subprocess +import sys +import time + +import psutil # Bundled immutable Python environment, not an application module. + +ROOT = Path(__file__).resolve().parents[2] + + +def read(path, default=None): + return json.loads(path.read_text(encoding='utf-8')) if path.exists() else default + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def atomic_copy(source, target): + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + '.recover-' + secrets.token_hex(6)) + try: + with source.open('rb') as reader, temporary.open('xb') as writer: + shutil.copyfileobj(reader, writer) + writer.flush() + os.fsync(writer.fileno()) + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + + +def safe(root, name): + if (not isinstance(name, str) or not name or '\\' in name or ':' in name or name.startswith('/') + or any(part in ('', '.', '..') or part.endswith((' ', '.')) for part in name.split('/'))): + raise RuntimeError('更新恢复路径无效。') + target = root.joinpath(*name.split('/')) + for path in (target, *target.parents): + if path == root: + break + if path.is_symlink() or (path.exists() and getattr(path.lstat(), 'st_file_attributes', 0) & 0x400): + raise RuntimeError('更新恢复目录包含无效链接。') + if not target.resolve().is_relative_to(root.resolve()): + raise RuntimeError('更新恢复路径越界。') + return target + + +def program(name): + if name == 'deploy/offline/update-public-key.pem' or name.startswith('deploy/bootstrap/'): + return False + path = Path(name) + return (name == 'app/backend.jar' or name.startswith('app/web/') + or name == 'backend/src/main/resources/analysis-runtime-policy.json' + or name.startswith('scripts/') and path.suffix == '.py' + or name.startswith('ai/nao-quality/') + or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md')) + + +def atomic_json(path, value): + temporary = path.with_name(path.name + '.stop-' + secrets.token_hex(6)) + try: + with temporary.open('x', encoding='utf-8') as stream: + json.dump(value, stream) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def same_path(left, right): + return os.path.normcase(str(Path(left).resolve())) == os.path.normcase(str(Path(right).resolve())) + + +def runtime_executable(root, role): + suffix = '.exe' if os.name == 'nt' else '' + relative = {'supervisor': 'runtimes/python/' + ('python.exe' if os.name == 'nt' else 'bin/python3'), + 'backend': 'runtimes/java/bin/java' + suffix, 'mysql': 'runtimes/mysql/bin/mysqld' + suffix}[role] + return root / relative + + +def bound_to_root(root, role, args): + if not isinstance(args, list) or not all(isinstance(arg, str) for arg in args): + return False + if role == 'supervisor': + return (len(args) in (2, 3) and same_path(args[0], root / 'deploy/offline/control.py') + and args[1] == 'serve' and (len(args) == 2 or args[2] == '--database-only')) + if role == 'backend': + return len(args) >= 2 and args[-2] == '-jar' and same_path(args[-1], root / 'app/backend.jar') + datadirs = [arg.split('=', 1)[1] for arg in args if arg.startswith('--datadir=')] + return len(datadirs) == 1 and same_path(datadirs[0], root / 'runtimedata/mysql') + + +def current_user(process): + if os.name == 'nt': + return process.username().casefold() == psutil.Process().username().casefold() + return process.uids().real == os.getuid() + + +def owned_process(root, role, info): + """Return a fully verified owned PID, None for a dead PID, never guess.""" + if (not isinstance(info, dict) or type(info.get('pid')) is not int or info['pid'] <= 0 + or not isinstance(info.get('created'), (int, float)) + or not isinstance(info.get('executable'), str) or not isinstance(info.get('args'), list)): + raise RuntimeError('本包服务进程记录无效,未停止任何不明进程。') + try: + process = psutil.Process(info['pid']) + if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE: + return None + forbidden = {os.getpid(), *(parent.pid for parent in psutil.Process().parents())} + if (process.pid in forbidden or abs(process.create_time() - info['created']) >= .01 + or not current_user(process) or not same_path(process.exe(), info['executable']) + or process.cmdline()[1:] != info['args'] + or not same_path(info['executable'], runtime_executable(root, role)) + or not bound_to_root(root, role, info['args'])): + raise RuntimeError('无法确认本包服务进程归属,未停止任何不明进程。') + return process + except psutil.NoSuchProcess: + return None + except psutil.Error: + raise RuntimeError('当前用户无法核验本包服务进程,恢复已安全停止。') from None + + +def root_markers(root, args): + """Detect unregistered package processes without inspecting unrelated data.""" + for arg in args: + if arg.startswith('--datadir='): + if same_path(arg.split('=', 1)[1], root / 'runtimedata/mysql'): + return True + elif Path(arg).is_absolute() and (same_path(arg, root / 'app/backend.jar') + or same_path(arg, root / 'deploy/offline/control.py')): + return True + return False + + +def refuse_unregistered(root, expected): + for process in psutil.process_iter(): + try: + if process.pid == os.getpid() or process.status() == psutil.STATUS_ZOMBIE: + continue + args = process.cmdline()[1:] + if not root_markers(root, args): + continue + if process.pid not in expected: + raise RuntimeError('发现未登记的本包服务进程,已保留现场,请先检查该进程。') + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + continue + + +def wait_owned(root, role, info, seconds): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if owned_process(root, role, info) is None: + return True + time.sleep(.05) + return owned_process(root, role, info) is None + + +def terminate_owned(root, role, info, seconds): + process = owned_process(root, role, info) + if process is None: + return + try: + process.terminate() + if wait_owned(root, role, info, seconds): + return + process = owned_process(root, role, info) + if process is not None: + process.kill() + if not wait_owned(root, role, info, 10): + raise RuntimeError('本包服务尚未退出,未替换任何程序文件。') + except psutil.NoSuchProcess: + return + except psutil.Error: + raise RuntimeError('当前用户无法停止本包服务,未替换任何程序文件。') from None + + +def shutdown_owned_mysql(root, info): + process = owned_process(root, 'mysql', info) + if process is None: + return + value = read(safe(root, '.runtime/deployment.json'), {}) + port = value.get('mysqlPort') + if (type(port) is not int or not 1024 <= port <= 65535 + or not re.fullmatch(r'[A-Za-z0-9_-]{20,100}', str(value.get('rootPassword', ''))) + or info['args'].count('--port=' + str(port)) != 1): + raise RuntimeError('无法核验本包数据库关闭配置,未替换任何程序文件。') + executable = root / 'runtimes/mysql/bin' / ('mysqladmin.exe' if os.name == 'nt' else 'mysqladmin') + env = os.environ.copy() + env['MYSQL_PWD'] = value['rootPassword'] + # Only the immutable bundled MySQL client is executed; no application code + # is loaded, and the current user's privileges are never elevated. + command = [str(executable), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1', + '--port=' + str(port), '--user=root', '--connect-timeout=3', 'shutdown'] + try: + subprocess.run(command, env=env, cwd=root, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20) + except (OSError, subprocess.SubprocessError): + raise RuntimeError('本包数据库无法安全关闭,未替换任何程序文件。') from None + if not wait_owned(root, 'mysql', info, 120): + raise RuntimeError('本包数据库仍在关闭,未替换任何程序文件,请稍后重新启动。') + + +def stop_owned_services(root, grace_seconds=200): + """Immutable implementation of service.json + stop-request token contract. + + A crashed worker can leave a healthy new supervisor/Java behind. Validate + every recorded identity before requesting shutdown; a live stale/reused PID + is an error rather than permission to terminate it. Java must be gone before + a JAR is replaced. An orphaned MySQL is shut down through its immutable + client, never forcibly killed on Windows. + """ + state = read(safe(root, '.runtime/service.json'), {}) + if not isinstance(state, dict) or not isinstance(state.get('children', []), list): + raise RuntimeError('本包服务状态无效,未替换任何程序文件。') + entries = [] + supervisor = state.get('supervisor') + if supervisor: + entries.append(('supervisor', supervisor)) + for info in state.get('children', []): + if not isinstance(info, dict) or info.get('name') not in ('backend', 'mysql'): + raise RuntimeError('本包服务子进程记录无效,未替换任何程序文件。') + entries.append((info['name'], info)) + if len(entries) > 20 or len({info.get('pid') for _, info in entries}) != len(entries): + raise RuntimeError('本包服务进程记录重复,未替换任何程序文件。') + # Validate all owners before the first stop request or signal is sent. + active = {(role, info['pid']): owned_process(root, role, info) for role, info in entries} + refuse_unregistered(root, {pid for (_, pid), process in active.items() if process is not None}) + if supervisor and active.get(('supervisor', supervisor['pid'])) is not None: + token = state.get('token') + if not isinstance(token, str) or not re.fullmatch(r'[a-f0-9]{48}', token): + raise RuntimeError('本包服务停止令牌无效,未停止任何进程。') + atomic_json(safe(root, '.runtime/stop-request.json'), {'token': token}) + if not wait_owned(root, 'supervisor', supervisor, grace_seconds): + terminate_owned(root, 'supervisor', supervisor, 10) + # A dead supervisor cannot finish its own finally block. Only independently + # verified package children may be stopped; unrelated/reused PIDs are refused. + for role, info in entries: + if role == 'backend': + terminate_owned(root, role, info, 35) + for role, info in entries: + if role == 'mysql': + shutdown_owned_mysql(root, info) + for role, info in entries: + if owned_process(root, role, info) is not None: + raise RuntimeError('本包服务尚未完全退出,未替换任何程序文件。') + refuse_unregistered(root, set()) + stop = safe(root, '.runtime/stop-request.json') + if isinstance(state.get('token'), str) and read(stop, {}).get('token') == state['token']: + stop.unlink(missing_ok=True) + + +def restore_program_before_import(root, directory, journal): + name = journal.get('backup', '') + if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name): + raise RuntimeError('更新恢复记录无效。') + backup = safe(directory, name) + manifest_file, inventory_file = backup / 'manifest.json', backup / 'program.json' + if (digest(manifest_file) != journal.get('backupManifestSha256') + or digest(inventory_file) != journal.get('programInventorySha256')): + raise RuntimeError('本机更新备份校验失败,请保留目录并联系管理员。') + manifest = read(manifest_file) + old = {entry['path']: entry for entry in manifest['files'] if program(entry['path'])} + inventory = read(inventory_file) + all_names, present = inventory['all'], set(inventory['present']) + if (not isinstance(all_names, list) or len(all_names) > 50000 + or len(set(all_names)) != len(all_names) or not present.issubset(all_names)): + raise RuntimeError('更新恢复清单无效。') + for name in all_names: + if not program(name): + raise RuntimeError('更新恢复清单包含受保护路径。') + target = safe(root, name) + if name in present: + source = safe(backup / 'program', name) + if name not in old or not source.is_file() or digest(source) != old[name]['sha256']: + raise RuntimeError('原程序备份缺失或损坏。') + elif name in old: + raise RuntimeError('原程序备份不完整。') + # All backups are validated before any application file is replaced. + stop_owned_services(root) + for name in all_names: + target = safe(root, name) + if name in present: + atomic_copy(safe(backup / 'program', name), target) + else: + target.unlink(missing_ok=True) + atomic_copy(manifest_file, root / 'manifest.json') + + +def live(info): + try: + process = psutil.Process(info['pid']) + return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE + and abs(process.create_time() - info['created']) < .01) + except (psutil.Error, ValueError, TypeError, KeyError): + return False + + +def recover(root=ROOT): + directory = safe(root, '.runtime/update') + journal = read(directory / 'journal.json', {}) + if journal.get('phase') in (None, 'complete', 'rolledBack'): + return {'recovered': False} + if live(read(directory / 'worker.json', {})): + raise RuntimeError('更新正在进行,请等待原窗口完成。') + with (directory / 'worker.lock').open('a+b') as lock: + try: + if os.name == 'nt': + import msvcrt + lock.seek(0) + msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + raise RuntimeError('已有更新恢复正在进行,请等待完成。') from None + identity = {'pid': os.getpid(), 'created': psutil.Process().create_time()} + worker = directory / 'worker.json' + temporary = worker.with_suffix('.bootstrap.tmp') + temporary.write_text(json.dumps(identity), encoding='utf-8') + temporary.replace(worker) + try: + journal = read(directory / 'journal.json', {}) + if journal.get('phase') == 'installing': + restore_program_before_import(root, directory, journal) + elif journal.get('phase') == 'stopping': + stop_owned_services(root) + else: + raise RuntimeError('更新恢复阶段无效,未替换任何程序文件。') + # This import occurs only after restoring a complete, verified old + # program set; no mutable application module was imported above. + path = safe(root, 'deploy/offline/update_manager.py') + spec = importlib.util.spec_from_file_location('quality_recovered_updater', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.recover_locked(root) + finally: + if read(worker, {}) == identity: + worker.unlink(missing_ok=True) + + +if __name__ == '__main__': + try: + print(json.dumps(recover(), ensure_ascii=False)) + except Exception as error: + print(json.dumps({'error': str(error) if isinstance(error, RuntimeError) else '更新恢复未完成,请保留本机备份并联系管理员。'}, ensure_ascii=False)) + raise SystemExit(1) diff --git a/deploy/offline/update_integration_mysql.py b/deploy/offline/update_integration_mysql.py new file mode 100644 index 0000000..9e87aa7 --- /dev/null +++ b/deploy/offline/update_integration_mysql.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Real, isolated MySQL application-update acceptance on the macOS build host. + +Never reads the source deployment configuration or connects to its MySQL port. +The test copies the application and creates its own database server and ports. +Only HTTP transport and the target-runtime verify action are adapted for the +host; signature verification, backup, file replacement, startup and rollback +are the production updater implementation. +""" +from __future__ import annotations + +import argparse +import base64 +from datetime import datetime, timezone +import hashlib +import http.cookiejar +import json +import os +from pathlib import Path +import secrets +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import traceback +from unittest.mock import patch +import urllib.request +import zipfile + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat + +import update_manager as update + +SOURCE = Path(__file__).resolve().parents[2] +OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def free_port(): + with socket.socket() as test: + test.bind(('127.0.0.1', 0)) + return test.getsockname()[1] + + +def backend_port(): + for port in (5285, 5284, 5283): + try: + with socket.socket() as test: + test.bind(('127.0.0.1', port)) + return port + except OSError: + continue + raise RuntimeError('No unused standalone test backend port is available; no existing instance was touched.') + + +def query(root, sql, database=None): + config = update.read(root / '.runtime/deployment.json') + command = [str(root / 'runtimes/mysql/bin/mysql'), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1', + '--port=' + str(config['mysqlPort']), '--user=root', '--connect-timeout=5', '--batch', '--skip-column-names', + '--default-character-set=utf8mb4'] + if database: + command.append(database) + env = os.environ.copy() + env['MYSQL_PWD'] = config['rootPassword'] + result = subprocess.run(command, input=sql, encoding='utf-8', capture_output=True, env=env, timeout=60) + if result.returncode: + (root / '.runtime/update/test-sql-error.log').write_text(result.stderr) + raise RuntimeError('SQL failed in the isolated test server; see its private test log.') + return result.stdout.strip() + + +def web(root, path='/'): + config = update.read(root / '.runtime/deployment.json') + with OPENER.open(f'http://127.0.0.1:{config["webPort"]}' + path, timeout=10) as response: + return response.read() + + +def login(root, username, password): + config = update.read(root / '.runtime/deployment.json') + base = f'http://127.0.0.1:{config["webPort"]}' + cookies = http.cookiejar.CookieJar() + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(cookies)) + with opener.open(base + '/api/auth/csrf', timeout=10) as response: + token = json.load(response)['token'] + request = urllib.request.Request(base + '/api/auth/login', data=json.dumps({'username': username, 'password': password}).encode(), + headers={'Content-Type': 'application/json', 'X-XSRF-TOKEN': token}) + with opener.open(request, timeout=15) as response: + assert response.status == 200 + with opener.open(base + '/api/auth/me', timeout=10) as response: + assert response.status == 200 + + +def prepare(work): + # MySQL's Unix socket has a 103-byte path limit on this host. + root = Path(tempfile.mkdtemp(prefix='qc-upd-', dir='/tmp')).resolve() + (work / 'isolated-root.txt').write_text(str(root) + '\n') + for name in ('app/web', 'scripts', 'deploy/offline', 'runtimes/python/bin', '.runtime/update', + 'runtimedata/logs', 'runtimedata/sources', 'runtimedata/archive', 'backend/src/main/resources'): + (root / name).mkdir(parents=True, exist_ok=True) + for name in ('control.py', 'gateway.py', 'update_manager.py'): + shutil.copy2(SOURCE / 'deploy/offline' / name, root / 'deploy/offline' / name) + shutil.copy2(SOURCE / 'deploy/offline/runtime_platform.py', root / 'scripts/runtime_platform.py') + # No AI processes run in this acceptance test; preserve supervisor cleanup's + # normal subprocess path while keeping it confined to the disposable root. + (root / 'scripts/nao_quality_runtime.py').write_text('"""Isolated test: no AI processes have been started."""\n') + shutil.copy2(SOURCE / 'backend/target/quality-platform-backend-0.1.0-SNAPSHOT.jar', root / 'app/backend.jar') + shutil.copy2(SOURCE / 'backend/src/main/resources/analysis-runtime-policy.json', root / 'backend/src/main/resources/analysis-runtime-policy.json') + (root / 'app/web/index.html').write_text('
isolated-update-v1
') + (root / 'app/web/obsolete.js').write_text('/* obsolete test application asset */') + (root / 'runtimes/java').symlink_to(SOURCE / '.runtime/jdk/Contents/Home', target_is_directory=True) + (root / 'runtimes/mysql').symlink_to(SOURCE / '.runtime/mysql', target_is_directory=True) + (root / 'runtimes/python/bin/python3').symlink_to(sys.executable) + key = Ed25519PrivateKey.generate() + (root / update.KEY).write_bytes(key.public_key().public_bytes(Encoding.PEM, PublicFormat.SubjectPublicKeyInfo)) + config = {'mysqlPort': free_port(), 'backendPort': backend_port(), 'webPort': free_port(), 'webBind': '127.0.0.1', + 'mysqlDatabase': 'quality_platform', 'archiveDirectory': 'runtimedata/archive', + 'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32), + 'adminUsername': 'update-test-admin', 'adminPassword': secrets.token_urlsafe(24)} + assert config['mysqlPort'] != 53306 + update.atomic(root / '.runtime/deployment.json', config) + (root / '.runtime/deployment.json').chmod(0o600) + private_marker = secrets.token_hex(20) + update.atomic(root / '.runtime/analysis-api-settings.json', {'testPrivateMarker': private_marker}) + manifest = {'platform': 'darwin', 'architecture': 'arm64', 'updateVersion': 'host-initial', 'updateSequence': 1, + 'runtimes': [{'name': 'java', 'hostTestRuntime': True}, {'name': 'mysql', 'hostTestRuntime': True}, + {'name': 'python', 'version': sys.version.split()[0], 'hostTestRuntime': True}], + 'pythonWheels': [], 'files': []} + for path in sorted(root.rglob('*')): + if path.is_file() and not path.is_symlink() and not path.relative_to(root).as_posix().startswith(('.runtime/', 'runtimedata/', 'runtimes/')): + manifest['files'].append({'path': path.relative_to(root).as_posix(), 'sha256': update.digest(path)}) + update.atomic(root / 'manifest.json', manifest) + return root, key, config, private_marker + + +def release_fixture(root, work, key, version, sequence, bad=False): + folder = work / ('fixture-' + version) + folder.mkdir() + program = {} + for entry in update.read(root / 'manifest.json')['files']: + name = entry['path'] + if update.program_path(name) and name != 'app/web/obsolete.js': + program[name] = root / name + index = folder / 'index.html' + index.write_text(f'
isolated-update-{version}
') + program['app/web/index.html'] = index + if bad: + bad_jar = folder / 'bad-backend.jar' + # The real new Java process runs Flyway, changes data, then fails. This + # proves rollback restores the dump, rather than merely restarting Java. + migration = ("UPDATE platform_metadata SET metadata_value='FAILED-UPDATE-DATA' WHERE metadata_key='update_integration_sentinel';\n" + "UPDATE app_user SET password_hash='FAILED-UPDATE-HASH' WHERE username='update-test-admin';\n" + "CREATE TABLE isolated_update_failure_marker(id INT PRIMARY KEY);\n" + "SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Intentional isolated updater integration failure';\n") + with zipfile.ZipFile(root / 'app/backend.jar') as source, zipfile.ZipFile(bad_jar, 'w') as target: + for member in source.infolist(): + target.writestr(member, source.read(member)) + target.writestr('BOOT-INF/classes/db/migration/V999__isolated_update_failure.sql', migration) + program['app/backend.jar'] = bad_jar + archive = folder / ('quality-update-' + version + '.zip') + with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED, compresslevel=1) as target: + for name, path in sorted(program.items()): + target.write(path, name) + release = {'schema': 1, 'platform': 'windows-x64', 'version': version, 'sequence': sequence, + 'runtimeFingerprint': update.runtime_fingerprint(update.read(root / 'manifest.json')), + 'archive': {'file': archive.name, 'size': archive.stat().st_size, 'sha256': update.digest(archive)}, + 'files': [{'path': name, 'size': path.stat().st_size, 'sha256': update.digest(path)} for name, path in sorted(program.items())]} + payload = update.canonical(release) + envelope = {'payload': base64.b64encode(payload).decode(), 'signature': base64.b64encode(key.sign(payload)).decode()} + (folder / 'latest.json').write_bytes(update.canonical(envelope)) + return release, envelope, archive + + +def run_update(root, release, envelope, archive): + real_controller = update.controller + verify_checks = [] + + def host_controller(target, action, database_only=False, timeout=1100): + assert target == root + if action != 'verify': + return real_controller(target, action, database_only, timeout) + # Host adaptation: this ARM Mac cannot satisfy Windows/x64 runtime probes. + # All installed program hashes and the native Java/MySQL executables are + # still checked. Real full application startup is checked below by serve. + for entry in update.read(root / 'manifest.json')['files']: + assert update.digest(root / entry['path']) == entry['sha256'], entry['path'] + for command in ([root / 'runtimes/java/bin/java', '-version'], + [root / 'runtimes/mysql/bin/mysqld', '--no-defaults', '--version']): + result = subprocess.run([str(a) for a in command], capture_output=True, timeout=30) + assert result.returncode == 0 + verify_checks.append(True) + + def transport(url, target=None, expected_size=None, expected_hash=None): + if url == update.LATEST and target is None: + return update.canonical(envelope) + assert url == update.BASE + archive.name and target is not None + assert expected_size == archive.stat().st_size and expected_hash == update.digest(archive) + with archive.open('rb') as reader, target.open('xb') as writer: + shutil.copyfileobj(reader, writer) + return b'' + + launcher = subprocess.Popen([sys.executable, '-c', 'pass']) + launcher.wait(timeout=10) + token = secrets.token_hex(16) + update.atomic(root / '.runtime/update/request.json', {'token': token, 'created': time.time()}) + # Patch only this test process. Production Windows/source enablement remains + # unchanged, including in the real backend processes started by control.py. + with patch.object(update, 'enabled', return_value=True), patch.object(update, 'download', side_effect=transport), \ + patch.object(update, 'controller', side_effect=host_controller): + update.worker(launcher.pid, token, root) + assert verify_checks, update.read(root / '.runtime/update/status.json').get('message') + return update.read(root / '.runtime/update/status.json') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--work', type=Path, required=True) + parser.add_argument('--expected-tables', type=int, default=83) + args = parser.parse_args() + work = args.work.resolve() + work.mkdir(parents=True, exist_ok=False) + root = None + report = {'startedAt': datetime.now(timezone.utc).isoformat(), 'passed': False, + 'environment': 'macOS ARM host, disposable native MySQL + Java + Python runtimes', + 'productionServicesTouched': False, 'productionDatabasePortUsed': False, + 'networkTransport': 'local signed fixtures substituted only in test process', + 'targetRuntimeVerify': 'host adapter: all manifest hashes plus native Java/MySQL probes; actual backend restarts', + 'windowsNativeProcessDetachmentValidated': False, 'windowsNativeRuntimeCompatibilityValidated': False, + 'checks': {}} + try: + print('Preparing isolated update acceptance root.', flush=True) + root, key, config, private_marker = prepare(work) + report['isolatedRoot'] = str(root) + report['ports'] = {k: config[k] for k in ('mysqlPort', 'backendPort', 'webPort')} + print('Starting isolated MySQL and Java; applying real Flyway schema.', flush=True) + process = update.controller(root, 'serve') + update.wait_ready(root, process) + database = config['mysqlDatabase'] + count = int(query(root, f"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='{database}';")) + assert count == args.expected_tables, f'Expected {args.expected_tables} tables; actual isolated schema has {count}.' + report['checks']['tableCount'] = count + sentinel = secrets.token_hex(24) + query(root, f"INSERT INTO platform_metadata(metadata_key,metadata_value) VALUES ('update_integration_sentinel','{sentinel}');", database) + account_hash = query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database) + assert account_hash.startswith('$2') + login(root, config['adminUsername'], config['adminPassword']) + report['checks']['initialLogin'] = True + config['restoredFromMigration'] = 'isolated-update-migration-fixture' + update.atomic(root / '.runtime/deployment.json', config) + update.atomic(root / '.runtime/migration-receipt.json', {'migrationId': config['restoredFromMigration'], + 'database': database, 'migrationManifestSha256': 'a' * 64}) + before_pid = update.read(root / '.runtime/service.json')['supervisor']['pid'] + print('Applying signed success release with actual dump, file replacement and service restart.', flush=True) + good = release_fixture(root, work, key, 'host-success', 2) + result = run_update(root, *good) + assert result['state'] == 'completed', result.get('message') + assert update.read(root / 'manifest.json')['updateVersion'] == 'host-success' + assert b'isolated-update-host-success' in web(root) + assert not (root / 'app/web/obsolete.js').exists() + assert update.read(root / '.runtime/service.json')['supervisor']['pid'] != before_pid + assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", database) == sentinel + assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database) == account_hash + login(root, config['adminUsername'], config['adminPassword']) + first_journal = update.read(root / '.runtime/update/journal.json') + first_dump = root / '.runtime/update' / first_journal['backup'] / 'database.sql' + assert first_dump.is_file() and first_dump.stat().st_size > 10000 + report['checks'].update(realMysqldump=True, successProgramSwitch=True, realServiceRestart=True, + obsoleteProgramRemoved=True, successSentinelPreserved=True, + successAccountHashPreserved=True, successLogin=True, + successDumpBytes=first_dump.stat().st_size) + print('Applying signed bad Java release: real Flyway mutates sentinel/hash then intentionally fails.', flush=True) + bad = release_fixture(root, work, key, 'host-bad', 3, bad=True) + result = run_update(root, *bad) + assert result['state'] == 'failed' and '已恢复原版本' in result['message'], result + restored = update.read(root / '.runtime/deployment.json') + restored_db = restored['mysqlDatabase'] + assert restored_db != database and restored_db.startswith('quality_restore_') + assert update.read(root / 'manifest.json')['updateVersion'] == 'host-success' + assert b'isolated-update-host-success' in web(root) + assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", restored_db) == sentinel + assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", restored_db) == account_hash + assert int(query(root, f"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='{restored_db}';")) == count + assert query(root, "SELECT metadata_value FROM platform_metadata WHERE metadata_key='update_integration_sentinel';", database) == 'FAILED-UPDATE-DATA', 'Failed original sentinel did not retain the migration write.' + assert query(root, "SELECT password_hash FROM app_user WHERE username='update-test-admin';", database) == 'FAILED-UPDATE-HASH', 'Failed original account hash did not retain the migration write.' + assert query(root, "SELECT COUNT(*) FROM flyway_schema_history WHERE version='999' AND success=0;", database) == '1' + assert query(root, "SELECT COUNT(*) FROM flyway_schema_history WHERE version='999';", restored_db) == '0' + assert update.read(root / '.runtime/analysis-api-settings.json')['testPrivateMarker'] == private_marker + assert restored['archiveDirectory'] == config['archiveDirectory'] + assert restored['rootPassword'] == config['rootPassword'] and restored['mysqlPassword'] == config['mysqlPassword'] + assert update.read(root / '.runtime/migration-receipt.json')['database'] == restored_db + login(root, config['adminUsername'], config['adminPassword']) + failed_journal = update.read(root / '.runtime/update/journal.json') + assert failed_journal['phase'] == 'rolledBack' + failed_dump = root / '.runtime/update' / failed_journal['backup'] / 'database.sql' + report['checks'].update(badReleaseActuallyMutatedDatabase=True, failedOriginalDatabasePreserved=True, + restoredIntoNewDatabase=True, restoredTableCount=count, rollbackSentinelRestored=True, + rollbackAccountHashRestored=True, rollbackLogin=True, oldProgramRestarted=True, + privateConfigurationPreserved=True, archiveDirectoryPreserved=True, + migrationReceiptDatabaseAligned=True, + failedFlywayHistoryPreserved=True, restoredFlywayHistoryClean=True, + rollbackDumpBytes=failed_dump.stat().st_size, + originalDatabase=database, rollbackDatabase=restored_db) + report['passed'] = True + print('Real MySQL success and failed-migration rollback acceptance passed.', flush=True) + except Exception as error: + report['error'] = str(error) + (work / 'test-failure-trace.txt').write_text(traceback.format_exc()) + print('Isolated acceptance failed: ' + str(error), flush=True) + finally: + if root is not None: + try: + update.controller(root, 'stop', timeout=180) + state = update.read(root / '.runtime/service.json', {}) + assert not update.live(state.get('supervisor', {})) + report['isolatedServicesStopped'] = True + except Exception as error: + report['isolatedServicesStopped'] = False + report['cleanupError'] = str(error) + report['passed'] = False + report['finishedAt'] = datetime.now(timezone.utc).isoformat() + (work / 'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') + print(json.dumps(report, ensure_ascii=False), flush=True) + return 0 if report['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/deploy/offline/update_manager.py b/deploy/offline/update_manager.py new file mode 100644 index 0000000..656cc3d --- /dev/null +++ b/deploy/offline/update_manager.py @@ -0,0 +1,646 @@ +#!/usr/bin/env python3 +"""Signed Windows standalone application updates. No caller-supplied URLs or paths. + +The release envelope signs the *decoded payload bytes* with Ed25519. Runtime +identity is SHA256(canonical({runtimes, pythonWheels})). Program files are a full +snapshot; private data and runtimes are never update destinations. +""" +from __future__ import annotations + +import argparse +import base64 +import contextlib +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import secrets +import shutil +import stat +import subprocess +import sys +import time +import zipfile +from urllib.request import ProxyHandler, HTTPRedirectHandler, build_opener + +import psutil +from cryptography.hazmat.primitives.serialization import load_pem_public_key +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +ROOT = Path(__file__).resolve().parents[2] +BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/' +LATEST = BASE + 'latest.json' +KEY = 'deploy/offline/update-public-key.pem' +MAX_MANIFEST = 16 * 1024 * 1024 +MAX_ARCHIVE = 2 * 1024**3 +MAX_EXPANDED = 4 * 1024**3 +ACTIVE = {'queued', 'downloading', 'backingUp', 'installing', 'restarting', 'rollingBack'} +SHA = re.compile(r'[a-f0-9]{64}') + + +class UpdateError(RuntimeError): + pass + + +class BusyUpdate(UpdateError): + pass + + +def canonical(value): + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8') + + +def unique_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise UpdateError('更新清单包含重复字段。') + value[key] = item + return value + + +def decode(raw): + return json.loads(raw, object_pairs_hook=unique_object) + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def atomic(path, value): + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = path.with_name(path.name + '.' + secrets.token_hex(6) + '.tmp') + with temporary.open('xb') as stream: + stream.write(canonical(value)) + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +def copy_atomic(source, target): + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + '.update-' + secrets.token_hex(6)) + try: + with source.open('rb') as reader, temporary.open('xb') as writer: + shutil.copyfileobj(reader, writer) + writer.flush() + os.fsync(writer.fileno()) + shutil.copystat(source, temporary) + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + + +def read(path, default=None): + return decode(path.read_bytes()) if path.exists() else default + + +def runtime_fingerprint(manifest): + return hashlib.sha256(canonical({key: manifest.get(key, []) for key in ('runtimes', 'pythonWheels')})).hexdigest() + + +def safe_path(name): + if (not isinstance(name, str) or not name or len(name) > 230 or '\\' in name + or ':' in name or '\x00' in name or name.startswith('/') + or any(ord(c) < 32 for c in name)): + raise UpdateError('更新文件路径无效。') + parts = name.split('/') + if any(p in ('', '.', '..') or p.endswith((' ', '.')) + or re.fullmatch(r'(?i)(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?', p) for p in parts): + raise UpdateError('更新文件路径无效。') + return PurePosixPath(name) + + +def program_path(name): + path = safe_path(name) + if name == KEY or name.startswith('deploy/bootstrap/'): + return False # immutable trust anchor, even in a correctly signed release + return (name == 'app/backend.jar' or name.startswith('app/web/') + or name == 'backend/src/main/resources/analysis-runtime-policy.json' + or name.startswith('scripts/') and path.suffix == '.py' + or name.startswith('ai/nao-quality/') + or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md')) + + +def destination(root, name): + safe_path(name) + target = root.joinpath(*name.split('/')) + for path in (target, *target.parents): + if path == root: + break + if path.is_symlink(): + raise UpdateError('更新目录包含符号链接,已停止更新。') + if os.name == 'nt' and path.exists() and getattr(path.lstat(), 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT: + raise UpdateError('更新目录包含重解析点,已停止更新。') + if not target.resolve().is_relative_to(root.resolve()): + raise UpdateError('更新文件超出程序目录。') + return target + + +def entry_map(entries, program_only=False): + if not isinstance(entries, list) or len(entries) > 50000: + raise UpdateError('更新文件清单无效。') + result, folded = {}, set() + for entry in entries: + if not isinstance(entry, dict): + raise UpdateError('更新文件清单无效。') + name = entry.get('path') + safe_path(name) + if name.casefold() in folded: + raise UpdateError('更新文件路径重复。') + folded.add(name.casefold()) + if program_only and not program_path(name): + raise UpdateError('更新包含不允许替换的文件。') + if entry.get('symlink') is not None or not SHA.fullmatch(str(entry.get('sha256', ''))): + raise UpdateError('更新文件校验信息无效。') + result[name] = entry + return result + + +def validate_release(envelope, root=ROOT): + try: + raw = base64.b64decode(envelope['payload'], validate=True) + signature = base64.b64decode(envelope['signature'], validate=True) + key = load_pem_public_key(destination(root, KEY).read_bytes()) + if not isinstance(key, Ed25519PublicKey): + raise ValueError('key type') + key.verify(signature, raw) + release = decode(raw) + except Exception as error: + raise UpdateError('更新签名验证失败。') from error + current = read(root / 'manifest.json') + if release.get('schema') != 1 or release.get('platform') != 'windows-x64': + raise UpdateError('更新包不适用于此系统。') + if not isinstance(release.get('version'), str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]{0,79}', release['version']): + raise UpdateError('更新版本无效。') + if type(release.get('sequence')) is not int or release['sequence'] <= 0: + raise UpdateError('更新版本序号无效。') + if release.get('runtimeFingerprint') != runtime_fingerprint(current): + raise UpdateError('运行环境版本不同,请使用完整安装包更新。') + archive = release.get('archive', {}) + if (not re.fullmatch(r'quality-update-[A-Za-z0-9][A-Za-z0-9._-]{0,100}\.zip', str(archive.get('file', ''))) + or type(archive.get('size')) is not int or not 0 < archive['size'] <= MAX_ARCHIVE + or not SHA.fullmatch(str(archive.get('sha256', '')))): + raise UpdateError('更新下载信息无效。') + files = entry_map(release.get('files'), True) + if not files or any(type(e.get('size')) is not int or e['size'] < 0 for e in files.values()) or sum(e['size'] for e in files.values()) > MAX_EXPANDED: + raise UpdateError('更新文件大小无效。') + if 'bundleManifest' in release: + raise UpdateError('更新清单不能包含部署数据。') + old_files = entry_map(current.get('files')) + required = {'app/backend.jar', 'deploy/offline/control.py', 'deploy/offline/update_manager.py', + 'backend/src/main/resources/analysis-runtime-policy.json'} + if not required.issubset(files) or not any(name.startswith('app/web/') for name in files): + raise UpdateError('更新缺少必要程序文件。') + if KEY not in old_files or digest(destination(root, KEY)) != old_files[KEY]['sha256']: + raise UpdateError('更新信任公钥校验失败。') + for name in set(old_files) | set(files): + destination(root, name) + return release + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise UpdateError('更新服务器返回了不允许的跳转。') + + +def download(url, target=None, expected_size=None, expected_hash=None): + if url != LATEST and not (url.startswith(BASE) and re.fullmatch(r'quality-update-[A-Za-z0-9][A-Za-z0-9._-]{0,100}\.zip', url[len(BASE):])): + raise UpdateError('更新下载地址无效。') + limit = expected_size if expected_size is not None else MAX_MANIFEST + hasher, count, chunks = hashlib.sha256(), 0, [] + with build_opener(ProxyHandler({}), NoRedirect()).open(url, timeout=30) as response: + if response.geturl() != url or response.status != 200: + raise UpdateError('更新服务器响应无效。') + with target.open('xb') if target is not None else contextlib.nullcontext() as stream: + while block := response.read(1024 * 1024): + count += len(block) + if count > limit: + raise UpdateError('更新下载大小不符合清单。') + hasher.update(block) + if stream is not None: + stream.write(block) + else: + chunks.append(block) + if expected_size is not None and (count != expected_size or hasher.hexdigest() != expected_hash): + raise UpdateError('更新下载校验失败。') + return b''.join(chunks) + + +def unpack(archive, stage, release): + entries = {e['path']: e for e in release['files']} + with zipfile.ZipFile(archive) as source: + seen = set() + for member in source.infolist(): + name = member.filename + if name not in entries or name in seen or member.is_dir() or member.flag_bits & 1: + raise UpdateError('更新压缩包文件清单不符。') + seen.add(name) + if stat.S_ISLNK(member.external_attr >> 16) or member.file_size != entries[name]['size']: + raise UpdateError('更新压缩包文件类型或大小不符。') + target = destination(stage, name) + target.parent.mkdir(parents=True, exist_ok=True) + count = 0 + with source.open(member) as reader, target.open('xb') as writer: + while block := reader.read(1024 * 1024): + count += len(block) + if count > entries[name]['size']: + raise UpdateError('更新解压大小不符。') + writer.write(block) + if digest(target) != entries[name]['sha256']: + raise UpdateError('更新解压校验失败。') + if seen != set(entries): + raise UpdateError('更新压缩包缺少文件。') + + +def enabled(root=ROOT): + return os.name == 'nt' and os.environ.get('QUALITY_UPDATE_ENABLED') == 'true' and (root / 'runtimes/python/python.exe').is_file() and (root / 'manifest.json').is_file() + + +def status(root=ROOT): + manifest = read(root / 'manifest.json', {}) + value = read(root / '.runtime/update/status.json', {}) + allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'} + result = {k: v for k, v in value.items() if k in allowed} + result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本'))) + result.setdefault('state', 'idle') + if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})): + pending = read(root / '.runtime/update/request.json', {}) + journal = read(root / '.runtime/update/journal.json', {}) + if time.time() - pending.get('created', 0) > 120: + result.update(state='failed', available=False, message='更新进程已中断,请重新运行软件启动入口恢复。' + if journal.get('phase') not in (None, 'complete', 'rolledBack') else '更新进程已结束,请重新检查更新。') + if not result['enabled']: + result['message'] = '此部署不支持软件内更新。' + return result + + +def write_status(state, message, root=ROOT, **extra): + destination(root, '.runtime/update/status.json') + value = read(root / '.runtime/update/status.json', {}) + value.update(extra, state=state, message=message) + atomic(root / '.runtime/update/status.json', value) + return status(root) + + +def check(root=ROOT): + release = validate_release(decode(download(LATEST)), root) + available = release['sequence'] > read(root / 'manifest.json').get('updateSequence', 0) + atomic(root / '.runtime/update/checked.json', release) + return write_status('available' if available else 'current', '发现新版本。' if available else '已是最新版本。', root, + version=release['version'], available=available, checkedAt=datetime.now(timezone.utc).isoformat()) + + +def process_options(): + return {'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_BREAKAWAY_FROM_JOB} if os.name == 'nt' else {'start_new_session': True} + + +@contextlib.contextmanager +def exclusive_update(root=ROOT): + directory = root / '.runtime/update' + destination(root, '.runtime/update') + directory.mkdir(parents=True, exist_ok=True) + if live(read(directory / 'worker.json', {})): + raise BusyUpdate('已有更新或恢复正在进行。') + sys.path.insert(0, str(root / 'scripts')) + from runtime_platform import fcntl + with (directory / 'worker.lock').open('a+b') as lock: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except (BlockingIOError, OSError): + raise BusyUpdate('已有更新或恢复正在进行。') from None + identity = {'pid': os.getpid(), 'created': psutil.Process().create_time()} + atomic(directory / 'worker.json', identity) + try: + yield + finally: + if read(directory / 'worker.json', {}) == identity: + (directory / 'worker.json').unlink(missing_ok=True) + + +def launch(root=ROOT): + with exclusive_update(root): + return launch_locked(root) + + +def launch_locked(root=ROOT): + directory = root / '.runtime/update' + destination(root, '.runtime/update') + directory.mkdir(parents=True, exist_ok=True) + # This short launcher MUST exit before its worker may stop Java. The worker + # verifies launcher disappearance; DETACHED alone does not remove ancestry. + pending = read(directory / 'request.json', {}) + if pending and time.time() - pending.get('created', 0) < 120: + raise BusyUpdate('已有更新正在启动。') + check(root) # refetch and verify; do not trust browser or cached JSON + if not status(root).get('available'): + return status(root) + token = secrets.token_hex(16) + atomic(directory / 'request.json', {'token': token, 'created': time.time()}) + write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False) + try: + with (directory / 'worker.log').open('ab') as log: + subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token], + cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options()) + except Exception: + (directory / 'request.json').unlink(missing_ok=True) + raise UpdateError('独立更新进程无法启动,服务仍在运行。') from None + return status(root) + + +def live(info): + try: + process = psutil.Process(info['pid']) + return process.is_running() and process.status() != psutil.STATUS_ZOMBIE and abs(process.create_time() - info['created']) < .01 + except (psutil.Error, KeyError, TypeError): + return False + + +def controller(root, action, database_only=False, timeout=1100): + python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3') + command = [str(python), str(root / 'deploy/offline/control.py'), action] + if database_only: + command += ['--database-only'] + if action == 'serve': + with (root / '.runtime/update/service.log').open('ab') as log: + return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options()) + result = subprocess.run(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + if result.returncode: + raise UpdateError('服务启停失败,请查看本机更新日志。') + return result + + +def wait_ready(root, process, database_only=False): + deadline = time.monotonic() + (180 if database_only else 1000) + while time.monotonic() < deadline: + state = read(root / '.runtime/service.json', {}) + if state.get('ready') and state.get('databaseOnly', False) == database_only and state.get('supervisor', {}).get('pid') == process.pid: + return + if process.poll() is not None: + raise UpdateError('更新后服务启动失败,正在恢复旧版本。') + time.sleep(.5) + raise UpdateError('更新后服务启动超时,正在恢复旧版本。') + + +def mysql(root, config, binary='mysql', database=None, stdin=None, stdout=subprocess.PIPE): + exe = '.exe' if os.name == 'nt' else '' + command = [str(root / 'runtimes/mysql/bin' / (binary + exe)), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1', + '--port=' + str(config['mysqlPort']), '--user=root', '--default-character-set=utf8mb4'] + if binary == 'mysqldump': + command += ['--single-transaction', '--routines', '--events', '--triggers', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', config.get('mysqlDatabase', 'quality_platform')] + elif database: + command.append(database) + if binary == 'mysql': + command.append('--connect-timeout=10') + env = os.environ.copy() + env['MYSQL_PWD'] = config['rootPassword'] + result = subprocess.run(command, cwd=root, env=env, stdin=stdin, stdout=stdout, stderr=subprocess.PIPE, timeout=3600) + if result.returncode: + error_log = destination(root, '.runtime/update/database-error.log') + with error_log.open('ab') as log: + log.write((binary + ' failed\n').encode() + result.stderr[-16384:] + b'\n') + raise UpdateError('数据库备份或恢复失败,已保留本机备份和原数据库。') + + +def sql(root, config, query): + # stdin must be a real file on Windows; do not place passwords in arguments. + path = root / '.runtime/update/query.sql' + try: + path.write_text(query, encoding='utf-8') + with path.open('rb') as source: + mysql(root, config, stdin=source) + finally: + path.unlink(missing_ok=True) + + +def backup_program(root, backup, release): + current = read(root / 'manifest.json') + names = sorted({e['path'] for e in current['files'] if program_path(e['path'])} | {e['path'] for e in release['files']}) + present = [] + for name in names: + source = destination(root, name) + if source.exists(): + target = destination(backup / 'program', name) + target.parent.mkdir(parents=True, exist_ok=True) + copy_atomic(source, target) + present.append(name) + copy_atomic(root / 'manifest.json', backup / 'manifest.json') + # All private configuration (AI keys, deployment credentials and bootstrap + # flags) is backed up locally, but never becomes a release payload. + shutil.copytree(root / '.runtime', backup / 'private', ignore=shutil.ignore_patterns('update', '*.lock', '*.log', 'service.json', 'stop-request.json'), symlinks=False) + for path in (backup / 'private').rglob('*'): + if path.is_file(): + with path.open('rb') as stream: + os.fsync(stream.fileno()) + atomic(backup / 'program.json', {'all': names, 'present': present}) + + +def replace_program(root, stage, backup, release): + current = read(root / 'manifest.json') + names = read(backup / 'program.json')['all'] + wanted = {e['path'] for e in release['files']} + for name in names: + target = destination(root, name) + if name in wanted: + copy_atomic(stage / name, target) + else: + target.unlink(missing_ok=True) + current['files'] = ([e for e in current['files'] if not program_path(e['path'])] + + [{'path': e['path'], 'sha256': e['sha256']} for e in release['files']]) + current.update(updateSequence=release['sequence'], updateVersion=release['version']) + atomic(root / 'manifest.json', current) + + +def restore_program(root, backup): + names = read(backup / 'program.json') + for name in names['all']: + target = destination(root, name) + if name in names['present']: + copy_atomic(backup / 'program' / name, target) + else: + target.unlink(missing_ok=True) + copy_atomic(backup / 'manifest.json', destination(root, 'manifest.json')) + for source in (backup / 'private').rglob('*'): + if source.is_file(): + relative = source.relative_to(backup / 'private').as_posix() + copy_atomic(source, destination(root, '.runtime/' + relative)) + + +def rollback(root, backup, database_backed_up): + write_status('rollingBack', '更新未完成,正在恢复原版本。', root) + controller(root, 'stop', timeout=180) + restore_program(root, backup) + if database_backed_up: + config = read(root / '.runtime/deployment.json') + process = controller(root, 'serve', True) + wait_ready(root, process, True) + database = 'quality_restore_' + secrets.token_hex(12) + sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;') + with (backup / 'database.sql').open('rb') as source: + mysql(root, config, database=database, stdin=source) + sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';") + controller(root, 'stop', timeout=180) + config['mysqlDatabase'] = database + atomic(root / '.runtime/deployment.json', config) + receipt_path = root / '.runtime/migration-receipt.json' + receipt = read(receipt_path, {}) + if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'): + receipt['database'] = database + atomic(receipt_path, receipt) + process = controller(root, 'serve') + wait_ready(root, process) + + +def worker(launcher, token, root=ROOT): + directory = root / '.runtime/update' + for _ in range(300): + if not psutil.pid_exists(launcher): + break + time.sleep(.1) + else: + raise UpdateError('更新启动器未退出,已安全停止。') + if any('java' in p.name().lower() for p in psutil.Process().parents()): + raise UpdateError('更新进程未独立启动,已安全停止。') + if read(directory / 'request.json', {}).get('token') != token: + raise UpdateError('更新请求已失效。') + with exclusive_update(root): + backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4)) + stopped = backed_up = program_saved = changed = False + try: + envelope = decode(download(LATEST)) + release = validate_release(envelope, root) + if release['sequence'] <= read(root / 'manifest.json').get('updateSequence', 0): + raise UpdateError('此更新版本已安装。') + backup.mkdir() + archive, stage = backup / 'update.zip', backup / 'stage' + expanded = sum(entry['size'] for entry in release['files']) + # Keep enough headroom for extraction, old program files and a + # logical dump; the dump itself must complete before replacement. + db_size = sum(p.stat().st_size for p in (root / 'runtimedata/mysql').rglob('*') if p.is_file()) + old_program_size = sum((root / entry['path']).stat().st_size for entry in read(root / 'manifest.json')['files'] if program_path(entry['path']) and (root / entry['path']).is_file()) + private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory)) + if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2: + raise UpdateError('磁盘剩余空间不足,请释放空间后重试。') + write_status('downloading', '正在下载并校验更新。', root) + info = release['archive'] + download(BASE + info['file'], archive, info['size'], info['sha256']) + unpack(archive, stage, release) + # Revalidate immediately before any service or disk mutation. + validate_release(envelope, root) + write_status('backingUp', '正在备份,服务即将重新启动。', root) + atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False}) + controller(root, 'stop', timeout=180) + stopped = True + backup_program(root, backup, release) + program_saved = True + config = read(root / '.runtime/deployment.json') + database = controller(root, 'serve', True) + wait_ready(root, database, True) + with (backup / 'database.sql').open('xb') as output: + mysql(root, config, 'mysqldump', stdout=output) + output.flush() + os.fsync(output.fileno()) + if (backup / 'database.sql').stat().st_size < 100: + raise UpdateError('数据库备份为空,已停止更新。') + backed_up = True + controller(root, 'stop', timeout=180) + # The actual dump can exceed the source's physical storage. Do not + # replace anything unless a separate rollback database still fits. + required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2 + if shutil.disk_usage(root).free < required_recovery: + raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。') + atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True, + 'backupManifestSha256': digest(backup / 'manifest.json'), + 'programInventorySha256': digest(backup / 'program.json')}) + write_status('installing', '正在安装更新。', root) + changed = True + replace_program(root, stage, backup, release) + controller(root, 'verify', timeout=180) + write_status('restarting', '正在重新启动,请稍候。', root) + process = controller(root, 'serve') + wait_ready(root, process) + atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'complete', 'databaseBackedUp': True}) + write_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False) + except Exception as error: + message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。' + if stopped: + try: + if program_saved and changed: + rollback(root, backup, backed_up) + else: + controller(root, 'stop', timeout=180) + process = controller(root, 'serve') + wait_ready(root, process) + atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'rolledBack', 'databaseBackedUp': backed_up}) + message += ' 已恢复原版本。' + except Exception: + message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。' + write_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat()) + finally: + (directory / 'request.json').unlink(missing_ok=True) + + +def recover(root=ROOT): + with exclusive_update(root): + return recover_locked(root) + + +def recover_locked(root=ROOT): + directory = root / '.runtime/update' + journal = read(directory / 'journal.json', {}) + if journal.get('phase') in (None, 'complete', 'rolledBack'): + return status(root) + name = journal.get('backup', '') + if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name): + raise UpdateError('更新恢复记录无效。') + backup = directory / name + if journal.get('phase') == 'installing': + if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256') + or digest(backup / 'program.json') != journal.get('programInventorySha256')): + raise UpdateError('更新恢复备份校验失败,已保留现场。') + rollback(root, backup, journal.get('databaseBackedUp') is True) + else: + controller(root, 'stop', timeout=180) + process = controller(root, 'serve') + wait_ready(root, process) + atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'}) + return write_status('failed', '上次更新中断,已恢复原版本。', root, available=False) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('status', 'check', 'launch', 'worker', 'recover')) + parser.add_argument('--launcher', type=int) + parser.add_argument('--token') + args = parser.parse_args() + if args.action == 'status': + return status() + if not enabled(): + raise UpdateError('此部署不支持软件内更新。') + if args.action == 'worker': + if not args.launcher or not re.fullmatch(r'[a-f0-9]{32}', args.token or ''): + raise UpdateError('更新请求无效。') + worker(args.launcher, args.token) + return status() + if args.action == 'recover': + return recover() + journal = read(ROOT / '.runtime/update/journal.json', {}) + if journal.get('phase') not in (None, 'complete', 'rolledBack'): + raise UpdateError('上次更新中断,请重新运行软件启动入口恢复。') + if live(read(ROOT / '.runtime/update/worker.json', {})): + raise BusyUpdate('已有更新正在进行。') + return check() if args.action == 'check' else launch() + + +if __name__ == '__main__': + try: + print(json.dumps(main(), ensure_ascii=False)) + except Exception as error: + message = str(error) if isinstance(error, UpdateError) else '更新服务暂不可用,请稍后重试。' + if enabled() and not isinstance(error, BusyUpdate) and sys.argv[1:2] != ['status'] and not live(read(ROOT / '.runtime/update/worker.json', {})): + write_status('failed', message) + print(json.dumps({'error': message}, ensure_ascii=False)) + raise SystemExit(1) diff --git a/deploy/offline/windows-prepare.ps1 b/deploy/offline/windows-prepare.ps1 new file mode 100644 index 0000000..9559e4c --- /dev/null +++ b/deploy/offline/windows-prepare.ps1 @@ -0,0 +1,7 @@ +param([ValidateRange(1024,65535)][int]$Port = 5280) + +$ErrorActionPreference = 'Stop' +Write-Host '此兼容脚本不执行任何系统级准备操作,也不需要管理员权限。' +Write-Host '运行库随离线包提供。请使用 windows-start.ps1 启动应用。' +Write-Host "网页端口为 $Port;其他设备能否访问取决于 Windows 当前防火墙和网络策略。" +exit 0 diff --git a/deploy/offline/windows-start.ps1 b/deploy/offline/windows-start.ps1 new file mode 100644 index 0000000..47b7e8c --- /dev/null +++ b/deploy/offline/windows-start.ps1 @@ -0,0 +1,64 @@ +$ErrorActionPreference = 'Stop' +$root = [System.IO.Path]::GetFullPath((Split-Path (Split-Path $PSScriptRoot -Parent) -Parent)) +try { + if (-not [Environment]::Is64BitOperatingSystem -or $env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { + throw '此离线包需要 Intel/AMD 64 位 Windows 电脑。' + } + + $manifestPath = Join-Path $root 'manifest.json' + if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) { + throw '未找到离线包清单 manifest.json。请解压完整离线包后再启动。' + } + $manifest = Get-Content -Raw -Encoding UTF8 $manifestPath | ConvertFrom-Json + $maxRelativePath = 0 + foreach ($entry in $manifest.files) { + if ($null -eq $entry.path -or $entry.path -isnot [string] -or [string]::IsNullOrWhiteSpace($entry.path)) { + throw '离线包清单包含无效文件路径。请重新解压完整离线包。' + } + if ($entry.path.Length -gt $maxRelativePath) { $maxRelativePath = $entry.path.Length } + } + if ($maxRelativePath -eq 0) { throw '离线包清单没有文件路径。请重新解压完整离线包。' } + $maxRootLength = 248 - $maxRelativePath + if ($root.Length -gt $maxRootLength) { + $recommended = if ($env:LOCALAPPDATA) { Join-Path $env:LOCALAPPDATA 'QualityCenter' } else { '当前用户可写的较短本地 NTFS 目录' } + throw "部署目录路径过长:根路径和清单最长相对路径不能超过 248 个字符。请尝试 $recommended;若该路径仍过长,请选择更短的当前用户可写 NTFS 目录。" + } + $driveRoot = [System.IO.Path]::GetPathRoot($root) + $drive = [System.IO.DriveInfo]::new($driveRoot) + if ($drive.DriveType -ne [System.IO.DriveType]::Fixed -or $drive.DriveFormat -ne 'NTFS') { + throw '请将离线包解压到本机固定 NTFS 磁盘上的当前用户可写目录。' + } + + $python = Join-Path $root 'runtimes\python\python.exe' + if (-not (Test-Path -LiteralPath $python -PathType Leaf)) { + throw '未找到随包提供的 Python 运行环境。请重新解压完整离线包。' + } + $env:PYTHONUTF8 = '1' + $env:PYTHONIOENCODING = 'utf-8' + $env:QUALITY_UPDATE_ENABLED = 'true' + + # Recover an interrupted app-local update before reading mutable deployment + # configuration or starting the local service processes. + & $python (Join-Path $root 'deploy\bootstrap\update-recovery.py') + if ($LASTEXITCODE -ne 0) { throw '更新恢复仍未完成。请保留本地更新备份后重试。' } + + $port = 5280 + $config = Join-Path $root '.runtime\deployment.json' + if (Test-Path -LiteralPath $config -PathType Leaf) { + $port = [int](Get-Content -Raw -Encoding UTF8 $config | ConvertFrom-Json).webPort + } + if ($port -lt 1024 -or $port -gt 65535) { throw '网页端口配置无效。' } + + if (Test-Path (Join-Path $root 'migration\manifest.json')) { + & $python (Join-Path $root 'deploy\offline\restore_migration.py') + if ($LASTEXITCODE -ne 0) { throw '数据恢复未完成。现有数据库已保留,请查看 runtimedata\logs。' } + } + + Write-Host '正在以当前用户启动本机应用进程;不会安装系统组件、创建 Windows 服务或更改防火墙规则。' + Write-Host "网页服务使用内网监听,端口为 $port。其他设备能否访问取决于 Windows 当前防火墙和网络策略。" + & $python (Join-Path $root 'deploy\offline\control.py') start --lan + if ($LASTEXITCODE -ne 0) { throw '启动未完成。请查看 runtimedata\logs\supervisor.log。' } +} catch { + Write-Host $_ -ForegroundColor Red + exit 1 +} diff --git a/web/src/ManagementPages.tsx b/web/src/ManagementPages.tsx index 34743ba..a6aa2a5 100644 --- a/web/src/ManagementPages.tsx +++ b/web/src/ManagementPages.tsx @@ -291,8 +291,8 @@ function SourcesPage({ onUnauthorized, user }: { onUnauthorized: (error: unknown return
{error && void load()}>重试} />} void load()} /> }, - { key: 'files', label: '来源文件', children: setPendingDeletion({ ids: [], releaseId: null })} onUnauthorized={onUnauthorized} onRefresh={() => void load()} /> }, + { key: 'intake', label: '数据接入', children: void load()} /> }, + { key: 'files', label: '来源文件', children: setPendingDeletion({ ids: [], releaseId: null })} onUnauthorized={onUnauthorized} onRefresh={() => void load()} /> }, { key: 'history', label: '导入记录', children: }, { key: 'audit', label: '来源审计', children: }, ]} /> diff --git a/web/src/NoCodeDataTypes.ts b/web/src/NoCodeDataTypes.ts index d02203f..7b56d35 100644 --- a/web/src/NoCodeDataTypes.ts +++ b/web/src/NoCodeDataTypes.ts @@ -7,9 +7,9 @@ export type WorkbookPreview = { preview: { sheetName: string; headerRow: number; headers: string[]; sampleRows: { rowNumber: number; values: Record }[] }; fields: Record; }; -export type PublishedSourceTarget = { documentId: string; revisionId: string; sourceSlot: string; stage: DataStage; year: number; month: number; scopeCode: string; sourceFile: string }; +export type PublishedSourceTarget = { documentId: string; revisionId: string; sourceSlot: string; stage: DataStage; year: number; month: number; scopeCode: string; sourceFile: string; fileId?: string | null; sourceMode?: 'MAIN' | 'SHARD_SNAPSHOT' | string }; export type SourceUploadOptions = { currentReleaseId: string | null; targets: PublishedSourceTarget[]; fields: Record; scopes: string[]; currentSourceIds?: string[]; currentSourcesComplete?: boolean }; -export type ConfigureSourceBody = { uploadId: string; stage: DataStage; year: number; month: number; sheetName: string; headerRow: number; fieldMap: Record; mode: 'ADD' | 'REPLACE'; expectedCurrentReleaseId: string | null; targetDocumentId?: string; targetRevisionId?: string; reason: string; scopeCode: string }; +export type ConfigureSourceBody = { uploadId: string; stage: DataStage; year: number; month: number; sheetName: string; headerRow: number; fieldMap: Record; mode: 'ADD' | 'APPEND' | 'REPLACE'; fileId?: string; expectedCurrentReleaseId: string | null; targetDocumentId?: string; targetRevisionId?: string; reason: string; scopeCode: string }; export type MasterWorkbookKind = 'departments' | 'groups' | 'products' | 'projectGroups' | 'categoryLookup' | 'projectGroupLookup' | 'departmentLookup' | 'productLookup' | 'standardBundle'; export type MasterWorkbookKeyFormat = 'SEPARATE_COLUMNS' | 'PDT_REGION_COMBINED'; export type MasterWorkbookPreview = { diff --git a/web/src/NoCodeDataWorkflow.tsx b/web/src/NoCodeDataWorkflow.tsx index 8dd36df..e957df1 100644 --- a/web/src/NoCodeDataWorkflow.tsx +++ b/web/src/NoCodeDataWorkflow.tsx @@ -14,6 +14,7 @@ import { usePageCache } from './pageCache'; import { useDelayedBusy, useDelayedIndicator } from './useDelayedBusy'; import { useVisiblePolling } from './useVisiblePolling'; import { runImportPollFollowup } from './importLifecycleGuards'; +import { configureSourceBatch } from './sourceBatchConfigure'; import './no-code-data-workflow.css'; type Row = Record; @@ -43,28 +44,28 @@ function WorkflowOperationFeedback({ operation, error, busy, onResume }: { opera const status = String(operation?.status || '').toUpperCase(); const errors = Array.isArray(operation?.errors) ? operation.errors.map(item => item.message).filter(Boolean) : []; return
- {error && 重试读取进度 : undefined} />} + {error && 重试读取进度 : undefined} />}
{workflowStageLabel[String(operation?.stage || operation?.status)] || (busy ? '正在处理' : status === 'COMPLETED' ? '报表已更新' : '操作状态')}
; } export type PendingSourceDeletion = { ids: string[]; releaseId: string | null }; -type SourceDraft = { id: string; file: File; workbook: WorkbookPreview; stage: DataStage; year: number; month: number; sheetName: string; headerRow: number; fieldMap: Record; mode: 'ADD' | 'REPLACE'; targetId: string; scopeCode: string }; +type SourceDraft = { id: string; file: File; workbook: WorkbookPreview; stage: DataStage; year: number; month: number; sheetName: string; headerRow: number; fieldMap: Record; mode: 'APPEND' | 'REPLACE'; fileId: string; targetId: string; scopeCode: string }; +type WizardFileDraft = { id: string; file: File; workbook: WorkbookPreview; sheetName: string; headerRow: number; fieldMap: Record; fileId: string }; -function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onDeleted, onBusyChange, onUnauthorized, onRefresh }: { options: SourceUploadOptions; workflowStatus: DataWorkflowStatus; pendingDeletion: PendingSourceDeletion; onDeleted: (ids: string[]) => void; onBusyChange: (busy: boolean) => void; onUnauthorized: (error: unknown) => boolean; onRefresh: () => void }) { +function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onBusyChange, onUnauthorized, onRefresh }: { options: SourceUploadOptions; workflowStatus: DataWorkflowStatus; pendingDeletion: PendingSourceDeletion; onBusyChange: (busy: boolean) => void; onUnauthorized: (error: unknown) => boolean; onRefresh: () => void }) { const confirmDelete = useDeleteConfirmation(); const { message } = App.useApp(); const input = useRef(null); const configured = useRef(new Map()); - const consumed = useRef(new Set()); const [drafts, setDrafts] = useState([]); const [activeId, setActiveId] = useState(''); const [reading, setReading] = useState<{ done: number; total: number } | null>(null); const [busy, setBusy] = useState(false); const [error, setError] = useState(''); const actionLock = useRef(false); - const uploadOperation = useDataWorkflowOperation('source-import', () => { setDrafts([]); setActiveId(''); configured.current.clear(); consumed.current.clear(); onRefresh(); message.success('所选来源已统一导入,报表已更新'); }); - const recalcOperation = useDataWorkflowOperation('recalculate', value => { if (Array.isArray(value.sourceIds)) onDeleted(value.sourceIds.map(String)); onRefresh(); message.success('报表已更新'); }); + const uploadOperation = useDataWorkflowOperation('source-import', () => { setDrafts([]); setActiveId(''); configured.current.clear(); onRefresh(); message.success('所选来源已统一导入,报表已更新'); }); + const recalcOperation = useDataWorkflowOperation('recalculate', () => { onRefresh(); message.success('报表已更新'); }); const active = drafts.find(item => item.id === activeId) || drafts[0]; const workbook = active?.workbook || null; const stage = active?.stage || 'black'; const year = active?.year || new Date().getFullYear(); const month = active?.month || 1; @@ -72,19 +73,28 @@ function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onDe const mode = active?.mode || 'ADD'; const targetId = active?.targetId || ''; const scopeCode = active?.scopeCode || options.scopes[0] || 'ALL'; const patch = (values: Partial) => { if (active) setDrafts(previous => previous.map(item => item.id === active.id ? { ...item, ...values } : item)); }; const setWorkbook = (value: WorkbookPreview) => patch({ workbook: value }); - const setStage = (value: DataStage) => patch({ stage: value }); const setYear = (value: number) => patch({ year: value }); const setMonth = (value: number) => patch({ month: value }); + const setStage = (value: DataStage) => patch({ stage: value, targetId: '' }); const setYear = (value: number) => patch({ year: value, targetId: '' }); const setMonth = (value: number) => patch({ month: value, targetId: '' }); const setSheetName = (value: string) => patch({ sheetName: value }); const setHeaderRow = (value: number) => patch({ headerRow: value }); - const setFieldMap = (value: Record) => patch({ fieldMap: value }); const setMode = (value: 'ADD' | 'REPLACE') => patch({ mode: value }); - const setTargetId = (value: string) => patch({ targetId: value }); const setScopeCode = (value: string) => patch({ scopeCode: value }); + const setFieldMap = (value: Record) => patch({ fieldMap: value }); const setMode = (value: 'APPEND' | 'REPLACE') => patch({ mode: value, targetId: '' }); + const setTargetId = (value: string) => patch({ targetId: value }); const setScopeCode = (value: string) => patch({ scopeCode: value, targetId: '' }); const fields = options.fields[stage] || []; const headers = workbook?.preview.headers || []; const missing = fields.filter(field => field.required && !fieldMap[field.name]); const duplicate = Object.values(fieldMap).length !== new Set(Object.values(fieldMap)).size; const importBusy = busy || uploadOperation.busy || recalcOperation.busy; useEffect(() => { onBusyChange(importBusy); return () => onBusyChange(false); }, [importBusy, onBusyChange]); const hasImportCapability = workflowStatus.capabilities?.canImport !== false; - const draftReady = (item: SourceDraft) => (options.fields[item.stage] || []).filter(field => field.required).every(field => item.fieldMap[field.name]) && Object.values(item.fieldMap).length === new Set(Object.values(item.fieldMap)).size && (item.mode !== 'REPLACE' || options.targets.some(target => target.documentId === item.targetId)); - const slots = drafts.map(item => `${item.year}:${item.month}:${item.stage}:${item.scopeCode}`); - const duplicateSlots = new Set(slots).size !== slots.length; - const canImport = hasImportCapability && drafts.length > 0 && drafts.every(draftReady) && !duplicateSlots && !pendingDeletion.ids.length; + const targetFor = (item: SourceDraft) => options.targets.find(target => target.documentId === item.targetId && target.year === item.year && target.month === item.month && target.stage === item.stage && target.scopeCode === item.scopeCode); + const hasMainSource = (item: Pick) => options.targets.some(target => !target.fileId && target.sourceMode !== 'MONTHLY_SNAPSHOT' && target.year === item.year && target.month === item.month && target.stage === item.stage && target.scopeCode === item.scopeCode); + const draftReady = (item: SourceDraft) => { + const target = targetFor(item); + const matched = (options.fields[item.stage] || []).filter(field => field.required).every(field => item.fieldMap[field.name]) && Object.values(item.fieldMap).length === new Set(Object.values(item.fieldMap)).size; + return matched && (item.mode === 'APPEND' ? !hasMainSource(item) : Boolean(target) && (target?.sourceMode !== 'APPEND_SNAPSHOT' || Boolean(target.fileId))); + }; + const replacementTargets = active ? options.targets.filter(item => item.year === year && item.month === month && item.stage === stage && item.scopeCode === scopeCode) : []; + const activeHasMainSource = active ? hasMainSource(active) : false; + const configuredActive = Boolean(active && configured.current.has(active.id)); + const configuredReplacement = configuredActive && mode === 'REPLACE'; + const repeatedReplacement = drafts.filter(item => item.mode === 'REPLACE' && item.targetId).length !== new Set(drafts.filter(item => item.mode === 'REPLACE' && item.targetId).map(item => item.targetId)).size; + const canImport = hasImportCapability && drafts.length > 0 && drafts.every(draftReady) && !repeatedReplacement && !pendingDeletion.ids.length; const reportError = (value: unknown) => { if (!onUnauthorized(value)) setError(failure(value)); }; const removeDraft = (id: string) => confirmDelete({title:`移除文件“${drafts.find(item=>item.id===id)?.workbook.sourceFile || "选中的文件"}”?`,description:"仅从本次导入列表移除,此文件的字段对应设置也将移除。电脑上的原文件及已接收来源保留。",confirmText:"确认移除",onConfirm:()=>{setDrafts(previous => previous.filter(item => item.id !== id));configured.current.delete(id);}}); const readFiles = async (files: File[]) => { @@ -102,7 +112,7 @@ function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onDe const { workbook: result, stage: detected } = await detectSourceWorkbook(uploaded, options.fields, api.sourceWorkbookPreview); const nextMonth = Number(file.name.match(/(?:^|\D)(0?[1-9]|1[0-2])月/)?.[1] || new Date().getMonth() + 1); const nextYear = Number(file.name.match(/(20\d{2})年/)?.[1] || new Date().getFullYear()); - return { id: crypto.randomUUID(), file, workbook: result, stage: detected, year: nextYear, month: nextMonth, sheetName: result.preview.sheetName, headerRow: result.preview.headerRow, fieldMap: matchFields(options.fields[detected] || [], result.preview.headers), mode: 'ADD' as const, targetId: '', scopeCode: options.scopes[0] || 'ALL' }; + return { id: crypto.randomUUID(), file, workbook: result, stage: detected, year: nextYear, month: nextMonth, sheetName: result.preview.sheetName, headerRow: result.preview.headerRow, fieldMap: matchFields(options.fields[detected] || [], result.preview.headers), mode: 'APPEND' as const, fileId: crypto.randomUUID(), targetId: '', scopeCode: options.scopes[0] || 'ALL' }; } catch (value) { if (!onUnauthorized(value)) failures.push(`${file.name}:${failure(value)}`); return null; } })); const ready = results.filter(item => item !== null); @@ -124,58 +134,50 @@ function SimplifiedDataWorkflow({ options, workflowStatus, pendingDeletion, onDe const importAndUpdate = async () => { if (!canImport || importBusy || actionLock.current) return; actionLock.current = true; setBusy(true); setError(''); try { - const sourceIds: string[] = []; - for (const draft of drafts) { - const target = options.targets.find(item => item.documentId === draft.targetId); - const body = { stage: draft.stage, year: draft.year, month: draft.month, sheetName: draft.sheetName, headerRow: draft.headerRow, fieldMap: draft.fieldMap, mode: draft.mode, expectedCurrentReleaseId: workflowStatus.currentReleaseId ?? options.currentReleaseId, ...(draft.mode === 'REPLACE' && target ? { targetDocumentId: target.documentId, targetRevisionId: target.revisionId } : {}), reason: '批量导入来源数据', scopeCode: draft.scopeCode }; - const signature = JSON.stringify(body); const prior = configured.current.get(draft.id); - if (prior?.signature === signature) { sourceIds.push(prior.sourceId); continue; } - let uploadId = draft.workbook.uploadId; - if (consumed.current.has(uploadId)) uploadId = (await api.previewSourceWorkbook(draft.file)).uploadId; - const result = await api.configureSourceWorkbook({ ...body, uploadId }); - consumed.current.add(uploadId); configured.current.set(draft.id, { signature, sourceId: result.source.id }); sourceIds.push(result.source.id); - } + const items = drafts.map(draft => { + const target = targetFor(draft); + return { id: draft.id, body: { uploadId: draft.workbook.uploadId, stage: draft.stage, year: draft.year, month: draft.month, sheetName: draft.sheetName, headerRow: draft.headerRow, fieldMap: draft.fieldMap, mode: draft.mode, ...(draft.mode === 'APPEND' ? { fileId: draft.fileId } : target?.fileId ? { fileId: target.fileId } : {}), expectedCurrentReleaseId: workflowStatus.currentReleaseId ?? options.currentReleaseId, ...(draft.mode === 'REPLACE' && target ? { targetDocumentId: target.documentId, targetRevisionId: target.revisionId } : {}), reason: draft.mode === 'APPEND' ? '按独立批次追加,不覆盖旧文件' : '替换所选来源文件', scopeCode: draft.scopeCode } }; + }); + const sourceIds = await configureSourceBatch(items, configured.current, body => api.configureSourceWorkbook(body)); await uploadOperation.start(() => api.importDataWorkflow({ sourceIds })); } catch (value) { reportError(value); } finally { actionLock.current = false; setBusy(false); } }; const performRecalculation = async () => { - if (actionLock.current) return; actionLock.current = true; + if (actionLock.current || pendingDeletion.ids.length) return; actionLock.current = true; setError(''); - try { await recalcOperation.start(() => pendingDeletion.ids.length ? api.deleteDataWorkflowSources({ sourceIds: pendingDeletion.ids, expectedCurrentReleaseId: pendingDeletion.releaseId }) : api.recalculateDataWorkflow()); } + try { await recalcOperation.start(() => api.recalculateDataWorkflow()); } catch (value) { reportError(value); } finally { actionLock.current = false; } }; - const recalculate = () => { - if (actionLock.current) return; - if (!pendingDeletion.ids.length) return performRecalculation(); - confirmDelete({title:`删除 ${pendingDeletion.ids.length} 个来源并更新报表?`,description:<>

将从当前查询结果移除以下来源并更新报表,原文件和处理记录保留。

    {pendingDeletion.ids.slice(0,10).map((id,index)=>
  • {options.targets.find(item=>item.documentId===id)?.sourceFile || `第 ${index+1} 个待删除来源`}
  • )}
{pendingDeletion.ids.length>10&&

另有 {pendingDeletion.ids.length-10} 个待删除来源。

},confirmText:'确认删除并更新',onConfirm:performRecalculation}); - }; return
-
测试数据维护

导入来源并更新报表

支持一次选择多个 Excel 文件,分别核对年月与阶段后统一导入。

{workflowStatus.capabilities?.canRecalculate !== false && }
+
测试数据维护

导入来源并更新报表

同一月份可分批添加多个文件;相同文件内容不会重复计入。

{workflowStatus.capabilities?.canRecalculate !== false && }
{!hasImportCapability && } {error && } void readFiles(Array.from(event.target.files || []))} aria-label="选择质量数据Excel文件" /> - {pendingDeletion.ids.length > 0 && } + {pendingDeletion.ids.length > 0 && } {reading &&
正在上传并识别文件 {reading.done} / {reading.total}
} {drafts.length > 0 && <>
待导入文件 · {drafts.length} 个
}, + { title: '文件', key: 'file', render: (_, item) => }, { title: '阶段', key: 'stage', render: (_, item) => stageName(item.stage) }, { title: '统计期间', key: 'period', render: (_, item) => `${item.year}年${item.month}月` }, { title: '识别结果', key: 'validation', render: (_, item) => draftReady(item) ? 已识别 : 需核对列或替换对象 }, - { title: '操作', key: 'action', render: (_, item) => }, + { title: '操作', key: 'action', render: (_, item) => configured.current.has(item.id) ? 已接收待导入 : }, ]} />} - {duplicateSlots && } {!workbook ?

选择 Excel 来源文件

可多选文件;系统自动识别阶段和月份,每个文件都可单独调整。

{hasImportCapability && }
: <>
{workbook.sourceFile}{hasImportCapability && }
- - - } - {mode === 'REPLACE' && + + {options.scopes.length > 1 && }
+ {mode === 'APPEND' && activeHasMainSource && } + {mode === 'REPLACE' && replacementTargets.length === 0 && } + {repeatedReplacement && } +

请确保文件范围内的记录不重叠;相同文件内容会被系统阻止,不同文件里的重复业务行无法自动识别。

列识别与预览
}, { title: '字段匹配', key: 'mapping', render: (_, item) => (options.fields[stage] || []).filter(field => field.required).every(field => item.fieldMap[field.name]) ? 已完成 : 需核对 }, { title: '操作', key: 'action', render: (_, item) => configuredByFile.current.has(item.id) ? 已接收待导入 : }]} />} + {step === 1 &&
{mode === 'REPLACE' ? { setStage(value); setWizardFiles(previous => previous.map(item => ({ ...item, fieldMap: matchFields(options.fields[value] || [], item.workbook.preview.headers) }))); }} /> { if (value) setYear(value); }} /> ({ value, label: value === 'ALL' ? '全部范围' : value }))} onChange={setScopeCode} />}}
{target &&
原文件{target.sourceFile} · {target.fileId ? `分批 ${target.fileId.slice(0, 8)}` : '整月文件'}新文件{workbook?.sourceFile}
} setReason(event.target.value)} />} {step === 2 && workbook && <>
`${row.dimension}:${row.key}:${index}`} pagination={{ pageSize: 15, showSizeChanger: false }} scroll={{ x: 900 }} columns={[{ title: '期间', width: 130, render: (_, row) => { const period = asRow(row.period); return `${txt(period.year)}年${period.month ? `${txt(period.month)}月` : period.quarter ? `第${txt(period.quarter)}季度` : '全年'}`; } }, { title: '维度 / 对象', width: 200, render: (_, row) => `${({ overall: '总体', department: '产业部', group: '业务分组', product: '产品', projectGroup: '项目组' } as Record)[String(row.dimension)] || txt(row.dimension)} / ${txt(row.key)}` }, { title: '原综合率', width: 120, render: (_, row) => ratio(asRow(row.before).compositeRate) }, { title: '新综合率', width: 120, render: (_, row) => ratio(asRow(row.after).compositeRate) }, { title: '差异', render: (_, row) => asRows(row.differences).map(item => `${metricLabel(item.path)}:${txt(item.before)} → ${txt(item.after)}`).join(';') || '一致' }]} /> : }{reconciliation?.status === 'REVIEW_REQUIRED' &&
{canPublish ? <> setDifferenceReason(event.target.value)} /> setConfirmDifferences(event.target.checked)}>我已核对上方差异,并确认新数据正确 :

候选已保存,请交由有数据发布权限的同事完成差异确认。

}
}{canPublish ?
setPublishConfirmed(event.target.checked)}>{purpose === 'WITHDRAW' ? '我确认发布此删除候选,从当前查询结果移除已批准的来源' : '我确认将此候选发布为当前查询数据'}{target ? `,替换${target.year}年${target.month}月${stageName(target.stage)}` : ''}{!publishReady && 来源校验和基线对账通过后可发布。}
: 当前账号没有数据发布权限。可复制处理链接交接,正式查询数据尚未更新。} />}} diff --git a/web/src/SystemUpdatePanel.tsx b/web/src/SystemUpdatePanel.tsx new file mode 100644 index 0000000..d092eaf --- /dev/null +++ b/web/src/SystemUpdatePanel.tsx @@ -0,0 +1,107 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { Alert, Button, Spin, Tag } from 'antd'; +import { ReloadOutlined } from '@ant-design/icons'; +import { api, type SystemUpdateStatus } from './api'; +import { configurationDateTime } from './configurationName'; +import { usePageCache } from './pageCache'; +import { useVisiblePolling } from './useVisiblePolling'; + +const runningStates = new Set(['queued', 'downloading', 'backingUp', 'installing', 'restarting', 'rollingBack']); + +function errorStatus(error: unknown): number | undefined { + return error && typeof error === 'object' && 'status' in error && typeof error.status === 'number' ? error.status : undefined; +} + +function reconnectable(error: unknown): boolean { + const status = errorStatus(error); + return status === 0 || status === 408 || (status !== undefined && status >= 500 && status <= 599); +} + +export function SystemUpdatePanel({ onUnauthorized }: { onUnauthorized: (error: unknown) => boolean }) { + const { active } = usePageCache(); + const [status, setStatus] = useState(); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(''); + const [reconnecting, setReconnecting] = useState(false); + const [permissionDenied, setPermissionDenied] = useState(false); + const inFlight = useRef(false); + const restarting = useRef(false); + const running = !!status && runningStates.has(status.state); + const load = useCallback(async () => { + try { + const next = await api.systemUpdateStatus(); + setStatus(next); + setError(''); + setReconnecting(false); + setPermissionDenied(false); + restarting.current = runningStates.has(next.state); + } catch (failure) { + const denied = [401, 403].includes(errorStatus(failure) ?? -1); + const handled = onUnauthorized(failure); + if (handled || denied) { + restarting.current = false; + setReconnecting(false); + setPermissionDenied(true); + if (!handled) setError(failure instanceof Error ? failure.message : '无权查看更新状态'); + } else if (restarting.current && reconnectable(failure)) { + setReconnecting(true); + } else { + restarting.current = false; + setReconnecting(false); + setError(failure instanceof Error ? failure.message : '更新状态读取失败'); + } + } + }, [onUnauthorized]); + useEffect(() => { if (active) void load(); }, [active, load]); + useVisiblePolling(load, running || reconnecting ? 4_000 : 30_000, active && !permissionDenied); + + const run = async (apply: boolean) => { + if (inFlight.current) return; + inFlight.current = true; + setBusy(true); + setError(''); + try { + if (apply) restarting.current = true; + const next = await (apply ? api.applySystemUpdate() : api.checkSystemUpdate()); + setStatus(next); + restarting.current = runningStates.has(next.state); + } catch (failure) { + // A successful updater intentionally takes the server offline. A lost + // response is resolved by polling the persisted task after restart. + if (apply && reconnectable(failure)) { + setReconnecting(true); + } else { + restarting.current = false; + const denied = [401, 403].includes(errorStatus(failure) ?? -1); + setPermissionDenied(denied); + if (!onUnauthorized(failure)) setError(failure instanceof Error ? failure.message : '更新操作失败'); + if (!denied) void load(); + } + } finally { + setBusy(false); + inFlight.current = false; + } + }; + + return
+
+ 系统更新 +
+ + {status?.available && !running && !reconnecting && } +
+
+ {!status && !error && } + {error && void load()}>重试} />} + {reconnecting ? } message="正在重新启动,完成后将自动连接。" /> + : status && <> + {status.enabled &&
+
当前版本{status.currentVersion || '—'}
+ {status.version &&
{status.available ? '可用版本' : '最近检查版本'}{status.version}
} + {status.checkedAt &&
检查时间{configurationDateTime(status.checkedAt)}
} +
} + {running && 更新中}{status.message || '点击检查更新获取最新版本。'}} /> + } +
; +} diff --git a/web/src/TasksMonitorPage.tsx b/web/src/TasksMonitorPage.tsx index e4e4ffe..163a0a9 100644 --- a/web/src/TasksMonitorPage.tsx +++ b/web/src/TasksMonitorPage.tsx @@ -12,6 +12,7 @@ import { configurationDateTime } from './configurationName'; import { backupSummary, isActive, isFailed, monitorLabel, orderedTasks, taskFailure, taskPercent, type MonitorRow } from './taskMonitorModel'; import { useDelayedBusy } from './useDelayedBusy'; import { useVisiblePolling } from './useVisiblePolling'; +import { SystemUpdatePanel } from './SystemUpdatePanel'; export function TasksMonitorPage({user,onUnauthorized}:{user:AuthUser;onUnauthorized:(error:unknown)=>boolean}) { const {message}=App.useApp(); @@ -72,7 +73,7 @@ export function TasksMonitorPage({user,onUnauthorized}:{user:AuthUser;onUnauthor {loading&&!backupLoaded?:backupLoaded&&
最近备份{backupSummary(latest)}
备份时间{configurationDateTime(latest.createdAt)||'—'}
恢复验证时间{configurationDateTime(latest.restoreVerifiedAt)||'—'}
自动备份{backup.planActive===true?'已启用':'未启用'}
} }, - ...(canManage ? [{ key:'monitor', label:'运行监控与告警', children: },{ key:'backups', label:'备份管理', children: }] : []), + ...(canManage ? [{ key:'monitor', label:'运行监控与告警', children: },{ key:'backups', label:'备份管理', children: },{ key:'systemUpdate', label:'系统更新', children: }] : []), ]} /> setSelected(null)}>{selected&&
任务
{monitorLabel(selected.type,'后台任务')}
状态
{monitorLabel(selected.status)}
阶段
{monitorLabel(selected.stage,monitorLabel(selected.status))}
{Number(selected.total)>0&&<>
已处理 / 总数
{String(selected.processed??0)} / {String(selected.total)}
}
提交时间
{configurationDateTime(selected.createdAt)}
{isFailed(selected)&&<>
失败原因
{taskFailure(selected)}
}
任务编号
{String(selected.id)}
}{selected?.sourceKind==='TASK'&&
{selected.status==='FAILED'&&!!selected.importRunId&&user.roles.some(role=>['SYSTEM_ADMIN', 'DATA_ADMIN'].includes(role))&&}{selected.status==='QUEUED'&&user.roles.some(role=>['SYSTEM_ADMIN', 'DATA_ADMIN'].includes(role))&&}
}{selected?.sourceKind==='ANALYSIS'&&!!selected.sessionId&&}
; diff --git a/web/src/api.ts b/web/src/api.ts index 18dcb44..503c4b9 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -29,6 +29,18 @@ import type { ConfigureSourceBody, DataStage, DataWorkflowOperation, DataWorkflo export { configureRequestScope, getRequestDiagnostics, invalidateReadResources, resetRequestScope }; +export interface SystemUpdateStatus { + enabled: boolean; + currentVersion?: string; + state: string; + message?: string; + checkedAt?: string; + version?: string; + available?: boolean; + startedAt?: string; + finishedAt?: string; +} + export class ApiError extends Error { status: number; details: unknown; @@ -206,6 +218,9 @@ async function mutateForm(path: string, body: FormData): Promise { } export const api = { + systemUpdateStatus() { return request('/api/system-update', {}, 12_000); }, + checkSystemUpdate() { return mutate('/api/system-update/check', undefined, 'POST', 55_000); }, + applySystemUpdate() { return mutate('/api/system-update/apply', undefined, 'POST', 55_000); }, exportSalesOrderTypes(signal?:AbortSignal) { return requestBlob('/api/sales-order-types/export',signal); }, salesOrderTypeStatus() { return request('/api/sales-order-types'); }, async salesOrderTypePreview(file:File,mode:string) { diff --git a/web/src/no-code-data-workflow.css b/web/src/no-code-data-workflow.css index a73bd2d..d52f2d5 100644 --- a/web/src/no-code-data-workflow.css +++ b/web/src/no-code-data-workflow.css @@ -16,6 +16,7 @@ .nc-workflow-modal { top: 24px; padding-bottom: 24px; } .nc-workflow-modal .ant-modal-container, .nc-workflow-modal .ant-modal-content, .nc-workflow-modal .ant-input, .nc-workflow-modal .ant-input-number, .nc-workflow-modal .ant-select-selector, .nc-workflow-modal .ant-btn, .nc-workflow-modal .ant-alert { border-radius: 0 !important; } .nc-workflow-modal .ant-modal-body { display: flex; flex-direction: column; gap: 18px; max-height: calc(var(--viewport-height) - 240px); overflow-y: auto; overflow-x: hidden; padding-right: 4px; color: var(--ink); } +.nc-workflow-modal .ant-modal-body > .quality-table-frame { flex-shrink: 0; } .nc-workflow-modal .ant-modal-title { font-size: var(--font-dialog-title); color: var(--ink); } .nc-workflow-modal h3 { font-size: var(--font-section); margin: 4px 0 12px; color: var(--ink); } .nc-workflow-modal .ant-form-item { margin-bottom: 16px; } @@ -156,3 +157,4 @@ @media (max-width: 1050px) { .nc-intake-start { grid-template-columns: 1fr; } .nc-primary-actions { margin-top: 20px; } .nc-intake-links { flex-wrap: wrap; gap: 0 16px; padding: 8px 0; } .nc-intake-links > span { margin-left: 0; width: 100%; padding: 5px 0; } .nc-column-matching { grid-template-columns: 1fr; } .nc-pending-strip { align-items: flex-start; flex-direction: column; } .nc-pending-strip .ant-select { min-width: 0; width: 100%; } } @media (max-width: 680px) { .nc-intake-start { padding: 20px 18px 0; } .nc-intake-intro h2 { font-size: var(--font-page-title); } .nc-primary-actions { flex-wrap: wrap; } .nc-form-grid { grid-template-columns: 1fr; } .nc-modal-footer { flex-direction: column; align-items: stretch; } .nc-modal-footer > div { justify-content: flex-end; flex-wrap: wrap; } .nc-file-summary { flex-wrap: wrap; } .nc-file-summary > span:last-child { margin-left: 0; } .nc-workbook-upload-intro { flex-direction: column; align-items: flex-start; } .nc-pending-strip > div:first-child { align-items: flex-start; flex-direction: column; gap: 6px; } .nc-review-facts { grid-template-columns: 1fr 1fr; } } @media (max-width: 680px) { .nc-source-delete-modal .nc-deletion-impact { grid-template-columns: 1fr 1fr; } } +.nc-source-maintenance-actions { display: flex; flex-wrap: wrap; gap: 10px; margin: 0 16px 16px; } diff --git a/web/src/sourceBatchConfigure.ts b/web/src/sourceBatchConfigure.ts new file mode 100644 index 0000000..1acb412 --- /dev/null +++ b/web/src/sourceBatchConfigure.ts @@ -0,0 +1,32 @@ +export type SourceConfigureBody = { uploadId: string; fileId?: string }; +export type ConfiguredSourceCandidate = { signature: string; sourceId: string }; + +type ConfigureItem
= { id: string; body: TBody }; +type ConfigureResult = { source: { id: string } }; + +/** Reuses confirmed candidate IDs and retries changed discovered candidates with the same upload/file identity. */ +export async function configureSourceBatch( + items: ConfigureItem[], + configured: Map, + configure: (body: TBody) => Promise, + onConfigured?: (candidate: ConfiguredSourceCandidate) => void, +): Promise { + const sourceIds: string[] = []; + for (const item of items) { + const signature = JSON.stringify(item.body); + const prior = configured.get(item.id); + if (prior?.signature === signature) { + sourceIds.push(prior.sourceId); + continue; + } + + // The server updates an existing DISCOVERED candidate in place when the same + // uploadId/fileId is sent again. Do not mint another upload or logical file ID. + const result = await configure(item.body); + const candidate = { signature, sourceId: result.source.id }; + configured.set(item.id, candidate); + onConfigured?.(candidate); + sourceIds.push(candidate.sourceId); + } + return sourceIds; +} diff --git a/web/tests/sourceBatchConfigure.test.ts b/web/tests/sourceBatchConfigure.test.ts new file mode 100644 index 0000000..ed33cb8 --- /dev/null +++ b/web/tests/sourceBatchConfigure.test.ts @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { configureSourceBatch } from '../src/sourceBatchConfigure.ts'; + +type Body = { uploadId: string; fileId: string; year: number; fieldMap: Record }; +type Candidate = { id: string; status: 'DISCOVERED' | 'RUNNING'; body: Body }; + +function candidateService() { + const byIdentity = new Map(); + const calls: Body[] = []; + const configure = async (body: Body) => { + calls.push(body); + const identity = `${body.uploadId}:${body.fileId}`; + const previous = byIdentity.get(identity); + if (previous && previous.status !== 'DISCOVERED') throw new Error('409 candidate is no longer editable'); + const candidate = previous || { id: `candidate-${body.fileId}`, status: 'DISCOVERED' as const, body }; + candidate.body = body; + byIdentity.set(identity, candidate); + return { source: { id: candidate.id } }; + }; + return { byIdentity, calls, configure }; +} + +const item = (id: string, fileId = id, year = 2026): { id: string; body: Body } => ({ + id, + body: { uploadId: `upload-${id}`, fileId, year, fieldMap: { weight: '重量' } }, +}); + +test('keeps the confirmed source ID when a later file fails, then retries only the failed file', async () => { + const service = candidateService(); + const configured = new Map(); + const files = [item('A'), item('B')]; + let bAttempts = 0; + const configure = async (body: Body) => { + if (body.fileId === 'B' && ++bAttempts === 1) throw new Error('temporary configure failure'); + return service.configure(body); + }; + + await assert.rejects(configureSourceBatch(files, configured, configure), /temporary configure failure/); + assert.equal(configured.get('A').sourceId, 'candidate-A'); + assert.equal(configured.has('B'), false); + + const sourceIds = await configureSourceBatch(files, configured, configure); + assert.deepEqual(sourceIds, ['candidate-A', 'candidate-B']); + assert.equal(service.calls.filter(body => body.fileId === 'A').length, 1); + assert.equal(service.calls.filter(body => body.fileId === 'B').length, 1); +}); + +test('reconfigures changed metadata with the original uploadId and fileId while preserving the candidate ID', async () => { + const service = candidateService(); + const configured = new Map(); + const first = item('A', 'logical-A', 2025); + const second = item('A', 'logical-A', 2026); + + const [initialId] = await configureSourceBatch([first], configured, service.configure); + const [updatedId] = await configureSourceBatch([second], configured, service.configure); + + assert.equal(initialId, updatedId); + assert.equal(service.calls.length, 2); + assert.equal(service.calls[0].uploadId, service.calls[1].uploadId); + assert.equal(service.calls[0].fileId, service.calls[1].fileId); + assert.equal(service.byIdentity.get('upload-A:logical-A')?.body.year, 2026); +}); + +test('does not replace the cached candidate when the server rejects edits after processing starts', async () => { + const service = candidateService(); + const configured = new Map(); + const first = item('A', 'logical-A', 2025); + const changed = item('A', 'logical-A', 2026); + + await configureSourceBatch([first], configured, service.configure); + const candidate = service.byIdentity.get('upload-A:logical-A'); + assert.ok(candidate); + candidate.status = 'RUNNING'; + + await assert.rejects(configureSourceBatch([changed], configured, service.configure), /409 candidate is no longer editable/); + assert.equal(configured.get('A').signature, JSON.stringify(first.body)); +}); diff --git a/web/verification/chart_entry_motion.test.mjs b/web/verification/chart_entry_motion.test.mjs index 76a7207..ac1f9ae 100644 --- a/web/verification/chart_entry_motion.test.mjs +++ b/web/verification/chart_entry_motion.test.mjs @@ -8,7 +8,7 @@ function fixture() { const end=source.includes('export function TrendTable(')?source.indexOf('export function TrendTable('):source.indexOf('function TrendTable('); const code=source.slice(source.indexOf('export function TrendChart('),end).replace('export function','function'); const slots=new Map(),effects=new Map(),observers=[],charts=[];let cursor=0,dirty=false,queued=[],active=true,visible=true; - const node={clientWidth:900,clientHeight:380}; + const node={clientWidth:900,clientHeight:380,closest(){return null;}}; const hooks={useRef(initial){const i=cursor++;if(!slots.has(i))slots.set(i,{current:initial});return slots.get(i);},useState(initial){const i=cursor++;if(!slots.has(i))slots.set(i,initial);return [slots.get(i),v=>{const next=typeof v==='function'?v(slots.get(i)):v;if(!Object.is(next,slots.get(i))){slots.set(i,next);dirty=true;}}];},useCallback(fn,deps){const i=cursor++;const old=slots.get(i);if(!old||deps.some((d,j)=>!Object.is(d,old.deps[j])))slots.set(i,{fn,deps});return slots.get(i).fn;},useEffect(fn,deps){const i=cursor++,old=effects.get(i);if(!old||deps.some((d,j)=>!Object.is(d,old.deps[j])))queued.push(()=>{old?.cleanup?.();effects.set(i,{deps,cleanup:fn()});});}}; hooks.useLayoutEffect=hooks.useEffect; const ec={init(_node,_theme,opts){const c={opts,options:[],resizes:[],disposed:false,getWidth:()=>c.opts.width??100,getHeight:()=>c.opts.height??380,on(){},resize(o){c.resizes.push(o);},setOption(o){c.options.push(o);},convertToPixel:()=>0,dispose(){c.disposed=true;}};charts.push(c);return c;}}; diff --git a/web/verification/defect_smooth_refresh.test.mjs b/web/verification/defect_smooth_refresh.test.mjs index 66b202c..0442a22 100644 --- a/web/verification/defect_smooth_refresh.test.mjs +++ b/web/verification/defect_smooth_refresh.test.mjs @@ -10,7 +10,7 @@ function fixture(){ const source=fs.readFileSync(process.env.QUERY_PARETO_SOURCE || new URL('../src/QueryPage.tsx',import.meta.url),'utf8'); const component=source.slice(source.indexOf('export function DefectParetoPanel('),source.indexOf('export function SectionHeading(')).replace('export function','function'); const requests=[],options=[],effects=new Map(),states=new Map(),refs=new Map();let cursor=0,dirty=false,queued=[],tree,creates=0,clears=0; - const node={clientWidth:900,clientHeight:470}; + const node={clientWidth:900,clientHeight:470,closest(){return null;}}; const hooks={useState(initial){const i=cursor++;if(!states.has(i))states.set(i,initial);return [states.get(i),v=>{states.set(i,typeof v==='function'?v(states.get(i)):v);dirty=true;}];},useRef(initial){const i=cursor++;if(!refs.has(i))refs.set(i,{current:initial});return refs.get(i);},useMemo(fn){cursor++;return fn();},useEffect(fn,deps){const i=cursor++,old=effects.get(i);if(!old||deps.some((x,j)=>!Object.is(x,old.deps[j])))queued.push(()=>{old?.cleanup?.();effects.set(i,{deps,cleanup:fn()});});}}; const api={fineDefectTop(params,signal){const d=deferred();requests.push({...d,params,signal});return d.promise;}}; const chart={setOption:(value,config)=>options.push({value,config}),clear(){clears++;},resize(){},dispose(){}}; diff --git a/web/verification/delete_confirmation.test.mjs b/web/verification/delete_confirmation.test.mjs new file mode 100644 index 0000000..1e80250 --- /dev/null +++ b/web/verification/delete_confirmation.test.mjs @@ -0,0 +1,32 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import ts from 'typescript'; +const deferred=()=>{let resolve,reject;const promise=new Promise((r,j)=>{resolve=r;reject=j;});return{promise,resolve,reject};}; +function fixture(){ + const source=fs.readFileSync(new URL('../src/useDeleteConfirmation.ts',import.meta.url),'utf8').replace(/^import .*;\n/gm,'').replace('export function','function'); + const dialogs=[],cleanups=[]; + const App={useApp:()=>({modal:{confirm(options){const dialog={options,updates:[],destroyed:false,update(value){this.updates.push(value);},destroy(){this.destroyed=true;options.afterClose();}};dialogs.push(dialog);return dialog;}}})}; + const compiled=ts.transpileModule(source,{compilerOptions:{target:ts.ScriptTarget.ES2022,module:ts.ModuleKind.None}}).outputText; + const open=new Function('App','useCallback','useEffect','useRef',compiled+'\nreturn useDeleteConfirmation();')(App,fn=>fn,effect=>cleanups.push(effect()),value=>({current:value})); + return{open,dialogs,unmount:()=>cleanups.forEach(fn=>fn?.())}; +} +test('opening or cancelling a deletion never changes data, including a late confirm callback',async()=>{ + const f=fixture(),rows=['a','b'];f.open({title:'删除a?',description:'仅修改草稿',onConfirm:()=>rows.splice(0,1)}); + assert.deepEqual(rows,['a','b']);const d=f.dialogs[0];assert.equal(d.options.autoFocusButton,'cancel');assert.equal(d.options.maskClosable,false);assert.equal(d.options.okButtonProps.danger,true); + d.options.onCancel();await d.options.onOk();assert.deepEqual(rows,['a','b']);d.destroy(); +}); +test('explicit confirmation performs an async action only once and disables cancellation while running',async()=>{ + const f=fixture(),work=deferred();let writes=0;f.open({title:'删除a?',description:'立即生效',onConfirm:()=>{writes++;return work.promise;}}); + const d=f.dialogs[0],first=d.options.onOk();await d.options.onOk();assert.equal(writes,1);assert.equal(d.updates.at(-1).cancelButtonProps.disabled,true);work.resolve();await first;d.destroy(); +}); +test('a failed confirmed request can be retried without losing the active confirmation',async()=>{ + const f=fixture();let writes=0;f.open({title:'删除a?',description:'立即生效',onConfirm:()=>{if(++writes===1)throw Error('request failed');}}); + const d=f.dialogs[0];await assert.rejects(d.options.onOk(),/request failed/);assert.equal(d.destroyed,false);assert.equal(d.updates.at(-1).cancelButtonProps.disabled,false);await d.options.onOk();assert.equal(writes,2); +}); +test('leaving the component closes its confirmation and cannot trigger deletion afterwards',async()=>{ + const f=fixture();let writes=0;f.open({title:'删除a?',description:'立即生效',onConfirm:()=>writes++});const d=f.dialogs[0];f.unmount();assert.equal(d.destroyed,true);await d.options.onOk();assert.equal(writes,0); +}); +test('repeated delete clicks show one confirmation, and closing it allows a different target',async()=>{ + const f=fixture();const writes=[];f.open({title:'a',description:'a',onConfirm:()=>writes.push('a')});f.open({title:'b',description:'b',onConfirm:()=>writes.push('b')});assert.equal(f.dialogs.length,1);f.dialogs[0].destroy();f.open({title:'b',description:'b',onConfirm:()=>writes.push('b')});assert.equal(f.dialogs.length,2);await f.dialogs[1].options.onOk();assert.deepEqual(writes,['b']); +}); diff --git a/web/verification/system_update_panel.test.mjs b/web/verification/system_update_panel.test.mjs new file mode 100644 index 0000000..b9ae018 --- /dev/null +++ b/web/verification/system_update_panel.test.mjs @@ -0,0 +1,64 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import ts from 'typescript'; +import React from 'react'; + +const tick = () => new Promise(resolve => setImmediate(resolve)); + +function fixture() { + const source = fs.readFileSync(new URL('../src/SystemUpdatePanel.tsx', import.meta.url), 'utf8') + .replace(/^import[\s\S]*?;\n/gm, '').replace(/export /g, ''); + let cursor = 0, dirty = false, queued = [], polling, pollingEnabled, tree, failure; + const states = new Map(), refs = new Map(), memo = new Map(), effects = new Map(), authHandled = []; + let server = { enabled: true, state: 'available', available: true, version: '2', currentVersion: '1' }; + const props = { onUnauthorized(error) { if ([401, 403].includes(error?.status)) authHandled.push(error.status); return error?.status === 401; } }; + const hooks = { + useState(initial) { const i = cursor++; if (!states.has(i)) states.set(i, initial); return [states.get(i), next => { states.set(i, next); dirty = true; }]; }, + useRef(initial) { const i = cursor++; if (!refs.has(i)) refs.set(i, { current: initial }); return refs.get(i); }, + useCallback(fn, deps) { const i = cursor++, old = memo.get(i); if (!old || deps.some((value, j) => value !== old.deps[j])) memo.set(i, { fn, deps }); return memo.get(i).fn; }, + useEffect(fn, deps) { const i = cursor++, old = effects.get(i); if (!old || deps.some((value, j) => value !== old.deps[j])) queued.push(() => { effects.set(i, { deps }); fn(); }); }, + }; + const api = { + async systemUpdateStatus() { if (failure !== undefined) { const status = failure; failure = undefined; throw Object.assign(Error('connection-or-session-error'), { status }); } return { ...server }; }, + async applySystemUpdate() { server = { ...server, state: 'queued', available: false }; return { ...server }; }, + async checkSystemUpdate() { return { ...server }; }, + }; + const prelude = 'const {useState,useRef,useCallback,useEffect}=hooks;'; + const compiled = ts.transpileModule(prelude + source + '\nreturn SystemUpdatePanel;', + { fileName: 'fixture.tsx', compilerOptions: { target: ts.ScriptTarget.ES2022, jsx: ts.JsxEmit.React } }).outputText; + const component = new Function('React', 'hooks', 'api', 'usePageCache', 'useVisiblePolling', 'configurationDateTime', + 'Alert', 'Button', 'Spin', 'Tag', 'ReloadOutlined', compiled)(React, hooks, api, () => ({ active: true }), + (callback, interval, enabled) => { polling = callback; pollingEnabled = enabled; }, value => value, 'Alert', 'Button', 'Spin', 'Tag', 'ReloadOutlined'); + const render = () => { let attempts = 0; do { cursor = 0; dirty = false; queued = []; tree = component(props); queued.forEach(fn => fn()); assert.ok(++attempts < 15); } while (dirty); return tree; }; + const buttons = () => { const result = []; function walk(node) { if (!node || typeof node !== 'object') return; if (node.type === 'Button') result.push(node); React.Children.forEach(node.props?.children, walk); } walk(tree); return result; }; + return { + authHandled, reconnecting: () => states.get(3), error: () => states.get(2), pollingEnabled: () => pollingEnabled, + async start() { render(); await tick(); render(); }, + async apply() { const button = buttons().find(node => node.props.children === '更新并重启'); assert.ok(button); button.props.onClick(); await tick(); render(); }, + async failPoll(status) { failure = status; await polling(); render(); }, + async complete() { server = { ...server, state: 'completed', message: '更新完成。', currentVersion: '2' }; await polling(); render(); }, + }; +} + +for (const status of [401, 403]) test(`restart polling forwards ${status} to authentication handling`, async () => { + const ui = fixture(); await ui.start(); await ui.apply(); await ui.failPoll(status); + assert.deepEqual(ui.authHandled, [status]); + assert.equal(ui.reconnecting(), false); + assert.equal(ui.pollingEnabled(), false); + if (status === 403) assert.equal(ui.error(), 'connection-or-session-error'); +}); + +test('temporary restart connection loss reconnects and clears after completion', async () => { + const ui = fixture(); await ui.start(); await ui.apply(); await ui.failPoll(503); + assert.equal(ui.reconnecting(), true); + assert.deepEqual(ui.authHandled, []); + await ui.complete(); + assert.equal(ui.reconnecting(), false); +}); + +test('a permanent route failure is displayed rather than treated as a restart connection loss', async () => { + const ui = fixture(); await ui.start(); await ui.apply(); await ui.failPoll(404); + assert.equal(ui.reconnecting(), false); + assert.equal(ui.error(), 'connection-or-session-error'); +}); diff --git a/web/verification/unified_workspace_loading.test.mjs b/web/verification/unified_workspace_loading.test.mjs index 4b06986..f452fa5 100644 --- a/web/verification/unified_workspace_loading.test.mjs +++ b/web/verification/unified_workspace_loading.test.mjs @@ -5,6 +5,7 @@ import ts from 'typescript'; import React from 'react'; import { readQueryLocation } from '../src/queryLoadingScope.ts'; import { pageUsesResources } from '../src/resourceInvalidation.ts'; +import { canAccessWorkspacePage } from '../src/userRoles.ts'; const tick = () => new Promise(resolve => setImmediate(resolve)); const deferred = () => { let resolve, reject; const promise = new Promise((a, b) => { resolve = a; reject = b; }); return { promise, resolve, reject }; }; @@ -15,8 +16,8 @@ function fixture() { const values = new Map(), refs = new Map(), effects = new Map(), events = new Map(), navigations = [], scopes = []; let cursor = 0, dirty = false, queued = [], tree; let state = { authRevision: 'a1', dataRevision: 'd1', currentReleaseId: 'r1' }, failState = false, failStatus, me, optionReads=0, optionDenied=false; - const props = { page: 'overall', user: { id: 'A' }, onUnauthorized: () => false, - onPageChange(page) { props.page = page; dirty = true; }, onUserChange(user) { props.user = user; dirty = true; } }; + const props = { page: 'overall', user: { id: 'A', roles: ['USER'] }, onUnauthorized: () => false, + onPageChange(page) { props.page = page; dirty = true; }, onUserChange(user) { props.user = { roles: ['USER'], ...user }; dirty = true; } }; const ref = { current: null }; const hooks = { useState(initial) { const i = cursor++; if (!values.has(i)) values.set(i, typeof initial === 'function' ? initial() : initial); return [values.get(i), next => { values.set(i, typeof next === 'function' ? next(values.get(i)) : next); dirty = true; }]; }, @@ -26,6 +27,7 @@ function fixture() { }; const api = { async options() { optionReads++; if(optionDenied) { events.get('quality-permission-recheck')?.(); throw Object.assign(Error('denied'),{status:403}); } return {}; }, async sourceColumnCatalog() { return {}; }, async workspaceState() { if (failState) { failState = false; throw Object.assign(Error('temporary-offline'), { status: failStatus }); } return { ...state }; }, async me() { return me ? me.promise : props.user; } }; const env = { + canAccessWorkspacePage, location: { search: '?page=overall', pathname: '/' }, window: { scrollY: 0, scrollTo() {}, matchMedia: () => ({ matches: false }), addEventListener: (name, fn) => events.set(name, fn), removeEventListener: name => events.delete(name) }, document: { visibilityState: 'visible', addEventListener: (name, fn) => events.set(name, fn), removeEventListener: name => events.delete(name), querySelector() {} }, @@ -43,7 +45,7 @@ function fixture() { const render = () => { let attempts = 0; do { cursor = 0; dirty = false; queued = []; tree = component(props, ref); queued.forEach(fn => fn()); assert.ok(++attempts < 15); } while (dirty); return tree; }; const pages = () => { const result = []; function walk(node) { if (!node || typeof node !== 'object') return; if (node.props?.['data-page']) result.push(node.props['data-page']); React.Children.forEach(node.props?.children, walk); } walk(tree); return result; }; return { render, pages, props, scopes, ref, async start() { render(); await tick(); render(); }, async settle() { await Promise.all(navigations); await tick(); render(); }, - getOptionReads:()=>optionReads,denyOptions(){optionDenied=true;},failNextState(status) { failState = true; failStatus = status; }, changeIdentity() { state = { ...state, authRevision: 'b2' }; me = deferred(); return me; }, + locationSearch:()=>env.location.search,getOptionReads:()=>optionReads,denyOptions(){optionDenied=true;},failNextState(status) { failState = true; failStatus = status; }, changeIdentity() { state = { ...state, authRevision: 'b2' }; me = deferred(); return me; }, async focus() { events.get('focus')?.(); await tick(); render(); }, cleanup() { effects.forEach(effect => effect.cleanup?.()); } }; } @@ -86,3 +88,20 @@ test('a forbidden options resource does not trigger repeated warm reads during p const ui=fixture();ui.denyOptions();await ui.start();await tick();ui.render();const count=ui.getOptionReads(); await ui.focus();await tick();ui.render();assert.equal(ui.getOptionReads(),count);assert.ok(count<=1);ui.cleanup(); }); + +test('sales order type survives cross-stage navigation and clearing without copying stage source filters',async()=>{ + const ui=fixture();await ui.start(); + ui.ref.current.navigate('blackProduct',{year:2026,view:'product',extraFilters:{salesOrderType:['样品销售订单']},columnFilters:[{field:'blackOnly',stages:['black'],operator:'in',values:['A']}]});await ui.settle(); + ui.ref.current.navigate('brightProduct');await ui.settle(); + const bright=readQueryLocation(ui.locationSearch());assert.deepEqual(bright.extraFilters,{salesOrderType:['样品销售订单']});assert.equal(bright.columnFilters,undefined); + ui.ref.current.navigate('brightProduct',{year:2026,view:'product',extraFilters:{}});await ui.settle(); + ui.ref.current.navigate('blackProduct');await ui.settle(); + const black=readQueryLocation(ui.locationSearch());assert.deepEqual(black.extraFilters,{});assert.equal(black.columnFilters[0].field,'blackOnly');ui.cleanup(); +}); + +test('sales order exclusions retain their mode on navigation',async()=>{ + const ui=fixture();await ui.start(); + ui.ref.current.navigate('overall',{year:2026,view:'overall',extraFilters:{salesOrderTypeNotIn:['样品销售订单']}});await ui.settle(); + ui.ref.current.navigate('blackProduct');await ui.settle(); + assert.deepEqual(readQueryLocation(ui.locationSearch()).extraFilters,{salesOrderTypeNotIn:['样品销售订单']});ui.cleanup(); +}); diff --git a/web/verification/user_roles.test.mjs b/web/verification/user_roles.test.mjs new file mode 100644 index 0000000..bd6f208 --- /dev/null +++ b/web/verification/user_roles.test.mjs @@ -0,0 +1,29 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import ts from 'typescript'; + +const source = ts.transpileModule(readFileSync(new URL('../src/userRoles.ts', import.meta.url), 'utf8'), { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, +}).outputText.replace(/export /g, ''); +const { USER_ROLE_OPTIONS, canAccessWorkspacePage } = new Function(`${source};return { USER_ROLE_OPTIONS, canAccessWorkspacePage };`)(); + +test('only three assigned roles are exposed with the agreed names', () => { + assert.deepEqual(USER_ROLE_OPTIONS, [{ value: 'USER', label: '用户' }, { value: 'DATA_ADMIN', label: '数据管理员' }, { value: 'SYSTEM_ADMIN', label: '系统管理员' }]); +}); + +test('higher roles inherit business pages while system settings stay system-only', () => { + for (const role of ['USER', 'DATA_ADMIN', 'SYSTEM_ADMIN']) { + const user = { roles: [role] }; + for (const page of ['overall', 'department', 'product', 'blackProduct', 'brightProduct', 'analysis']) + assert.equal(canAccessWorkspacePage(user, page), true, `${role}: ${page}`); + for (const page of ['sources', 'imports', 'rules', 'mappings', 'masterData', 'tasks']) + assert.equal(canAccessWorkspacePage(user, page), role !== 'USER', `${role}: ${page}`); + assert.equal(canAccessWorkspacePage(user, 'access'), role === 'SYSTEM_ADMIN'); + } +}); + +test('unexpected or multiple role assignments do not expose management pages', () => { + for (const roles of [[], ['QUALITY_VIEWER'], ['USER', 'SYSTEM_ADMIN']]) + assert.equal(canAccessWorkspacePage({ roles }, 'access'), false); +});