From dc5b9b76ec80156e5eae8dd1d9f66b0d2582bbac Mon Sep 17 00:00:00 2001 From: lyun Date: Tue, 10 Nov 2026 14:53:07 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=EF=BC=9ALinux=20Docker=20?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E4=B8=8E=E8=87=AA=E5=8A=A8=E5=9B=9E=E6=94=B6?= =?UTF-8?q?=E3=80=81=E5=A4=87=E4=BB=BD=E5=8F=8A=E5=AE=B9=E9=87=8F=E7=9B=91?= =?UTF-8?q?=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 38 + README.md | 2 + ...tionProtocolStoreMySqlConcurrencyTest.java | 4 +- .../AnalysisProtocolServiceMySqlTest.java | 4 +- .../AnalysisProtocolTurnBindingMySqlTest.java | 4 +- .../analysis/FineDefectRollupsMySqlTest.java | 4 +- .../filters/SalesOrderTypesMySqlTest.java | 4 +- .../filters/SourceColumnQueriesMySqlTest.java | 4 +- .../ClassificationPreviewMySqlTest.java | 4 +- .../ClassificationSourceMySqlTest.java | 4 +- .../UnclassifiedMappingRebuildMySqlTest.java | 4 +- .../SalesOrderMetricRollupsMySqlTest.java | 4 +- .../source/SourceUploadAppendMySqlTest.java | 5 +- .../testsupport/OwnedTestDatabase.java | 141 ++ deploy/docker/.env.example | 12 + deploy/docker/Dockerfile.app | 52 + deploy/docker/Dockerfile.web | 13 + deploy/docker/README.md | 76 ++ deploy/docker/app.py | 161 +++ deploy/docker/assets.lock.json | 20 + deploy/docker/compose.yaml | 108 ++ deploy/docker/fetch_runtimes.py | 27 + deploy/docker/manage.py | 65 + deploy/docker/nginx.conf | 21 + deploy/docker/requirements-linux.lock | 1178 +++++++++++++++++ deploy/docker/test_docker.py | 148 +++ deploy/docker/worker.py | 88 ++ deploy/macos/README.md | 94 ++ deploy/macos/logback-quality.xml | 19 + deploy/macos/operations.py | 276 ++++ deploy/macos/retention-requirements.txt | 23 + deploy/macos/retention.py | 525 ++++++++ deploy/macos/service.py | 351 +++++ deploy/macos/test_native_operations.py | 120 ++ deploy/macos/test_operations.py | 195 +++ deploy/macos/test_retention.py | 305 +++++ deploy/macos/test_revision_retention.py | 237 ++++ deploy/macos/test_service.py | 116 ++ scripts/backup.py | 96 +- scripts/backup_retention.py | 381 ++++++ scripts/backup_verifier.py | 112 ++ scripts/log_retention.py | 158 +++ scripts/managed_test_resources.py | 207 +++ scripts/project.py | 3 +- scripts/revision_retention.py | 213 +++ 45 files changed, 5591 insertions(+), 35 deletions(-) create mode 100644 .dockerignore create mode 100644 backend/src/test/java/cn/yunsheng/quality/testsupport/OwnedTestDatabase.java create mode 100644 deploy/docker/.env.example create mode 100644 deploy/docker/Dockerfile.app create mode 100644 deploy/docker/Dockerfile.web create mode 100644 deploy/docker/README.md create mode 100644 deploy/docker/app.py create mode 100644 deploy/docker/assets.lock.json create mode 100644 deploy/docker/compose.yaml create mode 100644 deploy/docker/fetch_runtimes.py create mode 100644 deploy/docker/manage.py create mode 100644 deploy/docker/nginx.conf create mode 100644 deploy/docker/requirements-linux.lock create mode 100644 deploy/docker/test_docker.py create mode 100644 deploy/docker/worker.py create mode 100644 deploy/macos/README.md create mode 100644 deploy/macos/logback-quality.xml create mode 100644 deploy/macos/operations.py create mode 100644 deploy/macos/retention-requirements.txt create mode 100644 deploy/macos/retention.py create mode 100644 deploy/macos/service.py create mode 100644 deploy/macos/test_native_operations.py create mode 100644 deploy/macos/test_operations.py create mode 100644 deploy/macos/test_retention.py create mode 100644 deploy/macos/test_revision_retention.py create mode 100644 deploy/macos/test_service.py create mode 100644 scripts/backup_retention.py create mode 100644 scripts/backup_verifier.py create mode 100644 scripts/log_retention.py create mode 100644 scripts/managed_test_resources.py create mode 100644 scripts/revision_retention.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9e16127 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,38 @@ +# 仅允许源码和构建输入进入上下文,避免把本机数据库、备份、密码或业务表格送入构建器。 +** +!.dockerignore +!backend/ +!backend/pom.xml +!backend/src/ +!backend/src/** +!web/ +!web/package.json +!web/pnpm-lock.yaml +!web/pnpm-workspace.yaml +!web/index.html +!web/tsconfig*.json +!web/vite.config.ts +!web/src/ +!web/src/** +!web/public/ +!web/public/** +!scripts/ +!scripts/** +!ai/ +!ai/** +!deploy/ +!deploy/docker/ +!deploy/docker/** +!deploy/macos/ +!deploy/macos/** +!runtime.lock.json +!requirements-nao.in +!requirements-nao.lock +**/__pycache__ +**/*.pyc +**/.env +**/.env.* +**/secrets +**/original-workbook-acceptance.json +**/*.log +!deploy/docker/.env.example diff --git a/README.md b/README.md index ddc5520..53328c8 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ ## 本机运行 +Linux 容器部署、自动重启、空间回收、备份保留及容量告警见 [Docker 部署说明](deploy/docker/README.md)。这些配置随源码进入 Gitea;构建镜像使用手动入口,不要求配置 Gitea 自动构建。Mac 测试服务的开机启动与守护见 [本机服务说明](deploy/macos/README.md)。 + 本机启动脚本面向 macOS 15 及以上的 Apple Silicon。先准备 Python 3.11 及以上、Node.js/pnpm 和 [uv](https://docs.astral.sh/uv/getting-started/installation/)。首次运行需要安装项目运行依赖并构建: python3 scripts/setup_runtime.py diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisExecutionProtocolStoreMySqlConcurrencyTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisExecutionProtocolStoreMySqlConcurrencyTest.java index 6e9149b..063fd9c 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisExecutionProtocolStoreMySqlConcurrencyTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisExecutionProtocolStoreMySqlConcurrencyTest.java @@ -39,7 +39,7 @@ class AnalysisExecutionProtocolStoreMySqlConcurrencyTest { assertTrue(baseUrl.contains("/?"), "test server URL must have an empty database path and query parameters"); admin = new JdbcTemplate(new DriverManagerDataSource(baseUrl, user, password)); database = "quality_protocol_test_" + UUID.randomUUID().toString().replace("-", ""); - admin.execute("CREATE DATABASE `" + database + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database); created = true; dataSource = new DriverManagerDataSource(baseUrl.replaceFirst("/\\?", "/" + database + "?"), user, password); jdbc = new JdbcTemplate(dataSource); @@ -52,7 +52,7 @@ class AnalysisExecutionProtocolStoreMySqlConcurrencyTest { @AfterEach void cleanup() { if (created && admin != null && database != null && database.matches("quality_protocol_test_[0-9a-f]{32}")) { - admin.execute("DROP DATABASE IF EXISTS `" + database + "`"); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database); } } diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolServiceMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolServiceMySqlTest.java index cf2a95f..0f40277 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolServiceMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolServiceMySqlTest.java @@ -41,7 +41,7 @@ class AnalysisProtocolServiceMySqlTest { String user=System.getenv("QUALITY_PROTOCOL_TEST_MYSQL_USER"),password=System.getenv("QUALITY_PROTOCOL_TEST_MYSQL_PASSWORD"); admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password)); database="quality_protocol_test_"+UUID.randomUUID().toString().replace("-",""); - admin.execute("CREATE DATABASE `"+database+"` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");created=true; + cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);created=true; var ds=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password); jdbc=new JdbcTemplate(ds);tx=new TransactionTemplate(new DataSourceTransactionManager(ds)); jdbc.execute("CREATE TABLE analysis_session(id CHAR(36) PRIMARY KEY,user_id CHAR(36),release_id CHAR(36)) ENGINE=InnoDB"); @@ -67,7 +67,7 @@ class AnalysisProtocolServiceMySqlTest { jdbc.update("INSERT INTO analysis_session VALUES(?,?,?)",session,actor.getId().toString(),release); tx.executeWithoutResult(s->service.initialize(session,Map.of("releaseId",release,"year",2026))); } - @AfterEach void cleanup(){if(created&&database.matches("quality_protocol_test_[0-9a-f]{32}"))admin.execute("DROP DATABASE `"+database+"`");} + @AfterEach void cleanup(){if(created&&database.matches("quality_protocol_test_[0-9a-f]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);} @Test void clarificationTrendClearAndCancellationUseOneAuthority() { String first=newTurn("今年18×15×4综合合格率变化趋势");var a=claim(first,false); diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolTurnBindingMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolTurnBindingMySqlTest.java index 673b86d..3136a2c 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolTurnBindingMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/AnalysisProtocolTurnBindingMySqlTest.java @@ -48,7 +48,7 @@ class AnalysisProtocolTurnBindingMySqlTest { String password = System.getenv("QUALITY_PROTOCOL_TEST_MYSQL_PASSWORD"); admin = new JdbcTemplate(new DriverManagerDataSource(url, dbUser, password)); database = "quality_protocol_test_" + UUID.randomUUID().toString().replace("-", ""); - admin.execute("CREATE DATABASE `" + database + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database); created = true; var dataSource = new DriverManagerDataSource(url.replaceFirst("/\\?", "/" + database + "?"), dbUser, password); jdbc = new JdbcTemplate(dataSource); @@ -73,7 +73,7 @@ class AnalysisProtocolTurnBindingMySqlTest { @AfterEach void cleanup() { if (created && database != null && database.matches("quality_protocol_test_[0-9a-f]{32}")) - admin.execute("DROP DATABASE IF EXISTS `" + database + "`"); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database); } private AnalysisExecutionProtocolStore.Execution start(String turnId, String userId, String key) { diff --git a/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java index 80d081f..05d5c4e 100644 --- a/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/analysis/FineDefectRollupsMySqlTest.java @@ -36,7 +36,7 @@ class FineDefectRollupsMySqlTest { String user = System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"), password = System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD"); admin = new JdbcTemplate(new DriverManagerDataSource(url, user, password)); database = "quality_fine_rollup_" + UUID.randomUUID().toString().replace("-", ""); - admin.execute("CREATE DATABASE " + database + " CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database); jdbc = spy(new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?", "/" + database + "?"), user, password))); jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY,source_revision_id CHAR(36))"); jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)"); @@ -161,7 +161,7 @@ class FineDefectRollupsMySqlTest { } @AfterEach void cleanup() { - if (admin != null && database != null && database.matches("quality_fine_rollup_[a-f0-9]{32}")) admin.execute("DROP DATABASE " + database); + if (admin != null && database != null && database.matches("quality_fine_rollup_[a-f0-9]{32}")) cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database); } @Test diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SalesOrderTypesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SalesOrderTypesMySqlTest.java index 3db0636..100bf57 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SalesOrderTypesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SalesOrderTypesMySqlTest.java @@ -26,7 +26,7 @@ class SalesOrderTypesMySqlTest { assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?")); String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD"); admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_sales_type_test_"+UUID.randomUUID().toString().replace("-",""); - admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4"); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database); var source=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(source); try(var connection=source.getConnection()){ScriptUtils.executeSqlScript(connection,new ClassPathResource("db/migration/V39__sales_order_types.sql"));} jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64),label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)"); @@ -41,7 +41,7 @@ class SalesOrderTypesMySqlTest { jdbc.update("INSERT INTO fact_black(id,build_id,source_revision_id,department,group_name,raw_values) VALUES(1,'black-build','black-rev','部门A','组A',JSON_OBJECT('销售订单号','0001')),(2,'black-build','black-rev','部门A','组A',JSON_OBJECT('销售订单号','absent')),(3,'black-build','black-rev','部门A','组A',JSON_OBJECT('销售订单号',NULL))"); jdbc.update("INSERT INTO fact_bright(id,build_id,source_revision_id,department,group_name,raw_values) VALUES(4,'bright-build','bright-rev','部门A','组A',JSON_OBJECT('销售订单','0001'))"); } - @AfterEach void cleanup(){RequestContextHolder.resetRequestAttributes();if(database!=null)admin.execute("DROP DATABASE "+database);} + @AfterEach void cleanup(){RequestContextHolder.resetRequestAttributes();if(database!=null)cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);} Map load(String key,String type,String mode) throws Exception { var service=new SalesOrderTypeService(jdbc);var file=SalesOrderTypeServiceTest.workbook(new String[]{key,type});var preview=service.preview(file,mode); return service.importFile(file,mode,(String)preview.get("expectedVersion"),(String)preview.get("sha256"),actor); diff --git a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java index 060af22..fea92cd 100644 --- a/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/filters/SourceColumnQueriesMySqlTest.java @@ -44,7 +44,7 @@ class SourceColumnQueriesMySqlTest { String password = System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD"); admin = new JdbcTemplate(new DriverManagerDataSource(url, user, password)); database = "quality_source_column_test_" + UUID.randomUUID().toString().replace("-", ""); - admin.execute("CREATE DATABASE " + database + " CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database); created = true; var dataSource = new DriverManagerDataSource(url.replaceFirst("/\\?", "/" + database + "?"), user, password); jdbc = new JdbcTemplate(dataSource); @@ -57,7 +57,7 @@ class SourceColumnQueriesMySqlTest { @AfterEach void cleanup() { if (created && database != null && database.matches("quality_source_column_test_[0-9a-f]{32}")) - admin.execute("DROP DATABASE " + database); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database); } @Test void catalogIsScopedAuthorizedAndSupportsThirdAndSingleStageColumns() { diff --git a/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewMySqlTest.java index bf2e4da..586dca7 100644 --- a/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationPreviewMySqlTest.java @@ -18,7 +18,7 @@ class ClassificationPreviewMySqlTest { @BeforeEach void create(){ String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?")); String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD"); - admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_classification_preview_"+UUID.randomUUID().toString().replace("-","");admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_classification_preview_"+UUID.randomUUID().toString().replace("-","");cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database); jdbc=new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password)); jdbc.execute("CREATE TABLE current_release(dataset_id VARCHAR(20),release_id VARCHAR(20))");jdbc.update("INSERT INTO current_release VALUES('quality','r')"); jdbc.execute("CREATE TABLE release_member(release_id VARCHAR(20),transform_build_id VARCHAR(20))");jdbc.update("INSERT INTO release_member VALUES('r','b'),('r','l')"); @@ -31,7 +31,7 @@ class ClassificationPreviewMySqlTest { seed("bright","l","A","R","小写区域","2026-04","0001","x",200); service=new ClassificationPreviewService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),mock(CandidateAcceptanceService.class)); } - @AfterEach void cleanup(){if(database!=null&&database.matches("quality_classification_preview_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);} + @AfterEach void cleanup(){if(database!=null&&database.matches("quality_classification_preview_[a-f0-9]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);} private void seed(String stage,String build,String department,String product,String group,String month,String code,String region,int weight){ Map components=new HashMap<>();components.put("sourcePdtCode",code);components.put("sourceSalesRegion",region); jdbc.update("INSERT INTO fact_"+stage+" VALUES(?,?,?,?,?,?,CAST(? AS JSON))",build,department,product,group,month,weight,JsonMapper.builder().build().writeValueAsString(components)); diff --git a/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationSourceMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationSourceMySqlTest.java index 1b70823..6acb96a 100644 --- a/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationSourceMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/governance/ClassificationSourceMySqlTest.java @@ -19,7 +19,7 @@ class ClassificationSourceMySqlTest { @BeforeEach void create(){ String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?")); String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD"); - admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_classification_source_"+UUID.randomUUID().toString().replace("-","");admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_classification_source_"+UUID.randomUUID().toString().replace("-","");cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database); jdbc=new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password)); jdbc.execute("CREATE TABLE current_release(dataset_id VARCHAR(20),release_id VARCHAR(20))");jdbc.update("INSERT INTO current_release VALUES('quality','r')"); jdbc.execute("CREATE TABLE data_release(id VARCHAR(20),status VARCHAR(20))");jdbc.update("INSERT INTO data_release VALUES('r','PUBLISHED'),('new','PUBLISHED')"); @@ -32,7 +32,7 @@ class ClassificationSourceMySqlTest { access=mock(CandidateAcceptanceService.class);fields=mock(GovernanceAccessService.class);when(fields.allowedFields(actor)).thenReturn(Set.of("客户")); service=new ClassificationSourceService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),access,fields); } - @AfterEach void cleanup(){if(database!=null&&database.matches("quality_classification_source_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);} + @AfterEach void cleanup(){if(database!=null&&database.matches("quality_classification_source_[a-f0-9]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);} @Test void exactSourceRowsPaginateFifteenAndStayPinnedWithoutPreviewOrTarget(){ var first=service.list(new ClassificationSourceService.Request("black",conditions,1,15,null),actor); assertEquals(18,first.get("total"));var rows=(List)first.get("rows");assertEquals(15,rows.size());var row=(Map)rows.getFirst();assertEquals("原始数据.xlsx",row.get("sourceFile"));assertEquals(1,row.get("rowNumber"));assertEquals(Map.of("客户","原始客户"),row.get("raw")); diff --git a/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java index a10c511..e35a300 100644 --- a/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/ingestion/UnclassifiedMappingRebuildMySqlTest.java @@ -21,7 +21,7 @@ class UnclassifiedMappingRebuildMySqlTest { Map pending(){return Map.of("id","classification-test","name","区域补分类","enabled",true,"onlyUnclassified",true,"priority",-9900,"stage","all","target","group","value","G","conditions",List.of(Map.of("field","@department","operator","EQUALS","value","A"),Map.of("field","销售片区","operator","EQUALS","value","X")));} @BeforeEach void setup()throws Exception{ String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL"),user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?")); - admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_unclassified_delta_"+UUID.randomUUID().toString().replace("-","");admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_unclassified_delta_"+UUID.randomUUID().toString().replace("-","");cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database); dataSource=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(dataSource); jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY,source_revision_id VARCHAR(100),build_key CHAR(64),parser_version VARCHAR(100),rule_version VARCHAR(100),mapping_version VARCHAR(100),status VARCHAR(32),row_count BIGINT,rejected_count BIGINT,component_summary JSON,error_summary JSON,started_at TIMESTAMP(6) NULL,completed_at TIMESTAMP(6) NULL,parser_config JSON,parser_config_hash VARCHAR(64))"); jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(190),status VARCHAR(32),parent_release_id CHAR(36),expected_current_release_id CHAR(36),rule_version VARCHAR(100),mapping_version VARCHAR(100),target_version_id VARCHAR(100),complete BOOLEAN,validation JSON,created_by CHAR(36))"); @@ -47,7 +47,7 @@ class UnclassifiedMappingRebuildMySqlTest { jdbc.update("INSERT INTO defect_component(build_id,stage,sheet_name,physical_row_no,defect_name,weight,source_field) VALUES(?,?,'S',?,'料废',?,'原字段')",build,stage,row,scrap); } UnclassifiedMappingRebuild.Candidate run(MetricsRollupService metrics){return new TransactionTemplate(new DataSourceTransactionManager(dataSource)).execute(status->new UnclassifiedMappingRebuild(jdbc,json,worker,metrics).create(UUID.randomUUID().toString(),parent,"new-map","rule",Map.of("matchRules",List.of(pending())),List.of(pending()),actor));} - @AfterEach void cleanup(){if(admin!=null&&database!=null&&database.matches("quality_unclassified_delta_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);} + @AfterEach void cleanup(){if(admin!=null&&database!=null&&database.matches("quality_unclassified_delta_[a-f0-9]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);} @Test void onlyEmptyRowsAreMatchedAndOldFactsWeightsAndExistingClassificationsRemainUnchanged(){ var candidate=run(rollups);assertEquals(3,candidate.scannedRows());assertEquals(2,candidate.classifiedRows()); assertEquals(2,jdbc.queryForObject("SELECT COUNT(*) FROM release_member m JOIN fact_black f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND f.group_name IS NULL",Integer.class,parent)); diff --git a/backend/src/test/java/cn/yunsheng/quality/metrics/SalesOrderMetricRollupsMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/metrics/SalesOrderMetricRollupsMySqlTest.java index 03074dd..4b8d2ea 100644 --- a/backend/src/test/java/cn/yunsheng/quality/metrics/SalesOrderMetricRollupsMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/metrics/SalesOrderMetricRollupsMySqlTest.java @@ -30,7 +30,7 @@ class SalesOrderMetricRollupsMySqlTest { String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD"); admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password)); database="quality_sales_metric_test_"+UUID.randomUUID().toString().replace("-",""); - admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database); var source=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(source); jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)"); jdbc.execute("CREATE TABLE data_release(id CHAR(36),name VARCHAR(100),status VARCHAR(32),rule_version VARCHAR(100))"); @@ -53,7 +53,7 @@ class SalesOrderMetricRollupsMySqlTest { when(governance.allowedFields(actor)).thenReturn(Set.of("*"));when(actor.getDepartments()).thenReturn(List.of("*"));when(actor.getGroups()).thenReturn(List.of()); metrics=new MetricsService(jdbc,governance);rollups=new SalesOrderMetricRollups(jdbc); } - @AfterEach void cleanup(){if(database!=null)admin.execute("DROP DATABASE "+database);} + @AfterEach void cleanup(){if(database!=null)cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);} Object normalized(Object value){return json.readValue(json.writeValueAsString(value),Object.class);} MetricsService.MetricsQuery query(String view, boolean blanks, Map> filters){return new MetricsService.MetricsQuery(2026,null,null,null,null,null,null,view,blanks,release,"month",Map.of(),filters);} diff --git a/backend/src/test/java/cn/yunsheng/quality/source/SourceUploadAppendMySqlTest.java b/backend/src/test/java/cn/yunsheng/quality/source/SourceUploadAppendMySqlTest.java index d6a5e5d..c47cbd0 100644 --- a/backend/src/test/java/cn/yunsheng/quality/source/SourceUploadAppendMySqlTest.java +++ b/backend/src/test/java/cn/yunsheng/quality/source/SourceUploadAppendMySqlTest.java @@ -74,10 +74,11 @@ class SourceUploadAppendMySqlTest { String user=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_USER"),password=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_PASSWORD"); admin=new JdbcTemplate(new DriverManagerDataSource(serverUrl,user,password)); database="quality_verify_multifile_"+UUID.randomUUID().toString().replace("-",""); - admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.createEmpty(admin.getDataSource(),database); dataSource=new DriverManagerDataSource(serverUrl.replaceFirst("/\\?","/"+database+"?"),user,password); Flyway.configure().dataSource(dataSource).locations("classpath:db/migration") .placeholders(Map.of("legacyTimestampOffsetSeconds","0")).load().migrate(); + cn.yunsheng.quality.testsupport.OwnedTestDatabase.seal(admin.getDataSource(),database); jdbc=new JdbcTemplate(dataSource);tx=new TransactionTemplate(new DataSourceTransactionManager(dataSource)); archiveRoot=tempDir.resolve("archive");Files.createDirectories(archiveRoot); adminId=UUID.randomUUID(); @@ -101,7 +102,7 @@ class SourceUploadAppendMySqlTest { } @AfterEach void cleanup(){ - if(admin!=null&&database!=null&&database.matches("quality_verify_multifile_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database); + if(admin!=null&&database!=null&&database.matches("quality_verify_multifile_[a-f0-9]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database); } @Test void mainAndTwoAppendFilesImportPublishReplaceWithdrawAndKeepHistory() throws Exception { diff --git a/backend/src/test/java/cn/yunsheng/quality/testsupport/OwnedTestDatabase.java b/backend/src/test/java/cn/yunsheng/quality/testsupport/OwnedTestDatabase.java new file mode 100644 index 0000000..7b79b96 --- /dev/null +++ b/backend/src/test/java/cn/yunsheng/quality/testsupport/OwnedTestDatabase.java @@ -0,0 +1,141 @@ +package cn.yunsheng.quality.testsupport; + +import java.nio.file.*; +import java.nio.file.attribute.PosixFilePermissions; +import java.sql.*; +import java.time.Instant; +import java.util.UUID; +import javax.sql.DataSource; + +/** Durable ownership for test schemas. Production classes never use this helper. */ +public final class OwnedTestDatabase { + private static final String MARKER = "__quality_test_owner"; + private OwnedTestDatabase() {} + + private static Path root() throws Exception { + for (Path p = Path.of("").toAbsolutePath().normalize(); p != null; p = p.getParent()) { + if (Files.isRegularFile(p.resolve(".runtime/local-config.json"))) return p.toRealPath(); + } + throw new IllegalStateException("Project test configuration not found"); + } + + private static void valid(String name) { + if (name == null || name.length() > 64 || !name.matches("quality_(?:classification_source|classification_preview|sales_type_test|unclassified_delta|sales_metric_test|source_column_test|protocol_test|fine_rollup|verify_multifile)_[a-f0-9]{32}")) + throw new IllegalArgumentException("Not a disposable test database"); + } + + private static void protect(Path root, String database) throws Exception { + var match = java.util.regex.Pattern.compile("\"mysqlDatabase\"\\s*:\\s*\"([^\"]+)\"") + .matcher(Files.readString(root.resolve(".runtime/local-config.json"))); + if (!match.find() || database.equals(match.group(1))) + throw new IllegalArgumentException("Active database is protected"); + } + + private static String quote(String text) { + StringBuilder s = new StringBuilder("\""); + for (char c : text.toCharArray()) { + if (c == '"' || c == '\\') s.append('\\').append(c); + else if (c < 32) s.append(String.format("\\u%04x", (int)c)); + else s.append(c); + } + return s.append('"').toString(); + } + + private static void write(Path path, String value) throws Exception { + Path temp = path.resolveSibling(path.getFileName() + "." + UUID.randomUUID() + ".tmp"); + Files.writeString(temp, value, StandardOpenOption.CREATE_NEW); + if (Files.getFileStore(temp).supportsFileAttributeView("posix")) + Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------")); + try (var stream = java.nio.channels.FileChannel.open(temp, StandardOpenOption.WRITE)) { stream.force(true); } + Files.move(temp, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + + private static Path record(Path root, String database) throws Exception { + Path base = root.resolve(".runtime/test-resources"); + if (Files.isSymbolicLink(root.resolve(".runtime")) || Files.isSymbolicLink(base)) + throw new IllegalStateException("Unsafe test registry"); + Files.createDirectories(base); + if (Files.getFileStore(base).supportsFileAttributeView("posix")) + Files.setPosixFilePermissions(base, PosixFilePermissions.fromString("rwx------")); + return base.resolve(database + ".json"); + } + + public static void create(DataSource source, String database) { + create(source, database, false); + } + + public static void createEmpty(DataSource source, String database) { + create(source, database, true); + } + + private static void create(DataSource source, String database, boolean empty) { + valid(database); + try { + Path root = root(), path = record(root, database); + protect(root, database); + String token = UUID.randomUUID().toString().replace("-", ""); + double now = Instant.now().toEpochMilli() / 1000.0; + double started = ProcessHandle.current().info().startInstant().orElseThrow().toEpochMilli() / 1000.0; + try (Connection c = source.getConnection(); Statement s = c.createStatement()) { + String server; + try (ResultSet rs = s.executeQuery("SELECT @@server_uuid")) { rs.next(); server = rs.getString(1); } + if (!server.matches("[a-fA-F0-9-]{36}")) throw new IllegalStateException("Server identity unavailable"); + String prefix = "{\"kind\":\"quality-owned-test-db\",\"formatVersion\":1,\"root\":" + quote(root.toString()) + + ",\"database\":" + quote(database) + ",\"token\":" + quote(token) + ",\"serverUuid\":" + quote(server) + + ",\"ownerPid\":" + ProcessHandle.current().pid() + ",\"ownerCreated\":" + started + + ",\"createdAt\":" + now + ",\"expiresAt\":" + (now + 86400) + ",\"children\":[],\"users\":[]"; + Files.createFile(path); write(path, prefix + ",\"created\":false}"); + try { s.execute("CREATE DATABASE `" + database + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); } + catch (SQLException error) { Files.deleteIfExists(path); throw error; } + write(path, prefix + ",\"created\":true}"); + if (!empty) { + s.execute("CREATE TABLE `" + database + "`.`" + MARKER + "` (token CHAR(32) PRIMARY KEY)"); + s.execute("INSERT INTO `" + database + "`.`" + MARKER + "` VALUES ('" + token + "')"); + write(path, prefix + ",\"created\":true,\"ready\":true}"); + } + } + } catch (Exception error) { throw new IllegalStateException("Owned test database creation failed", error); } + } + + public static void seal(DataSource source, String database) { + valid(database); + try { + Path root = root(); protect(root, database); Path path = record(root, database); + String value = Files.readString(path); + var match = java.util.regex.Pattern.compile("\"token\"\\s*:\\s*\"([a-f0-9]{32})\"").matcher(value); + if (!match.find()) throw new IllegalStateException("Test token unavailable"); + try (Connection c = source.getConnection(); Statement s = c.createStatement()) { + s.execute("CREATE TABLE `" + database + "`.`" + MARKER + "` (token CHAR(32) PRIMARY KEY)"); + s.execute("INSERT INTO `" + database + "`.`" + MARKER + "` VALUES ('" + match.group(1) + "')"); + } + write(path, value.substring(0, value.length()-1) + ",\"ready\":true}"); + } catch (Exception error) { throw new IllegalStateException("Test schema registration failed", error); } + } + + public static void drop(DataSource source, String database) { + valid(database); + try { + Path root = root(); protect(root, database); + Path path = record(root, database); + if (Files.isSymbolicLink(path)) throw new IllegalStateException("Unsafe test owner record"); + String record = Files.readString(path); + var tokenMatch = java.util.regex.Pattern.compile("\"token\"\\s*:\\s*\"([a-f0-9]{32})\"").matcher(record); + var serverMatch = java.util.regex.Pattern.compile("\"serverUuid\"\\s*:\\s*\"([a-fA-F0-9-]{36})\"").matcher(record); + if (!tokenMatch.find() || !serverMatch.find()) throw new IllegalStateException("Test identity unavailable"); + try (Connection c = source.getConnection(); Statement s = c.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT @@server_uuid")) { + rs.next(); if (!serverMatch.group(1).equals(rs.getString(1))) throw new IllegalStateException("Test server changed"); + } + if (record.contains("\"ready\":true")) { + try (ResultSet rs = s.executeQuery("SELECT token FROM `" + database + "`.`" + MARKER + "`")) { + if (!rs.next() || !tokenMatch.group(1).equals(rs.getString(1))) throw new IllegalStateException("Test schema identity changed"); + } + } else if (!record.contains("\"ownerPid\":" + ProcessHandle.current().pid() + ",")) { + throw new IllegalStateException("Unsealed test owner not confirmed"); + } + s.execute("DROP DATABASE `" + database + "`"); + Files.delete(path); + } + } catch (Exception error) { throw new IllegalStateException("Owned test database cleanup deferred", error); } + } +} diff --git a/deploy/docker/.env.example b/deploy/docker/.env.example new file mode 100644 index 0000000..b940a81 --- /dev/null +++ b/deploy/docker/.env.example @@ -0,0 +1,12 @@ +# 此文件只放非敏感选项,真实密码由 manage.py init 生成。 +HTTP_BIND=127.0.0.1 +HTTP_PORT=5280 +QUALITY_IMAGE_TAG=local +MYSQL_DATABASE=quality_platform +MYSQL_USER=quality_app +QUALITY_ADMIN_USERNAME=admin +JAVA_MAX_HEAP=2048m +# 从旧库迁移时必须核实旧版本会话的时区,不要猜测。 +MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS=UNCONFIRMED +QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED=false +QUALITY_ANALYSIS_AUTONOMOUS_ENABLED=false diff --git a/deploy/docker/Dockerfile.app b/deploy/docker/Dockerfile.app new file mode 100644 index 0000000..fa44127 --- /dev/null +++ b/deploy/docker/Dockerfile.app @@ -0,0 +1,52 @@ +# syntax=docker/dockerfile:1 +FROM maven:3.9-eclipse-temurin-21 AS java-build +WORKDIR /build +COPY backend/pom.xml backend/pom.xml +RUN --mount=type=cache,target=/root/.m2 mvn -B -f backend/pom.xml dependency:go-offline +COPY backend/src backend/src +# 默认执行不需要私有工作簿的测试;不把生产库传入构建。 +RUN --mount=type=cache,target=/root/.m2 mvn -B -f backend/pom.xml package + +FROM python:3.11-slim-bookworm AS runtimes +ARG TARGETARCH +RUN test "$TARGETARCH" = amd64 +WORKDIR /build +COPY deploy/docker/assets.lock.json deploy/docker/fetch_runtimes.py ./ +RUN python fetch_runtimes.py + +FROM debian:bookworm-slim AS app +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates tzdata tini procps libaio1 libnuma1 libncurses6 libtirpc3 libstdc++6 libgomp1 \ + libatomic1 libcrypt1 libzstd1 liblz4-1 libx11-6 libxext6 libxi6 libxrender1 libxtst6 libasound2 fontconfig fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -g 10001 quality && useradd -u 10001 -g quality -m quality +COPY --from=runtimes /runtimes/jre /opt/java +COPY --from=runtimes /runtimes/python /opt/python +COPY --from=runtimes /runtimes/mysql /opt/mysql +ENV JAVA_HOME=/opt/java PATH=/opt/nao/bin:/opt/java/bin:/opt/mysql/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 QUALITY_CONTAINER=1 TZ=Asia/Shanghai +COPY deploy/docker/requirements-linux.lock deploy/macos/retention-requirements.txt /tmp/requirements/ +RUN /opt/python/bin/python3 -m venv /opt/nao \ + && /opt/nao/bin/pip install --no-cache-dir --require-hashes -r /tmp/requirements/requirements-linux.lock -r /tmp/requirements/retention-requirements.txt \ + && /opt/nao/bin/python -c "import nao_core, pymysql, psutil; import importlib.metadata as m; assert m.version('nao-core') == '0.3.26'" \ + && /opt/java/bin/java -version && /opt/mysql/bin/mysqld --version \ + && rm -rf /tmp/requirements +WORKDIR /srv/quality +COPY .dockerignore runtime.lock.json requirements-nao.in requirements-nao.lock ./ +COPY backend/pom.xml backend/pom.xml +COPY backend/src backend/src +COPY web/package.json web/pnpm-lock.yaml web/index.html web/tsconfig.json web/vite.config.ts web/pnpm-workspace.yaml web/ +COPY web/src web/src +COPY web/public web/public +COPY scripts scripts +COPY ai ai +COPY deploy/macos deploy/macos +COPY deploy/docker deploy/docker +COPY --from=java-build /build/backend/target/quality-platform-backend-0.1.0-SNAPSHOT.jar app/backend.jar +RUN sha256sum app/backend.jar | cut -d ' ' -f 1 > app/backend.sha256 \ + && mkdir -p .runtime output/backups output/logs runtimedata/archive \ + && chown -R quality:quality .runtime output runtimedata +USER 10001:10001 +EXPOSE 5281 +ENTRYPOINT ["/usr/bin/tini", "--", "/opt/nao/bin/python", "deploy/docker/app.py"] +CMD ["serve"] diff --git a/deploy/docker/Dockerfile.web b/deploy/docker/Dockerfile.web new file mode 100644 index 0000000..5a39537 --- /dev/null +++ b/deploy/docker/Dockerfile.web @@ -0,0 +1,13 @@ +# syntax=docker/dockerfile:1 +FROM node:22-bookworm-slim AS build +RUN npm install --global pnpm@10.15.1 +WORKDIR /build +COPY web/package.json web/pnpm-lock.yaml web/pnpm-workspace.yaml ./ +RUN --mount=type=cache,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile +COPY web ./ +RUN pnpm build + +FROM nginx:stable-alpine +COPY deploy/docker/nginx.conf /etc/nginx/conf.d/default.conf +COPY --from=build /build/dist /usr/share/nginx/html +EXPOSE 8080 diff --git a/deploy/docker/README.md b/deploy/docker/README.md new file mode 100644 index 0000000..0e341c7 --- /dev/null +++ b/deploy/docker/README.md @@ -0,0 +1,76 @@ +# Linux Docker 部署 + +本目录是项目源码的一部分,随 Gitea 仓库同步。它提供手动构建与运行入口,不需要 Gitea Actions、Runner 或自动发布流水线。构建需要联网获取公共依赖;已构建镜像在启动时不运行 Maven、pnpm、pip 或下载运行时。AI 调用仍需要可用的模型服务。 + +当前准备目标为 **Linux x86_64 / Docker Engine / Compose V2(建议 2.24 以上)**。应用运行时固定为 Java 21.0.13、Python 3.11.15、MySQL 8.4.13、nao-core 0.3.26。原生运行时校验和在 `assets.lock.json`,Python 依赖使用带哈希的锁文件。基础系统镜像按发行系列获取安全更新;正式发布时应记录构建提交和镜像 digest。ARM64 不在本版本支持范围内。 + +## 服务与数据 + +| 服务 | 内容 | 数据位置 | +| --- | --- | --- | +| mysql | MySQL,关闭 binary log,保留事务 redo/undo | `mysql_data` 命名卷 | +| app | 后端、AI、自动回收、备份验证、容量监测 | `state`、`archives`、`backups`、`logs` 命名卷 | +| web | 前端与 API 代理,支持流式响应 | 镜像内静态文件 | + +只发布网页端口,默认 `127.0.0.1:5280`;MySQL 和后端没有宿主机端口映射。应用以 UID 10001 运行,镜像只读,不挂载 Docker socket。后端及所有维护子进程共享同一容器的进程空间,避免误判其他容器内的校验任务已经退出。不要同时启动第二个 app 副本访问同一组数据卷。 + +首次初始化后,密码存放于项目私有目录 `.runtime/docker/secrets/`,不是源码、镜像层或命令参数。Compose 从管理入口的环境变量读取密码,再作为 secret 文件交给容器。主机管理员仍能查看容器配置,需限制 Docker 管理权限。数据库初始化后不要直接重新生成密码;密码轮换必须同步修改数据库账户。 + +## 从仓库构建与启动 + +在项目根目录执行: + +```sh +python3 deploy/docker/manage.py init +python3 deploy/docker/manage.py config +python3 deploy/docker/manage.py build +python3 deploy/docker/manage.py up +python3 deploy/docker/manage.py status +``` + +`init` 不覆盖已有设置或密码。非敏感设置位于 `.runtime/docker/config.env`;内网访问时可把 `HTTP_BIND` 改为服务器内网 IP,配合防火墙限制访问范围。管理员初始密码位于私有文件 `.runtime/docker/secrets/admin_password`,登录后及时更改。首次克隆为空系统,业务数据需单独授权导入。 + +启用服务器上的 Docker 开机启动后,已创建且未手动停止的容器通过 `restart: unless-stopped` 随 Docker 自动启动;MySQL 就绪后才启动应用,应用就绪后再启动网页。后端异常退出会使 app 退出并由 Docker 重启;维护进程退出由入口进程重新拉起。健康检查用于报告就绪状态,不会因一次查询超时强杀正在工作的数据库。 + +```sh +sudo systemctl enable --now docker +python3 deploy/docker/manage.py logs +python3 deploy/docker/manage.py stop +``` + +`stop` 保留所有数据卷,并表示主动停服;之后用 `up` 恢复。不要执行 `docker compose down -v`,它会删除数据库、归档和备份卷。操作系统关机时仍会停服;要摆脱测试 Mac 的关机和网络影响,必须将整套服务部署在持续开机联网的服务器上。 + +## 自动控制空间增长 + +复用 `deploy/macos/retention.py` 中的平台通用回收逻辑和 `scripts/` 内的保留模块;macOS 的 launchd 安装代码不会在 Linux 执行。 + +- 每 5 分钟检查旧版本、终态无引用构建、过期上传草稿和无引用旧来源修订。当前快照、候选基线、保存的报表及分析引用均保护;删除前重新确认当前版本与任务状态。 +- 导入、分析、排队、重试或未知状态任务存在时延期,锁冲突也延期,不终止任务。无引用对象连续 30 分钟后才进入回收;上传草稿先经过 24 小时有效期。 +- 发现新正式版本或距上次备份超过 24 小时,在空闲时生成候选备份。新候选通过独立、无网络 MySQL 实例恢复、表行数、业务汇总、来源归档及应用文件校验后,才替换上一份有效备份。失败时保留旧备份;创建期间会短暂同时存在新旧两份及恢复校验空间。 +- 测试数据库按所有者、进程身份、服务器 UUID 和标记登记。正常完成即清理;异常遗留超过 24 小时且所有者及子进程已退出后再处理。不按名称前缀批量删除未知数据库。新增数据库集成测试应使用 `OwnedTestDatabase` 或 `managed_test_resources.py`,并在 finally 中释放。 +- Java 日志按日期及每段 10MB 滚动,保留 7 天,归档总量上限 80MB,为活动文件留出空间。关闭的其他项目日志每类最多 100MiB、最长 7 天;活动文件受保护。Docker 每个容器日志使用 `local` 驱动、10m×10 段限制。MySQL 错误日志进入受限制的容器日志;binary log 关闭后不支持历史时间点恢复,恢复依赖最新有效快照。 +- 容量每 5 分钟采样,保留 30 天日报。业务数据、测试/校验临时空间分别达到 30GiB 提醒、50GiB 严重提醒;磁盘余量低于 50/20GiB、一天增长超过 2GiB 且超过 20%、连续维护失败也告警。告警写入容器日志及 `state` 卷的 `operations/alerts.jsonl`,同类 6 小时内去重。阈值在 `state` 卷的 `operations/policy.json`,不应为压低统计而删除业务数据。 + +Linux 的 MySQL 用量来自 `information_schema` 表和索引估算,不包含系统表空间、redo/undo 和 Docker 镜像缓存;宿主机还应使用磁盘监测与 `docker system df` 监测这些空间。容器内报告的可用空间取各业务卷的最小值;MySQL 卷所在磁盘也要由主机监控。此版本不提供 Linux 桌面弹窗或外部通知推送,容器日志可接入主机现有告警系统。 + +手动生成并验证一份备份: + +```sh +python3 deploy/docker/manage.py backup +``` + +维护记录分别在 `state` 卷的 `retention/state.json`、`operations/monitor.json`。运行包或数据库结构改变时,回收会暂停;检查新版本兼容性后才执行 `python3 deploy/docker/manage.py review-retention`。此命令只更新兼容性标记,不取消任务保护或备份校验。 + +## 数据迁移与公网入口 + +当前测试 Mac 不会因本目录更新而切换运行包、迁移数据库或改变公网入口。源码中也不包含测试库、来源表格、现有备份和 API 密钥。 + +正式迁移采用逻辑备份及来源归档,不把 Mac 的 MySQL 数据目录直接复制到 Linux。先做 `backup.py create-and-verify`,保留原机;在新服务器的新数据库和独立目录恢复并重绑定归档绝对路径,核对用户、当前发布版本、月度汇总、归档摘要、登录和 AI,再切换域名。`backup.py restore --help` 提供新库/新目录的防覆盖恢复入口;它不是自动覆盖容器数据卷的导入命令。旧库的时间迁移参数必须从原部署核实。 + +长期入口建议为:域名 → 服务器 HTTPS 反向代理 → 本服务器 web → app → MySQL。公网只开放 443(按需开放 80 做跳转/证书),不再依赖测试 Mac 的 SSH 隧道。另一条路径是服务器应用连接托管 MySQL,另配对象存储及异地备份;该方案需要单独评估数据库权限、备份方式及来源路径,不能直接套用本地卷配置。 + +## 验收边界 + +本目录当前已准备源码与测试;没有 Linux Docker 主机时,不把静态检查或 Mac 上的测试称作容器启动成功。上主机后至少完成:构建三项服务、空库初始化、登录/导入/AI、后端与 MySQL 异常退出重启、主机重启、有效备份恢复、新旧数据对账、进行中任务保护、日志滚动及告警、HTTPS 与断开测试 Mac 后访问。容量规划还应为新旧备份和独立恢复校验预留余量。 + +参考:[Compose 启动顺序](https://docs.docker.com/compose/how-tos/startup-order/)、[Compose 服务与重启策略](https://docs.docker.com/reference/compose-file/services/)、[MySQL 官方镜像](https://hub.docker.com/_/mysql)、[Logback 日志保留](https://logback.qos.ch/manual/appenders-rolling.html)。 diff --git a/deploy/docker/app.py b/deploy/docker/app.py new file mode 100644 index 0000000..d5e51c5 --- /dev/null +++ b/deploy/docker/app.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""容器入口:固定运行包、私有配置、子进程守护和安全停止。""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')] + + +def secret(name): + path = Path('/run/secrets') / name + value = path.read_text().strip() + if not re.fullmatch(r'[A-Za-z0-9_-]{24,128}', value): + raise ValueError('Invalid secret: ' + name) + return value + + +def identifier(value): + if not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,47}', value): + raise ValueError('Invalid database identifier') + return value + + +def prepare(root=ROOT): + import backup_retention as br + root = Path(root) + for relative in ('.runtime/retention', '.runtime/operations', '.runtime/nao-users', + '.runtime/nao-trial', 'runtimedata/archive', 'output/backups', 'output/logs'): + (root / relative).mkdir(parents=True, exist_ok=True, mode=0o700) + for relative, destination in (('.runtime/mysql', '/opt/mysql'), ('.runtime/nao-trial/venv', '/opt/nao')): + path = root / relative + if path.is_symlink(): + if str(path.readlink()) != destination: raise ValueError('Unexpected runtime link') + elif path.exists(): raise ValueError('Runtime path already occupied') + else: path.symlink_to(destination, target_is_directory=True) + value = {'mysqlHost': 'mysql', 'mysqlPort': 3306, + 'mysqlDatabase': identifier(os.environ.get('MYSQL_DATABASE', 'quality_platform')), + 'mysqlUser': identifier(os.environ.get('MYSQL_USER', 'quality_app')), + 'mysqlPassword': secret('mysql_password'), 'rootPassword': secret('mysql_root_password'), + 'adminUsername': os.environ.get('QUALITY_ADMIN_USERNAME', 'admin'), + 'adminPassword': secret('admin_password'), 'databaseReady': True, 'binaryLogEnabled': False, + 'legacyTimestampOffsetSeconds': os.environ.get('MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS', 'UNCONFIRMED')} + config = root / '.runtime/local-config.json' + if config.exists(): + old = json.loads(config.read_text()) + # 更改持久数据库的密码需要专门轮换,不能在重启时默默写入不一致配置。 + for key in ('mysqlDatabase', 'mysqlUser', 'mysqlPassword', 'rootPassword'): + if old.get(key) != value[key]: raise ValueError('Persisted database settings changed: ' + key) + br.atomic(config, value) + cnf = root / '.runtime/mysql-root.cnf' + cnf.write_text('[client]\nuser=root\npassword=' + value['rootPassword'] + '\nhost=mysql\nport=3306\nprotocol=TCP\n') + cnf.chmod(0o600) + import operations + policy = root / '.runtime/operations/policy.json' + if not policy.exists(): br.atomic(policy, operations.DEFAULTS) + return value + + +def backend_environment(value): + env = os.environ.copy() + env.update({'MYSQL_URL': 'jdbc:mysql://mysql:3306/' + value['mysqlDatabase'] + + '?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true', + 'MYSQL_USER': value['mysqlUser'], 'MYSQL_PASSWORD': value['mysqlPassword'], + 'QUALITY_BOOTSTRAP_ADMIN_USERNAME': value['adminUsername'], + 'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': value['adminPassword'], + 'SERVER_ADDRESS': '0.0.0.0', 'QUALITY_PROJECT_ROOT': str(ROOT), + 'QUALITY_ARCHIVE_ROOT': str(ROOT / 'runtimedata/archive'), 'QUALITY_ALLOWED_ROOTS': '', + 'LOGGING_CONFIG': str(ROOT / 'deploy/macos/logback-quality.xml'), + 'QUALITY_LOG_DIR': str(ROOT / 'output/logs'), 'QUALITY_CONTAINER': '1'}) + return env + + +def ready(): + try: + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open('http://127.0.0.1:5281/actuator/health', timeout=3) as response: + return json.load(response).get('status') == 'UP' + except (OSError, ValueError): return False + + +def review_retention(first=False): + import backup_retention as br + from retention import Database + path = ROOT / '.runtime/retention/config.json' + if first and path.exists(): return + with br.lock(ROOT, timeout=0): + db = Database(ROOT) + try: + if db.busy(): raise RuntimeError('Application jobs are active; review deferred') + accepted = br.accepted_backend(ROOT) + if not accepted: raise RuntimeError('Pinned backend not found') + br.atomic(path, {'schemaSignature': db.schema_signature, 'acceptedJarSha256': accepted[1], + 'retirementGraceSeconds': 1800, 'intervalSeconds': 300}) + finally: db.conn.close() + + +def serve(): + import backup_retention as br + (ROOT / '.runtime').mkdir(parents=True, exist_ok=True, mode=0o700) + guard = (ROOT / '.runtime/container.lock').open('a') + fcntl.flock(guard, fcntl.LOCK_EX | fcntl.LOCK_NB) + value = prepare() + br.accepted_backend(ROOT) + heap = os.environ.get('JAVA_MAX_HEAP', '2048m') + if not re.fullmatch(r'[1-9][0-9]*[mMgG]', heap): raise ValueError('Invalid Java heap size') + env = backend_environment(value) + children = {'backend': subprocess.Popen(['java', '-Xms128m', '-Xmx' + heap, '-jar', str(ROOT / 'app/backend.jar')], + env=env, start_new_session=True)} + stopping = False + def stop(signum, frame): + nonlocal stopping + stopping = True + signal.signal(signal.SIGTERM, stop); signal.signal(signal.SIGINT, stop) + try: + while not stopping: + if children['backend'].poll() is not None: return 1 + if ready(): + try: review_retention(first=True) + except Exception as error: print('retention-initialization: ' + type(error).__name__, flush=True) + for role in ('retention', 'monitor'): + if role not in children or children[role].poll() is not None: + children[role] = subprocess.Popen([sys.executable, str(ROOT / 'deploy/docker/worker.py'), role], + env=env, start_new_session=True) + time.sleep(1) + return 0 + finally: + for process in children.values(): + if process.poll() is None: + try: os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: pass + deadline = time.monotonic() + 270 + for process in children.values(): + try: process.wait(timeout=max(.1, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL); process.wait() + guard.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=['serve', 'health', 'review-retention'], default='serve', nargs='?') + args = parser.parse_args() + if args.action == 'health': return 0 if ready() else 1 + if args.action == 'review-retention': review_retention(); return 0 + return serve() + + +if __name__ == '__main__': + try: raise SystemExit(main()) + except Exception as error: + print('Container startup failed: ' + type(error).__name__, file=sys.stderr) + raise SystemExit(1) diff --git a/deploy/docker/assets.lock.json b/deploy/docker/assets.lock.json new file mode 100644 index 0000000..6d5f476 --- /dev/null +++ b/deploy/docker/assets.lock.json @@ -0,0 +1,20 @@ +[ + { + "name": "jre", + "version": "21.0.13+9.0.LTS", + "url": "https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.13%2B9/OpenJDK21U-jre_x64_linux_hotspot_21.0.13_9.tar.gz", + "sha256": "a535a1df9c6284d9618872984e026ae0352d1f7e9e0605425c0e6114f47351dd" + }, + { + "name": "python", + "version": "3.11.15", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.11.15%2B20260623-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", + "sha256": "0604cd029b142dc223e131f17f5941c0c8d2d5074997c8178b515b19eea2a6c2" + }, + { + "name": "mysql", + "version": "8.4.13", + "url": "https://cdn.mysql.com/Downloads/MySQL-8.4/mysql-8.4.13-linux-glibc2.28-x86_64-minimal.tar.xz", + "sha256": "5dfbb764f731df40ef9e4f5d69aed32bdac8ba1af20bd96b7d40a942d83d4ed2" + } +] diff --git a/deploy/docker/compose.yaml b/deploy/docker/compose.yaml new file mode 100644 index 0000000..efd0373 --- /dev/null +++ b/deploy/docker/compose.yaml @@ -0,0 +1,108 @@ +name: quality-center + +x-logging: &bounded-logging + driver: local + options: + max-size: "10m" + max-file: "10" + +services: + mysql: + image: mysql:8.4.13 + platform: linux/amd64 + restart: unless-stopped + environment: + MYSQL_DATABASE: ${MYSQL_DATABASE:-quality_platform} + MYSQL_USER: ${MYSQL_USER:-quality_app} + MYSQL_ROOT_HOST: "%" + MYSQL_ROOT_PASSWORD_FILE: /run/secrets/mysql_root_password + MYSQL_PASSWORD_FILE: /run/secrets/mysql_password + TZ: Asia/Shanghai + secrets: [mysql_root_password, mysql_password] + command: + - --skip-log-bin + - --mysqlx=OFF + - --max-connections=60 + - --innodb-buffer-pool-size=512M + - --innodb-redo-log-capacity=128M + volumes: [mysql_data:/var/lib/mysql] + networks: [database] + healthcheck: + test: [CMD-SHELL, "MYSQL_PWD=$$(cat /run/secrets/mysql_root_password) mysql --user=root --batch --skip-column-names -e 'SELECT 1' >/dev/null"] + interval: 10s + timeout: 5s + retries: 30 + start_period: 60s + stop_grace_period: 2m + mem_limit: 2g + logging: *bounded-logging + + app: + image: quality-center-app:${QUALITY_IMAGE_TAG:-local} + build: + context: ../.. + dockerfile: deploy/docker/Dockerfile.app + platform: linux/amd64 + restart: unless-stopped + depends_on: + mysql: {condition: service_healthy} + environment: + MYSQL_DATABASE: ${MYSQL_DATABASE:-quality_platform} + MYSQL_USER: ${MYSQL_USER:-quality_app} + QUALITY_ADMIN_USERNAME: ${QUALITY_ADMIN_USERNAME:-admin} + MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS: ${MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS:-UNCONFIRMED} + QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED: ${QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED:-false} + QUALITY_ANALYSIS_AUTONOMOUS_ENABLED: ${QUALITY_ANALYSIS_AUTONOMOUS_ENABLED:-false} + JAVA_MAX_HEAP: ${JAVA_MAX_HEAP:-2048m} + secrets: [mysql_root_password, mysql_password, admin_password] + volumes: + - state:/srv/quality/.runtime + - archives:/srv/quality/runtimedata/archive + - backups:/srv/quality/output/backups + - logs:/srv/quality/output/logs + networks: [database, frontend] + read_only: true + tmpfs: [/tmp:mode=1777,size=536870912] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + healthcheck: + test: [CMD, /opt/nao/bin/python, deploy/docker/app.py, health] + interval: 15s + timeout: 5s + retries: 6 + start_period: 15m + stop_grace_period: 5m + mem_limit: 6g + logging: *bounded-logging + + web: + image: quality-center-web:${QUALITY_IMAGE_TAG:-local} + build: + context: ../.. + dockerfile: deploy/docker/Dockerfile.web + restart: unless-stopped + depends_on: + app: {condition: service_healthy} + ports: ["${HTTP_BIND:-127.0.0.1}:${HTTP_PORT:-5280}:8080"] + networks: [frontend] + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/healthz] + interval: 30s + timeout: 5s + retries: 3 + mem_limit: 256m + logging: *bounded-logging + +networks: + database: {internal: true} + frontend: {} +volumes: + mysql_data: {} + state: {} + archives: {} + backups: {} + logs: {} +secrets: + mysql_root_password: {environment: QUALITY_SECRET_MYSQL_ROOT} + mysql_password: {environment: QUALITY_SECRET_MYSQL_APP} + admin_password: {environment: QUALITY_SECRET_ADMIN} diff --git a/deploy/docker/fetch_runtimes.py b/deploy/docker/fetch_runtimes.py new file mode 100644 index 0000000..ad6e464 --- /dev/null +++ b/deploy/docker/fetch_runtimes.py @@ -0,0 +1,27 @@ +"""构建时下载并校验固定 Linux x86_64 运行时,运行阶段不再下载。""" +import hashlib +import json +from pathlib import Path +import shutil +import tarfile +import tempfile +import urllib.request + + +def install(lock, destination): + destination = Path(destination) + for asset in json.loads(Path(lock).read_text()): + with tempfile.TemporaryDirectory() as temporary: + temporary = Path(temporary); archive = temporary / 'runtime.tar' + with urllib.request.urlopen(asset['url'], timeout=120) as source, archive.open('wb') as output: + shutil.copyfileobj(source, output) + with archive.open('rb') as source: digest = hashlib.file_digest(source, 'sha256').hexdigest() + if digest != asset['sha256']: raise ValueError('Runtime checksum mismatch: ' + asset['name']) + unpacked = temporary / 'unpacked'; unpacked.mkdir() + with tarfile.open(archive) as source: source.extractall(unpacked, filter='data') + children = list(unpacked.iterdir()) + if len(children) != 1 or not children[0].is_dir(): raise ValueError('Unexpected archive layout') + shutil.copytree(children[0], destination / asset['name'], symlinks=True) + + +if __name__ == '__main__': install('/build/assets.lock.json', '/runtimes') diff --git a/deploy/docker/manage.py b/deploy/docker/manage.py new file mode 100644 index 0000000..c2d4920 --- /dev/null +++ b/deploy/docker/manage.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""统一 Docker Compose 入口;密码只从本机私有文件加载,不进入源码或命令参数。""" +import argparse +import os +from pathlib import Path +import re +import secrets +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +SECRET_NAMES = {'mysql_root_password': 'QUALITY_SECRET_MYSQL_ROOT', + 'mysql_password': 'QUALITY_SECRET_MYSQL_APP', 'admin_password': 'QUALITY_SECRET_ADMIN'} + + +def initialize(root=ROOT): + base = Path(root) / '.runtime/docker' + directory = base / 'secrets' + if base.is_symlink() or directory.is_symlink(): raise ValueError('Unsafe private settings path') + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + base.chmod(0o700); directory.chmod(0o700) + for name in SECRET_NAMES: + path = directory / name + if path.is_symlink(): raise ValueError('Unsafe secret file') + if path.exists(): continue + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + with os.fdopen(fd, 'w') as stream: stream.write(secrets.token_urlsafe(32) + '\n') + config = base / 'config.env' + if not config.exists(): + shutil.copyfile(Path(root) / 'deploy/docker/.env.example', config); config.chmod(0o600) + return base + + +def environment(root=ROOT): + env = os.environ.copy(); directory = Path(root) / '.runtime/docker/secrets' + for name, variable in SECRET_NAMES.items(): + path = directory / name + if path.is_symlink() or not path.is_file() or path.stat().st_mode & 0o077: + raise ValueError('Missing or unsafe secret file; run init first') + value = path.read_text().strip() + if not re.fullmatch(r'[A-Za-z0-9_-]{24,128}', value): raise ValueError('Invalid secret file') + env[variable] = value + return env + + +def command(action): + actions = {'config': ['config', '--quiet'], 'build': ['build', '--pull'], 'up': ['up', '-d', '--wait', '--wait-timeout', '1200'], + 'stop': ['stop'], 'status': ['ps'], 'logs': ['logs', '--tail', '100'], + 'backup': ['exec', '-T', 'app', '/opt/nao/bin/python', 'scripts/backup.py', 'create-and-verify'], + 'review-retention': ['exec', '-T', 'app', '/opt/nao/bin/python', 'deploy/docker/app.py', 'review-retention']} + return ['docker', 'compose', '-f', str(ROOT / 'deploy/docker/compose.yaml'), + '--env-file', str(ROOT / '.runtime/docker/config.env'), *actions[action]] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=['init', 'config', 'build', 'up', 'stop', 'status', 'logs', 'backup', 'review-retention']) + args = parser.parse_args() + if args.action == 'init': + base = initialize(); print('Private settings prepared: ' + str(base)); return + if not shutil.which('docker'): parser.error('Docker Engine and Compose V2 are required') + subprocess.run(command(args.action), cwd=ROOT, env=environment(), check=True) + + +if __name__ == '__main__': main() diff --git a/deploy/docker/nginx.conf b/deploy/docker/nginx.conf new file mode 100644 index 0000000..6f7fa19 --- /dev/null +++ b/deploy/docker/nginx.conf @@ -0,0 +1,21 @@ +server { + listen 8080; + server_name _; + root /usr/share/nginx/html; + client_max_body_size 260m; + resolver 127.0.0.11 valid=10s; + set $quality_backend http://app:5281; + location = /healthz { access_log off; proxy_pass $quality_backend/actuator/health; } + location /api/ { + proxy_pass $quality_backend$request_uri; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_read_timeout 1800s; + proxy_send_timeout 1800s; + } + location / { try_files $uri $uri/ /index.html; } +} diff --git a/deploy/docker/requirements-linux.lock b/deploy/docker/requirements-linux.lock new file mode 100644 index 0000000..cfb1579 --- /dev/null +++ b/deploy/docker/requirements-linux.lock @@ -0,0 +1,1178 @@ +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 +apscheduler==3.11.3 \ + --hash=sha256:bbeb2ec02d23d3c06a6c07ed7f0f3939ada6680eb121fae809a69bb42c537a30 \ + --hash=sha256:cd2fcc9330039a81a5893472ad49facf23a6d5604cbe1d918c835c6de7834d5a +atpublic==9.0.0 \ + --hash=sha256:449c3c4f0c74df79749d6fe225ba55e2a2fce34b303f0329211e4d6989ed6f6e \ + --hash=sha256:61ea62d8445d2aaa83b6dffaa3d90f99fcec10e16683ee9b13792cdcdafa0966 +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 +backoff==2.2.1 \ + --hash=sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba \ + --hash=sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8 +certifi==2026.10.19 \ + --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae \ + --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 +cryptography==50.0.2 \ + --hash=sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602 \ + --hash=sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2 \ + --hash=sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047 \ + --hash=sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c \ + --hash=sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42 \ + --hash=sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18 \ + --hash=sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51 \ + --hash=sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81 \ + --hash=sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856 \ + --hash=sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2 \ + --hash=sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de \ + --hash=sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7 \ + --hash=sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd \ + --hash=sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2 \ + --hash=sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be \ + --hash=sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45 \ + --hash=sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0 \ + --hash=sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e \ + --hash=sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c \ + --hash=sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5 \ + --hash=sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452 \ + --hash=sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48 \ + --hash=sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05 \ + --hash=sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1 \ + --hash=sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93 \ + --hash=sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04 \ + --hash=sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e \ + --hash=sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67 \ + --hash=sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7 \ + --hash=sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107 \ + --hash=sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079 \ + --hash=sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134 \ + --hash=sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227 \ + --hash=sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1 \ + --hash=sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539 \ + --hash=sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e \ + --hash=sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d \ + --hash=sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c \ + --hash=sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd \ + --hash=sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020 \ + --hash=sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd \ + --hash=sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94 \ + --hash=sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a \ + --hash=sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408 \ + --hash=sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37 \ + --hash=sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e \ + --hash=sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454 \ + --hash=sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c \ + --hash=sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc \ + --hash=sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37 \ + --hash=sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767 \ + --hash=sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a \ + --hash=sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5 \ + --hash=sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc \ + --hash=sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67 \ + --hash=sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8 \ + --hash=sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480 \ + --hash=sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb \ + --hash=sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b +cyclopts==5.2.0 \ + --hash=sha256:5da2a5d65164e03008621fc4795946b88a722b881398b5dfa58afa6218342cff \ + --hash=sha256:b63c1b1beaadf3ead19214385a0f90b990f152c4107c174e1724c45dc71e9541 +distro==1.9.0 \ + --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ + --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 +docstring-parser==0.18.0 \ + --hash=sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015 \ + --hash=sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b +dotenv==0.9.9 \ + --hash=sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9 +duckdb==1.5.2 \ + --hash=sha256:0b291786014df1133f8f18b9df4d004484613146e858d71a21791e0fcca16cf4 \ + --hash=sha256:2323c1195c10fb2bb982fc0218c730b43d1b92a355d61e68e3c5f3ac9d44c34f \ + --hash=sha256:2a1de4f4d454b8c97aec546c82003fc834d3422ce4bc6a19902f3462ef293bed \ + --hash=sha256:35579b8e3a064b5eaf15b0eafc558056a13f79a0a62e34cc4baf57119daecfec \ + --hash=sha256:376856066c65ccd55fcb3a380bbe33a71ce089fc4623d229ffc6e82251afdb6d \ + --hash=sha256:411ad438bd4140f189a10e7f515781335962c5d18bd07837dc6d202e3985253d \ + --hash=sha256:4420b3f47027a7849d0e1815532007f377fa95ee5810b47ea717d35525c12f79 \ + --hash=sha256:486c862bf7f163c0110b6d85b3e5c031d224a671cca468f12ebb1d3a348f6b39 \ + --hash=sha256:52a21823f3fbb52f0f0e5425e20b07391ad882464b955879499b5ff0b45a376b \ + --hash=sha256:5596bbfc31b1b259db69c8d847b42d036ce2c4804f9ccb28f9fc46a16de7bc53 \ + --hash=sha256:56d38b3c4e0ef2abb58898d0fd423933999ed535c45e75e9d9f72e1d5fed69b8 \ + --hash=sha256:638da0d5102b6cb6f7d47f83d0600708ac1d3cb46c5e9aaabc845f9ba4d69246 \ + --hash=sha256:63bf8687feefeed51adf45fa3b062ab8b1b1c350492b7518491b86bae68b1da1 \ + --hash=sha256:6b0fe75c148000f060aa1a27b293cacc0ea08cc1cad724fbf2143d56070a3785 \ + --hash=sha256:70631c847ca918ee710ec874241b00cf9d2e5be90762cbb2a0389f17823c08f7 \ + --hash=sha256:7f69164b048e498b9e9140a24343108a5ae5f17bfb3485185f55fdf9b1aa924d \ + --hash=sha256:81fc4fbf0b5e25840b39ba2a10b78c6953c0314d5d0434191e7898f34ab1bba3 \ + --hash=sha256:84b193aca20565dedb3172de15f843c659c3a6c773bf14843a9bd781c850e7db \ + --hash=sha256:8d599758b4e48bf12e18c9b960cf491d219f0c4972d19a45489c05cc5ab36f83 \ + --hash=sha256:8dbd7e31e5dc157bfe8803fa7d2652336265c6c19926c5a4a9b40f8222868d08 \ + --hash=sha256:98c0535cd6d901f61a5ea3c2e26a1fd28482953d794deb183daf568e3aa5dda6 \ + --hash=sha256:a9cd5e71702d446613750405cde03f66ed268f4c321da071b0472759dad19536 \ + --hash=sha256:bb42e6ed543902e14eae647850da24103a89f0bc2587dec5601b1c1f213bd2ed \ + --hash=sha256:be37680ddb380015cb37318e378c53511c45c4f0d8fac5599d22b7d092b9217a \ + --hash=sha256:c69907354ffee94ba8cf782daf0480dab7557f21ce27fffa6c0ea8f74ed4b8e2 \ + --hash=sha256:c99ef73a277c8921bc0a1f16dee38d924484251d9cfd20951748c20fcd5ed855 \ + --hash=sha256:c9f3e0b71b8a50fccfb42794899285d9d318ce2503782b9dd54868e5ecd0ad31 \ + --hash=sha256:ce0b8141a10d37ecef729c45bc41d334854013f4389f1488bd6035c5579aaac1 \ + --hash=sha256:ce17670bb392ea1b3650537db02bd720908776b5b95f6d2472d31a7de59d1dc1 \ + --hash=sha256:d72b8856b1839d35648f38301b058f6232f4d36b463fe4dc8f4d3fdff2df1a2e \ + --hash=sha256:d9b4f5430bf4f05d4c0dc4c55c75def3a5af4be0343be20fa2bfc577343fbfc9 \ + --hash=sha256:e6495b00cad16888384119842797c49316a96ae1cb132bb03856d980d95afee1 \ + --hash=sha256:ea58ff5b0880593a280cf5511734b17711b32ee1f58b47d726e8600848358160 \ + --hash=sha256:ef461bca07313412dc09961c4a4757a851f56b95ac01c58fac6007632b7b94f2 \ + --hash=sha256:fc85a5dbcbe6eccac1113c72370d1d3aacfdd49198d63950bdf7d8638a307f00 +fastapi==0.159.1 \ + --hash=sha256:49b94c29afec2fca1390e9c39be202c59755d5cda8e4f1148a133761e4313e7e \ + --hash=sha256:b45b1ec57f3aa4d2f80f09bfbcf068b46764a2ee892e56753f35bb3d9d845ce2 +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad +ibis-framework==12.0.0 \ + --hash=sha256:0bbd790f268da9cb87926d5eaad2b827a573927113c4ed3be5095efa89b9e512 \ + --hash=sha256:238624f2c14fdab8382ca2f4f667c3cdb81e29844cd5f8db8a325d0743767c61 +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba +nao-core==0.3.26 \ + --hash=sha256:280e6d717c58715455b621086f6edd8b26d050a6b7813495f93eef05a99d2696 \ + --hash=sha256:5b444b025a089777832dcc819fa598cb8f7f68b0b60c8063f000b5bea14cecfd \ + --hash=sha256:6a997550fed8641422f9df663ed89b9916c790b9e7f344573c5bb1fb299e3610 \ + --hash=sha256:750cfbba306d853fe2b29a6c3e54e5d222c2dd1ab5e63d5435ddd482556cd89c \ + --hash=sha256:e758d88fba2d05962f6df7469bc1ac6747c134a4eb605eb32270639ee3d454c3 +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 +opentelemetry-api==1.45.1 \ + --hash=sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75 \ + --hash=sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb +orjson==3.13.0 \ + --hash=sha256:0526a3456db67b264c6d661b5f090077f326b6cd074d0ef53a72763595dec5d7 \ + --hash=sha256:08bf722f923d2100bc5e5a5dcf72c656db557049c1bea26582fdd5dd9d5395a1 \ + --hash=sha256:1807c2fa49d393c7ee95fd1ef1b39cbb24aa3ccd81f30b84503ba59407666960 \ + --hash=sha256:1d84820b2ec4ac975cba482214032de5b0dbdd17046170c98e642ef9c4a4ee4b \ + --hash=sha256:2715c4808d1571029ed18fd07a82140bf3ba7def0dc89f8d015c416e3649bf87 \ + --hash=sha256:3ef75ed7e81dae34a3649f82df52cd85f9ac839a7d6ec78ab355b33b3b27ef7f \ + --hash=sha256:4329c19b8a25693f60a77b867c9d2a3ab637b20e36f5b7bea7f5acb492b44b15 \ + --hash=sha256:45e34deb3437509f4ec9888dd9ee5dc426cfe21be10f1eb4ea3a9e4d33034f9e \ + --hash=sha256:4e5c8175e1574dcbe446ee654275d353c1d78bbd9a0dc9f209bf35c9df72d171 \ + --hash=sha256:4ee06e53b998c71ce3eb93b86222912fdd9dcced685ac64d4525d36fac338ea4 \ + --hash=sha256:4f66eac85b072092e9941c3111882afd7527bf926cbc717038fa3654b582002b \ + --hash=sha256:50a5202ba388b3850ba24437951727d3aa6d79a21964a30ae8dc6a059a5fd34c \ + --hash=sha256:51d11525bc3ca736fa97ce4e4c7da9999cc00bf261522bede43b4e7531bd7965 \ + --hash=sha256:554948becd1110123ef9f6a6e1310fd92b2d07d2cbac6dbf65df3de75702e736 \ + --hash=sha256:58a9619d88f8818d9ab6b39d70d203789457ba13c1ed5d274f33ce9ae7e81a36 \ + --hash=sha256:5ef4d4157392a0439b74f7e49e5636b4ea43d9616bd0884effc0195fffcaa2d5 \ + --hash=sha256:637dbca1fccffe83780e806fbc0f17427c0c59bf822528eb0acc8f0aa9f19acb \ + --hash=sha256:64e8f345048d988c8b68d3882e5d41028fca1219a9939b32e4a77be34c8ae8e3 \ + --hash=sha256:65c4e0e106ccc7265b488385659117a6805c37d042f737558ecd68aa0c67ad8f \ + --hash=sha256:6adcaa85d79977659a448b4123a88eb33511a11ed2db243535ad7ea88a6668e0 \ + --hash=sha256:6c8bfe728b81b0fd58a3c7f3f9c5a113f87f2992c9948e0f28707aafd737c0bc \ + --hash=sha256:6d0684895b119ad167fb4ec05113639dc7f728022deec4756a710e838ed92e7a \ + --hash=sha256:6ff2a2c67f35202f7d823753d38ad371a9b7fc297567cdfff4420e763cb9f6f8 \ + --hash=sha256:7804dd1d6161da0e53b284c2aebf20f23e78eaac617300803e1467d1828d987f \ + --hash=sha256:78a12d4f8d740cc9ae197f5223682e5e960ba61b4fb2ce5a6a3bb54e83fde28e \ + --hash=sha256:7991921c5da527a963b6d4cffd0e4ea89c7e71d4be0c8be1bfe6edb223ce7d96 \ + --hash=sha256:7b3bc6b81835ce65f4729ae401607583d41139c6de95bc7453f450f1391d3e7b \ + --hash=sha256:83705c12b4afde10c62a5dd3fe6fdb21b7900bd0dcd5af1c85612ae94d0ee590 \ + --hash=sha256:84d87e322e1674408f85adea63f11aa19201eba082755aec20ebc217f493bbd2 \ + --hash=sha256:8594956a75223f657e1e68c568c0eeb3dd145f02cd6b78a47fd9a8095dbc4eae \ + --hash=sha256:89bcf2d4bc6c9a7e1763c8cf534f38712e66b76a0fefda7fb7785462f0d635e4 \ + --hash=sha256:89efecad02515df7f318d0613b5dfd6d2a1acd323a2b8294712789a715945525 \ + --hash=sha256:8c2ac5c09b017c484df1b4c68b2cf250b4e8ba08204cb58e7cd6cbbc71a9c902 \ + --hash=sha256:91d933e668ff0ffe164d7c2daec36beba6d1ce7fadb71538fbe142a71f8a1e6e \ + --hash=sha256:93c70a5e22bbbbdeafc7b273441e8452a196041d67fd4d9a9c450c66370a8486 \ + --hash=sha256:948bad47f2e2e43527f14248364a0e5dee26dd3184691010ec4a1ebeb0fd6771 \ + --hash=sha256:9825b954155b345c4759f24e5f8d652b9aec2261bb5d4e1abe06bba0a1200535 \ + --hash=sha256:a0377d6962fa431c93ecd78fdea771bb62ec545b24ee0c5d4e32acf2260af259 \ + --hash=sha256:a79cdc4934fe81f593072c94e13da3095e9d41c2deef8f6ff2901794ca1c5042 \ + --hash=sha256:a7bfc7db961c7d96cb75889dc6a1e4ae1e91d87ee61da564f582bd742b8dfeef \ + --hash=sha256:ac81530647c3423107cf61c3481e91f57134e9ddfb6ef83f5150ccbdcbc3a3ee \ + --hash=sha256:ae1d895cf7bbfd50ef34bb63bb727b14514f259f3e3f8dd010783bd38e864c6e \ + --hash=sha256:b081f0e7b600ff24513dec4ca75507fa05e904607847e386e8310d5b7b96b6c7 \ + --hash=sha256:b571236d8393edcd3236e07423f762bfcf571f852aad667a3bce9e7b755e0790 \ + --hash=sha256:b74c30e56346aad067937d766846ee74c231d1d18aad3f324e9b9261de3b2d5e \ + --hash=sha256:bceadfd314bd238f584fc229a4bbaf0e573597e7a026dec5429fbf29fd66c641 \ + --hash=sha256:c5e3ccaac3106e8fa6e2f2f6962449d7c757d7b067e41b395a19d6f0d6cec892 \ + --hash=sha256:c749ab3ac30b5ab1ffb7677f8b92eacfdfdc5260210baa398f845bc3714c05d8 \ + --hash=sha256:cbed5f4c4b88d94bcc36115f4c3bb3aa25da1563a5c3328aa3acebce2b083040 \ + --hash=sha256:d1de5eb04485110c5da4c657e49168995d55e076b1ce60f1a042e254f4186c4f \ + --hash=sha256:dd61e64802d51d1e4f16531c64536354fc3bc67932dc0cff254044f72bf0f187 \ + --hash=sha256:dd9d9a101bd8dbfad112170f009cd155e52bb8c936468821a0d03cbb96c0e426 \ + --hash=sha256:ded33b972cffdaf4ca0ac917338ab61d2bb10d68987dbcae641c313fbfdbf499 \ + --hash=sha256:e8e05549f3b30f9d8a8e28c5aba11cc2a4b90b90961ec685ca58444b0815fc09 \ + --hash=sha256:e9b61676116f755126b90e740a9cff36b91562f47ec330056cc88cc3b9f02f4b \ + --hash=sha256:efa160215c4630836d3b1250af4c7a305acd8239e0d75aff986b8088c2fcacb6 \ + --hash=sha256:f5c05a8fee59309f537590a1ff12d3c1009c485e96a50a9ac60dd085c09d0fc0 \ + --hash=sha256:fb8644dc6d705e1269ed2842bf4dbe2b4e50d670de503bf79d5cef3a5148a4c7 \ + --hash=sha256:fbbad6b9b1da43f25c1f5b20cd5a268e028a2fc95d5a8d1ade6059973bc71584 +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +pandas==3.0.6 \ + --hash=sha256:0704044b676496b8350e023b09f174a26772456c974a2b11c36bebb558c9490d \ + --hash=sha256:085e3786ae6b2e82b406266bce36690f72b9dc1421903ba9296b2981a9fcf586 \ + --hash=sha256:097090508a1dd335013d39106fc10b20f4fd4a171638e47b77d55798ed9dab6c \ + --hash=sha256:1bcb3e9ed29e74a7439cedff9e2aefd3ea65de84d7de9ccb6c194192541bd60e \ + --hash=sha256:1e7c0afdcaf6661d795fcefc2f647ddd1136f62cdc153fba177c685d97a87808 \ + --hash=sha256:1e92d9fa834c7d877130027cddc0cad8dcff97c1f6cca26bd6310f847228b658 \ + --hash=sha256:22172a92e7ee678ec0140c7af4fc9366b55413834a1cd86af78b3caa0b0574de \ + --hash=sha256:253e12cb9081b0afbac607920f6142975966bc315135e09de275fdbaa415d2de \ + --hash=sha256:265f562fdd1079f69f3de96dd425c3405224038c0af4f920c54bd240ee2c4640 \ + --hash=sha256:2a8fc94be2ee5f1d86f97aacd8cc566f81680b6498e76f3007421bb5d98151bf \ + --hash=sha256:2e5fa32ff162dfdbc280157d664f44d23049ae414725af9676df339c501d82cd \ + --hash=sha256:3ef908d28590b3f42d7070e7ad8f9b34b442b260b7f3c1afb57e0040c58cdb1b \ + --hash=sha256:429d9df32731ab01383ed98f2baa7a60368090d1a94fc06019a12062510e8630 \ + --hash=sha256:47121f9571503f724c9b93e297ab6254ac99c77adf5e9ed085ea419fd585c258 \ + --hash=sha256:4e25e2e1adee99ddfada6f7206a79ae8e9c8a8861b0e3eaaba165006d3eef18e \ + --hash=sha256:4ff44b2cb51cbd691c91f92c4ea6c71e34003f239ebd67c2e857dc898466b49c \ + --hash=sha256:50c44cbf5820b6b91a5f74aae04972472aefadd3cd9fbd1010409d85528bd570 \ + --hash=sha256:569e114072b24fc4970c12e2b4bab252671668a40b324318903380cab0254c0c \ + --hash=sha256:583be68728a31d0d750d5b8d9e00f02b153df0d4655f858bde93cb84cfc4227c \ + --hash=sha256:5e75072773c1b2f7cb63faa3a6f562aede11f3976f68ed34cb538bc091a28171 \ + --hash=sha256:5edd0a7abb0986ecce1ac81f56d99b6763f86aa6946dceb6c661224f90af5a19 \ + --hash=sha256:60d81f9e1799b36f3739e7fff44d1fbb2e8fd5a271b3863e03de9715fccda0fa \ + --hash=sha256:62f51d7f651c8054c5e82a69265c98082e795d1442df7ca6edc3a545d61214b1 \ + --hash=sha256:654aae059295dbba6ecd2328ca12712a2cf1676214c8699f1c29213f7ccf9c34 \ + --hash=sha256:66b07ef7315a31bfe1089cd3d71a7de781c9dca986762d0b4fe7c0ef17465d10 \ + --hash=sha256:6ff482fa91fa2bafd92e8fe66ce3645c851824310f295c1f0a2f96e928fc4541 \ + --hash=sha256:77ccbe5057aece6fc172b9b77f19c04335af6882bc2e10c8f3ee4e6bfb3da553 \ + --hash=sha256:7dac2d65e9087e8e7b5a45fe15c4920911a221df061ab629943ce016489145c7 \ + --hash=sha256:83e91d15738d7783c050197cef2f2cf82fc6353dae9865aa87ed1fa16aa4d55a \ + --hash=sha256:86fa853a12e0b70927e2b1ee00d56d2224ec9cbb4b9d58348b5ad52d2f21150e \ + --hash=sha256:8fe77b408d82e2615674dfed62533b95e18a03610573877422aada4f625d4947 \ + --hash=sha256:963ca21199097a84c7827c4678b04e30833084fbf8ef44fde3fa7180a29f8fa0 \ + --hash=sha256:97274c9adf6255bb48c620cd6959805efa7f09ea2167f0e0ae006a448cd2fca7 \ + --hash=sha256:994a79608263fe1c14cc48ffa7300e2b834b7d1cb406ffe96a08828cb0cdd79b \ + --hash=sha256:9ae8073aed8e21d1a7fe263dcdc6840743549722a6738198a0a46000fa9476f2 \ + --hash=sha256:9dab635a549e58a053c7b0fa054dc0bd7be22f0ed9a720f4a85d5fb993276172 \ + --hash=sha256:9e492cd4bdba6778de4fe0df7f4590c012161ebcf9902dce01b01dc683105514 \ + --hash=sha256:a3a22e07fe75347eaacc75b0e85297947af4fba6b4aae23916bd8b6828d0bba3 \ + --hash=sha256:a4dbd4dc65cbe645b92b8785d0f96dd7311010dc6606cf620e51b07b8788a12a \ + --hash=sha256:a77a1a44e4d88f1c6a2a64d3eb12efec8420875722e14279800b173a7c7c2804 \ + --hash=sha256:b27c8d890e4aa2171437ae2a39de1d215e674158e4865c4023a8b31c932513b2 \ + --hash=sha256:bd75ed0c840f709fc2ae26ddd9534ac77ca1a48ac0cce521a74acaa85f3340a7 \ + --hash=sha256:c6e4aae3e9bea26c6c9a20d88d96c86ec4a99b4db5fd516bcb4e829ab2c0ee36 \ + --hash=sha256:c826e9babb7790142c399f58599d8de679bea059d7b39c5b6efa2096fac37266 \ + --hash=sha256:cc39303913e2ea129915670de5d1c9fbd647f543bb72e5543bac8baa94e9e42f \ + --hash=sha256:d7564d86a94c2eb8ab290b07f63ddaae5c032fa53897c29a2ff2197d43aee8af \ + --hash=sha256:d7dcd21238cbb4828ff148481ba01cac8946dc5121457b5aeba28636f8f99a60 \ + --hash=sha256:db7ec631f26223beee8e5c9e0b8f23c24d8197bbd1d982421d4e3188bea51965 \ + --hash=sha256:e3dccb584123b399c07562ac4d62543e90ede49ddf8ce3c13ffc64cbe828c281 \ + --hash=sha256:e7c1905ef02c3d6d43d9dbd5b6ccb4da4870a0b0c821bbc103fbdb6f3ad2707b \ + --hash=sha256:eb6900de08ac85f93ac4948aa6b80842eba555875337b8359035ac9c43e92d34 \ + --hash=sha256:ee913a91669056c1de1a6b733fbfeab711de9e54e3bee2dfa5fe79d9457247d1 \ + --hash=sha256:ef738d71d1059245b6bb03e312be06d8b3821326a83486c1ad03b9aba3710e44 \ + --hash=sha256:f3ce8a6968045481e91a3990e797e348ce13db45ee164a7095bbc824e26c09dd \ + --hash=sha256:f4e7c52eb108d752e7592268108fd3e98efd76d83a3125cdd06c621c2e44359b \ + --hash=sha256:f8029ec0f1f89e4f985929ce1f6626dabf3140d61a4e9c1215afdab34eaf9a5d \ + --hash=sha256:fb625f426b375bcc96e3a04c5d5d266cd7be6ae5d6866e0e703382ab5164068c \ + --hash=sha256:ff51a4459ed036e93d1eb1bb5e6e7b28685d3cb6b7c12b91c05b31024e234729 +parsy==2.2 \ + --hash=sha256:5e981613d9d2d8b68012d1dd0afe928967bea2e4eefdb76c2f545af0dd02a9e7 \ + --hash=sha256:e943147644a8cf0d82d1bcb5c5867dd517495254cea3e3eb058b1e421cb7561f +posthog==7.75.0 \ + --hash=sha256:068e179f6591546146a886e51719559833349145363588e93fd7bb9f04a046a6 \ + --hash=sha256:0bdf1de16b3a6e1bdfaa8d9b77746e06be21fca30bcbd9dac5d9d24bd0df8c1e +prompt-toolkit==3.0.53 \ + --hash=sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2 \ + --hash=sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6 +psutil==7.2.2 \ + --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \ + --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ + --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \ + --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \ + --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \ + --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \ + --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \ + --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \ + --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \ + --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \ + --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \ + --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \ + --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \ + --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \ + --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \ + --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \ + --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ + --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 +pyarrow==26.0.0 \ + --hash=sha256:01c863a18bd9c8412453dd0d92de6d0ee7b2b3d6fb079d9734a4b2a3c8bd4453 \ + --hash=sha256:0cccd36e00ea3afeb52ded61f2721ce71f604853d70c45365c58324eb773d6ae \ + --hash=sha256:106bb9290fc6fd9a84138a9440038ef184bac86463543c5ff099229cb30d996c \ + --hash=sha256:13b0972a3dc71b642050d1bc72664a3916e14f59c943d8c1368154d6e4b0c2d5 \ + --hash=sha256:210cc9b83888b87cdc8f793eebb264f22b20d0dedbedefc73b9687a7047b4747 \ + --hash=sha256:240bd18a7487f8767616a948a69dd4e740a8bc36a1c9da49e4dc9a32c5c2faed \ + --hash=sha256:24f892fdf1ae1942d69d3f7742e2f49960ec95277cfb1a70b8a1d91f4a96d935 \ + --hash=sha256:290a74c48e9491b436fd5edacfadf357943f82aa45c81110bd83a69aab33d1cf \ + --hash=sha256:2b5fcd69c0e1107b79e55839877db5a6ed04651b73fd6fec581d09e230bed5e4 \ + --hash=sha256:2e4a413046eba9896e632925066c74095182200ba32e19ff0166bf64d2f936ac \ + --hash=sha256:3a4d235876f14b4136b4d616ec42eb469ea0d6ead336cae631aa1dd29b21c962 \ + --hash=sha256:3de30a7432b48b98b9decbd9e25a53bb9251d202c2e6c5a29a50869592ccb117 \ + --hash=sha256:41dd3661ef40790a78870052ad7a58ad827b27c67a4511f06962eb9e9b74d19b \ + --hash=sha256:4a5fa8dc70dd50808990ff36faf44088e357b353d86c7682dd92d4b78d4c97d5 \ + --hash=sha256:4bcba83299cb2b8f8e443d36c6ba6269a5034431879015fb0719495df8a14de2 \ + --hash=sha256:515a10dae2a1d236bc9c9209d0317acb6746ea63cd4f98704904af7156d90ed1 \ + --hash=sha256:5780d487ff6c6ed7b42298609680d87fe0036e529a9dc2e1105364bce9697f50 \ + --hash=sha256:5b827650e874f1f9f9392524ea3e9e3e8a245de5ba64acca1f81ab188090afb9 \ + --hash=sha256:5d5768d03426abe6526d5274adefa00abf00a7f81118c46e98b5a46390f5549e \ + --hash=sha256:645917e976671debabf854abab6e2b75c571ca4f82adc33a2d338697f7c27d93 \ + --hash=sha256:68cd662e9e2b00876a131950cf32336ace2d0865e1f9418763e3d3be8481dfa4 \ + --hash=sha256:6a628922ba20705fa964ca73e4ef959c2fb2f14b9bbec5589a6a1e68e6257c85 \ + --hash=sha256:6e89dee53aaeb50505ed6152ea55bc7ddfd4f4df264f5427ea255288d8f0e580 \ + --hash=sha256:6e949744dcfc2d379808f7013c5f9cafaf0f817656dff7d46c6931528dd1784b \ + --hash=sha256:734312d3d99088d9ec28c5b17bad40389bd8373a1afc10acb60b83fd217af087 \ + --hash=sha256:7aa12ab8e236789b1ecd2d6ecaef036b4e63d675ddf1864a43c6799d18f2d028 \ + --hash=sha256:7c3fda041e7078802589cf257750323ee3d0cd1e56e53a9b20ec845697fb3d28 \ + --hash=sha256:879331ddea2a26479fa18fade71e6facf684a6cf19f67daec3775c871569e8e5 \ + --hash=sha256:8e8e28c464552b5ca03e30d4504168c4425ce383884f8611b00e972f9fd933fc \ + --hash=sha256:90ddaf7c625307ad52f31a9b25c34fe5e4897c7529ee3481135822b2b6842ff1 \ + --hash=sha256:954d971b363b16ee41f89389a4053315dc71265f2ce5c2468eb0a910b1166268 \ + --hash=sha256:9db18a9dc0af52135c9eac549d80a7a882696efbe5406cf882b044525d4ecc2e \ + --hash=sha256:a0e4e92eeb088f1d7c2c04d6c7de8434c75abb4b4ccf0bbcd045aa7164c68d93 \ + --hash=sha256:a6ca849f90cf73fe361f08a5762c783ead9671e4548c1f558cc637b54c9103f2 \ + --hash=sha256:ab6914db225d7f399652ae1f08588dfbc9efe617612715701e3d9d5cfa5ca19f \ + --hash=sha256:c2ba350957076b1b3a22f549261dc3e9c67ca20816d8bd5f79d7b9c69be4c4c2 \ + --hash=sha256:ca77c43ca55bfc9a4eeb1f0cd5f093f08731b77c24cdba0829035f084959b0bb \ + --hash=sha256:cc903e1069e9dd5e9dcf780324c0112e27e051e422ecfaff574fb33ed65d9160 \ + --hash=sha256:ce28748cbeb0f29c3ce9603782979c7117580fc76f16aa3ca448b38a22281adb \ + --hash=sha256:d58798c4d8d629700058e9afc1e16b9801023f3ce4dc1c92d945e79b5ffe4e98 \ + --hash=sha256:e2a1856e9565fe2679863b372478c681806aebbf7d0a6e72f33e77f804e647d6 \ + --hash=sha256:e3b190ba1d3d22a5a8758597f797111b77d433473744352a184a5ee0a42d672e \ + --hash=sha256:e890816e5ee89c74a0f8b9379fe8b5ba83f46132b2a0bbb9b1c21359ec30dfda \ + --hash=sha256:eaf9e7cc7ab59f6c760232bbde18f64d559bbc50544841303bfb32be53533297 \ + --hash=sha256:ee341973f78a0b46e073d065e88e75026a9c584051e97f98a0d05d96c6bac7dd \ + --hash=sha256:f1c1b4263fd13abbc339a16f2bf19f3a5cbf2a620853d812b1256f03c5342cb8 \ + --hash=sha256:f7444ea6975c49a857c68f9bd8fa11acae96dede63d120ffb3bf0a603ea82516 \ + --hash=sha256:f800e9e722c145ccd18012d82a864cb21bfee4ba4ceffde77100d25eced511a9 \ + --hash=sha256:fcdd1e04982637c6042337d3e24d472f938f01fdc502e2b994844b726d12c3f4 \ + --hash=sha256:ff1e816af7abff71f289242e109217036723ce36aca74ad6691e52d964a74afa +pyarrow-hotfix==0.7 \ + --hash=sha256:3236f3b5f1260f0e2ac070a55c1a7b339c4bb7267839bd2015e283234e758100 \ + --hash=sha256:59399cd58bdd978b2e42816a4183a55c6472d4e33d183351b6069f11ed42661d +pycparser==3.11 \ + --hash=sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80 \ + --hash=sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc +pydantic==2.14.1 \ + --hash=sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454 \ + --hash=sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26 +pydantic-core==2.50.1 \ + --hash=sha256:0036473f5583e6a60e50b8b21651511564277a3f05cc5dab8cf579f552cd5f6c \ + --hash=sha256:0048b6dddc8ef4b64fccaad878bd143b0c3882ea9936279dc11d613f6b7dd1bc \ + --hash=sha256:009634b83993777ddcd69cad0ffcace43dabde692109528e35f0fde91e386a8b \ + --hash=sha256:028e2f212273d4a39b1ec1e0de8166b1165a65fc0f1111452a9d94fc7c625c63 \ + --hash=sha256:062e891facce5ca296a1c37098e5e466780457f86413894b399f0cf22934f769 \ + --hash=sha256:06e01fbbfdb9be777b316a71b6c49efaf4a08b615d0a98d678cda3023f79d019 \ + --hash=sha256:06ead20d39ffd6f2f6f2a8f8a6de67ff8bb1b4f14a8a30e058502514ee2ac685 \ + --hash=sha256:0b3a6f334c6a2345ca15318ff894502a90012536404b37c844a976c76c846e0b \ + --hash=sha256:0c003c3b7f49debb893d2d85ae099ac5959c9839e2f330fadb1fcdf7a6594482 \ + --hash=sha256:131059670f1d2444269b8585cb888963994871932447c08b39ac6a51fcfef658 \ + --hash=sha256:132529c83901437ff642f585216831bf5fd7a91df66829907e155192ead62498 \ + --hash=sha256:17e722e156d0444ecaefbe640bdb60928752bf2013e2b7a11cdb099aaae19bec \ + --hash=sha256:1a9006395dece0e32e704c315eff8a00bede494f6108546cfc5539c89fef4f9a \ + --hash=sha256:1c8632d4ac04e6f91128fca584b3a8a507d81604c24eeaaad00d4be42765c32b \ + --hash=sha256:1c96fd793b73d1b92e65570132505498fe7b21eaef73cdf74e67e5dfba7ac9e4 \ + --hash=sha256:1cf41f1ae3fa155cf167a72689ad044bcc1e3c97e064123677149bdfb5dafc4a \ + --hash=sha256:1deeacb112d14d3f4fcb16b165f7dbaf76c70ba6e82f37ba042bdab51970a0b8 \ + --hash=sha256:1ef800dd7d85bcdadf4c3076e4c94e43939493558a3b69a1ea830c706d4617bb \ + --hash=sha256:1fa4c8bc12c1354c5550c0c35c1852c8c1901e89e06561724e03f8d0342e1f87 \ + --hash=sha256:2005207aafe1231315718bf6ed5d064a7300fb4772754af35ee72fc68159492e \ + --hash=sha256:23923ab9292c40da026330b1ecf4dc2618c8e86e0422e5d1fbf50d94d64ca4f8 \ + --hash=sha256:23edad659e8dbd8ca7e4e877fe6c81573abbdf215bd25a68b53e1272f58b80c7 \ + --hash=sha256:2ab756b72bd5054e4c7ef3ded331b35786cbd3cf931531a508f79a9537517064 \ + --hash=sha256:2cbd1b75b09e976ed0d6b6ca297675632ca35df86130088457cdc60ef36970ae \ + --hash=sha256:2cf91809d0721ab81592ba67bea7694821679c10b1a2e3c3460082b286c1918a \ + --hash=sha256:2df1ff41884de2bc4b307bafd7c40a691094fad2ff8e767e5b45a319257bcf4e \ + --hash=sha256:2eb75304506894a281d346220a4f7481a1b8729577c5ed2a05395991966a8396 \ + --hash=sha256:2eedf82ee4753cdab8e50044c6bd569577eebc3859b11fecf4eb9223761ff966 \ + --hash=sha256:30ddf019d082c117b5d309e5b86710c2a78909907ec1a9381feec3eec02eca0b \ + --hash=sha256:325c23f3e35cfbf0fe3486fa5f7260d1e45885173002d30a28ca019994124255 \ + --hash=sha256:32fad3a91e51b6d2039c572db04a5a873260b399f6bd62c3552671fa7a4a2899 \ + --hash=sha256:36c426eac0af8d1529ff8467e612b933346caec1fdc0d774f78f67a1a11e16c1 \ + --hash=sha256:3a5fce22f1e87d181e924e12da7d81cfe031fb3881a5ddf26ad28f141756ca43 \ + --hash=sha256:3aa9de446b793de2beb6fa2d9d0961803126c4e2a99c2f25ab59b9fd6ea125c0 \ + --hash=sha256:3e46a9eb0a0901dd6275e6b06ac3a464885ef350ec4121fe486869de8053e4bb \ + --hash=sha256:3fde4fdc6487a58d944ca87cf5adc95d5f266e872c19599f5f4c0a8a1b1f9f9f \ + --hash=sha256:409e0ea40ec30d9158f33574fd758e689f6045a0f2596701828c27816ca9687d \ + --hash=sha256:40f523349960fa30f3ea51404308ff50f9997a90df639590f47a057c1f32b415 \ + --hash=sha256:41bc8237121bd8dc8d888dfd6279fc166ffc88c1f1bf3a8bf00869680533ca4c \ + --hash=sha256:42b54c2c90ad348b5e3a85e03e715d572c1fde357ef104cdfe3b03b697a404ea \ + --hash=sha256:455a773617b5913bf5c20d0692e5787b119e52c4d40ea644ca31f5758fd31be2 \ + --hash=sha256:45b11cac094aa25725581d9304eee93c9028516b9ea80dd9e175e13a5a2c840e \ + --hash=sha256:45c6266d071c241f2a168d45bf8c54344f0effce35e7e6b73afdec11f3687568 \ + --hash=sha256:46b3301d3b5c886f77de7546e47274a5842c622ea2020b8c6524c6b66913b4a6 \ + --hash=sha256:476f6ed8e43cd1e0b460920e23571700872b284e77331cb30c4faf459cf48a4b \ + --hash=sha256:48569b0ade9edfbe065cad1d700175546592aebbb42f02adcebcc26e75b896fe \ + --hash=sha256:49c2cbb2397fe4d0987e84606e691af6cb87bc0ee1bd3e7b737f7e10b4c142f9 \ + --hash=sha256:4a53d13cdfbedbfa87f08b83c1a0a5efcc767d785a4b41934fa9cb672670493a \ + --hash=sha256:4be846f55c9477f5f3ddde8f2ce941137e16862a56d018ed885d422bb6ae02f2 \ + --hash=sha256:4df197990c15b5a37c5a277d131d9f2c67de6133f2e5dafd80d9bba4b99f46f9 \ + --hash=sha256:4e834f6a8e4ff772dcc34f58ef5504147a3ea5b0f4eeb13b0f8eb2ca75ac57f1 \ + --hash=sha256:57f51b31ff826e2859120cf4737c5a758a48d96f3e97da40ccee1796d58078ff \ + --hash=sha256:5958c72adb417c39b12ac87525ac60b0d73315fcdc59e21f44ee4a5e2512c9ef \ + --hash=sha256:5dfe41f232befddb9c4377f6cfc702b51595e2d78ed082672adf8758d2c4619f \ + --hash=sha256:5f3cae32fc46121f787cb2486de9cf95a8bf72aec5cc78f64c606fa1735a6ef5 \ + --hash=sha256:64f6047f62a6c5ae08d0a6afb035667aa2d97c3d20d69762e034c5ea144d92a5 \ + --hash=sha256:6a733778df2f7087ec1100ed0b41533e4f3001976e99570fa34f57c66e7f8e3e \ + --hash=sha256:6b20a4bffabdad0db2927ac034ae3b8a681b1f7a0182f3e60b479ad2fde21ebb \ + --hash=sha256:6dbcbee53bf17196a7f745aa9bf5a9603953a1e365b1f020be3207c676a3e7c4 \ + --hash=sha256:6ed4f3cef55164b026fefb41341b7754cc6b624c75dfe7142d2ecceb5ad21c87 \ + --hash=sha256:704075d10b74f2f3c6e15407c696d88701df35fc8953f434a431add0d0074db0 \ + --hash=sha256:739dc730e6be3bd5ec2f4ab5cfc7eb047cc45fc1497b3bafec74ff2ed07df597 \ + --hash=sha256:7456d699b13954e9c0164dcb267250a10ae0dfb03e6e26d6796ab0d46e189c84 \ + --hash=sha256:756d669f04e62ec4148ecfe22be6a4484d9b1181a6ef32e205ebfd200540858b \ + --hash=sha256:7689580e72a642ab5ec64d5f55b2e33636fa43b4ebe63c0c2c965ef307c7d1aa \ + --hash=sha256:76e2e83fa6ec8cdc972d438dafc2522b3a47bee4ec0ae668b29cfb1977ab5242 \ + --hash=sha256:7816e98acc08119dc0f340ab167048ecc54126316330c1f0caf7c6756c88e28f \ + --hash=sha256:79490e33c4c0fcb933bbbcfc3a62184d8803b99f535863dfbb925e1bcb6945ad \ + --hash=sha256:7f476456ac2bb0d937f75191494a09c83a30765fea4f70f3b404942fe25f6cdf \ + --hash=sha256:844b869f118e22a41a091bdcedda8a71bc1b0f62c38d1a0c3211cece47e1d8fc \ + --hash=sha256:84bc765b282a9d5b7fe0348b8648904f25a6a04b2139da52b1dd30c8ac3a2c8f \ + --hash=sha256:84f34323a61a365b4e9295de6028474754829aaddd59c7bf1a040e7487ef8f3c \ + --hash=sha256:8812592c85d0edf423f10eadcef42716d71e8219085ad9e85b775057b7306133 \ + --hash=sha256:88e492e8b9d0312e7dc13667c30222abf284dc3b79b5302b3607b41a5784ce61 \ + --hash=sha256:8a6791afa2245e6c6b180122d105941644f5bd410bb18623b408808cc41a3102 \ + --hash=sha256:8b4c3df25bd323bf1d36a648d563cf1fc69d717451569927151bdad7cad07a77 \ + --hash=sha256:8daa7ee75245d43ad7d747e5c9ecc1b1d06552f72b14887e9276f787d57375f4 \ + --hash=sha256:93ba4e9d8210d941c200431a56b2c0400b131865947903937ed3ec5404307d2e \ + --hash=sha256:94845ff54dc5193f228cab81b2662a04bfbb892e95bdc15edf7399000ce57d54 \ + --hash=sha256:94be440c03fede26969a5ce75468e0e6a9927a1b46d9b679ee8adc1b057b0350 \ + --hash=sha256:9572c1369e9c9da2d64a7b7992c786d90ff295abc93964cfe3125e4290768070 \ + --hash=sha256:983a662de2571cb2502fc8ff47b6770b03d025d2eb314c92f77b3f07c74720ed \ + --hash=sha256:992c3514ec891fa7858099183e4d64e6bd5a5d4ff452fae29df22faa77a006bb \ + --hash=sha256:99ba9bc2b8062ea0c326a990f7f00e6530c23579de66dd246e72c4cafef950a5 \ + --hash=sha256:9d1bed94af6a63835461f3cf7502058eb166c58c4778e11d0f433cfb1bd69e19 \ + --hash=sha256:9e4472072de0137ee0d8e72d6620e85939c271d2f90f6bbb4b15c24638b79f92 \ + --hash=sha256:a27c09d86600f1bf2fe3f37e1ae697faf3143931c09322cd799da94deee923b5 \ + --hash=sha256:a29a061fec0b4e2d714f277e70a3a18125ecff803f2fea6eade2f2e53711d112 \ + --hash=sha256:a3cda0e538208e5d722bbf3698b24f19c0a7d05bc8d5f8a7f9b121ea7fa243d9 \ + --hash=sha256:a44101320cfe99432db74237545a63057dc7a88dfe792cbcad0647f2af56cb81 \ + --hash=sha256:a4aaaa791bdae1c972a7e81765f4f3571c926b8e0b9b6e47346499fb80079665 \ + --hash=sha256:a51eee75939cf811ac09b278745a6cee7dc873ccfbc8b9af3cc88fe4b7ce25b5 \ + --hash=sha256:a7c58106de36ac6a56314182958de20db8d3a29dfd5db527192cc754e4f8e7fb \ + --hash=sha256:aa8224f10880d9bf1b5993988ba153d42a8b4f3f4f511f93b1f09c93ff613c72 \ + --hash=sha256:acbf31f37c53a5ac0c34706c80b4f5107ba20b05fdd3816124bf236ef0c57dd2 \ + --hash=sha256:adc06d218a1cadfd2ec4628424d7d79ce4eba69c2965e7e7b55106f0da5208c8 \ + --hash=sha256:ae28183297fb0d2b8dc46a1f01d51f5e45825fc5afe76a835a6cb7fb34821295 \ + --hash=sha256:b0135bcdcaa0f23573f286e4cb5e0fd2962700964ed13df085b85f2b97aeab9e \ + --hash=sha256:b087b1c5be7ac687cf22eabfe4b6b608d40df23610651e93611e1f49118baf84 \ + --hash=sha256:b0d955195bbbe489ad343fcc956eacea9357b79cb22192c66cacdefcbc14b32f \ + --hash=sha256:b281a3b0f0822618fe5e3e0d8a2048b6356b14388505dc9374ccffeb69989713 \ + --hash=sha256:b6d0c2183008c188e19f4906d426b293bdc4f67ab17df8e180fe16cda208fa71 \ + --hash=sha256:bbce99252ba3167b2b6277f1829d5bf4b43b754524bddf7f944707c3db7d2253 \ + --hash=sha256:bc1f08f68dac9f9e83845a8039880aba2ab553eb9b2259c3243a313182c253fe \ + --hash=sha256:bc94f474417604bd383d2cd445d071b07dd55fedceed3ce33407bf1fcc107290 \ + --hash=sha256:bed5163e03b98bc1fa2eb05d74c63d9c5c95d8ed6254985481640fbf5e237dea \ + --hash=sha256:c17799a62c142d61b8a3c51752a7cbc87fe2ad4ccfab10e628a77b405075c662 \ + --hash=sha256:c18db21573bd2c6489f9a544b7499f0df2853958c568e5e783536ee1f690af41 \ + --hash=sha256:c3ede305158e75510be50869b319550ab072008c13d64d4ab1e094fb286b6f44 \ + --hash=sha256:c516cc5367ca3448995d42cb994bf3f4c9002d2a7c22eac9622551269ad1b807 \ + --hash=sha256:c531166c42ea7bdfecc8c50049581f05dd1993b09cc7c52bb36a14e96deaec7d \ + --hash=sha256:c73622ef819328873b53109ee4f77ceb598bffedd02daf916102be3228866b78 \ + --hash=sha256:c8dce1f1e0e5358b682a6ad3fa5e31b31d4560997b8e61417e9217c8d60f8a0c \ + --hash=sha256:cb57f304525a5e3c13333b772bf9a473f36326e9c821b2e8e1b2fd36f80ae2c3 \ + --hash=sha256:ce8c25ca38cc0e3d7753ba180808de2c0c8cb24eae0df64491e40921454e9831 \ + --hash=sha256:ceff0acc940be2715bd6ad17b24c0e5304abf44f6efd0f81ee8499e640f9dc86 \ + --hash=sha256:cf356f70551d40374eaffb1aa63f1eb6d2006681cbd7a9faea173ce0f4dd7cd2 \ + --hash=sha256:d2d82aa62521c55ddfb000ae70f88cdd8de974078f6024e821dfe5addd0c818f \ + --hash=sha256:d32f3acc081cc3923386d88f422cde8892335e95f034e0104bb4cf9310d9915f \ + --hash=sha256:d4193206b6587047437f6f11d7e776df23e1c1e23af2a54d9347275614791e10 \ + --hash=sha256:d5c0e32fdbce7f1e8ef4d11f655694bf5f4175c757a9f1dc2be09b8864e5bcf5 \ + --hash=sha256:d5e062c01286d861fd6a1c4ff6e063547b3e713067f2df033c0ff97ac2ca006b \ + --hash=sha256:d8f9e8a6c4ab04b78d61f78627370d834eb004b2869dcb28cfffa647b4ea1980 \ + --hash=sha256:d939de9c82e2126f7f48a7e658f8a85ed46d57662d53f44c49b8895fe94a3eb7 \ + --hash=sha256:de531ce1e2a3364e8767878b58f4ff728a434b4fde089781fe30b1e08e2396e0 \ + --hash=sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09 \ + --hash=sha256:e5faeaee74a57d32b3ab3aebad2e348f06d3ba946fc5d28c1728455f00a3d13a \ + --hash=sha256:e6f0cc1bb9900dc558960894adeb30b0c083366fc1d69b856209fb2ca5c36fe5 \ + --hash=sha256:e8e1d6ce820aa23317e8209a86bd65a540973c12dc7552b48a4f6c8e9926815e \ + --hash=sha256:ed1e728b39a383c81035b2459cfcb35d99dfb01f7d6ebe3a913bc1cc5b81e459 \ + --hash=sha256:ee6db2fbed51a7991302e8fac498cd67e336246026d0dfa84cf5166ce1412760 \ + --hash=sha256:efbecf43d321f7b9281441f1f213f7c21c66988b0e06c2730ba13ed47a46bb08 \ + --hash=sha256:f2c634642694e6a0dad2ab1d375589fa671fd442edd5caf7d9737b8f6ca22906 \ + --hash=sha256:f3377c8c2b3ce898423c5e5dd94c7982e30aa7717a7e6ab2470b9de364963709 \ + --hash=sha256:f5187624823423e1d1b82b1072ac41dc837389e18d3d0572cc19bbee46cd550a \ + --hash=sha256:f77ac30b19221cd9bd3fcfa3d4614eff93140d0572ab730cded17b64adca05f3 \ + --hash=sha256:fe90228920fd8ff2be62622b6bb8a2b11acd65046d50c6b130614b5879605a20 +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c +pyngrok==9.0.2 \ + --hash=sha256:1229f56e5cca7e41cc34b6a1569bdcbca071e71ff914b7af908ad2892694b694 \ + --hash=sha256:c4aeb236d6c9b22da747b550a2753f8e73aff48ecba883729f3c5acef212d19e +python-dateutil==3.0.0 \ + --hash=sha256:3c79630202e4152c6add678446984236efe4f57e84d8e892aab0637bab82e9fd \ + --hash=sha256:7ebcb3c29a92f3ceafe79a7a48a5c45677e9e004bbeec5742ac0bbb00560b7a2 +python-dotenv==1.2.4 \ + --hash=sha256:42269a8a5b3fd54ffa6f3d84b18abed50064717576b4ecf03dc4a55d8aa04fdc \ + --hash=sha256:f0d53e69935a851c0dcc78f3ab7aaccd8cabef0b92382b576b824212902873c0 +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 +questionary==2.1.1 \ + --hash=sha256:3d7e980292bb0107abaa79c68dd3eee3c561b83a0f89ae482860b181c8bd412d \ + --hash=sha256:a51af13f345f1cdea62347589fbb6df3b290306ab8930713bfae4d475a7d4a59 +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 +rich-rst==2.2.0 \ + --hash=sha256:1ea1c43813dc4a8d86475fce27ffb74e0ada3d4656c9f8130ba2a59868055fe9 \ + --hash=sha256:b1e6a67f8f694a6f36035624bf73e2b1a0a4be13edaf3ba5e654d9758b61073a +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 +sqlglot==30.24.0 \ + --hash=sha256:6451aabd1c9994d0f73355c962a5eaf9258381b3f2554e53db916cd9a605552a \ + --hash=sha256:9488828cc018453b7a3ba01e65d2749c9a731314c14583695163ca1f358203fe +starlette==1.9.0 \ + --hash=sha256:3d931de246ed870e35ce605974440816ef895e592f255d1396cfdf191d4c4318 \ + --hash=sha256:fd3f159936e882d06c1108f84688f7cda2cf85fc6a592a50edbd175e882c2ddc +toolz==1.2.0 \ + --hash=sha256:890f820b1cb8152785aaf9386d8707770110809035800985ca65cb24ce1120ef \ + --hash=sha256:9667a038e9d6ecba37995e26cb2f59ec6420b6ad8dd9677de59db9b956b08490 +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 +typing-inspection==0.4.4 \ + --hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \ + --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 +tzdata==2026.5 \ + --hash=sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7 \ + --hash=sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac +tzlocal==5.4.4 \ + --hash=sha256:8dbb8660838688a7b6ba4fed31d18dedf842afb4d47ca050d6d891c2c15f3be4 \ + --hash=sha256:aae09f0126a8a86fa736be266eb4a471380d26a0de3bc14844e7821fee3e2a15 +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 +uvicorn==0.54.0 \ + --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf \ + --hash=sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620 +wcwidth==0.9.3 \ + --hash=sha256:16d09df90b3a8f799ea14c726a41d9b906c3104f95f46d158e1ebe45cf4d547b \ + --hash=sha256:2f875a4c9e62df2ecfa70d99b24c551f2245609858e179c3a40e74631b2d08a4 \ + --hash=sha256:3b119ef60177a92b4f298b80e98464c2922a1fba3ed77fe9a2bcb69edf1b9af4 \ + --hash=sha256:3e3e31d6c17d6da3051f7eb854d5e4bebdc52a08a7a52adc2da6f15af51304e8 \ + --hash=sha256:457f986e4867111cb037ccbf0d97ae100b1ecad030d43a020e7207f1e29649d5 \ + --hash=sha256:52a13869704d43a4add59c68ebc892198e53063883114783bf0b7457e250e3f4 \ + --hash=sha256:537ea00b6792bb243cd09582322c54dbc848879e5d34ea4c5fee0147529685ab \ + --hash=sha256:569a0cdb826d3097b6dbacfaa45aa0002d72a2ba6fa8a63251ea43ac4dd7b6a4 \ + --hash=sha256:5d3b0b6c37f40e8d8298941ec373a426a892bb9a514328abcc9226cdc60ae6eb \ + --hash=sha256:658dcbce546e7f28ee361f88e4a8e591dd56f46bff6084e11b4ad8310ed99946 \ + --hash=sha256:6f3e855d3a09af8b7b6d56e76bc270681032f8a37d527a8d5707a441958e3730 \ + --hash=sha256:70a2ca239b85a219324968993c76c117fe423d4e1a7c9d860c999c66192f2570 \ + --hash=sha256:7a95d8c784cbccbbcb808eaadfd156cd282b2bdd95e89b32505e3443381ce61d \ + --hash=sha256:80557db1f77148b78e1b3c2a9cc4ebc4feabef42aaf69a555565057031c32367 \ + --hash=sha256:9470169fcb4e5340269175ebb3642dd5bfc8f82da8c86f29fc1fefc6c2b9bc1b \ + --hash=sha256:9ab8bfee107e08de09e15f9f6afb9fe0f64c8d64fd296d0d8557bafe71ea3f93 \ + --hash=sha256:9e1cbe18c18d152e1e68e4e2561dab6c2bc988f8fd337a1fb2e1e0809c3fa568 \ + --hash=sha256:a44acbb23b525070a0b8c343dd7cab7a4020aff73b38804f7a66d6a66e16e3fc \ + --hash=sha256:a7c0405a3bf3c564e481d6b2d55c99cc231bb68cf8f4a0995944bbce5ebf1a5d \ + --hash=sha256:aab90ab489850f87633922c3064f6d0442d8f6ae5407a62f82bdb1f424179c05 \ + --hash=sha256:c12abd75ffcbad417e7770a6feb82413c102ccce5c435068c39a82e2218b00b3 \ + --hash=sha256:c1bbc8ebf133422cadfefea95a9f89d6c9e9a1ae19930bef7862a9b59b20bbad \ + --hash=sha256:c4531d93344441a7948e665854f0c40e3a8342ee106436d1531efa7a9fa7641e \ + --hash=sha256:dfe7cfbd4cb59db9fe0931bd6091ab4a95b3acc5d56bfb50fa9fef0ccba5dc5c \ + --hash=sha256:e3c9069c54a2da0bf4f6c7e65a735cb1da0f47601a15cb5fef7e4b6ffca4cf60 \ + --hash=sha256:e9362be098c43d076aa43754face3cc0180fa81cea9ba304240b15490f2617e8 \ + --hash=sha256:eef39f3260991d8767189d296f0116fa4b45ad9c3a9a16d5af9187fb66b27564 \ + --hash=sha256:f11789e520ade8017fa93b01a4b8985571bb433eaefc2f857b57e36875e2e1d8 \ + --hash=sha256:f2f943c4ef14e9baba730740265f4e5bf5fc8473550e36cd92828cd5663561a6 diff --git a/deploy/docker/test_docker.py b/deploy/docker/test_docker.py new file mode 100644 index 0000000..9c44edf --- /dev/null +++ b/deploy/docker/test_docker.py @@ -0,0 +1,148 @@ +import importlib.util +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')] + + +def module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + result = importlib.util.module_from_spec(spec); spec.loader.exec_module(result) + return result + + +app = module('container_app', Path(__file__).with_name('app.py')) +manage = module('container_manage', Path(__file__).with_name('manage.py')) +import backup +import backup_retention as br + + +class ContainerOperationsTests(unittest.TestCase): + def test_retirement_rename_stays_on_the_backup_volume_and_resumes_cleanup(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory).resolve();folder=root/'output/backups/20261110T010000Z-12345678' + folder.mkdir(parents=True);(folder/'snapshot').write_bytes(b'old backup') + keep=folder.parent/'unrelated';keep.mkdir() + br.retire_folder(root,folder,'backup') + self.assertFalse(folder.exists()) + record=next((root/'.runtime/retention/retired').glob('*.json')) + value=json.loads(record.read_text());self.assertEqual(value['storage'],'backup-volume') + self.assertTrue((root/'output/backups/.retired'/value['target']).is_dir()) + self.assertEqual(br.clean_retired(root),1);self.assertTrue(keep.is_dir()) + self.assertFalse(record.exists()) + + def test_secret_initialization_is_private_idempotent_and_not_in_command_arguments(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); (root / 'deploy/docker').mkdir(parents=True) + (root / 'deploy/docker/.env.example').write_text('HTTP_BIND=127.0.0.1\n') + base = manage.initialize(root) + before = {p.name: p.read_bytes() for p in (base / 'secrets').iterdir()} + manage.initialize(root) + self.assertEqual(before, {p.name: p.read_bytes() for p in (base / 'secrets').iterdir()}) + for path in (base / 'secrets').iterdir(): self.assertEqual(path.stat().st_mode & 0o077, 0) + env = manage.environment(root) + self.assertEqual(len(set(env[k] for k in manage.SECRET_NAMES.values())), 3) + args = manage.command('up') + for value in before.values(): self.assertNotIn(value.decode().strip(), str(args)) + self.assertNotIn('-v', manage.command('stop')) + + def test_config_uses_tcp_and_refuses_uncoordinated_secret_changes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with patch.object(app, 'secret', return_value='a' * 32), patch.dict(os.environ, {}, clear=True): + first = app.prepare(root) + self.assertEqual(first['mysqlHost'], 'mysql'); self.assertFalse(first['binaryLogEnabled']) + self.assertEqual((root / '.runtime/local-config.json').stat().st_mode & 0o077, 0) + env = app.backend_environment(first) + self.assertIn('jdbc:mysql://mysql:3306/', env['MYSQL_URL']) + self.assertEqual(env['SERVER_ADDRESS'], '0.0.0.0') + with patch.object(app, 'secret', return_value='b' * 32): + with self.assertRaises(ValueError): app.prepare(root) + + def test_backend_exit_causes_container_failure_instead_of_leaving_a_dead_api(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); (root / '.runtime').mkdir() + child = Mock(pid=777); child.poll.return_value = 2 + with patch.object(app, 'ROOT', root), patch.object(app, 'prepare', return_value={}), \ + patch.object(app, 'backend_environment', return_value={}), patch.object(br, 'accepted_backend'), \ + patch.object(app.subprocess, 'Popen', return_value=child), patch.object(app.signal, 'signal'): + self.assertEqual(app.serve(), 1) + child.wait.assert_called_once() + + def test_shutdown_signals_backend_and_workers_before_waiting(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); (root / '.runtime').mkdir() + children = [Mock(pid=100 + i) for i in range(3)] + for child in children: child.poll.return_value = None + handlers = {} + def register(signum, handler): handlers[signum] = handler + def tick(_): handlers[app.signal.SIGTERM](app.signal.SIGTERM, None) + with patch.object(app, 'ROOT', root), patch.object(app, 'prepare', return_value={}), \ + patch.object(app, 'backend_environment', return_value={}), patch.object(br, 'accepted_backend'), \ + patch.object(app.subprocess, 'Popen', side_effect=children), patch.object(app.signal, 'signal', side_effect=register), \ + patch.object(app, 'ready', return_value=True), patch.object(app, 'review_retention'), \ + patch.object(app.time, 'sleep', side_effect=tick), patch.object(app.os, 'killpg') as kill: + self.assertEqual(app.serve(), 0) + self.assertEqual(kill.call_count, 3) + for child in children: child.wait.assert_called_once() + + def test_container_pinned_jar_rejects_corruption(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {'QUALITY_CONTAINER': '1'}): + root = Path(directory); (root / 'app').mkdir() + jar = root / 'app/backend.jar'; jar.write_bytes(b'accepted') + (root / 'app/backend.sha256').write_text(br.sha(jar)) + self.assertEqual(br.accepted_backend(root)[0], jar.resolve()) + jar.write_bytes(b'changed') + with self.assertRaises(ValueError): br.accepted_backend(root) + + def test_backup_uses_owned_private_instance_for_container_and_owns_temporary_files(self): + from contextlib import contextmanager + import backup_verifier + configuration = Path('/tmp/owned-verifier/client.cnf') + @contextmanager + def instance(root): yield configuration + seen = [] + def verify(folder): + seen.append((backup.client_config(), backup.verification_temp_root())) + return 'verified' + with tempfile.TemporaryDirectory() as directory, patch.object(backup, 'ROOT', Path(directory)), \ + patch.dict(os.environ, {'QUALITY_CONTAINER': '1'}), patch.object(backup_verifier, 'instance', instance), \ + patch.object(backup, '_verify', side_effect=verify): + self.assertEqual(backup.verify('unused'), 'verified') + self.assertEqual(seen, [(configuration, configuration.parent)]) + self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get()) + + def test_compose_preserves_volumes_bounded_logs_secrets_and_private_networks(self): + try: import yaml + except ImportError: self.skipTest('Install PyYAML for Compose contract checks') + compose = yaml.safe_load((ROOT / 'deploy/docker/compose.yaml').read_text()) + services = compose['services'] + self.assertNotIn('ports', services['mysql']); self.assertNotIn('ports', services['app']) + self.assertEqual(services['mysql']['command'][0], '--skip-log-bin') + self.assertEqual(len(compose['volumes']), 5) + self.assertEqual(services['app']['depends_on']['mysql']['condition'], 'service_healthy') + for service in services.values(): + self.assertEqual(service['restart'], 'unless-stopped') + self.assertEqual(service['logging']['options']['max-size'], '10m') + self.assertTrue(services['app']['read_only']) + self.assertNotIn('docker.sock', json.dumps(compose)) + self.assertFalse((ROOT / '.gitea/workflows').exists()) + + def test_build_context_excludes_live_data_and_private_baseline(self): + ignore = (ROOT / '.dockerignore').read_text() + self.assertEqual(next(line for line in ignore.splitlines() if line and not line.startswith('#')), '**') + for allowed in ('!.runtime', '!runtimedata', '!output', '!质量数据', '!apikey.txt'): + self.assertNotIn(allowed, ignore) + self.assertIn('**/original-workbook-acceptance.json', ignore) + nginx = (ROOT / 'deploy/docker/nginx.conf').read_text() + self.assertIn('proxy_buffering off', nginx) + self.assertIn('resolver 127.0.0.11', nginx) + + +if __name__ == '__main__': unittest.main() diff --git a/deploy/docker/worker.py b/deploy/docker/worker.py new file mode 100644 index 0000000..d69504c --- /dev/null +++ b/deploy/docker/worker.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""与后端共享 PID 命名空间的低优先级维护任务;不向容器暴露 Docker socket。""" +import json +import logging +import os +from pathlib import Path +import signal +import sys +import time +import shutil + +ROOT = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')] +import backup_retention as br +import operations +import retention + + +def console_notify(title, message): + print(json.dumps({'event': 'capacity-alert', 'title': title, 'message': message}, ensure_ascii=False), flush=True) + return {'submitted': True, 'channel': 'container-log'} + + +def snapshot(root, support=None, now=None): + import backup + import managed_test_resources as resources + import log_retention + root = Path(root); now = time.time() if now is None else now + cfg = json.loads((root / '.runtime/local-config.json').read_text()) + rows = backup.client('SELECT TABLE_SCHEMA,COALESCE(SUM(DATA_LENGTH+INDEX_LENGTH),0) FROM information_schema.tables GROUP BY TABLE_SCHEMA;') + estimates = {parts[0]: int(parts[1]) for line in rows.splitlines() if len(parts := line.split('\t')) == 2} + registered = 0; overdue = 0 + for path in resources.registry(root).glob('*.json'): + item = resources.load(root, path.stem) + if not item or not item.get('created'): continue + registered += estimates.get(item['database'], 0) + if now >= item.get('expiresAt', now + 1): overdue += 1 + private = sum(operations.allocated(p) for p in (root / '.runtime/retention').glob('verify-*') if p.is_dir() and not p.is_symlink()) + mysql = estimates.get(cfg['mysqlDatabase'], 0) + archive = operations.allocated(root / 'runtimedata/archive'); backups = operations.allocated(root / 'output/backups') + native = operations.allocated(root / '.runtime/nao-users') + logs = sum(p.stat().st_blocks * 512 for p in log_retention.candidates(root, root / '.runtime/operations')) + core = mysql + archive + backups + native + logs + free = min(shutil.disk_usage(root / p).free for p in ('.runtime', 'output/backups', 'output/logs', 'runtimedata/archive')) + return {'at': now, 'mysqlBytes': mysql, 'mysqlMeasurement': 'information_schema_estimate_excludes_system_redo_undo', + 'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': native, + 'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private, + 'coreBytes': core, 'artifactBytes': 0, 'permanentBytes': core, 'freeBytes': free, + 'overdueTestResources': overdue} + + +def once(role): + if role == 'monitor': + operations.snapshot = snapshot + return operations.run(ROOT, support=ROOT / '.runtime/operations', notifier=console_notify) + return retention.run(ROOT) + + +def main(): + role = sys.argv[1] + if role not in ('retention', 'monitor'): raise ValueError('Unknown worker role') + try: os.nice(10) + except OSError: pass + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') + stopping = False + def stop(signum, frame): + nonlocal stopping + stopping = True + signal.signal(signal.SIGTERM, stop); signal.signal(signal.SIGINT, stop) + while not stopping: + try: + once(role) + except (br.Busy, retention.Busy): pass + except Exception as error: + if role == 'monitor': operations.failed_run(ROOT, error, notifier=console_notify) + else: + path = ROOT / '.runtime/retention/state.json' + state = json.loads(path.read_text()) if path.exists() else {} + state['consecutiveFailures'] = min(1000, state.get('consecutiveFailures', 0) + 1) + state['lastError'] = {'time': time.time(), 'type': type(error).__name__} + br.atomic(path, state) + print(json.dumps({'event': 'maintenance-error', 'role': role, 'errorType': type(error).__name__}), flush=True) + for _ in range(300): + if stopping: break + time.sleep(1) + + +if __name__ == '__main__': main() diff --git a/deploy/macos/README.md b/deploy/macos/README.md new file mode 100644 index 0000000..75e3750 --- /dev/null +++ b/deploy/macos/README.md @@ -0,0 +1,94 @@ +# 本机服务自动启动 + +本机使用 `com.lyunlab.quality-center-service` LaunchAgent:解锁并登录当前 macOS 用户后启动,持续守护现有 MySQL 和已验收的后端 JAR。SSH 隧道由已有的 `com.lyunlab.quality-center-tunnel` 独立守护。 + +启用 FileVault 的电脑在解锁登录前无法运行用户级守护。退出用户会话、电脑关机、休眠或断网期间,不能依靠这台 Mac 持续提供公网服务。需要全天候服务时,按 [Docker 部署说明](../docker/README.md) 将后台和数据库部署到常在线服务器。 + +## 管理 + +首次安装维护环境时运行 `python3 -m venv .runtime/retention-env`,再使用 `.runtime/retention-env/bin/pip install --require-hashes -r deploy/macos/retention-requirements.txt` 安装固定依赖。该步骤只安装维护工具,不初始化业务数据库。 + +在项目目录执行: + +```sh +python3 deploy/macos/service.py status +python3 deploy/macos/service.py pause +python3 deploy/macos/service.py stop +python3 deploy/macos/service.py start +``` + +- `status`:显示守护、进程身份、数据库和接口就绪状态。 +- `pause`:停用自动启动和守护,保留当前 MySQL / 后端进程,适合维护前使用。 +- `stop`:停用守护,按后端、MySQL 的顺序正常停服;超时不会强制结束数据库。 +- `start`:重新启用自动启动和守护,先确认数据库可查询,再启动后端。 + +维护期间先暂停守护,再操作服务。直接运行旧的停止脚本后,仍启用的守护会把进程重新拉起。 + +## 实现与边界 + +- 安装时登记已验收 JAR 的 SHA-256,从 `.runtime/backend-builds/` 启动对应的不可变运行包。 +- 只使用 `.runtime/local-config.json` 指定的已初始化数据库;缺少数据标记或运行包摘要不符时停止启动,不初始化、不导入数据库。 +- 每 3 秒检查进程;异常退出后等待 10 秒重启。数据库可查询后才启动后端。端口被未知进程占用或进程身份无法确认时保留现场。 +- 守护自身由 launchd 的 `KeepAlive` 负责重启。重新启动守护时会核对启动时间、用户、程序和参数,接管仍存活的服务,避免重复启动。 +- 只对已经退出的进程自动重启。健康检查暂时失败时不强行终止正在导入或查询的数据库。 +- 守护脚本按摘要复制到用户的 Application Support 目录。项目源文件的后续编辑不会立即改变已安装脚本。 +- MySQL 的 `binaryLogEnabled` 配置为 false 时,使用 `--skip-log-bin` 启动。事务重做日志和 undo 文件继续保留。历史时间点恢复不可用,恢复依赖最新完整快照。 + +启动配置:`~/Library/LaunchAgents/com.lyunlab.quality-center-service.plist`。 + +状态和滚动日志:`~/Library/Application Support/QualityCenterService/`。 + +进程业务日志:项目的 `output/logs/`。 + +重新安装或升级守护脚本时使用 `service.py install --root <项目绝对路径> --jar <已验收JAR绝对路径>`。部署配置中只记录路径与摘要,密码仍从原有私有配置读取。 + +## 验证 + +```sh +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/macos -p 'test_service.py' -v +``` + +隔离进程检查覆盖 MySQL 与后端异常退出重启、数据库启动顺序、已有进程接管、未知进程保护和运行包/数据目录前置校验。实际重启电脑和 FileVault 解锁后的启动仍需单独验收。 + +## 自动数据保留 + +`com.lyunlab.quality-center-retention` 是独立的低优先级维护任务,解锁登录后运行,每5分钟检查一次。它使用当前已验收 JAR,独立运行维护脚本;不重新构建业务后端。 + +- **旧版本**:当前正式版本、候选版本及基线、待配置来源的基线、分析会话、已保存报表/快照/清单、巡检等业务引用均保护。其他已发布版本连续30分钟未被引用,且当前版本已有恢复验证通过的备份后,才进入回收。 +- **旧构建**:只处理已无版本引用的终态构建。分批清理,每批最多1000行;先撤销其可复用状态和唯一构建键,避免新导入复用正在清理的半成品。仍有未知业务引用时保留。 +- **旧来源修订**:每5分钟检查,已完成修订连续30分钟没有任何版本、构建、明细、来源清单、替换/撤销/恢复记录或保存内容引用后才回收。修订身份变化或重新出现引用会重置保护期;未知状态、缺失或校验异常的文件保留。SHA-256、大小、路径边界与文件身份全部通过后才删除修订记录,原件另行核对共享路径和保存内容引用。共享文件保留;文件删除意图先持久登记,意外退出后可续清,不扫描删除未登记的孤立文件。 +- **上传草稿**:24小时过期后再留30分钟保护期。已入库文件先核对正式归档摘要,再重绑定来源路径、清理临时副本;待导入来源仍引用的文件保留。无数据库记录的上传临时文件超过24小时才检查回收。 +- **备份**:检测到新正式版本或距离上次备份超过24小时,在任务空闲时生成新备份。新备份必须通过独立 MySQL 实例恢复、表行数、正式数据汇总、应用文件和来源归档校验,才原子更新最新备份入口并回收旧备份。失败时保留旧有效备份,失败重试间隔至少1小时。 +- **任务保护**:导入、数据工作流、分析、报告、巡检、排队任务及待自动重试任务存在时,自动维护延期。每一批删除都重新检查任务状态和当前版本;短暂取得数据库排他访问失败时延期,不终止业务查询或任务。 +- **互斥与失败恢复**:手动备份、恢复和自动维护共享 `.runtime/data-maintenance.lock`。恢复校验使用临时、仅本机私有 socket 可访问的独立 MySQL,完成后正常关闭并删除;异常遗留只根据所有者记录及进程身份处理,不按数据库名前缀批量删除。 +- **维护暂停**:`service.py pause/stop` 同时写入维护暂停标记;`service.py start/install` 解除。执行中的备份允许安全完成,后续删除批次会停下。 +- **升级保护**:当前 JAR 摘要或数据库结构发生变化时暂停自动清理,重新检查兼容性后执行维护安装命令。未知任务状态也按进行中处理。 +- **空间余量**:开始新备份前预留至少三倍数据库表/索引容量、归档空间和额外安全余量;空间不足时保留现有有效备份并延期。失败的新候选只清理本次登记的文件,未知目录不会删除。 + +最新备份记录在 `.runtime/retention/latest.json`,实际目录在 `output/backups/`,界面“任务与监控”的备份列表可查看。 + +维护任务状态与滚动日志分别在 `.runtime/retention/state.json` 和 `~/Library/Application Support/QualityCenterService/logs/retention.log`。 + +来源修订回收计数为 `sourceRevisionsRemoved`、`sourceArchiveFilesRemoved` 和 `sourceArchiveBytesRemoved`,待解除文件引用的登记数为 `sourceFilesPending`。登记目录为 `.runtime/retention/source-files/`。来源清单中仍可重新导入的记录(包括已导入状态)属于有效引用,不能作为无引用修订删除。 + +```sh +.runtime/retention-env/bin/python3 deploy/macos/retention.py status --root "$PWD" +.runtime/retention-env/bin/python3 deploy/macos/retention.py install --root "$PWD" +PYTHONDONTWRITEBYTECODE=1 .runtime/retention-env/bin/python3 -m unittest discover -s deploy/macos -p 'test_retention.py' -v +``` + +维护依赖 PyMySQL 固定版本及 SHA-256 在 `retention-requirements.txt` 中。私有 Python 环境与依赖不修改全局安装。日常删除释放的 InnoDB 表内空间由后续导入复用;必要的物理表空间整理另在低峰进行。 + +## 容量、测试资源与日志 + +`operations.py` 安装独立的 5 分钟检查任务。容量阈值为 30/50GiB,余量阈值为 50/20GiB;监测业务数据、交付产物、临时校验空间及约 24 小时增长,保留 30 天日报。告警通过 macOS 系统通知发送,并记录在 `.runtime/operations/`,相同告警 6 小时内去重。 + +注册过的测试数据库在正常退出时回收;异常遗留超过 24 小时,确认所有者和子进程退出、标记一致且无连接后才删除。未登记数据库继续保留。日志按 7 天、每类 100MiB 限制;Java 原生日志滚动配置在 `logback-quality.xml`,MySQL 通过重新打开错误日志完成滚动,不重启数据库,不截断活动文件。 + +```sh +.runtime/retention-env/bin/python3 deploy/macos/operations.py install --root "$PWD" +.runtime/retention-env/bin/python3 deploy/macos/operations.py status --root "$PWD" +.runtime/retention-env/bin/python3 deploy/macos/operations.py notify-test --root "$PWD" +``` + +安装日志配置后,后端需在任务空闲时正常重启一次才能加载;不要为加载日志配置中断导入。Linux 复用同一回收逻辑,采用容器日志告警,见 Docker 说明。 diff --git a/deploy/macos/logback-quality.xml b/deploy/macos/logback-quality.xml new file mode 100644 index 0000000..93bd2d9 --- /dev/null +++ b/deploy/macos/logback-quality.xml @@ -0,0 +1,19 @@ + + + + + + ${QUALITY_LOG_DIR}/backend.log + true + + ${QUALITY_LOG_DIR}/backend.%d{yyyy-MM-dd}.%i.log.gz + 10MB + 7 + 80MB + true + + ${FILE_LOG_PATTERN}UTF-8 + + + + diff --git a/deploy/macos/operations.py b/deploy/macos/operations.py new file mode 100644 index 0000000..247f042 --- /dev/null +++ b/deploy/macos/operations.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Independent capacity monitoring, registered-test cleanup and log retention.""" +import argparse +import fcntl +from datetime import datetime +import hashlib +import json +import logging +from logging.handlers import RotatingFileHandler +import os +from pathlib import Path +import plistlib +import shutil +import subprocess +import sys +import time +from zoneinfo import ZoneInfo + +LABEL = 'com.lyunlab.quality-center-operations' +SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService' +GIB = 1024**3 +DEFAULTS = {'intervalSeconds': 300, 'warningBytes': 30 * GIB, 'criticalBytes': 50 * GIB, + 'freeWarningBytes': 50 * GIB, 'freeCriticalBytes': 20 * GIB, + 'growthBytes': 2 * GIB, 'growthFraction': .2, 'notificationCooldownSeconds': 21600, + 'logMaxAgeSeconds': 604800, 'logClassMaxBytes': 100 * 1024**2, + 'testMaxAgeSeconds': 86400} + + +def allocated(path): + path = Path(path) + if not path.exists() or path.is_symlink(): return 0 + result = subprocess.run(['/usr/bin/du', '-sk', str(path)], capture_output=True, text=True, timeout=45) + lines = result.stdout.splitlines() + if result.returncode or not lines: raise RuntimeError('Capacity scan incomplete') + return int(lines[-1].split()[0]) * 1024 + + +def snapshot(root, support=SUPPORT, now=None): + import managed_test_resources as resources + import log_retention + root = Path(root).resolve(); now = time.time() if now is None else now + mysql = allocated(root / '.runtime/mysql-data') + archives = allocated(root / 'runtimedata/archive'); backups = allocated(root / 'output/backups') + runtime = allocated(root / '.runtime'); outputs = allocated(root / 'output') + logs = log_retention.candidates(root, support) + log_bytes = sum(p.stat().st_blocks * 512 for p in logs) + output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output')) + runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime')) + native = allocated(root / '.runtime/nao-users') + registered = 0; overdue = 0 + for p in resources.registry(root).glob('*.json'): + value = resources.load(root, p.stem) + if not value or not value.get('created'): continue + if now >= value.get('expiresAt', now + 1): overdue += 1 + registered += allocated(root / '.runtime/mysql-data' / value['database']) + private = 0 + for p in (root / '.runtime/retention').glob('verify-*'): + if p.is_symlink(): continue + try: owner = json.loads((p / 'owner.json').read_text()) + except (OSError, ValueError): continue + if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p) + # Core service data is the same scope as the earlier 11.86GiB measurement. + # Software/build/delivery artifacts are visible separately and included in growth checks. + core = max(0, mysql - registered) + archives + backups + log_bytes + native + artifacts = max(0, runtime - mysql - private - native - runtime_logs) + max(0, outputs - backups - output_logs) + return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups, + 'logBytes': log_bytes, 'nativeDataBytes': native, 'registeredTestBytes': registered, + 'privateVerifierBytes': private, 'temporaryBytes': registered + private, + 'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts, + 'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue} + + +def conditions(sample, policy, maintenance, history): + alerts = {} + if sample['coreBytes'] >= policy['criticalBytes']: + alerts['capacity'] = {'severity': 'critical', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB,请检查增长来源。"} + elif sample['coreBytes'] >= policy['warningBytes']: + alerts['capacity'] = {'severity': 'warning', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB。"} + for key, label, size in [('temporary', '测试及校验临时空间', sample.get('temporaryBytes', 0)), + ('artifacts', '交付及构建产物', sample.get('artifactBytes', max(0, sample['permanentBytes'] - sample['coreBytes'])))]: + if size >= policy['warningBytes']: + alerts[key] = {'severity': 'critical' if size >= policy['criticalBytes'] else 'warning', + 'message': f'{label}已占用{size/GIB:.1f}GiB,请核对;正在运行的任务会继续保护。'} + if sample['freeBytes'] < policy['freeCriticalBytes']: + alerts['disk'] = {'severity': 'critical', 'message': f"磁盘仅余{sample['freeBytes']/GIB:.1f}GiB。"} + elif sample['freeBytes'] < policy['freeWarningBytes']: + alerts['disk'] = {'severity': 'warning', 'message': f"磁盘剩余{sample['freeBytes']/GIB:.1f}GiB,请预留备份空间。"} + if maintenance.get('consecutiveFailures', 0) >= 3: + alerts['maintenance'] = {'severity': 'critical', 'message': '数据维护已连续失败3次,请检查维护记录。'} + if sample['overdueTestResources']: + alerts['tests'] = {'severity': 'warning', 'message': f"有{sample['overdueTestResources']}个测试资源已超过24小时;正在运行或身份未确认的资源将继续保留。"} + old = [v for v in history if 21 * 3600 <= sample['at'] - v['at'] <= 27 * 3600] + if old: + baseline = min(old, key=lambda x: abs(sample['at'] - x['at'] - 86400)) + changes = [] + for label, key in [('业务数据', 'coreBytes'), ('交付及构建产物', 'artifactBytes')]: + before = baseline.get(key, baseline['permanentBytes'] - baseline['coreBytes']) + after = sample.get(key, sample['permanentBytes'] - sample['coreBytes']) + delta = after - before + if delta > max(policy['growthBytes'], before * policy['growthFraction']): + changes.append(f'{label}增长{delta/GIB:.1f}GiB') + if changes: + alerts['growth'] = {'severity': 'warning', 'message': '约24小时' + ','.join(changes) + ',请核对。'} + return alerts + + +def notify(title, message): + script = 'on run argv\ndisplay notification (item 2 of argv) with title (item 1 of argv)\nend run\n' + try: + p = subprocess.run(['/usr/bin/osascript', '-', title, message], input=script, text=True, + capture_output=True, timeout=10) + return {'submitted': p.returncode == 0, 'channel': 'macOS', 'exitCode': p.returncode} + except (OSError, subprocess.TimeoutExpired): + return {'submitted': False, 'channel': 'macOS'} + + +def dispatch(alerts, previous, now, policy, notifier=notify): + updated = {}; events = [] + for key, value in alerts.items(): + prior = previous.get(key, {}) + due = (not prior or not prior.get('lastSubmittedAt') or prior.get('severity') != value['severity'] or + now - prior.get('lastSubmittedAt', 0) >= policy['notificationCooldownSeconds']) + entry = {**value, 'since': prior.get('since', now), 'lastSubmittedAt': prior.get('lastSubmittedAt', 0)} + if due: + result = notifier('质量平台容量告警', value['message']) + entry['notification'] = result + # Failed submission retries on the next check; duplicates are limited after success. + if result.get('submitted'): entry['lastSubmittedAt'] = now + events.append({'kind': 'alert', 'key': key, **value, 'at': now, 'notification': result}) + updated[key] = entry + for key in set(previous) - set(alerts): + events.append({'kind': 'resolved', 'key': key, 'at': now}) + return updated, events + + +def janitor(root, support, policy): + import backup_retention as br + import backup_verifier + import managed_test_resources as resources + import log_retention + from retention import Database, Busy + root = Path(root).resolve() + if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'} + try: + with br.lock(root, timeout=0): + db = Database(root) + try: + config = json.loads((root / '.runtime/retention/config.json').read_text()) + accepted = br.accepted_backend(root) + if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]: + return {'deferred': 'Application version changed'} + db.current() + if db.busy(): return {'deferred': 'Application jobs are active'} + # Use a short-lived CLI connection so no observer holds a test schema open. + import backup + result = resources.cleanup(root, backup.client) + result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root) + result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br) + result.update(log_retention.retain(root, support, + max_bytes=policy['logClassMaxBytes'], max_age=policy['logMaxAgeSeconds'])) + return result + finally: db.conn.close() + except (br.Busy, Busy): return {'deferred': 'Another maintenance or application operation is active'} + + +def run(root, support=SUPPORT, notifier=notify): + import backup_retention as br + root = Path(root).resolve(); base = root / '.runtime/operations' + if base.is_symlink() or (root / '.runtime').is_symlink(): raise ValueError('Unsafe operations state path') + base.mkdir(parents=True, exist_ok=True, mode=0o700) + policy = {**DEFAULTS, **json.loads((base / 'policy.json').read_text())} + state_path = base / 'monitor.json'; previous = json.loads(state_path.read_text()) if state_path.exists() else {} + history_path = base / 'samples.json'; history = json.loads(history_path.read_text()) if history_path.exists() else [] + now = time.time(); sample = snapshot(root, support, now) + maintenance_path = root / '.runtime/retention/state.json' + maintenance = json.loads(maintenance_path.read_text()) if maintenance_path.exists() else {} + alerts = conditions(sample, policy, maintenance, history) + cleanup = {}; failures = previous.get('cleanupFailures', 0) + try: + cleanup = janitor(root, support, policy) + if 'deferred' not in cleanup: failures = 0 + except Exception as error: + failures += 1; cleanup = {'errorType': type(error).__name__} + if failures >= 3: alerts['cleanup'] = {'severity': 'warning', 'message': '日志或测试环境清理已连续失败3次,请检查维护记录。'} + active, events = dispatch(alerts, previous.get('alerts', {}), now, policy, notifier) + state = {'checkedAt': now, 'sample': sample, 'cleanup': cleanup, 'cleanupFailures': failures, 'alerts': active} + br.atomic(state_path, state) + (base / 'failure.json').unlink(missing_ok=True) + history = [v for v in history if now - v['at'] <= 27 * 3600] + history.append(sample); br.atomic(history_path, history[-330:]) + daily_path = base / 'daily.json'; daily = json.loads(daily_path.read_text()) if daily_path.exists() else {} + day = datetime.fromtimestamp(now, ZoneInfo('Asia/Shanghai')).date().isoformat(); daily[day] = sample + br.atomic(daily_path, {k: daily[k] for k in sorted(daily)[-30:]}) + if events: + handler = RotatingFileHandler(base / 'alerts.jsonl', maxBytes=1024**2, backupCount=2) + handler.setFormatter(logging.Formatter('%(message)s')) + logger = logging.getLogger('quality-operations-alerts'); logger.addHandler(handler); logger.setLevel(logging.INFO); logger.propagate = False + try: + for event in events: logger.info(json.dumps(event, ensure_ascii=False)) + finally: logger.removeHandler(handler); handler.close() + return state + + +def failed_run(root, error, notifier=notify): + import backup_retention as br + base = Path(root).resolve() / '.runtime/operations' + if base.is_symlink() or base.parent.is_symlink(): return + path = base / 'failure.json'; value = json.loads(path.read_text()) if path.exists() else {} + now = time.time(); value.update({'count': value.get('count', 0) + 1, 'time': now, 'type': type(error).__name__}) + if value['count'] >= 3 and now - value.get('lastSubmittedAt', 0) >= DEFAULTS['notificationCooldownSeconds']: + result = notifier('质量平台监测异常', '容量监测已连续失败3次,请查看监测状态。') + value['notification'] = result + if result.get('submitted'): value['lastSubmittedAt'] = now + br.atomic(path, value) + + +def install(root): + import backup_retention as br + root = Path(root).resolve(); base = root / '.runtime/operations' + base.mkdir(parents=True, exist_ok=True, mode=0o700) + policy_path = base / 'policy.json' + policy = {**DEFAULTS, **(json.loads(policy_path.read_text()) if policy_path.exists() else {})} + br.atomic(policy_path, policy) + files = [Path(__file__).resolve(), root / 'deploy/macos/retention.py'] + [root / 'scripts' / p for p in + ('backup_retention.py', 'revision_retention.py', 'backup_verifier.py', 'managed_test_resources.py', 'log_retention.py')] + digest = hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest() + version = SUPPORT / 'operations' / digest; version.mkdir(parents=True, exist_ok=True, mode=0o700) + for p in files: shutil.copy2(p, version / p.name); (version / p.name).chmod(0o600) + log_config = root / 'deploy/macos/logback-quality.xml' + shutil.copy2(log_config, base / 'logback-quality.xml'); (base / 'logback-quality.xml').chmod(0o600) + logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True, mode=0o700) + value = {'Label': LABEL, 'ProgramArguments': [str(root / '.runtime/retention-env/bin/python3'), str(version / 'operations.py'), 'run', '--root', str(root)], + 'RunAtLoad': True, 'StartInterval': 300, 'WorkingDirectory': str(root), 'ProcessType': 'Background', + 'Nice': 10, 'LowPriorityIO': True, 'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'}, + 'StandardOutPath': str(logs / 'operations-launchd.log'), 'StandardErrorPath': str(logs / 'operations-launchd-error.log')} + domain = 'gui/' + str(os.getuid()); plist = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist') + subprocess.run(['launchctl', 'bootout', domain + '/' + LABEL], capture_output=True) + with plist.open('wb') as f: plistlib.dump(value, f) + plist.chmod(0o600); subprocess.run(['launchctl', 'enable', domain + '/' + LABEL], check=True) + for _ in range(50): + p = subprocess.run(['launchctl', 'bootstrap', domain, str(plist)], capture_output=True) + if p.returncode == 0: break + time.sleep(.2) + else: raise RuntimeError('Operations job registration did not complete') + print('Installed independent five-minute capacity and housekeeping checks') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=['install', 'run', 'status', 'notify-test']); parser.add_argument('--root', required=True) + args = parser.parse_args(); root = Path(args.root).resolve(); sys.path.append(str(root / 'scripts')) + import backup_retention as br + if args.action == 'install': + with br.lock(root, timeout=0): install(root) + elif args.action == 'status': + print((root / '.runtime/operations/monitor.json').read_text()) + elif args.action == 'notify-test': + print(json.dumps(notify('质量平台通知测试', '容量监测已启用。这是测试通知,当前没有容量告警。'), ensure_ascii=False)) + else: + logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True) + handler = RotatingFileHandler(logs / 'operations.log', maxBytes=2 * 1024**2, backupCount=2) + handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) + logging.getLogger().addHandler(handler); logging.getLogger().setLevel(logging.INFO) + with (root / '.runtime/operations/monitor.lock').open('a') as monitor_lock: + fcntl.flock(monitor_lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + state = run(root) + logging.info('operations complete coreGiB=%.3f temporaryGiB=%.3f artifactsGiB=%.3f cleanup=%s alerts=%s', + state['sample']['coreBytes'] / GIB, state['sample']['temporaryBytes'] / GIB, + state['sample']['artifactBytes'] / GIB, json.dumps(state['cleanup']), list(state['alerts'])) + except Exception as error: + logging.error('operations failed safely: %s', type(error).__name__) + failed_run(root, error); raise SystemExit(1) + + +if __name__ == '__main__': main() diff --git a/deploy/macos/retention-requirements.txt b/deploy/macos/retention-requirements.txt new file mode 100644 index 0000000..a03d18e --- /dev/null +++ b/deploy/macos/retention-requirements.txt @@ -0,0 +1,23 @@ +PyMySQL==1.2.3 --hash=sha256:14f1c68e2ed859243ae5ca41ffbe677027fc46bc136a9f0be8a4e928e5e7415a +psutil==7.2.2 \ + --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \ + --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ + --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \ + --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \ + --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \ + --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \ + --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \ + --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \ + --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \ + --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \ + --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \ + --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \ + --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \ + --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \ + --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \ + --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \ + --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ + --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 diff --git a/deploy/macos/retention.py b/deploy/macos/retention.py new file mode 100644 index 0000000..6d9c031 --- /dev/null +++ b/deploy/macos/retention.py @@ -0,0 +1,525 @@ +#!/usr/bin/env python3 +"""Conservative, online retention for this Mac's accepted Quality Center runtime.""" +import argparse +from contextlib import contextmanager +from datetime import datetime +import hashlib +import importlib +import json +import logging +from logging.handlers import RotatingFileHandler +import os +from pathlib import Path +import plistlib +import re +import shutil +import subprocess +import sys +import time +import urllib.request + +import pymysql +from pymysql.cursors import DictCursor + +LABEL = 'com.lyunlab.quality-center-retention' +SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService' +TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED') +BUILD_CHILDREN = ('defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup', + 'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build', + 'fact_black', 'fact_bright') +OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion', + 'release_action', 'candidate_reconciliation') +MUTABLE_RELEASE_REFS = ('data_release', 'import_run', 'release_action', 'candidate_reconciliation') + + +def ident(value): + if not re.fullmatch(r'[A-Za-z0-9_]+', value): + raise ValueError('Invalid SQL identifier') + return '`' + value + '`' + + +class Busy(RuntimeError): + pass + + +def safe_file(path, boundary): + path, boundary = Path(path), Path(boundary).absolute() + if not path.is_absolute(): + return False + if not path.is_relative_to(boundary) or not path.resolve().is_relative_to(boundary.resolve()): + return False + cursor = path + while cursor != boundary: + if cursor.is_symlink(): + return False + cursor = cursor.parent + return not boundary.is_symlink() and path.is_file() + + +def fingerprint(path): + s = Path(path).stat() + return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns) + + +class Database: + def __init__(self, root, database=None): + self.root = Path(root).resolve() + self.config = json.loads((self.root / '.runtime/local-config.json').read_text()) + self.name = database or self.config['mysqlDatabase'] + ident(self.name) + endpoint = ({'host': self.config['mysqlHost'], 'port': int(self.config['mysqlPort'])} + if self.config.get('mysqlHost') else {'unix_socket': str(self.root / '.runtime/mysql.sock')}) + self.conn = pymysql.connect(**endpoint, user='root', + password=self.config['rootPassword'], database=self.name, charset='utf8mb4', + autocommit=True, cursorclass=DictCursor, connect_timeout=3, read_timeout=10, write_timeout=5) + self.execute("SET SESSION time_zone='+00:00'") + self.execute('SET SESSION lock_wait_timeout=1') + self.execute('SET SESSION innodb_lock_wait_timeout=1') + rows = self.rows('SELECT TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE FROM information_schema.columns WHERE TABLE_SCHEMA=%s', (self.name,)) + self.columns = {} + self.column_types = {} + for row in rows: + self.columns.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['IS_NULLABLE'] == 'YES' + self.column_types.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['DATA_TYPE'] + self.tables = sorted(self.columns) + self.fks = self.rows('SELECT TABLE_NAME,COLUMN_NAME,REFERENCED_TABLE_NAME,REFERENCED_COLUMN_NAME FROM information_schema.key_column_usage WHERE TABLE_SCHEMA=%s AND REFERENCED_TABLE_NAME IS NOT NULL', (self.name,)) + self.schema_signature=hashlib.sha256(json.dumps({'columns':self.columns,'fks':sorted(self.fks,key=lambda r:json.dumps(r,sort_keys=True))},sort_keys=True).encode()).hexdigest() + self.longest_exclusion=0.0 + if not {'current_release','data_release','release_member','transform_build','source_upload_draft'}.issubset(self.columns): + raise ValueError('Unexpected application schema') + + def rows(self, sql, args=()): + with self.conn.cursor() as cursor: + cursor.execute(sql, args) + return list(cursor.fetchall()) + + def execute(self, sql, args=()): + with self.conn.cursor() as cursor: + return cursor.execute(sql, args) + + def current(self): + rows = self.rows("SELECT release_id FROM current_release WHERE dataset_id='quality'") + if len(rows) != 1 or not rows[0]['release_id']: + raise Busy('Current release not confirmed') + fields='status,complete' if 'complete' in self.columns['data_release'] else 'status' + release=self.rows('SELECT '+fields+' FROM data_release WHERE id=%s',(rows[0]['release_id'],)) + if len(release)!=1 or release[0]['status']!='PUBLISHED' or release[0].get('complete',True) not in (True,1): + raise Busy('Current release is not a complete publication') + return rows[0]['release_id'] + + def busy(self): + for table in ('task','import_run','data_workflow_operation','analysis_turn','analysis_protocol_execution','quality_inspection_run'): + if table not in self.columns or 'status' not in self.columns[table]: + continue + sql = 'SELECT 1 FROM ' + ident(table) + ' WHERE (status IS NULL OR status NOT IN (' + ','.join(['%s']*len(TERMINAL)) + '))' + if table == 'task' and {'next_retry_at','retry_count','max_retries'}.issubset(self.columns[table]): + sql += " OR (status='FAILED' AND next_retry_at IS NOT NULL AND retry_count= grace: + eligible.append(rid) + if not eligible: + return 0 + rid = sorted(eligible, key=lambda x: seen[x])[0] + # Clear optional historical links only. Unknown/business references were + # included in protected above and are never cascaded away. + for fk in self.fks: + t,c = fk['TABLE_NAME'],fk['COLUMN_NAME'] + if fk['REFERENCED_TABLE_NAME'] == 'data_release' and t in MUTABLE_RELEASE_REFS and self.columns[t][c]: + self.execute('UPDATE ' + ident(t) + ' SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,)) + if 'data_workflow_operation' in self.columns: + for c in ('release_id','expected_current_release_id'): + if c in self.columns['data_workflow_operation']: + self.execute('UPDATE data_workflow_operation SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,)) + for t in OWNED_RELEASE: + if t in self.columns: + self.execute('DELETE FROM ' + ident(t) + ' WHERE release_id=%s', (rid,)) + self.execute('DELETE FROM data_release WHERE id=%s', (rid,)) + seen.pop(rid, None) + return 1 + + def build_step(self, state, grace, now, batch): + referenced = {r['transform_build_id'] for r in self.rows('SELECT DISTINCT transform_build_id FROM release_member')} + builds = self.rows('SELECT id,status,build_key FROM transform_build') + seen = state.setdefault('builds', {}) + ids = {r['id'] for r in builds} + for stale in set(seen) - ids: + seen.pop(stale) + for row in builds: + bid = row['id'] + unknown_reference = any(self.rows('SELECT 1 FROM ' + ident(f['TABLE_NAME']) + ' WHERE ' + ident(f['COLUMN_NAME']) + '=%s LIMIT 1', (bid,)) + for f in self.fks if f['REFERENCED_TABLE_NAME'] == 'transform_build' and f['TABLE_NAME'] not in (*BUILD_CHILDREN, 'release_member')) + if bid in referenced or unknown_reference or row['status'] not in ('COMPLETED','FAILED','CANCELLED','RETIRED'): + seen.pop(bid, None) + continue + seen.setdefault(bid, now) + if row['status'] != 'RETIRED' and now - seen[bid] < grace: + continue + if row['status'] != 'RETIRED': + # Free the unique reuse key before deleting chunks. A new import + # can build a fresh result, never reuse this partially emptied one. + tombstone = hashlib.sha256(('retired:'+bid+':'+row['build_key']).encode()).hexdigest() + self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid)) + for t in BUILD_CHILDREN: + if t in self.columns: + count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,)) + if count: + return count, 0 + self.execute('DELETE FROM transform_build WHERE id=%s', (bid,)) + seen.pop(bid, None) + return 0, 1 + return 0, 0 + + def file_referenced(self, path): + for table, cols in self.columns.items(): + for col in cols: + if col.endswith('_path') and self.rows('SELECT 1 FROM ' + ident(table) + ' WHERE ' + ident(col) + '=%s LIMIT 1', (str(path),)): + return True + return False + + def expired(self, limit=10): + return self.rows('SELECT * FROM source_upload_draft WHERE expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6)) ORDER BY expires_at LIMIT ' + str(int(limit))) + + def promotion(self, draft): + if not draft.get('candidate_id') or not {'source_revision','source_document','source_candidate'}.issubset(self.columns): + return None + rows = self.rows("SELECT * FROM source_candidate WHERE id=%s AND status='IMPORTED'", (draft['candidate_id'],)) + if not rows: + return None + c = rows[0] + docs = {d['id']:d for d in self.rows('SELECT id,source_slot FROM source_document')} + for rev in self.rows('SELECT source_document_id,archive_path,sha256,parser_config_hash,file_size FROM source_revision WHERE sha256=%s AND parser_config_hash=%s', (c['sha256'],c['parser_config_hash'])): + if docs.get(rev['source_document_id'],{}).get('source_slot') == c['source_slot']: + p=Path(rev['archive_path']) + if safe_file(p,self.root/'runtimedata/archive/source-revisions') and p.stat().st_size==rev['file_size']: + return c,rev,p + return None + + def clean_owned_scratch(self, br): + import psutil + server=self.rows('SELECT @@server_uuid AS value')[0]['value'] + directory=self.root/'.runtime/retention/scratch' + removed=0 + for path in directory.glob('*.json'): + if path.is_symlink():continue + value=json.loads(path.read_text());name=value.get('database','') + if value.get('kind')!='quality-restore-check' or value.get('server')!=server or not value.get('created') or name==self.name or not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',name):continue + if time.time()-value['createdAt']<300:continue + alive=False + for pid_key,created_key in [('ownerPid','ownerCreated'),('childPid','childCreated')]: + if pid_key not in value:continue + try: + process=psutil.Process(value[pid_key]) + if abs(process.create_time()-value[created_key])<.01 and process.is_running() and process.status()!=psutil.STATUS_ZOMBIE:alive=True + except psutil.NoSuchProcess:pass + except psutil.Error:alive=True + if alive or self.rows('SELECT 1 FROM information_schema.processlist WHERE DB=%s LIMIT 1',(name,)):continue + self.execute('DROP DATABASE IF EXISTS '+ident(name));path.unlink();removed+=1 + return removed + + +def cleanup_drafts(db, current, state, now, br): + removed=0; bytes_removed=0 + # Checksum expensive files outside the brief database exclusion window. + for draft in db.expired(): + path=Path(draft['source_path']); boundary=db.root/'runtimedata/archive/upload-drafts' + info=None + if safe_file(path,boundary): + info=fingerprint(path) + if info[2]!=draft['file_size'] or br.sha(path)!=draft['sha256']: + continue + promotion=db.promotion(draft) + if promotion: + c,rev,target=promotion + target_stamp=fingerprint(target) + if br.sha(target)!=rev['sha256']: + continue + with db.frozen(current): + live=db.rows('SELECT * FROM source_upload_draft WHERE id=%s AND expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6))',(draft['id'],)) + if not live or live[0]!=draft: + continue + if info and (not safe_file(path,boundary) or fingerprint(path)!=info): + continue + if promotion and safe_file(target,db.root/'runtimedata/archive/source-revisions') and fingerprint(target)==target_stamp: + # The WHERE identity prevents overwriting a source reconfigured + # between preflight hashing and acquiring the database exclusion. + db.execute("UPDATE source_candidate SET source_path=%s,archive_path=%s WHERE id=%s AND status='IMPORTED' AND source_path=%s AND sha256=%s AND parser_config_hash=%s", + (str(target),str(target),c['id'],c['source_path'],c['sha256'],c['parser_config_hash'])) + db.execute('DELETE FROM source_upload_draft WHERE id=%s',(draft['id'],)) + if info: + with db.frozen(current): + if safe_file(path,boundary) and fingerprint(path)==info and not db.file_referenced(path): + bytes_removed+=path.stat().st_size + path.unlink() + removed+=1 + # Interrupted uploads can leave no database row at all. Their age and every + # direct path reference are rechecked under the same exclusion before unlink. + boundary=db.root/'runtimedata/archive/upload-drafts' + for path in list(boundary.glob('upload-*.xlsx'))[:100]: + if not safe_file(path,boundary) or now-path.stat().st_mtime<86400: + continue + info=fingerprint(path) + with db.frozen(current): + if fingerprint(path)==info and not db.file_referenced(path): + size=path.stat().st_size;path.unlink();bytes_removed+=size + return removed,bytes_removed + + +def run(root, max_batches=30, grace=1800): + root=Path(root).resolve();sys.path.append(str(root/'scripts')) + import backup_retention as br + import backup + import backup_verifier + import revision_retention + if (root/'.runtime/retention/paused').exists():raise Busy('Service maintenance is paused') + readiness=urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with readiness.open('http://127.0.0.1:5281/actuator/health',timeout=2) as response: + if json.load(response).get('status')!='UP':raise Busy('Backend is not ready') + except (OSError,ValueError) as error:raise Busy('Backend is not ready') from error + state_path=root/'.runtime/retention/state.json' + state=json.loads(state_path.read_text()) if state_path.exists() else {} + result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0} + with br.lock(root,timeout=0): + result['retiredBackupFoldersRemoved']=br.clean_retired(root) + result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root) + db=Database(root) + try: + policy_path=root/'.runtime/retention/config.json' + if policy_path.exists(): + policy=json.loads(policy_path.read_text());accepted=br.accepted_backend(root) + if policy['schemaSignature']!=db.schema_signature or not accepted or policy['acceptedJarSha256']!=accepted[1]: + raise Busy('Application version changed; retention compatibility review required') + result['scratchRemoved']=db.clean_owned_scratch(br) + current=db.current() + if db.busy(): + raise Busy('An application job is active') + latest=br.latest(root) + # Recover a restore-verified candidate left between verification and + # pointer publication by a previous crash, without creating a duplicate. + if not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current: + candidates=[] + for folder in (root/'output/backups').iterdir() if (root/'output/backups').exists() else []: + candidate=br.verified(folder,checksums=False) + if candidate and candidate.get('queryEvidence',{}).get('releaseId')==current:candidates.append((candidate.get('createdAt',''),folder)) + if candidates: + selected=max(candidates)[1] + if br.verified(selected):br.rotate(root,selected);latest=br.latest(root) + if latest: + state['lastBackupAt']=max(state.get('lastBackupAt',0),datetime.fromisoformat(latest[1]['createdAt']).timestamp()) + due=not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current + due=due or time.time()-state.get('lastBackupAt',0)>86400 + due=due or (state.get('dirtySince') and not state.get('builds') and time.time()-state['dirtySince']>300) + if due: + database_bytes=int(db.rows('SELECT COALESCE(SUM(DATA_LENGTH+INDEX_LENGTH),0) AS size FROM information_schema.tables WHERE TABLE_SCHEMA=%s',(db.name,))[0]['size']) + archive_bytes=sum(p.stat().st_size for p in (root/'runtimedata/archive').rglob('*') if p.is_file()) + required_free=max(10*1024**3,3*database_bytes+2*archive_bytes+5*1024**3) + reserves = [root / '.runtime', root / 'output/backups'] + for reserve in reserves: reserve.mkdir(parents=True, exist_ok=True) + if min(shutil.disk_usage(p).free for p in reserves)deadline or db.longest_exclusion>.5: + break + time.sleep(.05) + drafts,size=cleanup_drafts(db,current,state,time.time(),br) + result['draftsRemoved']=drafts;result['bytesRemoved']=size + sources=revision_retention.Cleaner(db,br,safe_file,fingerprint,Busy) + result.update(sources.collect(current,state,time.time(),grace)) + result['maxExclusiveSeconds']=round(db.longest_exclusion,4) + if any(result[k] for k in ('releasesRemoved','rowsRemoved','buildsRemoved','draftsRemoved','bytesRemoved','sourceRevisionsRemoved','sourceArchiveBytesRemoved')): + state.setdefault('dirtySince',time.time()) + state['lastResult']=result;state.pop('lastError',None) + state['consecutiveFailures']=0 + br.atomic(state_path,state) + return result + except Busy as error: + state['lastDeferred']={'time':time.time(),'reason':str(error)} + state['lastResult']=result;br.atomic(state_path,state) + raise + finally: + db.conn.close() + + +def install(root): + root=Path(root).resolve();sys.path.insert(0,str(root/'scripts'));import backup_retention as br + python=root/'.runtime/retention-env/bin/python3' + if not python.is_file(): + raise ValueError('Private maintenance runtime missing') + files=[Path(__file__).resolve(),root/'scripts/backup_retention.py',root/'scripts/revision_retention.py'] + db=Database(root) + try: + accepted=br.accepted_backend(root) + if not accepted:raise ValueError('Accepted backend runtime is not registered') + br.atomic(root/'.runtime/retention/config.json',{'schemaSignature':db.schema_signature, + 'acceptedJarSha256':accepted[1],'retirementGraceSeconds':1800,'intervalSeconds':300, + 'backupIntervalSeconds':86400,'draftExpiryGraceSeconds':1800,'sourceRevisionGraceSeconds':1800}) + finally:db.conn.close() + digest=hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest() + version=SUPPORT/'retention'/digest;version.mkdir(parents=True,exist_ok=True,mode=0o700) + for p in files: + shutil.copy2(p,version/p.name);(version/p.name).chmod(0o600) + legacy=root/'.runtime/current-snapshot-20261109' + if legacy.is_dir() and not legacy.is_symlink() and (legacy/'manifest.json').is_file(): + br.atomic(root/'.runtime/retention/legacy.json',{'path':'.runtime/current-snapshot-20261109','manifestSha256':br.sha(legacy/'manifest.json')}) + logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700) + plist=Path.home()/'Library/LaunchAgents'/(LABEL+'.plist') + value={'Label':LABEL,'ProgramArguments':[str(python),str(version/'retention.py'),'run','--root',str(root)], + 'WorkingDirectory':str(root),'RunAtLoad':True,'StartInterval':300,'ProcessType':'Background','Nice':10, + 'LowPriorityIO':True,'ExitTimeOut':300,'EnvironmentVariables':{'PYTHONDONTWRITEBYTECODE':'1'}, + 'StandardOutPath':str(logs/'retention-launchd.log'),'StandardErrorPath':str(logs/'retention-launchd-error.log')} + subprocess.run(['/bin/launchctl','bootout','gui/'+str(os.getuid())+'/'+LABEL],capture_output=True) + with plist.open('wb') as out:plistlib.dump(value,out) + plist.chmod(0o600) + subprocess.run(['/bin/launchctl','enable','gui/'+str(os.getuid())+'/'+LABEL],check=True) + for attempt in range(50): + loaded=subprocess.run(['/bin/launchctl','bootstrap','gui/'+str(os.getuid()),str(plist)],capture_output=True) + if loaded.returncode==0:break + time.sleep(.2) + else:raise RuntimeError('Maintenance registration did not complete') + print('Installed five-minute idle maintenance and daily/publication backup checks') + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('action',choices=['run','install','status']) + parser.add_argument('--root',required=True) + args=parser.parse_args();root=Path(args.root).resolve();sys.path.append(str(root/'scripts'));import backup_retention as br + if args.action=='install': + with br.lock(root,timeout=0):install(root) + return + if args.action=='status': + p=root/'.runtime/retention/state.json';print(p.read_text() if p.exists() else '{}');return + logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700) + handler=RotatingFileHandler(logs/'retention.log',maxBytes=2*1024*1024,backupCount=2) + handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) + logging.getLogger().addHandler(handler);logging.getLogger().setLevel(logging.INFO) + try: + result=run(root);logging.info('maintenance complete %s',json.dumps(result)) + except (Busy,br.Busy) as error: + logging.info('maintenance deferred: %s',str(error)) + except Exception as error: + p=root/'.runtime/retention/state.json';state=json.loads(p.read_text()) if p.exists() else {} + state['lastError']={'time':time.time(),'type':type(error).__name__};br.atomic(p,state) + state['consecutiveFailures']=min(1000,state.get('consecutiveFailures',0)+1);br.atomic(p,state) + logging.error('maintenance failed safely: %s',type(error).__name__) + raise SystemExit(1) + + +if __name__=='__main__':main() diff --git a/deploy/macos/service.py b/deploy/macos/service.py new file mode 100644 index 0000000..457a2c8 --- /dev/null +++ b/deploy/macos/service.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +"""Pinned macOS service guardian. Never initializes or imports a database.""" +import argparse +import fcntl +import hashlib +import json +import logging +from logging.handlers import RotatingFileHandler +import os +from pathlib import Path +import plistlib +import shutil +import signal +import socket +import subprocess +import sys +import time +import urllib.request + +import psutil + +LABEL = 'com.lyunlab.quality-center-service' +HOME = Path.home() / 'Library/Application Support/QualityCenterService' +CONFIG = HOME / 'service.json' +PLIST = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist') +DOMAIN = 'gui/' + str(os.getuid()) + + +def atomic(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = path.with_name(path.name + '.tmp-' + str(os.getpid())) + with temporary.open('w') as stream: + os.chmod(temporary, 0o600) + json.dump(value, stream, ensure_ascii=False, indent=2) + temporary.replace(path) + + +def stamp(pid): + return subprocess.run(['/bin/ps', '-p', str(pid), '-o', 'lstart='], + capture_output=True, text=True).stdout.strip() + + +def matches(record, command=None): + """True=confirmed identity, False=gone/reused, None=unknown identity.""" + if not record: + return False + try: + p = psutil.Process(record['pid']) + if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE: + return False + if 'created' in record and abs(p.create_time() - record['created']) > .01: + return False + if 'started' in record and stamp(p.pid) != record['started']: + return False + expected = command or record['command'] + if p.uids().real != os.getuid(): + return None + if Path(p.exe()).resolve() != Path(expected[0]).resolve() or p.cmdline() != expected: + return None + return True + except psutil.NoSuchProcess: + return False + except (psutil.Error, KeyError, ValueError, OSError): + return None + + +def record(pid, command): + return {'pid': pid, 'created': psutil.Process(pid).create_time(), + 'command': command, 'started': stamp(pid)} + + +class Guardian: + def __init__(self, config, home=HOME): + self.config = config + self.home = Path(home) + self.root = Path(config['root']).resolve() + self.runtime = self.root / '.runtime' + self.records = {} + self.children = {} + self.next_start = {'mysql': 0, 'backend': 0} + self.restarts = {'mysql': 0, 'backend': 0} + self.stopping = False + self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def preflight(self): + self.value = json.loads((self.runtime / 'local-config.json').read_text()) + if not self.value.get('databaseReady') or not (self.runtime / 'mysql-data/auto.cnf').is_file(): + raise RuntimeError('Existing initialized database not confirmed; refusing bootstrap') + self.jar = self.runtime / 'backend-builds' / (self.config['jarSha256'] + '.jar') + if hashlib.sha256(self.jar.read_bytes()).hexdigest() != self.config['jarSha256']: + raise RuntimeError('Accepted backend digest mismatch') + for binary in [self.runtime / 'mysql/bin/mysqld', self.runtime / 'jdk/Contents/Home/bin/java']: + if not os.access(binary, os.X_OK): + raise RuntimeError('Required runtime missing') + + def command(self, role): + r = self.runtime + if role == 'backend': + return [str(r / 'jdk/Contents/Home/bin/java'), '-Xms128m', '-Xmx2048m', '-jar', str(self.jar)] + return [str(r / 'mysql/bin/mysqld'), '--no-defaults', '--basedir=' + str(r / 'mysql'), + '--datadir=' + str(r / 'mysql-data'), '--port=' + str(self.value['mysqlPort']), + '--bind-address=127.0.0.1', '--socket=' + str(r / 'mysql.sock'), + '--pid-file=' + str(r / 'mysql-server.pid'), + '--log-error=' + str(self.root / 'output/logs/mysql-server.log'), '--mysqlx=OFF', + '--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M', + *(['--skip-log-bin'] if self.value.get('binaryLogEnabled', True) is False else [])] + + def environment(self): + v, r = self.value, self.runtime + env = os.environ.copy() + env.update({ + 'JAVA_HOME': str(r / 'jdk/Contents/Home'), + 'PATH': str(r / 'jdk/Contents/Home/bin') + ':' + self.config['path'], + 'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{v["mysqlPort"]}/{v["mysqlDatabase"]}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true', + 'MYSQL_USER': v['mysqlUser'], 'MYSQL_PASSWORD': v['mysqlPassword'], + 'MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS': str(v.get('legacyTimestampOffsetSeconds', 'UNCONFIRMED')), + 'QUALITY_BOOTSTRAP_ADMIN_USERNAME': v['adminUsername'], + 'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': v['adminPassword'], + 'QUALITY_ARCHIVE_ROOT': str(self.root / 'runtimedata/archive'), + 'QUALITY_ALLOWED_ROOTS': str(self.root / '质量数据/系统原始数据示例'), + 'QUALITY_DATA_WORKFLOW_TEST_MODE': 'true', + 'QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED': str(v.get('analysisProtocolV1Enabled', False)).lower(), + 'QUALITY_ANALYSIS_AUTONOMOUS_ENABLED': str(v.get('analysisAutonomousEnabled', False)).lower(), + 'QUALITY_PROJECT_ROOT': str(self.root), 'QUALITY_DEFAULT_SOURCE_YEAR': '2026', + 'SERVER_ADDRESS': '127.0.0.1', 'PORT': '5281', 'PYTHONDONTWRITEBYTECODE': '1', + }) + log_config = r / 'operations/logback-quality.xml' + if log_config.is_file() and not log_config.is_symlink(): + env['LOGGING_CONFIG'] = str(log_config) + env['QUALITY_LOG_DIR'] = str(self.root / 'output/logs') + return env + + def port(self, role): + return self.value['mysqlPort'] if role == 'mysql' else 5281 + + def occupied(self, role): + try: + with socket.create_connection(('127.0.0.1', self.port(role)), timeout=.5): + return True + except OSError: + return False + + def db_ready(self): + result = subprocess.run([str(self.runtime / 'mysql/bin/mysql'), + '--defaults-file=' + str(self.runtime / 'mysql-root.cnf'), '--connect-timeout=2', + '--batch', '--skip-column-names', '-e', 'SELECT 1;'], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4) + return result.returncode == 0 and result.stdout.strip() == b'1' + + def backend_ready(self): + try: + with self.opener.open('http://127.0.0.1:5281/actuator/health', timeout=2) as response: + return json.load(response).get('status') == 'UP' + except (OSError, ValueError): + return False + + def adopt(self, role): + path = self.runtime / (role + '.process.json') + if not path.exists(): + return False + info = json.loads(path.read_text()) + identity = matches(info, self.command(role)) + if identity is None: + raise RuntimeError(role + ': existing process identity cannot be confirmed') + if identity: + self.records[role] = record(info['pid'], self.command(role)) + logging.info('%s adopted pid=%s', role, info['pid']) + return bool(identity) + + def ensure(self, role): + child = self.children.get(role) + if child is not None: + child.poll() # Reap exited children before checking PID identity. + info = self.records.get(role) + state = matches(info, self.command(role)) + if state is None: + raise RuntimeError(role + ': process identity is unknown; no replacement started') + if state: + return True + if info: + self.records.pop(role) + self.next_start[role] = time.monotonic() + 10 + logging.warning('%s exited; restart scheduled in 10 seconds', role) + if time.monotonic() < self.next_start[role]: + return False + if self.adopt(role): + return True + if self.occupied(role): + raise RuntimeError(role + ': port occupied by an unconfirmed process') + command = self.command(role) + logs = self.root / 'output/logs' + logs.mkdir(parents=True, exist_ok=True) + log_name = ('backend-console.log' if role == 'backend' and + (self.runtime / 'operations/logback-quality.xml').is_file() else role + '.log') + with (logs / log_name).open('ab') as log: + child = subprocess.Popen(command, cwd=self.root, env=self.environment(), + stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True) + self.children[role] = child + self.records[role] = info = record(child.pid, command) + (self.runtime / (role + '.pid')).write_text(str(child.pid)) + atomic(self.runtime / (role + '.process.json'), {'pid': child.pid, 'started': info['started']}) + self.restarts[role] += 1 + logging.info('%s started pid=%s', role, child.pid) + return True + + def tick(self): + mysql_alive = self.ensure('mysql') + ready = mysql_alive and self.db_ready() + if ready: + self.ensure('backend') + atomic(self.home / 'state.json', {'updatedAt': time.time(), 'supervisorPid': os.getpid(), + 'processes': self.records, 'starts': self.restarts, + 'databaseReady': bool(ready), 'backendReady': self.backend_ready()}) + + def run(self): + self.preflight() + with (self.runtime / 'quality-service.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: setattr(self, 'stopping', True)) + logging.info('guardian started; accepted backend=%s', self.config['jarSha256']) + while not self.stopping: + try: + self.tick() + except Exception as error: + # Never log exception arguments: database credentials may be embedded. + detail = str(error) if type(error) is RuntimeError else type(error).__name__ + logging.error('check failed: %s; leaving existing processes intact', detail) + atomic(self.home / 'error.json', {'time': time.time(), 'type': type(error).__name__}) + for _ in range(30): + if self.stopping: + break + time.sleep(.1) + logging.info('guardian stopped; service processes remain available for maintenance') + + +def ctl(*args, check=True): + return subprocess.run(['/bin/launchctl', *args], check=check, capture_output=True, text=True) + + +def install(root, jar, clear_maintenance_pause=True): + root, jar = Path(root).resolve(), Path(jar).resolve() + cfg = {'root': str(root), 'jarSha256': hashlib.sha256(jar.read_bytes()).hexdigest(), + 'path': '/opt/homebrew/bin:/usr/local/bin:/Library/Frameworks/Python.framework/Versions/3.13/bin:/usr/bin:/bin:/usr/sbin:/sbin'} + Guardian(cfg).preflight() + HOME.mkdir(parents=True, exist_ok=True, mode=0o700) + HOME.chmod(0o700) + script = Path(__file__).resolve() + digest = hashlib.sha256(script.read_bytes()).hexdigest() + target = HOME / 'versions' / digest / 'service.py' + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + shutil.copyfile(script, target) + target.chmod(0o600) + # Remove only the previous guardian; adopted MySQL/backend processes keep running. + ctl('bootout', DOMAIN + '/' + LABEL, check=False) + atomic(CONFIG, cfg) + logs = HOME / 'logs' + logs.mkdir(mode=0o700, exist_ok=True) + plist = {'Label': LABEL, 'ProgramArguments': [str(Path(sys.executable).resolve()), str(target), 'run'], + 'WorkingDirectory': str(HOME), 'RunAtLoad': True, 'KeepAlive': True, + 'ThrottleInterval': 10, 'ProcessType': 'Background', 'ExitTimeOut': 30, + 'AbandonProcessGroup': True, + 'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'}, + 'StandardOutPath': str(logs / 'launchd.log'), 'StandardErrorPath': str(logs / 'launchd-error.log')} + PLIST.parent.mkdir(exist_ok=True) + with PLIST.open('wb') as stream: + plistlib.dump(plist, stream) + PLIST.chmod(0o600) + ctl('enable', DOMAIN + '/' + LABEL) + for attempt in range(50): + loaded=ctl('bootstrap', DOMAIN, str(PLIST), check=False) + if loaded.returncode==0:break + time.sleep(.2) + else:raise RuntimeError('Service registration did not complete; existing processes preserved') + if clear_maintenance_pause: + (root/'.runtime/retention/paused').unlink(missing_ok=True) + print('Installed login startup and continuous guardian:', PLIST) + + +def status(): + loaded = ctl('print', DOMAIN + '/' + LABEL, check=False) + state_path = HOME / 'state.json' + state = json.loads(state_path.read_text()) if state_path.exists() else {} + output = {'launchdLoaded': loaded.returncode == 0, 'stateFresh': time.time() - state.get('updatedAt', 0) < 20, + 'databaseReady': state.get('databaseReady'), 'backendReady': state.get('backendReady'), + 'supervisorPid': state.get('supervisorPid'), 'starts': state.get('starts')} + output['processes'] = {k: {'pid': v['pid'], 'confirmed': matches(v)} for k, v in state.get('processes', {}).items()} + print(json.dumps(output, ensure_ascii=False, indent=2)) + + +def stop_services(): + cfg = json.loads(CONFIG.read_text()) + guardian = Guardian(cfg) + guardian.preflight() + for role in ('backend', 'mysql'): + if not guardian.adopt(role): + continue + info = guardian.records[role] + if matches(info) is not True: + raise RuntimeError('Cannot confirm process identity') + psutil.Process(info['pid']).terminate() + deadline = time.monotonic() + 300 + while matches(info) is True and time.monotonic() < deadline: + time.sleep(.2) + if matches(info) is not False: + raise RuntimeError('Graceful shutdown incomplete; no forced termination performed') + print(role + ' stopped gracefully') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=['install', 'run', 'status', 'start', 'pause', 'stop']) + parser.add_argument('--root') + parser.add_argument('--jar') + args = parser.parse_args() + if args.action == 'install': + if not args.root or not args.jar: + parser.error('install requires --root and --jar') + install(args.root, args.jar) + elif args.action == 'status': + status() + elif args.action in ('pause', 'stop'): + cfg = json.loads(CONFIG.read_text()) + atomic(Path(cfg['root'])/'.runtime/retention/paused', {'since':time.time()}) + ctl('disable', DOMAIN + '/' + LABEL) + ctl('bootout', DOMAIN + '/' + LABEL, check=False) + if args.action == 'stop': + stop_services() + print('Automatic restart disabled until the start command is used') + elif args.action == 'start': + cfg = json.loads(CONFIG.read_text()) + (Path(cfg['root'])/'.runtime/retention/paused').unlink(missing_ok=True) + ctl('enable', DOMAIN + '/' + LABEL) + if ctl('print', DOMAIN + '/' + LABEL, check=False).returncode: + ctl('bootstrap', DOMAIN, str(PLIST)) + else: + ctl('kickstart', DOMAIN + '/' + LABEL) + status() + else: + logs = HOME / 'logs' + logs.mkdir(parents=True, exist_ok=True, mode=0o700) + handler = RotatingFileHandler(logs / 'guardian.log', maxBytes=5*1024*1024, backupCount=4) + handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel(logging.INFO) + Guardian(json.loads(CONFIG.read_text())).run() + + +if __name__ == '__main__': + main() diff --git a/deploy/macos/test_native_operations.py b/deploy/macos/test_native_operations.py new file mode 100644 index 0000000..7d188e9 --- /dev/null +++ b/deploy/macos/test_native_operations.py @@ -0,0 +1,120 @@ +"""Verify the actual JDK, accepted logging libraries, JDBC and cross-language registry.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +import uuid +import zipfile + +ROOT=Path(__file__).resolve().parents[2] +sys.path.insert(0,str(ROOT/'scripts')) +import backup +import backup_retention as br +import managed_test_resources as resources + + +HARNESS=r''' +import java.sql.*; +import javax.sql.DataSource; +import cn.yunsheng.quality.testsupport.OwnedTestDatabase; +public class TestDbProbe { + static class DS implements DataSource { + public Connection getConnection() throws SQLException {return DriverManager.getConnection(System.getenv("TEST_DB_URL"),"root",System.getenv("TEST_DB_PASSWORD"));} + public Connection getConnection(String u,String p)throws SQLException{return DriverManager.getConnection(System.getenv("TEST_DB_URL"),u,p);} + public java.io.PrintWriter getLogWriter(){return null;} public void setLogWriter(java.io.PrintWriter w){} + public void setLoginTimeout(int t){} public int getLoginTimeout(){return 5;} + public java.util.logging.Logger getParentLogger(){return java.util.logging.Logger.getGlobal();} + public T unwrap(Class c)throws SQLException{throw new SQLException();} public boolean isWrapperFor(Class c){return false;} + } + public static void main(String[] a)throws Exception{ + Class.forName("com.mysql.cj.jdbc.Driver");DS ds=new DS();String name=a[1]; + if(a[0].equals("normal")){OwnedTestDatabase.create(ds,name);OwnedTestDatabase.drop(ds,name);} + else if(a[0].equals("empty")){ + OwnedTestDatabase.createEmpty(ds,name); + try(Connection c=ds.getConnection();Statement s=c.createStatement();ResultSet rs=s.executeQuery("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='"+name+"'")){ + rs.next();if(rs.getInt(1)!=0)throw new AssertionError("Fresh schema is not empty");} + OwnedTestDatabase.seal(ds,name);OwnedTestDatabase.drop(ds,name); + }else OwnedTestDatabase.create(ds,name); + System.out.println("PROBE_OK"); + } +} +''' + +LOG_PROBE=r''' +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.status.Status; +import java.nio.file.*; +public class LogProbe { + public static void main(String[] args)throws Exception{ + LoggerContext c=new LoggerContext();c.setMDCAdapter(new ch.qos.logback.classic.util.LogbackMDCAdapter());c.putProperty("QUALITY_LOG_DIR",args[1]); + JoranConfigurator config=new JoranConfigurator();config.setContext(c);config.doConfigure(args[0]); + for(Status s:c.getStatusManager().getCopyOfStatusList())if(s.getLevel()>=Status.ERROR)throw new AssertionError(s.toString()); + var logger=c.getLogger("probe"); + for(int i=0;i<12000;i++)logger.info("{} {}",java.util.UUID.randomUUID(),java.util.UUID.randomUUID()); + Thread.sleep(1500); + for(Status s:c.getStatusManager().getCopyOfStatusList())if(s.getLevel()>=Status.ERROR)throw new AssertionError(s.toString()); + c.stop();System.out.println("LOG_PROBE_OK"); + } +} +''' + + +class NativeOperationsTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Accepted local JDK/MySQL runtime is not configured') + cls.temp=tempfile.TemporaryDirectory();cls.build=Path(cls.temp.name).resolve();cls.libs=cls.build/'lib';cls.libs.mkdir() + cfg=json.loads((Path.home()/'Library/Application Support/QualityCenterService/service.json').read_text()) + jar=ROOT/'.runtime/backend-builds'/(cfg['jarSha256']+'.jar') + with zipfile.ZipFile(jar) as z: + for n in z.namelist(): + if n.startswith('BOOT-INF/lib/') and any(x in Path(n).name for x in ['logback-','slf4j-api-','spring-boot-4.','spring-core-','mysql-connector-']): + (cls.libs/Path(n).name).write_bytes(z.read(n)) + (cls.build/'TestDbProbe.java').write_text(HARNESS);(cls.build/'LogProbe.java').write_text(LOG_PROBE) + cls.java=ROOT/'.runtime/jdk/Contents/Home/bin/java';cls.javac=cls.java.with_name('javac') + cls.classes=cls.build/'classes';cls.classes.mkdir() + helper=ROOT/'backend/src/test/java/cn/yunsheng/quality/testsupport/OwnedTestDatabase.java' + p=subprocess.run([str(cls.javac),'-cp',str(cls.libs/'*'),'-d',str(cls.classes),str(helper),str(cls.build/'TestDbProbe.java'),str(cls.build/'LogProbe.java')],text=True,capture_output=True) + if p.returncode:raise RuntimeError(p.stderr) + @classmethod + def tearDownClass(cls):cls.temp.cleanup() + + def test_java_registration_normal_drop_fresh_schema_and_expired_orphan_cleanup(self): + cfg=json.loads((ROOT/'.runtime/local-config.json').read_text()) + with tempfile.TemporaryDirectory() as directory: + root=Path(directory).resolve();br.atomic(root/'.runtime/local-config.json',{'mysqlDatabase':cfg['mysqlDatabase']}) + env=os.environ.copy();env.update({'TEST_DB_URL':f"jdbc:mysql://127.0.0.1:{cfg['mysqlPort']}/?useSSL=false&allowPublicKeyRetrieval=true",'TEST_DB_PASSWORD':cfg['rootPassword']}) + names=[] + try: + for mode in ['normal','empty','orphan']: + name='quality_sales_type_test_'+uuid.uuid4().hex;names.append(name) + p=subprocess.run([str(self.java),'-cp',str(self.classes)+os.pathsep+str(self.libs/'*'),'TestDbProbe',mode,name],cwd=root,env=env,text=True,capture_output=True,timeout=30) + self.assertEqual(p.returncode,0,p.stderr);self.assertIn('PROBE_OK',p.stdout) + if mode!='orphan':self.assertIsNone(resources.load(root,name)) + else: + record=resources.load(root,name);self.assertTrue(record['ready']) + self.assertFalse(resources.active_process(record['ownerPid'],record['ownerCreated'])) + result=resources.cleanup(root,backup.client,record['expiresAt']+1) + self.assertEqual(result['testDatabasesRemoved'],1);self.assertIsNone(resources.load(root,name)) + finally: + for name in names:backup.client('DROP DATABASE IF EXISTS `'+name+'`;') + + def test_actual_logback_configuration_rolls_and_bounds_files(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);xml=(ROOT/'deploy/macos/logback-quality.xml').read_text() + # Exercise identical policy at a smaller scale, retaining asynchronous-roll headroom. + self.assertIn('10MB',xml);self.assertIn('80MB',xml) + config=root/'logback.xml';config.write_text(xml.replace('10MB','10KB').replace('80MB','80KB')) + p=subprocess.run([str(self.java),'-cp',str(self.classes)+os.pathsep+str(self.libs/'*'),'LogProbe',str(config),str(root)],text=True,capture_output=True,timeout=30) + self.assertEqual(p.returncode,0,p.stderr+p.stdout);self.assertIn('LOG_PROBE_OK',p.stdout) + archives=list(root.glob('backend.*.log.gz'));self.assertTrue(archives) + self.assertLessEqual(sum(f.stat().st_size for f in archives),90*1024) + self.assertLessEqual(sum(f.stat().st_size for f in archives)+(root/'backend.log').stat().st_size,100*1024) + self.assertLessEqual((root/'backend.log').stat().st_size,12*1024) + + +if __name__=='__main__':unittest.main() diff --git a/deploy/macos/test_operations.py b/deploy/macos/test_operations.py new file mode 100644 index 0000000..ea82161 --- /dev/null +++ b/deploy/macos/test_operations.py @@ -0,0 +1,195 @@ +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest +import uuid +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts')) +import backup +import backup_retention as br +import log_retention as logs +import managed_test_resources as resources +spec = importlib.util.spec_from_file_location('quality_operations', Path(__file__).with_name('operations.py')) +ops = importlib.util.module_from_spec(spec); spec.loader.exec_module(ops) + + +class OperationsTests(unittest.TestCase): + def sample(self, **extra): + return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB, + 'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra} + + def test_capacity_levels_free_space_failures_and_growth_are_separate(self): + self.assertEqual(ops.conditions(self.sample(), ops.DEFAULTS, {}, []), {}) + self.assertEqual(ops.conditions(self.sample(coreBytes=30*ops.GIB), ops.DEFAULTS, {}, [])['capacity']['severity'], 'warning') + self.assertIn('temporary',ops.conditions(self.sample(temporaryBytes=30*ops.GIB),ops.DEFAULTS,{},[])) + self.assertIn('artifacts',ops.conditions(self.sample(artifactBytes=50*ops.GIB),ops.DEFAULTS,{},[])) + found = ops.conditions(self.sample(coreBytes=50*ops.GIB, freeBytes=19*ops.GIB, overdueTestResources=1), + ops.DEFAULTS, {'consecutiveFailures': 3}, []) + self.assertEqual(set(found), {'capacity', 'disk', 'maintenance', 'tests'}) + old = self.sample(at=200000-86400, permanentBytes=30*ops.GIB) + self.assertIn('growth', ops.conditions(self.sample(), ops.DEFAULTS, {}, [old])) + self.assertNotIn('maintenance', ops.conditions(self.sample(), ops.DEFAULTS, {'lastDeferred': {'reason': 'active'}}, [])) + old = self.sample(at=200000-86400) + self.assertIn('growth',ops.conditions(self.sample(coreBytes=15*ops.GIB,permanentBytes=43*ops.GIB),ops.DEFAULTS,{},[old])) + + def test_monitor_failure_notifies_after_three_attempts_without_repeated_noise(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d).resolve();sent=[] + def notify(*_):sent.append(True);return {'submitted':True} + for _ in range(4):ops.failed_run(root,RuntimeError('not logged'),notify) + self.assertEqual(len(sent),1) + self.assertEqual(json.loads((root/'.runtime/operations/failure.json').read_text())['count'],4) + + def test_alert_deduplication_escalation_recovery_and_failed_delivery_retry(self): + sent = [] + def notify(title, message): sent.append(message); return {'submitted': True} + warning = {'capacity': {'severity': 'warning', 'message': 'warning'}} + first, events = ops.dispatch(warning, {}, 100, ops.DEFAULTS, notify) + second, events = ops.dispatch(warning, first, 200, ops.DEFAULTS, notify) + self.assertEqual(len(sent), 1); self.assertFalse(events) + critical = {'capacity': {'severity': 'critical', 'message': 'critical'}} + third, _ = ops.dispatch(critical, second, 300, ops.DEFAULTS, notify) + self.assertEqual(len(sent), 2) + _, events = ops.dispatch({}, third, 400, ops.DEFAULTS, notify) + self.assertEqual(events[0]['kind'], 'resolved') + failed, _ = ops.dispatch(warning, {}, 500, ops.DEFAULTS, lambda *_: {'submitted': False}) + ops.dispatch(warning, failed, 600, ops.DEFAULTS, notify) + self.assertEqual(len(sent), 3) + + def test_registered_and_private_temporary_storage_is_not_permanent_growth(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d).resolve(); support = root / 'support' + name = 'quality_retention_test_' + uuid.uuid4().hex[:12] + for path in [root/'.runtime/mysql-data/current/data.ibd', root/'.runtime/mysql-data'/name/'sample.ibd', + root/'output/backups/current/file.gz', root/'runtimedata/archive/source.xlsx']: + path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(b'x'*8192) + br.atomic(resources.registry(root)/(name+'.json'), {'kind':'quality-owned-test-db','root':str(root), + 'database':name,'token':'1'*32,'created':True,'expiresAt':time.time()+86400}) + private=root/'.runtime/retention/verify-fixture';private.mkdir(parents=True) + (private/'data').write_bytes(b'x'*16384) + br.atomic(private/'owner.json',{'kind':'private-quality-restore','directory':str(private)}) + sample=ops.snapshot(root,support) + self.assertGreater(sample['registeredTestBytes'],0); self.assertGreater(sample['privateVerifierBytes'],0) + self.assertLess(sample['coreBytes'],sample['mysqlBytes']+sample['backupBytes']+sample['archiveBytes']) + + def test_monitor_still_collects_when_housekeeping_is_deferred_and_history_is_bounded(self): + with tempfile.TemporaryDirectory() as d: + root=Path(d).resolve();base=root/'.runtime/operations';base.mkdir(parents=True) + br.atomic(base/'policy.json',ops.DEFAULTS) + history=[self.sample(at=time.time()-i*100) for i in range(450)] + br.atomic(base/'samples.json',history) + with patch.object(ops,'snapshot',return_value=self.sample(at=time.time())), \ + patch.object(ops,'janitor',return_value={'deferred':'Application jobs are active'}): + result=ops.run(root,root/'support',lambda *_:{'submitted':True}) + self.assertIn('sample',result);self.assertIn('deferred',result['cleanup']) + self.assertLessEqual(len(json.loads((base/'samples.json').read_text())),330) + + +class LogRetentionTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve();self.support=self.root/'support' + def tearDown(self):self.temp.cleanup() + def file(self,name,age=0,size=100): + p=self.root/'output/logs'/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(b'line\n'*(size//5)) + os.utime(p,(time.time()-age,time.time()-age));return p + + def test_old_closed_logs_removed_but_active_symlinks_and_business_files_preserved(self): + old=self.file('old.log',age=8*86400);active=self.file('backend.log',age=8*86400) + other=self.file('worker.log',age=8*86400);outside=self.root/'important.xlsx';outside.write_bytes(b'keep') + link=old.with_name('link.log');link.symlink_to(outside) + result=logs.retain(self.root,self.support,open_paths={str(other)}) + self.assertEqual(result['logsRemoved'],1);self.assertFalse(old.exists()) + self.assertTrue(active.exists());self.assertTrue(other.exists());self.assertTrue(outside.exists());self.assertTrue(link.is_symlink()) + + def test_per_class_cap_prunes_old_archives_and_bounds_a_closed_console_tail(self): + old=self.file('worker.log.1',age=100,size=100);new=self.file('worker.log',age=0,size=100) + logs.retain(self.root,self.support,open_paths=set(),max_bytes=150) + self.assertFalse(old.exists());self.assertTrue(new.exists()) + big=self.file('huge.log',size=1000) + logs.retain(self.root,self.support,open_paths=set(),max_bytes=100) + self.assertLessEqual(big.stat().st_size,100) + + def test_mysql_rotation_uses_reopen_and_recovers_after_interruption(self): + source=self.file('mysql-server.log',age=86400,size=100);original=source.read_bytes() + with self.assertRaises(RuntimeError): + logs.rotate_mysql(self.root,lambda *_:(_ for _ in ()).throw(RuntimeError('interrupted')),br) + self.assertFalse(source.exists());self.assertTrue((self.root/'.runtime/operations/mysql-log-rotation.json').exists()) + calls=[] + def reopen(sql,_):calls.append(sql);source.write_bytes(b'new log') + self.assertEqual(logs.rotate_mysql(self.root,reopen,br),1) + self.assertEqual(calls,['FLUSH ERROR LOGS;']) + archived=list(source.parent.glob('mysql-server.roll-*.log'));self.assertEqual(archived[0].read_bytes(),original) + self.assertEqual(source.read_bytes(),b'new log') + + +class ManagedResourceTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured') + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve() + cfg=json.loads((ROOT/'.runtime/local-config.json').read_text()) + br.atomic(self.root/'.runtime/local-config.json',{'mysqlDatabase':cfg['mysqlDatabase']}) + self.names=[];self.users=[] + def tearDown(self): + for n in self.names:backup.client('DROP DATABASE IF EXISTS `'+n+'`;') + for u in self.users:backup.client("DROP USER IF EXISTS '"+u+"'@'127.0.0.1';") + self.temp.cleanup() + def create(self,empty=False): + name='quality_retention_test_'+uuid.uuid4().hex[:12];self.names.append(name) + resources.create(self.root,name,backup.client,empty=empty);return name + + def test_successful_create_and_drop_unregisters_only_its_schema(self): + name=self.create();self.assertTrue(resources.load(self.root,name)['ready']) + resources.drop(self.root,name,backup.client);self.assertIsNone(resources.load(self.root,name)) + + def test_expired_dead_owner_is_cleaned_but_live_owner_is_protected(self): + name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1 + self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0) + with patch.object(resources,'active_process',return_value=False): + self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],1) + self.assertIsNone(resources.load(self.root,name)) + + def test_unregistered_schema_wrong_server_and_wrong_marker_are_never_dropped(self): + name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1 + value['serverUuid']='different';br.atomic(resources.registry(self.root)/(name+'.json'),value) + with patch.object(resources,'active_process',return_value=False): + self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0) + with self.assertRaises(ValueError):resources.drop(self.root,name,backup.client) + value['serverUuid']=backup.client('SELECT @@server_uuid;').strip();br.atomic(resources.registry(self.root)/(name+'.json'),value) + backup.client('UPDATE `'+name+'`.`'+resources.MARKER+"` SET token='different';") + with patch.object(resources,'active_process',return_value=False): + self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0) + with self.assertRaises(ValueError):resources.create(self.root,'quality_platform',backup.client) + + def test_empty_schema_stays_empty_until_migrations_finish_then_seals(self): + name=self.create(empty=True) + self.assertEqual(backup.client('SHOW TABLES;',name).strip(),'') + backup.client('CREATE TABLE business_data(id INT);',name) + resources.seal(self.root,name,backup.client) + self.assertTrue(resources.marker_matches(resources.load(self.root,name),backup.client)) + resources.drop(self.root,name,backup.client) + + def test_empty_failed_schema_can_be_dropped_by_its_current_owner(self): + name=self.create(empty=True);resources.drop(self.root,name,backup.client) + self.assertIsNone(resources.load(self.root,name)) + + def test_registered_test_account_is_cleaned_after_expired_schema(self): + name=self.create();user='qverify_'+uuid.uuid4().hex[:12];self.users.append(user) + backup.client("CREATE USER '"+user+"'@'127.0.0.1' IDENTIFIED BY 'disposable-test-account';") + backup.client('GRANT ALL ON `'+name+"`.* TO '"+user+"'@'127.0.0.1';") + resources.attach_user(self.root,name,user) + with patch.object(resources,'active_process',return_value=False): + result=resources.cleanup(self.root,backup.client,resources.load(self.root,name)['expiresAt']+1) + self.assertEqual(result['testDatabasesRemoved'],1);self.assertEqual(result['testAccountsRemoved'],1) + self.assertIsNone(resources.load(self.root,name)) + + +if __name__=='__main__':unittest.main() diff --git a/deploy/macos/test_retention.py b/deploy/macos/test_retention.py new file mode 100644 index 0000000..3f33398 --- /dev/null +++ b/deploy/macos/test_retention.py @@ -0,0 +1,305 @@ +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import tarfile +import gzip +from contextlib import contextmanager +import unittest +import uuid +from unittest.mock import patch + +ROOT=Path(__file__).resolve().parents[2] +sys.path.insert(0,str(ROOT/'scripts')) +import backup_retention as br +spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py')) +ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret) + + +def good_backup(root, suffix, created='2026-11-09T01:00:00+00:00'): + folder=root/'output/backups'/('20261109T010000Z-'+suffix) + folder.mkdir(parents=True) + m={'formatVersion':3,'status':'ISOLATED_RESTORE_VERIFIED','createdAt':created, + 'restoreVerifiedAt':created,'queryEvidence':{'releaseId':'current'}, + 'verification':{k:True for k in br.CHECKS}} + for name,key in br.ARTIFACTS.items(): + (folder/name).write_bytes(name.encode());m[key]=br.sha(folder/name) + br.atomic(folder/'manifest.json',m) + return folder.resolve() + + +class BackupRotationTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name) + def tearDown(self):self.temp.cleanup() + + def test_failed_or_corrupted_new_backup_keeps_previous(self): + old=good_backup(self.root,'11111111');br.rotate(self.root,old) + new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00') + (new/'database.sql.gz').write_bytes(b'corrupted') + with self.assertRaises(ValueError):br.rotate(self.root,new) + self.assertTrue(old.is_dir());self.assertEqual(br.latest(self.root)[0],old) + + def test_rotation_keeps_one_and_verifying_older_never_rolls_back_latest(self): + old=good_backup(self.root,'11111111');br.rotate(self.root,old) + new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00');br.rotate(self.root,new) + self.assertFalse(old.exists());self.assertEqual(br.latest(self.root)[0],new) + older=good_backup(self.root,'33333333');br.rotate(self.root,older) + self.assertEqual(br.latest(self.root)[0],new);self.assertFalse(older.exists()) + + def test_unknown_and_active_incomplete_folders_are_preserved(self): + old=good_backup(self.root,'11111111');unknown=old.parent/'unrelated';unknown.mkdir() + incomplete=old.parent/'20261109T010000Z-abcdefab';incomplete.mkdir() + br.atomic(incomplete/'.owner.json',{'kind':'quality-backup','root':str(self.root),'id':incomplete.name}) + new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00');br.rotate(self.root,new) + self.assertTrue(unknown.exists());self.assertTrue(incomplete.exists()) + + def test_shared_lock_excludes_another_process(self): + code='import sys;from pathlib import Path;sys.path.insert(0,sys.argv[1]);import backup_retention as b\ntry:\n with b.lock(Path(sys.argv[2]),timeout=0):pass\nexcept b.Busy:sys.exit(17)' + with br.lock(self.root): + child=subprocess.run([sys.executable,'-c',code,str(ROOT/'scripts'),str(self.root)],capture_output=True) + self.assertEqual(child.returncode,17) + + def test_managed_backup_root_cannot_be_redirected_outside_project(self): + with tempfile.TemporaryDirectory() as external: + folder=good_backup(Path(external),'11111111') + (self.root/'output').mkdir();(self.root/'output/backups').symlink_to(folder.parent,target_is_directory=True) + with self.assertRaises(ValueError):br.rotate(self.root,folder) + self.assertTrue(folder.is_dir()) + + def test_interrupted_old_backup_removal_can_resume_without_touching_latest(self): + old=good_backup(self.root,'11111111');br.rotate(self.root,old) + new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00') + with patch.object(br.shutil,'rmtree',side_effect=OSError('simulated removal failure')): + br.rotate(self.root,new) + self.assertEqual(br.latest(self.root)[0],new) + self.assertEqual(len(list((self.root/'.runtime/retention/retired').glob('*.json'))),1) + br.clean_retired(self.root) + self.assertFalse(list((self.root/'.runtime/retention/retired').glob('*.json'))) + self.assertTrue(new.is_dir()) + + +class DatabaseRetentionTests(unittest.TestCase): + """Real MySQL locking/FK tests in a newly owned, tiny disposable database.""" + @classmethod + def setUpClass(cls): + if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured') + cfg=json.loads((ROOT/'.runtime/local-config.json').read_text()) + cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True) + cls.name='quality_retention_test_'+uuid.uuid4().hex[:12] + with cls.admin.cursor() as c: + c.execute('CREATE DATABASE '+ret.ident(cls.name)) + c.execute('USE '+ret.ident(cls.name)) + ddl=[ + 'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))', + 'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))', + 'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)', + 'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))', + 'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))', + 'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))', + 'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)', + 'CREATE TABLE import_run(id int PRIMARY KEY,status varchar(24),expected_current_release_id varchar(36),result_release_id varchar(36),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id),FOREIGN KEY(result_release_id) REFERENCES data_release(id))', + 'CREATE TABLE release_action(id int PRIMARY KEY,release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))', + 'CREATE TABLE candidate_reconciliation(id int PRIMARY KEY,release_id varchar(36),baseline_release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(baseline_release_id) REFERENCES data_release(id))', + 'CREATE TABLE analysis_session(id int PRIMARY KEY,release_id varchar(36))', + 'CREATE TABLE saved_business_object(id int PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))', + 'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))', + 'CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),parser_config_hash varchar(64),file_size bigint)', + 'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),source_slot varchar(100),source_path varchar(1000),archive_path varchar(1000),sha256 char(64),parser_config_hash varchar(64),expected_current_release_id varchar(36))', + 'CREATE TABLE source_upload_draft(id varchar(36) PRIMARY KEY,source_file varchar(100),source_path varchar(1000),sha256 char(64),file_size bigint,candidate_id varchar(36),created_at datetime,expires_at datetime)'] + for statement in ddl:c.execute(statement+' ENGINE=InnoDB') + + @classmethod + def tearDownClass(cls): + with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name)) + cls.admin.close() + + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name) + self.db=ret.Database(ROOT,self.name);self.db.root=self.root + self.db.execute('SET FOREIGN_KEY_CHECKS=0') + for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table)) + self.db.execute('SET FOREIGN_KEY_CHECKS=1') + self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')") + self.db.execute("INSERT INTO current_release VALUES('quality','current')") + self.db.execute("INSERT INTO transform_build VALUES('shared','COMPLETED',%s),('obsolete','COMPLETED',%s)",('1'*64,'2'*64)) + self.db.execute("INSERT INTO release_member VALUES('current','shared'),('old','shared'),('old','obsolete')") + self.db.execute("INSERT INTO fact_black VALUES(1,'shared','current data'),(2,'obsolete','old data'),(3,'obsolete','old data')") + + def tearDown(self):self.db.conn.close();self.temp.cleanup() + + def retire_old(self): + state={'releases':{'old':1}} + with self.db.frozen('current'):self.assertEqual(self.db.release_step('current',state,1800,4000),1) + + def test_running_queued_retry_and_unknown_tasks_prevent_cleanup(self): + for status,retry in [('RUNNING',False),('QUEUED',False),('FAILED',True),('FUTURE_STATE',False)]: + self.db.execute('DELETE FROM task') + self.db.execute('INSERT INTO task VALUES(1,%s,%s,0,3)',(status,datetime_from_future() if retry else None)) + with self.assertRaises(ret.Busy): + with self.db.frozen('current'):self.fail('busy job was ignored') + self.assertEqual(len(self.db.rows('SELECT * FROM data_release')),2) + + def test_pending_candidate_and_saved_business_references_are_protected(self): + self.db.execute("INSERT INTO data_release VALUES('candidate','CANDIDATE','old','old')") + with self.db.frozen('current'): + self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0) + self.db.execute("DELETE FROM data_release WHERE id='candidate'") + self.db.execute("INSERT INTO saved_business_object VALUES(1,'old')") + with self.db.frozen('current'): + self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0) + + def test_analysis_session_reference_is_protected_without_database_fk(self): + self.db.execute("INSERT INTO analysis_session VALUES(1,'old')") + with self.db.frozen('current'): + self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0) + + def test_pending_source_configuration_preserves_its_old_baseline(self): + self.db.execute("INSERT INTO source_candidate VALUES('pending','DISCOVERED','slot','path',NULL,%s,'legacy','old')",('3'*64,)) + with self.db.frozen('current'): + self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0) + + def test_unknown_business_build_reference_is_preserved(self): + self.retire_old();self.db.execute("INSERT INTO unknown_build_use VALUES(1,'obsolete')") + with self.db.frozen('current'): + self.assertEqual(self.db.build_step({'builds':{'obsolete':1}},1800,4000,1),(0,0)) + self.assertEqual(self.db.rows("SELECT status FROM transform_build WHERE id='obsolete'")[0]['status'],'COMPLETED') + + def test_changed_publication_and_grace_period_prevent_cleanup(self): + with self.assertRaises(ret.Busy): + with self.db.frozen('old'):pass + with self.db.frozen('current'): + self.assertEqual(self.db.release_step('current',{'releases':{'old':3900}},1800,4000),0) + + def test_old_release_removed_but_current_membership_and_rows_preserved(self): + self.db.execute("INSERT INTO import_run VALUES(1,'COMPLETED','old','old')") + self.db.execute("INSERT INTO release_action VALUES(1,'old',NULL)") + self.db.execute("INSERT INTO candidate_reconciliation VALUES(1,'current','old')") + self.retire_old() + self.assertEqual(self.db.rows("SELECT payload FROM fact_black WHERE build_id='shared'")[0]['payload'],'current data') + self.assertEqual(self.db.rows("SELECT * FROM release_member"),[{'release_id':'current','transform_build_id':'shared'}]) + self.assertIsNone(self.db.rows('SELECT * FROM import_run')[0]['expected_current_release_id']) + self.assertIsNone(self.db.rows("SELECT * FROM data_release WHERE id='current'")[0]['parent_release_id']) + + def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self): + self.retire_old();state={'builds':{'obsolete':1}} + with self.db.frozen('current'): + self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0)) + old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0] + self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64) + self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,)) + for _ in range(4): + with self.db.frozen('current'):self.db.build_step(state,1800,4000,1) + self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")) + self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'")) + self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'")) + + def add_draft(self,candidate=None,where=None,age=25): + path=where or self.root/'runtimedata/archive/upload-drafts/upload-fixture.xlsx' + path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'workbook bytes') + self.db.execute('INSERT INTO source_upload_draft VALUES(%s,%s,%s,%s,%s,%s,TIMESTAMPADD(HOUR,-30,CURRENT_TIMESTAMP),TIMESTAMPADD(HOUR,%s,CURRENT_TIMESTAMP))',('draft','input.xlsx',str(path),br.sha(path),path.stat().st_size,candidate,-age)) + return path + + def test_expired_unbound_upload_removed_but_fresh_preview_is_kept(self): + path=self.add_draft() + count,size=ret.cleanup_drafts(self.db,'current',{},time.time(),br) + self.assertEqual(count,1);self.assertGreater(size,0);self.assertFalse(path.exists()) + self.add_draft(age=0) + self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br),(0,0)) + + def test_upload_still_referenced_by_pending_candidate_is_kept(self): + path=self.add_draft(candidate='pending') + self.db.execute("INSERT INTO source_candidate VALUES('pending','DISCOVERED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path))) + self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br)[0],1) + self.assertTrue(path.exists()) + + def test_imported_upload_is_rebound_to_verified_archive_before_cleanup(self): + path=self.add_draft(candidate='imported');archive=self.root/'runtimedata/archive/source-revisions/formal.xlsx';archive.parent.mkdir();archive.write_bytes(path.read_bytes()) + self.db.execute("INSERT INTO source_document VALUES('doc','slot')") + self.db.execute("INSERT INTO source_revision VALUES('revision','doc',%s,%s,'legacy',%s)",(str(archive),br.sha(archive),archive.stat().st_size)) + self.db.execute("INSERT INTO source_candidate VALUES('imported','IMPORTED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path))) + self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br)[0],1) + self.assertFalse(path.exists());self.assertTrue(archive.exists()) + self.assertEqual(self.db.rows('SELECT source_path FROM source_candidate')[0]['source_path'],str(archive)) + + def test_outside_file_is_never_deleted(self): + path=self.add_draft(where=self.root/'unmanaged/private.xlsx') + ret.cleanup_drafts(self.db,'current',{},time.time(),br) + self.assertTrue(path.exists()) + + def test_symlink_is_not_followed_or_deleted(self): + outside=self.root/'important.xlsx';outside.write_bytes(b'keep this') + link=self.root/'runtimedata/archive/upload-drafts/upload-link.xlsx';link.parent.mkdir(parents=True);link.symlink_to(outside) + self.db.execute('INSERT INTO source_upload_draft VALUES(%s,%s,%s,%s,%s,NULL,TIMESTAMPADD(HOUR,-30,CURRENT_TIMESTAMP),TIMESTAMPADD(HOUR,-25,CURRENT_TIMESTAMP))',('draft','link.xlsx',str(link),br.sha(outside),outside.stat().st_size)) + ret.cleanup_drafts(self.db,'current',{},time.time(),br) + self.assertTrue(outside.exists());self.assertTrue(link.is_symlink()) + + def test_file_changed_after_checksum_is_not_deleted(self): + path=self.add_draft();original=ret.fingerprint;calls=0 + def changed(p): + nonlocal calls + calls+=1 + if calls==2:path.write_bytes(b'new file contents') + return original(p) + with patch.object(ret,'fingerprint',side_effect=changed): + count,size=ret.cleanup_drafts(self.db,'current',{},time.time(),br) + self.assertEqual((count,size),(0,0));self.assertTrue(path.exists()) + self.assertTrue(self.db.rows('SELECT * FROM source_upload_draft')) + + def test_new_reference_before_file_unlink_is_rechecked(self): + path=self.add_draft();freeze=self.db.frozen;calls=0 + @contextmanager + def insert_reference(expected): + nonlocal calls + calls+=1 + with freeze(expected): + if calls==2:self.db.execute("INSERT INTO source_candidate VALUES('new','DISCOVERED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path))) + yield + with patch.object(self.db,'frozen',side_effect=insert_reference): + ret.cleanup_drafts(self.db,'current',{},time.time(),br) + self.assertTrue(path.exists());self.assertFalse(self.db.rows('SELECT * FROM source_upload_draft')) + + def test_long_running_sql_is_not_interrupted_for_cleanup(self): + other=ret.Database(ROOT,self.name) + try: + other.conn.autocommit(False);other.rows('SELECT * FROM fact_black WHERE id=1 FOR UPDATE') + with self.assertRaises(ret.Busy): + with self.db.frozen('current'):pass + self.assertTrue(other.rows('SELECT 1'));other.conn.rollback() + finally:other.conn.close() + + +def datetime_from_future(): + from datetime import datetime,timedelta,timezone + return datetime.now(timezone.utc).replace(tzinfo=None)+timedelta(hours=1) + + +class PrivateVerifierTests(unittest.TestCase): + def test_small_backup_restores_on_private_server_and_cleans_up(self): + import backup + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary).resolve();(root/'.runtime').mkdir() + (root/'.runtime/mysql').symlink_to(ROOT/'.runtime/mysql',target_is_directory=True) + base=root/'output/backups';folder=base/'20261109T020000Z-12345678';folder.mkdir(parents=True) + source=root/'source';source.mkdir();(source/'config.json').write_text('{}') + with tarfile.open(folder/'application.tar.gz','w:gz') as tar:tar.add(source/'config.json',arcname='config.json') + with tarfile.open(folder/'archive.tar.gz','w:gz'):pass + sql="CREATE TABLE current_release(dataset_id varchar(20),release_id varchar(36)); INSERT INTO current_release VALUES('quality',NULL); CREATE TABLE source_revision(id varchar(36),archive_path varchar(1024),sha256 char(64),file_size bigint);" + with gzip.open(folder/'database.sql.gz','wt') as out:out.write(sql) + manifest={'formatVersion':3,'createdAt':'2026-11-09T02:00:00+00:00','files':{},'archiveSourceRoot':str(root/'runtimedata/archive'), + 'applicationFiles':{'config.json':br.sha(source/'config.json')},'requiredApplicationFiles':['config.json'], + 'tableCounts':{'current_release':1,'source_revision':0},'queryEvidence':{'releaseId':None,'months':[]}} + for name,key in br.ARTIFACTS.items():manifest[key]=br.sha(folder/name) + br.atomic(folder/'manifest.json',manifest) + with patch.object(backup,'ROOT',root),patch.object(backup,'BACKUP_ROOT',base),patch.object(backup,'REQUIRED_APPLICATION_FILES',('config.json',)): + result=backup.verify(folder) + self.assertEqual(result['status'],'ISOLATED_RESTORE_VERIFIED') + self.assertFalse(list((root/'.runtime/retention').glob('verify-*'))) + self.assertFalse(list((root/'.runtime/retention/scratch').glob('*.json'))) + + +if __name__=='__main__':unittest.main() diff --git a/deploy/macos/test_revision_retention.py b/deploy/macos/test_revision_retention.py new file mode 100644 index 0000000..ee95b87 --- /dev/null +++ b/deploy/macos/test_revision_retention.py @@ -0,0 +1,237 @@ +"""Exercise source retirement against an owned disposable MySQL schema and files.""" +from contextlib import contextmanager +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import time +import unittest +import uuid +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts')) +import backup_retention as br +import revision_retention as revisions +spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py')) +ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret) + + +class SourceRevisionTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured') + cfg = json.loads((ROOT / '.runtime/local-config.json').read_text()) + cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root', + password=cfg['rootPassword'], autocommit=True) + cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12] + ddl = [ + 'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))', + 'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))', + 'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))', + "CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)", + 'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', + 'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', + 'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))', + 'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))', + 'CREATE TABLE source_upload_draft(id varchar(36) PRIMARY KEY,source_path varchar(1000))', + 'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),payload json)', + 'CREATE TABLE source_change_request(id int PRIMARY KEY,original_revision_id varchar(36),status varchar(24))', + 'CREATE TABLE saved_context(id int PRIMARY KEY,context_json json,attachment_path varchar(1000))', + 'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))'] + try: + with cls.admin.cursor() as c: + c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name)) + for statement in ddl: + c.execute(statement + ' ENGINE=InnoDB') + except BaseException: + with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name)) + cls.admin.close(); raise + + @classmethod + def tearDownClass(cls): + with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name)) + cls.admin.close() + + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve() + self.db = ret.Database(ROOT, self.name); self.db.root = self.root + self.db.execute('SET FOREIGN_KEY_CHECKS=0') + for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table)) + self.db.execute('SET FOREIGN_KEY_CHECKS=1') + self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')") + self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')") + self.db.execute("INSERT INTO current_release VALUES('quality','current')") + self.current_file = self.add_revision('current-rev', b'current workbook') + self.db.execute("INSERT INTO release_member VALUES('current','current-rev')") + self.db.execute("INSERT INTO transform_build VALUES('current-build','current-rev')") + self.cleaner = revisions.Cleaner(self.db, br, ret.safe_file, ret.fingerprint, ret.Busy) + self.state = {} + + def tearDown(self): + self.db.conn.close(); self.temp.cleanup() + + def add_revision(self, rid='old-rev-unique', content=b'old workbook'): + digest = __import__('hashlib').sha256(content).hexdigest() + path = self.root / 'runtimedata/archive/source-revisions/black/2026/01' / (digest + '.xlsx') + path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(content) + self.db.execute('INSERT INTO source_revision(id,source_document_id,archive_path,sha256,file_size) VALUES(%s,%s,%s,%s,%s)', + (rid, 'doc', str(path), digest, len(content))) + return path + + def collect(self, now): + return self.cleaner.collect('current', self.state, now) + + def present(self, rid='old-rev-unique'): + return bool(self.db.rows('SELECT id FROM source_revision WHERE id=%s', (rid,))) + + def test_unreferenced_revision_waits_grace_then_removes_only_its_owned_file(self): + path = self.add_revision() + self.assertEqual(self.collect(100)['sourceRevisionsRemoved'], 0) + result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 1) + self.assertEqual(result['sourceArchiveFilesRemoved'], 1) + self.assertEqual(result['sourceArchiveBytesRemoved'], len(b'old workbook')) + self.assertFalse(self.present()); self.assertFalse(path.exists()) + self.assertTrue(self.present('current-rev')); self.assertTrue(self.current_file.exists()) + self.assertTrue(self.db.rows('SELECT * FROM source_document')) + + def test_any_release_build_fact_or_unknown_foreign_key_protects_revision(self): + for rid, statement in [ + ('candidate', "INSERT INTO release_member VALUES('candidate','candidate')"), + ('build', "INSERT INTO transform_build VALUES('old-build','build')"), + ('fact', "INSERT INTO fact_black VALUES(1,'fact',JSON_OBJECT(),JSON_OBJECT())"), + ('future', "INSERT INTO unknown_reference VALUES(1,'future')")]: + self.add_revision(rid, rid.encode()); self.db.execute(statement) + self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0) + for rid in ('candidate', 'build', 'fact', 'future'): self.assertTrue(self.present(rid)) + + def test_replacement_withdrawal_and_saved_json_ids_without_fks_are_protected(self): + for rid in ('replacement', 'withdrawal', 'lineage', 'saved'): self.add_revision(rid, rid.encode()) + self.db.execute("INSERT INTO source_candidate VALUES('candidate','DISCOVERED',NULL,NULL,NULL,'replacement')") + self.db.execute("INSERT INTO source_change_request VALUES(1,'withdrawal','APPROVED')") + self.db.execute("UPDATE source_revision SET replaces_revision_id='lineage' WHERE id='current-rev'") + self.db.execute("INSERT INTO saved_context VALUES(1,JSON_OBJECT('sourceRevisionId','saved'),NULL)") + self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0) + + def test_catalog_hash_protects_original_even_with_external_path(self): + path = self.add_revision(); digest = br.sha(path) + self.db.execute("INSERT INTO source_candidate VALUES('catalog','IMPORTED',%s,'/external/original.xlsx',NULL,NULL)", (digest,)) + self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0) + self.assertTrue(self.present()); self.assertTrue(path.exists()) + + def test_new_reference_resets_continuous_grace(self): + self.add_revision(); self.collect(100) + self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500) + self.db.execute('DELETE FROM unknown_reference') + self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0) + self.assertEqual(self.collect(3801)['sourceRevisionsRemoved'], 1) + + def test_revision_metadata_change_restarts_grace(self): + self.add_revision(); self.collect(100) + self.db.execute("UPDATE source_revision SET parser_config=JSON_OBJECT('changed',TRUE) WHERE id='old-rev-unique'") + self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0) + self.assertEqual(self.collect(3801)['sourceRevisionsRemoved'], 1) + + def test_shared_archive_stays_until_its_other_revision_is_unreferenced(self): + path = self.add_revision(content=b'current workbook') + self.collect(100); result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 1) + self.assertEqual(result['sourceArchiveFilesRemoved'], 0) + self.assertTrue(path.exists()); self.assertTrue(self.present('current-rev')) + + def test_saved_file_path_protects_file_after_metadata_removal(self): + path = self.add_revision() + self.db.execute('INSERT INTO saved_context VALUES(1,JSON_OBJECT(\'archivePath\',%s),NULL)', (str(path),)) + self.collect(100); result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 1) + self.assertEqual(result['sourceFilesPending'], 1); self.assertTrue(path.exists()) + self.db.execute('DELETE FROM saved_context') + self.assertEqual(self.cleaner.drain('current')[0], 1); self.assertFalse(path.exists()) + + def test_external_symlink_mismatched_or_incomplete_files_are_kept(self): + for rid in ('outside', 'symlink', 'damaged', 'incomplete'): self.add_revision(rid, rid.encode()) + external = self.root / 'important.xlsx'; external.write_bytes(b'outside') + self.db.execute("UPDATE source_revision SET archive_path=%s WHERE id='outside'", (str(external),)) + row = self.db.rows("SELECT archive_path FROM source_revision WHERE id='symlink'")[0] + link = Path(row['archive_path']); link.unlink(); link.symlink_to(external) + row = self.db.rows("SELECT archive_path FROM source_revision WHERE id='damaged'")[0] + Path(row['archive_path']).write_bytes(b'changed') + self.db.execute("UPDATE source_revision SET status='RECEIVED' WHERE id='incomplete'") + self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0) + for rid in ('outside', 'symlink', 'damaged', 'incomplete'): self.assertTrue(self.present(rid)) + self.assertTrue(external.exists()); self.assertTrue(link.is_symlink()) + + def test_reference_added_before_database_delete_is_rechecked(self): + path = self.add_revision(); self.collect(100); original = self.db.frozen + @contextmanager + def add_reference(current): + with original(current): + self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')") + yield + with patch.object(self.db, 'frozen', side_effect=add_reference): result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 0) + self.assertTrue(path.exists()); self.assertTrue(self.present()) + + def test_file_changed_after_hash_is_kept(self): + path = self.add_revision(); self.collect(100); original = self.db.frozen + @contextmanager + def change_file(current): + path.write_bytes(b'file edited during preflight') + with original(current): yield + with patch.object(self.db, 'frozen', side_effect=change_file): result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 0) + self.assertTrue(path.exists()); self.assertTrue(self.present()) + + def test_crash_after_database_commit_resumes_registered_file_removal(self): + path = self.add_revision(); self.collect(100) + with patch.object(self.cleaner, 'drain', side_effect=RuntimeError('interrupted')): + with self.assertRaises(RuntimeError): self.collect(2000) + self.assertFalse(self.present()); self.assertTrue(path.exists()) + self.assertEqual(self.cleaner.drain('current')[0], 1) + self.assertFalse(path.exists()); self.assertTrue(self.current_file.exists()) + + def test_database_rollback_keeps_file_and_discards_only_journal(self): + path = self.add_revision(); self.collect(100); original = self.db.execute + def rollback(sql, args=()): + result = original(sql, args) + if sql.startswith('DELETE FROM source_revision'): raise RuntimeError('rollback') + return result + with patch.object(self.db, 'execute', side_effect=rollback): + with self.assertRaises(RuntimeError): self.collect(2000) + self.assertTrue(self.present()); self.assertTrue(path.exists()) + self.assertEqual(self.cleaner.drain('current'), (0, 0)) + self.assertFalse(list(self.cleaner.journals.glob('*.json'))) + + def test_new_path_reference_before_unlink_protects_file(self): + path = self.add_revision(); self.collect(100); original = self.cleaner.drain + def add_reference(current, limit=5): + self.db.execute('INSERT INTO source_upload_draft VALUES(\'new\',%s)', (str(path),)) + return original(current, limit) + with patch.object(self.cleaner, 'drain', side_effect=add_reference): result = self.collect(2000) + self.assertEqual(result['sourceRevisionsRemoved'], 1) + self.assertEqual(result['sourceArchiveFilesRemoved'], 0); self.assertTrue(path.exists()) + + def test_running_task_or_changed_publication_defers_without_deleting(self): + path = self.add_revision(); self.collect(100) + self.db.execute("INSERT INTO task VALUES(1,'RUNNING',NULL)") + with self.assertRaises(ret.Busy): self.collect(2000) + self.assertTrue(self.present()); self.assertTrue(path.exists()) + self.db.execute('DELETE FROM task') + self.db.execute("UPDATE current_release SET release_id='different'") + with self.assertRaises(ret.Busy): self.collect(2000) + self.assertTrue(self.present()); self.assertTrue(path.exists()) + + def test_unsafe_registry_and_unknown_stray_files_are_not_deleted(self): + path = self.add_revision(); self.collect(100) + self.cleaner.journals.mkdir(parents=True) + br.atomic(self.cleaner.journals / ('1' * 32 + '.json'), {'kind': 'quality-source-retirement'}) + unknown = path.with_name('unregistered.xlsx'); unknown.write_bytes(b'keep') + self.cleaner.drain('current'); self.assertTrue(unknown.exists()); self.assertTrue(path.exists()) + with self.cleaner.quick_checks(): + self.cleaner.reference_deadline = time.monotonic() - 1 + with self.assertRaises(ret.Busy): self.cleaner.exists('source_revision', 'id=%s', ('old-rev-unique',)) + + +if __name__ == '__main__': unittest.main() diff --git a/deploy/macos/test_service.py b/deploy/macos/test_service.py new file mode 100644 index 0000000..6248447 --- /dev/null +++ b/deploy/macos/test_service.py @@ -0,0 +1,116 @@ +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest + +spec = importlib.util.spec_from_file_location('quality_macos_service', Path(__file__).with_name('service.py')) +service = importlib.util.module_from_spec(spec) +spec.loader.exec_module(service) + + +class Fixture(service.Guardian): + def command(self, role): + return ['/bin/sleep', '90' if role == 'mysql' else '91'] + + def occupied(self, role): + return False + + def environment(self): + return os.environ.copy() + + def db_ready(self): + return self.ready + + def backend_ready(self): + return service.matches(self.records.get('backend')) is True + + +class GuardianTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + (self.root / '.runtime').mkdir() + self.g = Fixture({'root': str(self.root)}, self.root / 'guardian') + self.g.value = {'mysqlPort': 59998} + self.g.ready = False + self.external = [] + + def tearDown(self): + for p in list(self.g.children.values()) + self.external: + if p.poll() is None: + p.terminate() + p.wait(timeout=10) + self.temp.cleanup() + + def test_database_must_be_ready_before_backend_starts(self): + self.g.tick() + self.assertIn('mysql', self.g.records) + self.assertNotIn('backend', self.g.records) + self.g.ready = True + self.g.tick() + self.assertTrue(service.matches(self.g.records['backend'])) + + def test_abnormal_child_exit_is_restarted_without_duplicate_mysql(self): + self.g.ready = True + self.g.tick() + mysql = self.g.records['mysql']['pid'] + old = self.g.children['backend'] + old.kill() + old.wait() + self.g.tick() + self.assertNotIn('backend', self.g.records) + self.assertEqual(self.g.records['mysql']['pid'], mysql) + self.g.next_start['backend'] = 0 + self.g.tick() + self.assertNotEqual(self.g.records['backend']['pid'], old.pid) + self.assertEqual(self.g.restarts, {'mysql': 1, 'backend': 2}) + + def test_database_exit_is_restarted_before_missing_backend(self): + self.g.tick() + old = self.g.children['mysql'] + old.kill() + old.wait() + self.g.tick() + self.assertNotIn('mysql', self.g.records) + self.g.next_start['mysql'] = 0 + self.g.tick() + self.assertNotEqual(self.g.records['mysql']['pid'], old.pid) + self.assertNotIn('backend', self.g.records) + + def test_existing_process_is_adopted_and_unknown_process_is_not_replaced(self): + command = self.g.command('mysql') + p = subprocess.Popen(command, start_new_session=True) + self.external.append(p) + info = service.record(p.pid, command) + service.atomic(self.root / '.runtime/mysql.process.json', {'pid': p.pid, 'started': info['started']}) + self.assertTrue(self.g.ensure('mysql')) + self.assertEqual(self.g.restarts['mysql'], 0) + self.assertIsNone(service.matches(info, self.g.command('backend'))) + self.assertIs(p.poll(), None) + + def test_missing_data_marker_never_initializes_database(self): + service.atomic(self.root / '.runtime/local-config.json', {'databaseReady': True}) + real = service.Guardian({'root': str(self.root), 'jarSha256': '0'*64}, self.root / 'guardian') + with self.assertRaises(RuntimeError): + real.preflight() + self.assertFalse((self.root / '.runtime/mysql-data').exists()) + + def test_pinned_jar_mismatch_is_rejected(self): + r = self.root / '.runtime' + (r / 'mysql-data').mkdir() + (r / 'mysql-data/auto.cnf').write_text('existing marker') + (r / 'backend-builds').mkdir() + (r / 'backend-builds' / ('0'*64+'.jar')).write_bytes(b'unaccepted') + service.atomic(r / 'local-config.json', {'databaseReady': True}) + real = service.Guardian({'root': str(self.root), 'jarSha256': '0'*64}, self.root / 'guardian') + with self.assertRaises(RuntimeError): + real.preflight() + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/backup.py b/scripts/backup.py index 31d4215..3b9bbd2 100644 --- a/scripts/backup.py +++ b/scripts/backup.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 """Create and verify a local backup using an isolated database and archive tree.""" import argparse +from contextlib import contextmanager +from contextvars import ContextVar from datetime import datetime, timezone import gzip import hashlib @@ -10,11 +12,13 @@ from pathlib import Path, PurePosixPath import re import shutil import subprocess +import sys import tarfile import tempfile import time import uuid import project +import backup_retention ROOT = project.ROOT BACKUP_ROOT = ROOT / 'output/backups' @@ -55,6 +59,17 @@ SAFE_FAILURES = { ('CLEANUP', 'SCRATCH_CLEANUP_FAILED'): '隔离恢复完成后,临时数据库清理失败。', ('PROTOCOL', 'BACKUP_INTERNAL_ERROR'): '备份操作未完成,请检查本机运维状态。', } +CLIENT_CONFIG_OVERRIDE = ContextVar('quality_backup_client_config', default=None) + + +def client_config(): + return CLIENT_CONFIG_OVERRIDE.get() or project.RUNTIME / 'mysql-root.cnf' + + +def verification_temp_root(): + # 校验临时文件归属于独立 MySQL 的所有权目录;异常退出后可一并安全回收。 + configuration = CLIENT_CONFIG_OVERRIDE.get() + return configuration.parent if configuration is not None else None class BackupFailure(RuntimeError): @@ -101,14 +116,14 @@ def application_files(): if missing: fail('INPUTS', 'APPLICATION_INPUT_MISSING') paths = set() - for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist'): + for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist', 'deploy/macos', 'deploy/docker'): folder = ROOT / relative for path in folder.rglob('*'): if path.is_symlink(): fail('INPUTS', 'APPLICATION_INPUT_UNSAFE') if path.is_file() and '__pycache__' not in path.parts: paths.add(path) - for relative in (*REQUIRED_APPLICATION_FILES, '启动项目.command', '停止项目.command'): + for relative in (*REQUIRED_APPLICATION_FILES, '.dockerignore', 'app/backend.sha256', '启动项目.command', '停止项目.command'): path = ROOT / relative if path.exists(): if path.is_symlink() or not path.is_file(): @@ -124,6 +139,10 @@ def application_files(): if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()): fail('INPUTS', 'APPLICATION_INPUT_UNSAFE') paths.add(path) + accepted = backup_retention.accepted_backend(ROOT) + if accepted: + paths.add(accepted[0]) + paths.update(backup_retention.native_files(ROOT)) if not set(required_paths).issubset(paths): fail('INPUTS', 'APPLICATION_INPUT_MISSING') return sorted(paths) @@ -139,6 +158,13 @@ def archive_application(folder): if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()): fail('INPUTS', 'APPLICATION_INPUT_UNSAFE') relative = path.relative_to(ROOT).as_posix() + if path.name == 'nao.sqlite': + with tempfile.TemporaryDirectory(prefix='quality-native-snapshot-') as temporary: + copied = Path(temporary) / 'nao.sqlite' + backup_retention.snapshot_sqlite(path, copied) + files[relative] = file_sha256(copied) + tar.add(copied, arcname=relative, recursive=False) + continue before = file_sha256(path) tar.add(path, arcname=relative, recursive=False) if before != file_sha256(path): @@ -153,7 +179,7 @@ def archive_application(folder): def client(sql, database=None): - args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), + args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()), '--batch', '--skip-column-names'] if database: if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', database): @@ -181,7 +207,7 @@ def counts(database): def query_evidence(database): """Reproduce release membership and monthly formal input components in SQL.""" current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip() - if not current: + if not current or current == 'NULL': return {'releaseId': None, 'months': []} if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current): fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED') @@ -332,25 +358,29 @@ def _snapshot_counts(sql): return snapshot_counts +@backup_retention.serialized def create(): # Validate the full, fixed application input set before creating any backup files. selected_application = application_files() config = database_config() verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive', ROOT / 'runtimedata/archive') before_queries = query_evidence(config['mysqlDatabase']) + before_counts = counts(config['mysqlDatabase']) folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8]) folder.mkdir(parents=True, exist_ok=False) folder.chmod(0o700) + backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()), + 'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()}) sql = folder / 'database.sql.gz' command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks', - '--skip-extended-insert', config['mysqlDatabase']] + '--extended-insert', '--net-buffer-length=1048576', config['mysqlDatabase']] process = None try: with tempfile.TemporaryFile() as errors, sql.open('wb') as output: process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors) try: - with gzip.GzipFile(fileobj=output, mode='wb') as compressed: + with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed: shutil.copyfileobj(process.stdout, compressed) process.stdout.close() if process.wait() != 0: @@ -373,7 +403,11 @@ def create(): sql.chmod(0o600) if query_evidence(config['mysqlDatabase']) != before_queries: fail('DATABASE_EXPORT', 'DATABASE_CHANGED') - table_counts = _snapshot_counts(sql) + # The dump is a single consistent transaction. Restore verification below + # independently checks these counts, including a net-zero concurrent race. + if counts(config['mysqlDatabase']) != before_counts: + fail('DATABASE_EXPORT', 'DATABASE_CHANGED') + table_counts = before_counts archive = folder / 'archive.tar.gz' source = ROOT / 'runtimedata/archive' files = {} @@ -413,6 +447,9 @@ def create(): 'archiveReferencesVerified': False, 'applicationStartupVerified': False}, 'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'}, 'tableCounts': table_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'} + accepted = backup_retention.accepted_backend(ROOT) + if accepted: + manifest['acceptedBackend'] = {'path': accepted[0].relative_to(ROOT).as_posix(), 'sha256': accepted[1]} try: project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n') except Exception: @@ -420,7 +457,7 @@ def create(): return folder -def verify(folder): +def _verify(folder): started = time.monotonic() folder = Path(folder) if folder.is_symlink(): @@ -451,7 +488,7 @@ def verify(folder): try: if file_sha256(app_archive) != manifest['applicationSha256']: fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH') - with tempfile.TemporaryDirectory(prefix='quality-restore-application-') as temp: + with tempfile.TemporaryDirectory(prefix='quality-restore-application-', dir=verification_temp_root()) as temp: Path(temp).chmod(0o700) actual_app = extract_verified_tar(app_archive, Path(temp) / 'files', expected_app) application_verified = actual_app == expected_app @@ -465,14 +502,17 @@ def verify(folder): fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID') scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8] created_scratch = False + scratch_registration = backup_retention.scratch_register(ROOT, scratch, client('SELECT @@server_uuid;').strip()) try: client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;') created_scratch = True - args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--database=' + scratch, '--init-command=SET SESSION autocommit=0'] + backup_retention.scratch_created(scratch_registration) + args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()), '--database=' + scratch, '--init-command=SET SESSION autocommit=0'] process = None try: with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors: process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors) + backup_retention.scratch_child(scratch_registration, process.pid) try: shutil.copyfileobj(data, process.stdin) process.stdin.write(b'\nCOMMIT;\n') @@ -494,7 +534,7 @@ def verify(folder): fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH') if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']: fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH') - with tempfile.TemporaryDirectory(prefix='quality-restore-archive-') as temp: + with tempfile.TemporaryDirectory(prefix='quality-restore-archive-', dir=verification_temp_root()) as temp: relocated_root = Path(temp) / 'restored' / 'runtimedata' / 'archive' relocated_root.parent.mkdir(parents=True, exist_ok=True) actual_archives = _safe_extract_tree(archive, relocated_root, manifest.get('files', {})) @@ -534,10 +574,27 @@ def verify(folder): if created_scratch: try: client('DROP DATABASE `' + scratch + '`;') + scratch_registration.unlink(missing_ok=True) except Exception: fail('CLEANUP', 'SCRATCH_CLEANUP_FAILED') + else: + scratch_registration.unlink(missing_ok=True) +@backup_retention.serialized +def verify(folder): + if sys.platform == 'darwin' or os.environ.get('QUALITY_CONTAINER') == '1': + import backup_verifier + with backup_verifier.instance(ROOT) as configuration: + token = CLIENT_CONFIG_OVERRIDE.set(configuration) + try: + return _verify(folder) + finally: + CLIENT_CONFIG_OVERRIDE.reset(token) + return _verify(folder) + + +@backup_retention.serialized def restore(folder, target, database): """Restore a verified format-v3 backup to a new project directory and new database. @@ -709,13 +766,14 @@ def restore(folder, target, database): fail('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED') -def main(argv=None): +def _main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=['create', 'verify', 'create-and-verify', 'restore']) parser.add_argument('--directory', type=Path) parser.add_argument('--target', type=Path) parser.add_argument('--database') args = parser.parse_args(argv) + previous_backups = backup_retention.backup_ids(ROOT) try: if args.action == 'verify': if not args.directory: @@ -730,17 +788,31 @@ def main(argv=None): result = {'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'} if args.action == 'create-and-verify': result = verify(folder) + if args.action in ('verify', 'create-and-verify'): + selected = args.directory if args.action == 'verify' else folder + backup_retention.rotate(ROOT, selected) print(json.dumps({'event': 'completed', 'action': args.action, **result}, ensure_ascii=False, separators=(',', ':')), flush=True) return 0 except BackupFailure as error: + backup_retention.discard_failed_new(ROOT, previous_backups) emit_failure(error) return 20 + sorted(SAFE_FAILURES).index((error.stage, error.code)) except Exception: + backup_retention.discard_failed_new(ROOT, previous_backups) error = BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR') emit_failure(error) return 1 +def main(argv=None): + try: + with backup_retention.lock(ROOT): + return _main(argv) + except backup_retention.Busy: + emit_failure(BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR')) + return 1 + + if __name__ == '__main__': raise SystemExit(main()) diff --git a/scripts/backup_retention.py b/scripts/backup_retention.py new file mode 100644 index 0000000..22a8c43 --- /dev/null +++ b/scripts/backup_retention.py @@ -0,0 +1,381 @@ +"""Shared backup/maintenance exclusion and verified-only backup rotation.""" +from contextlib import contextmanager +from datetime import datetime, timezone +from functools import wraps +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import threading +import time +import uuid + +_thread_lock = threading.RLock() +_held = {} +ID = re.compile(r'^\d{8}T\d{6}Z-[a-f0-9]{8}$') +CHECKS = ('databaseRestoreVerified', 'applicationFilesVerified', 'requiredApplicationInputsVerified', + 'archiveFilesVerified', 'archiveReferencesVerified') +ARTIFACTS = {'database.sql.gz': 'databaseDumpSha256', 'archive.tar.gz': 'archiveSha256', + 'application.tar.gz': 'applicationSha256'} + + +class Busy(RuntimeError): + pass + + +def sha(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(block) + return digest.hexdigest() + + +def atomic(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + tmp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp') + with tmp.open('w') as output: + os.chmod(tmp, 0o600) + json.dump(value, output, ensure_ascii=False, indent=2) + output.flush() + os.fsync(output.fileno()) + tmp.replace(path) + if os.name != 'nt': + fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +@contextmanager +def lock(root, timeout=1800): + """One lock for UI backups, restores, scheduled backups and garbage collection.""" + root = Path(root).resolve() + key = str(root) + with _thread_lock: + if key in _held: + yield + return + path = root / '.runtime/data-maintenance.lock' + path.parent.mkdir(parents=True, exist_ok=True) + if path.parent.is_symlink() or path.is_symlink(): + raise ValueError('Unsafe maintenance lock path') + stream = path.open('a+b') + os.chmod(path, 0o600) + deadline = time.monotonic() + timeout + try: + while True: + try: + if os.name == 'nt': + import msvcrt + stream.seek(0) + if not path.stat().st_size: + stream.write(b'0'); stream.flush() + stream.seek(0) + msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except OSError: + if time.monotonic() >= deadline: + raise Busy('Another backup or maintenance operation is active') + time.sleep(.2) + _held[key] = stream + try: + yield + finally: + _held.pop(key, None) + if os.name == 'nt': + import msvcrt + stream.seek(0); msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(stream, fcntl.LOCK_UN) + finally: + stream.close() + + +def serialized(function): + @wraps(function) + def wrapped(*args, **kwargs): + with lock(function.__globals__['ROOT']): + return function(*args, **kwargs) + return wrapped + + +def verified(folder, checksums=True): + folder = Path(folder) + if folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): + return None + try: + if (folder / 'manifest.json').is_symlink(): + return None + manifest = json.loads((folder / 'manifest.json').read_text()) + if manifest.get('formatVersion') != 3 or manifest.get('status') != 'ISOLATED_RESTORE_VERIFIED': + return None + if not all(manifest.get('verification', {}).get(c) is True for c in CHECKS): + return None + for name, field in ARTIFACTS.items(): + p = folder / name + if p.is_symlink() or not p.is_file() or not manifest.get(field): + return None + if checksums and sha(p) != manifest[field]: + return None + return manifest + except (OSError, ValueError, TypeError): + return None + + +def _rotate(root, selected): + """Publish the new verified backup first. A failed candidate never deletes old backups.""" + root = Path(root).resolve() + selected = Path(selected) + if selected.is_symlink(): + raise ValueError('Backup directory cannot be a symbolic link') + selected = selected.resolve() + base = root / 'output/backups' + if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root): + raise ValueError('Unsafe managed backup root') + if selected.parent.resolve() != base.resolve(): + raise ValueError('Backup is outside the managed backup directory') + manifest = verified(selected) + if manifest is None: + raise ValueError('Backup has not passed restore and artifact verification') + previous = latest(root) + if previous and previous[0] != selected and previous[1].get('createdAt', '') > manifest.get('createdAt', ''): + selected, manifest = previous + for name in (*ARTIFACTS, 'manifest.json'): + with (selected / name).open('r+b') as stream: + stream.flush(); os.fsync(stream.fileno()) + if os.name!='nt': + for directory in (selected,base,base.parent): + fd=os.open(directory,os.O_RDONLY) + try:os.fsync(fd) + finally:os.close(fd) + pointer = {'id': selected.name, 'manifestSha256': sha(selected / 'manifest.json'), + 'releaseId': manifest.get('queryEvidence', {}).get('releaseId'), + 'verifiedAt': manifest.get('restoreVerifiedAt'), 'publishedAt': time.time()} + atomic(root / '.runtime/retention/latest.json', pointer) + removed = [] + for folder in base.iterdir(): + if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name): + continue + # A restore-verified older folder is safe to retire under the shared lock. + # Incomplete candidates are kept for 24h and never treated as good backups. + older = verified(folder, checksums=False) + try: + owner = json.loads((folder / '.owner.json').read_text()) + owned = owner.get('kind') == 'quality-backup' and owner.get('root') == str(root) and owner.get('id') == folder.name + except (OSError, ValueError): + owned = False + stale = owned and time.time() - folder.stat().st_mtime > 86400 + if older is not None or stale: + contents = list(folder.rglob('*')) + if any(p.is_symlink() for p in contents): + continue + try: + retire_folder(root,folder,'backup') + removed.append(folder.name) + except OSError: + pass # Latest stays usable; a later maintenance pass retries retirement. + # A one-off snapshot from the previous manual cleanup is retired only when + # explicitly registered and its complete original file set still matches. + registration = root / '.runtime/retention/legacy.json' + if registration.is_file(): + legacy_info = json.loads(registration.read_text()) + if not legacy_info.get('retired') and legacy_info.get('path') == '.runtime/current-snapshot-20261109' and manifest.get('acceptedBackend'): + legacy = root / legacy_info['path'] + if legacy.is_dir() and not legacy.is_symlink() and sha(legacy/'manifest.json') == legacy_info['manifestSha256']: + old = json.loads((legacy/'manifest.json').read_text()) + expected = old.get('filesSha256', {}) + actual = {p.relative_to(legacy).as_posix() for p in legacy.rglob('*') if p.is_file()} + if actual == set(expected)|{'manifest.json'} and all(not (legacy/k).is_symlink() and sha(legacy/k)==v for k,v in expected.items()): + evidence = root/'.runtime/retention/setup-evidence' + evidence.mkdir(parents=True,exist_ok=True,mode=0o700) + for name in ('cleanup-report.json','guardian-verification.json','api-verification.json'): + if (legacy/name).is_file(): + shutil.copy2(legacy/name,evidence/name) + retire_folder(root,legacy,'legacy') + legacy_info['retired']=True + atomic(registration,legacy_info) + clean_retired(root) + return pointer, removed + + +def retire_folder(root, folder, kind): + """Atomic rename makes a crash during removal safely resumable.""" + root=Path(root).resolve();folder=Path(folder).resolve() + if kind=='backup': + if folder.parent!=root/'output/backups' or not ID.fullmatch(folder.name):raise ValueError('Unsafe retirement source') + elif kind=='legacy': + if folder!=root/'.runtime/current-snapshot-20261109':raise ValueError('Unsafe legacy source') + else:raise ValueError('Unknown retirement kind') + area=root/'.runtime/retention/retired';area.mkdir(parents=True,exist_ok=True,mode=0o700) + if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')): + raise ValueError('Unsafe backup retirement directory') + token=uuid.uuid4().hex + # Docker 的状态卷与备份卷可能是不同挂载点,重命名必须留在备份所在文件系统。 + storage = root/'output/backups/.retired' if kind=='backup' else area + if storage.is_symlink() or not storage.resolve().is_relative_to(root):raise ValueError('Unsafe retirement storage') + storage.mkdir(parents=True,exist_ok=True,mode=0o700) + atomic(area/(token+'.json'),{'kind':'quality-retired-backup','root':str(root),'target':token, + 'origin':str(folder.relative_to(root)),'storage':'backup-volume' if kind=='backup' else 'state-volume'}) + folder.rename(storage/token) + if os.name!='nt': + fd=os.open(storage,os.O_RDONLY) + try:os.fsync(fd) + finally:os.close(fd) + + +def clean_retired(root): + root=Path(root).resolve();area=root/'.runtime/retention/retired';count=0 + if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):return 0 + for record in area.glob('*.json'): + try: + if record.is_symlink():continue + value=json.loads(record.read_text());token=value.get('target','') + if value.get('kind')!='quality-retired-backup' or value.get('root')!=str(root) or not re.fullmatch(r'[a-f0-9]{32}',token):continue + storage=value.get('storage','state-volume') + if storage not in ('state-volume','backup-volume'):continue + base=root/'output/backups/.retired' if storage=='backup-volume' else area + if any(p.is_symlink() for p in (base,base.parent,base.parent.parent)) or not base.resolve().is_relative_to(root):continue + folder=base/token + if folder.is_symlink():continue + if folder.exists(): + if any(p.is_symlink() for p in folder.rglob('*')):continue + shutil.rmtree(folder);count+=1 + record.unlink() + except (OSError,ValueError):continue + return count + + +def rotate(root, selected): + with lock(root): + return _rotate(root, selected) + + +def latest(root, checksums=True): + root = Path(root).resolve() + try: + base=root/'output/backups' + if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):return None + pointer = json.loads((root / '.runtime/retention/latest.json').read_text()) + if not ID.fullmatch(pointer['id']): + return None + folder = root / 'output/backups' / pointer['id'] + if sha(folder / 'manifest.json') != pointer['manifestSha256']: + return None + manifest = verified(folder, checksums=checksums) + return (folder, manifest) if manifest else None + except (OSError, ValueError, KeyError, TypeError): + return None + + +def backup_ids(root): + base=Path(root)/'output/backups' + return {p.name for p in base.iterdir()} if base.exists() else set() + + +def discard_failed_new(root, before): + """Delete only this call's registered, incomplete candidate after it has stopped.""" + root=Path(root).resolve();base=root/'output/backups' + for folder in base.iterdir() if base.exists() else []: + if folder.name in before or not ID.fullmatch(folder.name) or folder.is_symlink() or not folder.is_dir():continue + try: + owner=json.loads((folder/'.owner.json').read_text()) + if owner.get('root')!=str(root) or owner.get('ownerPid')!=os.getpid() or owner.get('id')!=folder.name:continue + if verified(folder,checksums=False) is not None:continue + if any(p.is_symlink() for p in folder.rglob('*')):continue + shutil.rmtree(folder) + except (OSError,ValueError):continue + + +def accepted_backend(root): + """The running pinned jar is part of a valid application backup on this Mac.""" + root = Path(root).resolve() + # 容器的只读镜像内保存运行包及构建时摘要,不依赖 macOS 服务注册表。 + if os.environ.get('QUALITY_CONTAINER') == '1': + path = root / 'app/backend.jar' + digest = (root / 'app/backend.sha256').read_text().strip() + if path.is_symlink() or not re.fullmatch(r'[a-f0-9]{64}', digest) or sha(path) != digest: + raise ValueError('Container backend digest mismatch') + return path, digest + config = Path.home() / 'Library/Application Support/QualityCenterService/service.json' + if not config.exists(): + return None + value = json.loads(config.read_text()) + if Path(value['root']).resolve() != root: + return None + digest = value['jarSha256'] + if not re.fullmatch(r'[a-f0-9]{64}', digest): + raise ValueError('Invalid accepted backend digest') + path = root / '.runtime/backend-builds' / (digest + '.jar') + if path.is_symlink() or sha(path) != digest: + raise ValueError('Accepted backend digest mismatch') + return path, digest + + +def native_files(root): + """Retain restorable native settings/conversations; exclude live access tokens.""" + for relative in ('.runtime/operations/policy.json','.runtime/operations/logback-quality.xml'): + path=Path(root)/relative + if path.is_symlink():raise ValueError('Unsafe operations configuration link') + if path.is_file():yield path + base=Path(root)/'.runtime/nao-users' + ignored={'logs','backups','__pycache__'} + ephemeral={'access.json','cookies.txt','sessions.json','current-session.json','instance.json'} + for path in base.rglob('*') if base.exists() else []: + if path.is_symlink(): + raise ValueError('Unsafe native workspace link') + if not path.is_file() or any(p in ignored for p in path.relative_to(base).parts): + continue + if path.name in ephemeral or path.suffix in ('.lock','.pid') or path.name.endswith(('-wal','-shm')): + continue + yield path + + +def snapshot_sqlite(source, target): + original=sqlite3.connect(Path(source).resolve().as_uri()+'?mode=ro',uri=True) + copied=sqlite3.connect(target) + try: + original.backup(copied) + copied.execute('PRAGMA foreign_keys=OFF') + copied.execute('PRAGMA secure_delete=ON') + tables={r[0] for r in copied.execute("SELECT name FROM sqlite_master WHERE type='table'")} + for name in ('session','verification','api_key','jwks','oauth_access_token','oauth_refresh_token', + 'oauth_consent','oauth_client','mcp_user_token','mcp_oauth_client','sandbox_secret','log','mcp_call_log'): + if name in tables: + copied.execute('DELETE FROM "'+name+'"') + copied.commit();copied.execute('VACUUM') + if copied.execute('PRAGMA integrity_check').fetchone()[0]!='ok': + raise ValueError('Native SQLite backup integrity failed') + finally: + original.close();copied.close() + + +def scratch_register(root, database, server): + import psutil + if not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',database): + raise ValueError('Unexpected verification database name') + path=Path(root)/'.runtime/retention/scratch'/(database+'.json') + value={'kind':'quality-restore-check','database':database,'server':server, + 'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time(),'created':False} + atomic(path,value) + return path + + +def scratch_created(path): + value=json.loads(Path(path).read_text());value['created']=True;atomic(path,value) + + +def scratch_child(path, pid): + import psutil + value=json.loads(Path(path).read_text()) + value.update({'childPid':pid,'childCreated':psutil.Process(pid).create_time()}) + atomic(path,value) diff --git a/scripts/backup_verifier.py b/scripts/backup_verifier.py new file mode 100644 index 0000000..45963e0 --- /dev/null +++ b/scripts/backup_verifier.py @@ -0,0 +1,112 @@ +"""Private, socket-only MySQL instance for low-priority restore verification.""" +from contextlib import contextmanager +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import uuid + +import psutil +import backup_retention as br + + +def stop_owned(value): + if value.get('initPid'): + try: + init=psutil.Process(value['initPid']) + if abs(init.create_time()-value['initCreated'])<.01 and init.is_running() and init.status()!=psutil.STATUS_ZOMBIE:return False + except psutil.NoSuchProcess:pass + except psutil.Error:return False + pid=value.get('serverPid') + if not pid: + p=Path(value['directory'])/'server.pid' + if not p.exists():return True + pid=int(p.read_text()) + try: + process=psutil.Process(pid) + if value.get('serverCreated') and abs(process.create_time()-value['serverCreated'])>.01:return True + command=process.cmdline() + if process.uids().real!=os.getuid() or Path(process.exe()).resolve()!=Path(value['binary']).resolve() or '--datadir='+value['directory']+'/data' not in command or '--skip-networking' not in command: + return False + process.terminate() + try:process.wait(timeout=180) + except psutil.TimeoutExpired:return False + return True + except psutil.NoSuchProcess:return True + except psutil.Error:return False + + +def remove_owned(root, directory, value): + directory=Path(directory);base=Path(root)/'.runtime/retention' + if directory.parent!=base or not directory.name.startswith('verify-') or directory.is_symlink(): + return False + if not stop_owned(value):return False + socket=Path(value['socketDirectory']) + if socket.parent.resolve()!=Path('/tmp').resolve() or not socket.name.startswith('qret-') or socket.is_symlink():return False + shutil.rmtree(socket,ignore_errors=True) + shutil.rmtree(directory) + return True + + +def clean_abandoned(root): + base=Path(root)/'.runtime/retention';removed=0 + for directory in base.glob('verify-*'): + path=directory/'owner.json' + if directory.is_symlink() or not path.is_file() or path.is_symlink():continue + value=json.loads(path.read_text()) + if value.get('kind')!='private-quality-restore' or value.get('directory')!=str(directory):continue + try: + p=psutil.Process(value['ownerPid']) + if abs(p.create_time()-value['ownerCreated'])<.01 and p.is_running() and p.status()!=psutil.STATUS_ZOMBIE:continue + except psutil.NoSuchProcess:pass + except psutil.Error:continue + if time.time()-value['createdAt']<300:continue + if remove_owned(root,directory,value):removed+=1 + return removed + + +@contextmanager +def instance(root): + root=Path(root).resolve();base=root/'.runtime/retention';base.mkdir(parents=True,exist_ok=True,mode=0o700) + directory=base/('verify-'+uuid.uuid4().hex);directory.mkdir(mode=0o700) + socket=Path(tempfile.mkdtemp(prefix='qret-',dir='/tmp')).resolve();socket.chmod(0o700) + binary=root/'.runtime/mysql/bin/mysqld' + value={'kind':'private-quality-restore','directory':str(directory),'socketDirectory':str(socket), + 'binary':str(binary),'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time()} + br.atomic(directory/'owner.json',value) + process=None + try: + with (directory/'initialize.log').open('ab') as output: + init=subprocess.Popen([str(binary),'--no-defaults','--initialize-insecure','--basedir='+str(binary.parent.parent), + '--datadir='+str(directory/'data'),'--lower-case-table-names='+('2' if sys.platform == 'darwin' else '0'),'--skip-log-bin'],stdout=output,stderr=output) + value.update({'initPid':init.pid,'initCreated':psutil.Process(init.pid).create_time()});br.atomic(directory/'owner.json',value) + try:code=init.wait(timeout=180) + except subprocess.TimeoutExpired: + init.kill();init.wait();raise + if code:raise RuntimeError('Private restore server initialization failed') + args=[str(binary),'--no-defaults','--basedir='+str(binary.parent.parent),'--datadir='+str(directory/'data'), + '--socket='+str(socket/'mysql.sock'),'--pid-file='+str(directory/'server.pid'), + '--log-error='+str(directory/'server.log'),'--skip-networking','--mysqlx=OFF','--skip-log-bin', + '--lower-case-table-names='+('2' if sys.platform == 'darwin' else '0'),'--innodb-buffer-pool-size=128M','--innodb-redo-log-capacity=64M','--max-connections=8'] + with (directory/'stdout.log').open('ab') as output: + process=subprocess.Popen(args,stdout=output,stderr=output,stdin=subprocess.DEVNULL) + value.update({'serverPid':process.pid,'serverCreated':psutil.Process(process.pid).create_time()});br.atomic(directory/'owner.json',value) + config=directory/'client.cnf' + config.write_text('[client]\nuser=root\nsocket='+str(socket/'mysql.sock')+'\n');config.chmod(0o600) + deadline=time.monotonic()+90 + while time.monotonic()= depth - 1: + dirs[:] = [] + else: + dirs[:] = [d for d in dirs if not (parent / d).is_symlink() and d not in + ('backups', 'mysql-data', 'mysql', 'jdk', 'm2', 'backend-builds', 'node_modules')] + for name in files: + if re.fullmatch(r'.+\.(?:log|out|err)(?:\.[a-zA-Z0-9_-]+)*(?:\.gz)?', name): + p = parent / name + if not p.is_symlink() and p.is_file() and p.stat().st_nlink == 1: + paths.add(p) + return sorted(paths) + + +def opened_files(): + """Unknown access is a reason to defer deletion, never a reason to assume closed.""" + paths = set() + for process in psutil.process_iter(['uids']): + try: + if process.info['uids'].real != os.getuid(): + continue + for f in process.open_files(): + paths.add(str(Path(f.path).absolute())) + except psutil.NoSuchProcess: + continue + except psutil.Error as error: + # macOS restricts system/GUI agents unrelated to these application log + # directories. A denied project runtime/interpreter still defers cleanup. + try: + executable = Path(process.exe()).resolve() + interpreter = re.fullmatch(r'(?:python.*|Python|java|node|bash|zsh|sh|uv|mvn)', executable.name) + if executable.is_relative_to(Path.cwd().resolve()) or interpreter: + raise RuntimeError('Project writer identity check unavailable') from error + except psutil.NoSuchProcess: + continue + except psutil.Error: + raise RuntimeError('Writer identity check unavailable') from error + return paths + + +def fingerprint(path): + s = path.stat() + return s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns + + +def category(path): + name = path.name.lstrip('.') + name = re.sub(r'\.\d{4}-\d{2}-\d{2}.*', '', name) + name = re.sub(r'\.roll-.*', '', name) + name = re.sub(r'\.(?:log|out|err)(?:\..*)?$', '', name) + return name + + +def rotate_mysql(root, execute, br, now=None): + root = Path(root).resolve(); now = time.time() if now is None else now + source = root / 'output/logs/mysql-server.log' + journal = root / '.runtime/operations/mysql-log-rotation.json' + if any(p.is_symlink() for p in (root / 'output', source.parent, source, root / '.runtime', journal.parent, journal)): + raise ValueError('Unsafe MySQL log path') + pending = None + if journal.exists(): + pending = json.loads(journal.read_text()) + if (pending.get('root') != str(root) or pending.get('kind') != 'quality-mysql-log-rotation' or + not re.fullmatch(r'mysql-server\.roll-\d+-[a-f0-9]{8}\.log', pending.get('archive', ''))): + raise ValueError('Unknown MySQL log rotation intent') + elif source.is_file() and source.stat().st_size: + old_day = time.localtime(source.stat().st_mtime)[:3] + if source.stat().st_size < 10 * 1024**2 and old_day == time.localtime(now)[:3]: + return 0 + pending = {'kind': 'quality-mysql-log-rotation', 'root': str(root), + 'archive': 'mysql-server.roll-' + str(int(now)) + '-' + uuid.uuid4().hex[:8] + '.log', + 'fingerprint': list(fingerprint(source))} + br.atomic(journal, pending) + if pending is None: + return 0 + target = source.with_name(pending['archive']) + if target.is_symlink(): + raise ValueError('Unsafe MySQL rotated log') + if not target.exists(): + if not source.is_file() or list(fingerprint(source)) != pending['fingerprint']: + journal.unlink(); return 0 + source.rename(target) + # MySQL closes the old descriptor and reopens its configured path. No truncation. + execute('FLUSH ERROR LOGS;', None) + if not source.is_file() or source.is_symlink() or source.stat().st_ino == target.stat().st_ino: + raise RuntimeError('MySQL error log reopening not confirmed') + source.chmod(0o600) + target.chmod(0o600) + journal.unlink() + return 1 + + +def retain(root, support, now=None, open_paths=None, max_bytes=MAX_BYTES, max_age=MAX_AGE): + now = time.time() if now is None else now + opened = opened_files() if open_paths is None else set(open_paths) + groups = {}; result = {'logsRemoved': 0, 'logBytesRemoved': 0, 'logsDeferred': 0} + for p in candidates(root, support): + groups.setdefault(category(p), []).append(p) + for files in groups.values(): + files.sort(key=lambda p: p.stat().st_mtime) + total = sum(p.stat().st_size for p in files) + for p in files: + s = p.stat(); stamp = fingerprint(p) + if now - s.st_mtime <= max_age and total <= max_bytes: + continue + # Native appenders own these live files. The monitor handles archives; + # MySQL is reopened by its dedicated rotation function above. + if ((p.parent == Path(root) / 'output/logs' and p.name in ('backend.log', 'mysql-server.log', 'backend-console.log')) or + (p.parent == Path(support) / 'logs' and p.name in ('guardian.log', 'retention.log', 'operations.log'))): + result['logsDeferred'] += 1; continue + if str(p.absolute()) in opened: + result['logsDeferred'] += 1; continue + # Recheck descriptors immediately before mutation. Data/config/archives + # never enter this allowlist, and active writers are never truncated. + if p.is_symlink() or fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened): + result['logsDeferred'] += 1; continue + if s.st_size > max_bytes and now - s.st_mtime <= max_age and len(files) == 1: + # Keep the newest bounded tail of a closed oversized console log. + temp = p.with_name(p.name + '.' + uuid.uuid4().hex + '.tmp') + try: + with p.open('rb') as source, temp.open('xb') as output: + temp.chmod(0o600); source.seek(max(0, s.st_size - max_bytes)) + source.readline() # Drop a possible partial leading line. + while block := source.read(1024 * 1024): output.write(block) + output.flush(); os.fsync(output.fileno()) + if fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened): + result['logsDeferred'] += 1; continue + os.utime(temp, ns=(s.st_atime_ns, s.st_mtime_ns)); temp.replace(p) + removed = s.st_size - p.stat().st_size + total -= removed; result['logBytesRemoved'] += removed + finally: + temp.unlink(missing_ok=True) + else: + p.unlink(); total -= s.st_size + result['logsRemoved'] += 1; result['logBytesRemoved'] += s.st_size + return result diff --git a/scripts/managed_test_resources.py b/scripts/managed_test_resources.py new file mode 100644 index 0000000..eb9907d --- /dev/null +++ b/scripts/managed_test_resources.py @@ -0,0 +1,207 @@ +"""Ownership records for disposable test schemas; never infer ownership from a prefix.""" +from pathlib import Path +import json +import os +import re +import time +import uuid + +import psutil +import backup_retention as br + +PREFIXES = ('quality_verify_', 'quality_scale_', 'quality_retention_test_', + 'quality_classification_source_', 'quality_classification_preview_', + 'quality_sales_type_test_', 'quality_unclassified_delta_', 'quality_sales_metric_test_', + 'quality_source_column_test_', 'quality_protocol_test_', 'quality_fine_rollup_') +MARKER = '__quality_test_owner' + + +def valid_name(name): + return (isinstance(name, str) and len(name) <= 64 and + re.fullmatch(r'[a-z0-9_]+_[a-f0-9]{8,32}', name) is not None and + name.startswith(PREFIXES)) + + +def registry(root): + root = Path(root).resolve(); base = root / '.runtime/test-resources' + for path in (root / '.runtime', base): + if path.is_symlink() or not path.resolve().is_relative_to(root): + raise ValueError('Unsafe test registry path') + return base + + +def protect(root, name): + cfg = json.loads((Path(root) / '.runtime/local-config.json').read_text()) + if not valid_name(name) or name == cfg['mysqlDatabase']: + raise ValueError('Not a disposable test database') + + +def active_process(pid, created): + try: + p = psutil.Process(int(pid)) + return (abs(p.create_time() - float(created)) < .02 and p.is_running() and + p.status() != psutil.STATUS_ZOMBIE) + except psutil.NoSuchProcess: + return False + except (psutil.Error, TypeError, ValueError): + return True + + +def load(root, name): + path = registry(root) / (name + '.json') + if not path.is_file() or path.is_symlink(): + return None + try: + value = json.loads(path.read_text()) + if (not isinstance(value, dict) or value.get('kind') != 'quality-owned-test-db' or + value.get('root') != str(Path(root).resolve()) or value.get('database') != name or + not valid_name(name) or not re.fullmatch(r'[a-f0-9]{32}', value.get('token', ''))): + return None + return value + except (OSError, ValueError, TypeError): + return None + + +def create(root, name, execute, ddl=None, empty=False): + root = Path(root).resolve(); protect(root, name) + base = registry(root); base.mkdir(parents=True, exist_ok=True, mode=0o700) + path = base / (name + '.json') + # Exclusive registration also prevents a second process claiming an existing name. + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600); os.close(fd) + value = {'kind': 'quality-owned-test-db', 'formatVersion': 1, 'root': str(root), + 'database': name, 'token': uuid.uuid4().hex, 'createdAt': time.time(), + 'expiresAt': time.time() + 86400, 'ownerPid': os.getpid(), + 'ownerCreated': psutil.Process().create_time(), 'children': [], 'users': [], + 'serverUuid': execute('SELECT @@server_uuid;', None).strip(), 'created': False} + br.atomic(path, value) + try: + result = execute(ddl or 'CREATE DATABASE `' + name + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;', None) + except BaseException: + path.unlink(missing_ok=True); raise + value['created'] = True; br.atomic(path, value) + directory = root / '.runtime/mysql-data' / name + if directory.is_dir() and not directory.is_symlink(): + stat = directory.stat() + value['directoryIdentity'] = [stat.st_dev, stat.st_ino] + br.atomic(path, value) + if not empty: + seal(root, name, execute) + return result + + +def seal(root, name, execute): + value = load(root, name) + if value is None or value['ownerPid'] != os.getpid(): + raise ValueError('Test resource owner not confirmed') + execute('CREATE TABLE `' + name + '`.`' + MARKER + '` (token CHAR(32) PRIMARY KEY);' + 'INSERT INTO `' + name + '`.`' + MARKER + "` VALUES ('" + value['token'] + "');", None) + value['ready'] = True; br.atomic(registry(root) / (name + '.json'), value) + + +def same_directory(root, value): + p = Path(root) / '.runtime/mysql-data' / value['database'] + if p.is_symlink() or not p.is_dir(): return False + stat = p.stat() + return value.get('directoryIdentity') == [stat.st_dev, stat.st_ino] + + +def attach_child(root, name, pid): + value = load(root, name) + if value is None or value['ownerPid'] != os.getpid(): + raise ValueError('Test resource owner not confirmed') + value.setdefault('children', []).append({'pid': pid, 'created': psutil.Process(pid).create_time()}) + br.atomic(registry(root) / (name + '.json'), value) + + +def attach_user(root, name, user): + if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user): + raise ValueError('Unsafe test account') + value = load(root, name) + if value is None or value['ownerPid'] != os.getpid(): + raise ValueError('Test resource owner not confirmed') + value.setdefault('users', []).append(user) + br.atomic(registry(root) / (name + '.json'), value) + + +def marker_matches(value, execute): + if not value.get('ready') or execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']: + return False + name = value['database'] + found = execute("SELECT COUNT(*) FROM information_schema.tables WHERE TABLE_SCHEMA='" + name + + "' AND TABLE_NAME='" + MARKER + "';", None).strip() + return found == '1' and execute('SELECT token FROM `' + name + '`.`' + MARKER + '`;', None).strip() == value['token'] + + +def drop(root, name, execute): + protect(root, name); value = load(root, name) + if value is None: + raise ValueError('Registered test schema identity not confirmed') + same_server = execute('SELECT @@server_uuid;', None).strip() == value['serverUuid'] + own_process = (value['ownerPid'] == os.getpid() and active_process(value['ownerPid'], value['ownerCreated'])) + if not same_server or not (marker_matches(value, execute) or + (value.get('created') and not value.get('ready') and (own_process or same_directory(root, value)))): + raise ValueError('Registered test schema identity not confirmed') + result = execute('DROP DATABASE `' + name + '`;', None) + value['databaseDropped'] = True + path = registry(root) / (name + '.json') + if value.get('users'): + br.atomic(path, value) + else: + path.unlink() + return result + + +def user_dropped(root, user): + for path in registry(root).glob('*.json'): + value = load(root, path.stem) + if value is None or value['ownerPid'] != os.getpid() or user not in value.get('users', []): + continue + value['users'].remove(user) + if value.get('databaseDropped') and not value['users']: + path.unlink() + else: + br.atomic(path, value) + + +def cleanup(root, execute, now=None, limit=3): + """Called while maintenance owns its shared lock and the business application is idle.""" + now = time.time() if now is None else now + result = {'testDatabasesRemoved': 0, 'testAccountsRemoved': 0, 'testResourcesDeferred': 0} + for path in sorted(registry(root).glob('*.json')): + if result['testDatabasesRemoved'] >= limit: + break + value = load(root, path.stem) + if value is None or not value.get('created') or now < value.get('expiresAt', now + 1): + continue + protect(root, value['database']) + if (active_process(value.get('ownerPid'), value.get('ownerCreated')) or + any(active_process(p.get('pid'), p.get('created')) for p in value.get('children', []))): + result['testResourcesDeferred'] += 1; continue + if execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']: + result['testResourcesDeferred'] += 1; continue + name = value['database'] + if execute("SELECT COUNT(*) FROM information_schema.processlist WHERE DB='" + name + "';", None).strip() != '0': + result['testResourcesDeferred'] += 1; continue + if not value.get('databaseDropped'): + if not marker_matches(value, execute) and not (not value.get('ready') and same_directory(root, value)): + result['testResourcesDeferred'] += 1; continue + drop(root, name, execute); result['testDatabasesRemoved'] += 1 + value['databaseDropped'] = True + for user in list(value.get('users', [])): + if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user): + continue + grantee = "'" + user + "'@'127.0.0.1'" + literal = "CONVERT(0x" + grantee.encode().hex() + " USING utf8mb4)" + unsafe = execute("SELECT (SELECT COUNT(*) FROM information_schema.USER_PRIVILEGES WHERE GRANTEE=" + literal + + " AND PRIVILEGE_TYPE<>'USAGE')+(SELECT COUNT(*) FROM information_schema.SCHEMA_PRIVILEGES WHERE GRANTEE=" + literal + + " AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.TABLE_PRIVILEGES WHERE GRANTEE=" + literal + + " AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.processlist WHERE USER='" + user + "');", None).strip() + if unsafe != '0': + result['testResourcesDeferred'] += 1; continue + execute("DROP USER IF EXISTS '" + user + "'@'127.0.0.1';", None) + value['users'].remove(user); result['testAccountsRemoved'] += 1 + if not value.get('users'): + path.unlink(missing_ok=True) + else: + br.atomic(path, value) + return result diff --git a/scripts/project.py b/scripts/project.py index 63608ba..6bc6a80 100644 --- a/scripts/project.py +++ b/scripts/project.py @@ -145,7 +145,8 @@ def start_mysql(): '--datadir=' + str(data), '--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1', '--socket=' + str(RUNTIME / 'mysql.sock'), '--pid-file=' + str(RUNTIME / 'mysql-server.pid'), '--log-error=' + str(OUTPUT / 'logs/mysql-server.log'), '--mysqlx=OFF', - '--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M']) + '--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M', + *(['--skip-log-bin'] if value.get('binaryLogEnabled', True) is False else [])]) client_config = RUNTIME / 'mysql-root.cnf' save_private(client_config, '[client]\nuser=root\npassword=' + value['rootPassword'] + '\nsocket=' + str(RUNTIME / 'mysql.sock') + '\n') for _ in range(120): diff --git a/scripts/revision_retention.py b/scripts/revision_retention.py new file mode 100644 index 0000000..6714c50 --- /dev/null +++ b/scripts/revision_retention.py @@ -0,0 +1,213 @@ +"""Reclaim only unreferenced, completed source revisions and their owned archives.""" +import hashlib +import json +from contextlib import contextmanager +from pathlib import Path +import re +import time +import uuid + + +# These are measured business values, not documents that refer to source IDs. +VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'), + ('fact_bright', 'raw_values'), ('fact_bright', 'components'), + ('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')} +TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'} + + +class Cleaner: + def __init__(self, db, br, safe_file, fingerprint, defer): + self.db, self.br, self.safe_file, self.fingerprint, self.defer = db, br, safe_file, fingerprint, defer + self.root = db.root.resolve() + self.boundary = self.root / 'runtimedata/archive/source-revisions' + self.journals = self.root / '.runtime/retention/source-files' + self.reference_deadline = None + + @contextmanager + def quick_checks(self): + self.reference_deadline = time.monotonic() + .25 + try: + yield + finally: + self.reference_deadline = None + + @staticmethod + def ident(value): + if not re.fullmatch(r'[A-Za-z0-9_]+', value): + raise ValueError('Invalid SQL identifier') + return '`' + value + '`' + + def exists(self, table, condition, args): + if self.reference_deadline and time.monotonic() > self.reference_deadline: + raise self.defer('Source reference checks exceeded the brief exclusion budget') + try: + result = bool(self.db.rows('SELECT /*+ MAX_EXECUTION_TIME(100) */ 1 FROM ' + + self.ident(table) + ' WHERE ' + condition + ' LIMIT 1', args)) + if self.reference_deadline and time.monotonic() > self.reference_deadline: + raise self.defer('Source reference checks exceeded the brief exclusion budget') + return result + except Exception as error: + if error.args and error.args[0] in (1317, 3024): + raise self.defer('Source reference check exceeded its short query budget') from error + raise + + def metadata_mentions(self, needles, excluded=None): + for table, columns in self.db.column_types.items(): + for column, kind in columns.items(): + if kind not in TEXT_TYPES or (table, column) in VALUE_JSON: + continue + conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles] + args = list(needles) + if excluded and table == excluded[0]: + conditions = ['(' + ' OR '.join(conditions) + ')', self.ident('id') + '<>%s'] + args.append(excluded[1]) + condition = ' AND '.join(conditions) + else: + condition = ' OR '.join(conditions) + if self.exists(table, condition, args): + return True + return False + + def referenced(self, revision): + rid = revision['id'] + for fk in self.db.fks: + if fk['REFERENCED_TABLE_NAME'] == 'source_revision' and self.exists( + fk['TABLE_NAME'], self.ident(fk['COLUMN_NAME']) + '=%s', (rid,)): + return True + # Replacement, withdrawal and restore records include IDs without FKs. + for table, columns in self.db.columns.items(): + for column in columns: + if column.endswith('revision_id'): + condition = self.ident(column) + '=%s' + args = [rid] + if table == 'source_revision': + condition += ' AND id<>%s'; args.append(rid) + if self.exists(table, condition, args): + return True + # A catalog source can be imported again. Retain its original even when + # its current source_path points to an external workbook rather than archive. + if 'source_candidate' in self.db.columns and 'sha256' in self.db.columns['source_candidate']: + if self.exists('source_candidate', 'sha256=%s', (revision['sha256'],)): + return True + return self.metadata_mentions([rid], ('source_revision', rid)) + + def file_referenced(self, path, exclude_revision=None): + for table, columns in self.db.columns.items(): + for column in columns: + if column.endswith('_path'): + condition = self.ident(column) + '=%s'; args = [str(path)] + if table == 'source_revision' and exclude_revision: + condition += ' AND id<>%s'; args.append(exclude_revision) + if self.exists(table, condition, args): + return True + # Search both raw and JSON-escaped spelling, including saved snapshots. + needles = list(dict.fromkeys([str(path), json.dumps(str(path), ensure_ascii=False)[1:-1]])) + return self.metadata_mentions(needles, ('source_revision', exclude_revision) if exclude_revision else None) + + def owned_file(self, revision): + path = Path(revision['archive_path']) + if not self.safe_file(path, self.boundary): + return None + relative = path.relative_to(self.boundary).parts + if (len(relative) != 4 or relative[0] not in ('black', 'bright') or + not re.fullmatch(r'\d{4}', relative[1]) or + not re.fullmatch(r'0[1-9]|1[0-2]', relative[2]) or + not re.fullmatch(r'[a-f0-9]{64}', revision['sha256']) or + relative[3] != revision['sha256'] + '.xlsx'): + return None + if path.stat().st_size != revision['file_size'] or path.stat().st_nlink != 1: + return None + return path + + def safe_journals(self): + for p in (self.root / '.runtime', self.root / '.runtime/retention', self.journals): + if p.is_symlink() or not p.resolve().is_relative_to(self.root): + raise ValueError('Unsafe source file journal directory') + + def drain(self, current, limit=5): + """Resume only registered deletions after database commit; never scan stray files.""" + self.safe_journals() + removed = size = 0 + for record in sorted(self.journals.glob('*.json'))[:limit]: + if record.is_symlink(): + continue + try: + value = json.loads(record.read_text()) + except (OSError, ValueError): + continue + token = value.get('token', '') + if (value.get('kind') != 'quality-source-retirement' or value.get('root') != str(self.root) or + value.get('database') != self.db.name or not isinstance(token, str) or + not re.fullmatch(r'[a-f0-9]{32}', token) or token + '.json' != record.name): + continue + revision = value.get('revision') + if (not isinstance(revision, dict) or not all(isinstance(revision.get(k), str) for k in ('id', 'archive_path', 'sha256')) or + not isinstance(revision.get('file_size'), int) or not isinstance(value.get('fingerprint'), list)): + continue + path = Path(revision['archive_path']) + if self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)): + record.unlink() # An interrupted/rolled-back delete still owns its file. + continue + if not path.exists() and not path.is_symlink(): + record.unlink(); continue + if self.owned_file(revision) != path or list(self.fingerprint(path)) != value['fingerprint']: + continue + if self.br.sha(path) != revision['sha256']: + continue + with self.db.frozen(current), self.quick_checks(): + if (self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)) or + self.owned_file(revision) != path or + list(self.fingerprint(path)) != value['fingerprint'] or self.file_referenced(path)): + continue + size += path.stat().st_size + path.unlink(); removed += 1 + record.unlink() + return removed, size + + def collect(self, current, state, now, grace=1800, limit=3): + self.safe_journals() + seen = state.setdefault('sourceRevisions', {}) + result = {'sourceRevisionsRemoved': 0, 'sourceArchiveFilesRemoved': 0, 'sourceArchiveBytesRemoved': 0} + # All registered/current revisions remain readable until both ownership and + # dependency checks pass. Unknown states and missing/damaged files are kept. + revisions = self.db.rows('SELECT * FROM source_revision') + present = {row['id'] for row in revisions} + for stale in set(seen) - present: + seen.pop(stale, None) + deadline = time.monotonic() + 5 + for revision in revisions: + if time.monotonic() > deadline or result['sourceRevisionsRemoved'] >= limit: + break + rid = revision['id'] + if revision.get('status') != 'TRANSFORMED' or self.referenced(revision): + seen.pop(rid, None); continue + identity = hashlib.sha256(json.dumps(revision, sort_keys=True, default=str).encode()).hexdigest() + if seen.get(rid, {}).get('identity') != identity: + seen[rid] = {'identity': identity, 'since': now} + if now - seen[rid]['since'] < grace: + continue + path = self.owned_file(revision) + if path is None: + continue + stamp = self.fingerprint(path) + if self.br.sha(path) != revision['sha256'] or self.fingerprint(path) != stamp: + continue + # Durable intent first: a crash after row deletion must not leave an + # unregistered orphan. The journal is harmless if the row still exists. + self.journals.mkdir(parents=True, exist_ok=True, mode=0o700) + token = uuid.uuid4().hex; record = self.journals / (token + '.json') + self.br.atomic(record, {'kind': 'quality-source-retirement', 'root': str(self.root), + 'database': self.db.name, 'token': token, 'createdAt': now, + 'revision': {k: revision[k] for k in ('id', 'archive_path', 'sha256', 'file_size')}, + 'fingerprint': stamp}) + with self.db.frozen(current), self.quick_checks(): + live = self.db.rows('SELECT * FROM source_revision WHERE id=%s', (rid,)) + if (not live or live[0] != revision or self.referenced(revision) or + self.owned_file(revision) != path or self.fingerprint(path) != stamp): + record.unlink(); seen.pop(rid, None); continue + self.db.execute('DELETE FROM source_revision WHERE id=%s', (rid,)) + result['sourceRevisionsRemoved'] += 1; seen.pop(rid, None) + files, size = self.drain(current) + result['sourceArchiveFilesRemoved'] = files; result['sourceArchiveBytesRemoved'] = size + result['sourceFilesPending'] = sum(1 for _ in self.journals.glob('*.json')) + return result