新增:Linux Docker 部署与自动回收、备份及容量监测
This commit is contained in:
@@ -0,0 +1,38 @@
|
|||||||
|
# 仅允许源码和构建输入进入上下文,避免把本机数据库、备份、密码或业务表格送入构建器。
|
||||||
|
**
|
||||||
|
!.dockerignore
|
||||||
|
!backend/
|
||||||
|
!backend/pom.xml
|
||||||
|
!backend/src/
|
||||||
|
!backend/src/**
|
||||||
|
!web/
|
||||||
|
!web/package.json
|
||||||
|
!web/pnpm-lock.yaml
|
||||||
|
!web/pnpm-workspace.yaml
|
||||||
|
!web/index.html
|
||||||
|
!web/tsconfig*.json
|
||||||
|
!web/vite.config.ts
|
||||||
|
!web/src/
|
||||||
|
!web/src/**
|
||||||
|
!web/public/
|
||||||
|
!web/public/**
|
||||||
|
!scripts/
|
||||||
|
!scripts/**
|
||||||
|
!ai/
|
||||||
|
!ai/**
|
||||||
|
!deploy/
|
||||||
|
!deploy/docker/
|
||||||
|
!deploy/docker/**
|
||||||
|
!deploy/macos/
|
||||||
|
!deploy/macos/**
|
||||||
|
!runtime.lock.json
|
||||||
|
!requirements-nao.in
|
||||||
|
!requirements-nao.lock
|
||||||
|
**/__pycache__
|
||||||
|
**/*.pyc
|
||||||
|
**/.env
|
||||||
|
**/.env.*
|
||||||
|
**/secrets
|
||||||
|
**/original-workbook-acceptance.json
|
||||||
|
**/*.log
|
||||||
|
!deploy/docker/.env.example
|
||||||
@@ -11,6 +11,8 @@
|
|||||||
|
|
||||||
## 本机运行
|
## 本机运行
|
||||||
|
|
||||||
|
Linux 容器部署、自动重启、空间回收、备份保留及容量告警见 [Docker 部署说明](deploy/docker/README.md)。这些配置随源码进入 Gitea;构建镜像使用手动入口,不要求配置 Gitea 自动构建。Mac 测试服务的开机启动与守护见 [本机服务说明](deploy/macos/README.md)。
|
||||||
|
|
||||||
本机启动脚本面向 macOS 15 及以上的 Apple Silicon。先准备 Python 3.11 及以上、Node.js/pnpm 和 [uv](https://docs.astral.sh/uv/getting-started/installation/)。首次运行需要安装项目运行依赖并构建:
|
本机启动脚本面向 macOS 15 及以上的 Apple Silicon。先准备 Python 3.11 及以上、Node.js/pnpm 和 [uv](https://docs.astral.sh/uv/getting-started/installation/)。首次运行需要安装项目运行依赖并构建:
|
||||||
|
|
||||||
python3 scripts/setup_runtime.py
|
python3 scripts/setup_runtime.py
|
||||||
|
|||||||
+2
-2
@@ -39,7 +39,7 @@ class AnalysisExecutionProtocolStoreMySqlConcurrencyTest {
|
|||||||
assertTrue(baseUrl.contains("/?"), "test server URL must have an empty database path and query parameters");
|
assertTrue(baseUrl.contains("/?"), "test server URL must have an empty database path and query parameters");
|
||||||
admin = new JdbcTemplate(new DriverManagerDataSource(baseUrl, user, password));
|
admin = new JdbcTemplate(new DriverManagerDataSource(baseUrl, user, password));
|
||||||
database = "quality_protocol_test_" + UUID.randomUUID().toString().replace("-", "");
|
database = "quality_protocol_test_" + UUID.randomUUID().toString().replace("-", "");
|
||||||
admin.execute("CREATE DATABASE `" + database + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);
|
||||||
created = true;
|
created = true;
|
||||||
dataSource = new DriverManagerDataSource(baseUrl.replaceFirst("/\\?", "/" + database + "?"), user, password);
|
dataSource = new DriverManagerDataSource(baseUrl.replaceFirst("/\\?", "/" + database + "?"), user, password);
|
||||||
jdbc = new JdbcTemplate(dataSource);
|
jdbc = new JdbcTemplate(dataSource);
|
||||||
@@ -52,7 +52,7 @@ class AnalysisExecutionProtocolStoreMySqlConcurrencyTest {
|
|||||||
|
|
||||||
@AfterEach void cleanup() {
|
@AfterEach void cleanup() {
|
||||||
if (created && admin != null && database != null && database.matches("quality_protocol_test_[0-9a-f]{32}")) {
|
if (created && admin != null && database != null && database.matches("quality_protocol_test_[0-9a-f]{32}")) {
|
||||||
admin.execute("DROP DATABASE IF EXISTS `" + database + "`");
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -41,7 +41,7 @@ class AnalysisProtocolServiceMySqlTest {
|
|||||||
String user=System.getenv("QUALITY_PROTOCOL_TEST_MYSQL_USER"),password=System.getenv("QUALITY_PROTOCOL_TEST_MYSQL_PASSWORD");
|
String user=System.getenv("QUALITY_PROTOCOL_TEST_MYSQL_USER"),password=System.getenv("QUALITY_PROTOCOL_TEST_MYSQL_PASSWORD");
|
||||||
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));
|
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));
|
||||||
database="quality_protocol_test_"+UUID.randomUUID().toString().replace("-","");
|
database="quality_protocol_test_"+UUID.randomUUID().toString().replace("-","");
|
||||||
admin.execute("CREATE DATABASE `"+database+"` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");created=true;
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);created=true;
|
||||||
var ds=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);
|
var ds=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);
|
||||||
jdbc=new JdbcTemplate(ds);tx=new TransactionTemplate(new DataSourceTransactionManager(ds));
|
jdbc=new JdbcTemplate(ds);tx=new TransactionTemplate(new DataSourceTransactionManager(ds));
|
||||||
jdbc.execute("CREATE TABLE analysis_session(id CHAR(36) PRIMARY KEY,user_id CHAR(36),release_id CHAR(36)) ENGINE=InnoDB");
|
jdbc.execute("CREATE TABLE analysis_session(id CHAR(36) PRIMARY KEY,user_id CHAR(36),release_id CHAR(36)) ENGINE=InnoDB");
|
||||||
@@ -67,7 +67,7 @@ class AnalysisProtocolServiceMySqlTest {
|
|||||||
jdbc.update("INSERT INTO analysis_session VALUES(?,?,?)",session,actor.getId().toString(),release);
|
jdbc.update("INSERT INTO analysis_session VALUES(?,?,?)",session,actor.getId().toString(),release);
|
||||||
tx.executeWithoutResult(s->service.initialize(session,Map.of("releaseId",release,"year",2026)));
|
tx.executeWithoutResult(s->service.initialize(session,Map.of("releaseId",release,"year",2026)));
|
||||||
}
|
}
|
||||||
@AfterEach void cleanup(){if(created&&database.matches("quality_protocol_test_[0-9a-f]{32}"))admin.execute("DROP DATABASE `"+database+"`");}
|
@AfterEach void cleanup(){if(created&&database.matches("quality_protocol_test_[0-9a-f]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);}
|
||||||
|
|
||||||
@Test void clarificationTrendClearAndCancellationUseOneAuthority() {
|
@Test void clarificationTrendClearAndCancellationUseOneAuthority() {
|
||||||
String first=newTurn("今年18×15×4综合合格率变化趋势");var a=claim(first,false);
|
String first=newTurn("今年18×15×4综合合格率变化趋势");var a=claim(first,false);
|
||||||
|
|||||||
+2
-2
@@ -48,7 +48,7 @@ class AnalysisProtocolTurnBindingMySqlTest {
|
|||||||
String password = System.getenv("QUALITY_PROTOCOL_TEST_MYSQL_PASSWORD");
|
String password = System.getenv("QUALITY_PROTOCOL_TEST_MYSQL_PASSWORD");
|
||||||
admin = new JdbcTemplate(new DriverManagerDataSource(url, dbUser, password));
|
admin = new JdbcTemplate(new DriverManagerDataSource(url, dbUser, password));
|
||||||
database = "quality_protocol_test_" + UUID.randomUUID().toString().replace("-", "");
|
database = "quality_protocol_test_" + UUID.randomUUID().toString().replace("-", "");
|
||||||
admin.execute("CREATE DATABASE `" + database + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);
|
||||||
created = true;
|
created = true;
|
||||||
var dataSource = new DriverManagerDataSource(url.replaceFirst("/\\?", "/" + database + "?"), dbUser, password);
|
var dataSource = new DriverManagerDataSource(url.replaceFirst("/\\?", "/" + database + "?"), dbUser, password);
|
||||||
jdbc = new JdbcTemplate(dataSource);
|
jdbc = new JdbcTemplate(dataSource);
|
||||||
@@ -73,7 +73,7 @@ class AnalysisProtocolTurnBindingMySqlTest {
|
|||||||
|
|
||||||
@AfterEach void cleanup() {
|
@AfterEach void cleanup() {
|
||||||
if (created && database != null && database.matches("quality_protocol_test_[0-9a-f]{32}"))
|
if (created && database != null && database.matches("quality_protocol_test_[0-9a-f]{32}"))
|
||||||
admin.execute("DROP DATABASE IF EXISTS `" + database + "`");
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);
|
||||||
}
|
}
|
||||||
|
|
||||||
private AnalysisExecutionProtocolStore.Execution start(String turnId, String userId, String key) {
|
private AnalysisExecutionProtocolStore.Execution start(String turnId, String userId, String key) {
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ class FineDefectRollupsMySqlTest {
|
|||||||
String user = System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"), password = System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
String user = System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"), password = System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
||||||
admin = new JdbcTemplate(new DriverManagerDataSource(url, user, password));
|
admin = new JdbcTemplate(new DriverManagerDataSource(url, user, password));
|
||||||
database = "quality_fine_rollup_" + UUID.randomUUID().toString().replace("-", "");
|
database = "quality_fine_rollup_" + UUID.randomUUID().toString().replace("-", "");
|
||||||
admin.execute("CREATE DATABASE " + database + " CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);
|
||||||
jdbc = spy(new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?", "/" + database + "?"), user, password)));
|
jdbc = spy(new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?", "/" + database + "?"), user, password)));
|
||||||
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY,source_revision_id CHAR(36))");
|
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY,source_revision_id CHAR(36))");
|
||||||
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)");
|
jdbc.execute("CREATE TABLE source_revision(id CHAR(36) PRIMARY KEY)");
|
||||||
@@ -161,7 +161,7 @@ class FineDefectRollupsMySqlTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@AfterEach void cleanup() {
|
@AfterEach void cleanup() {
|
||||||
if (admin != null && database != null && database.matches("quality_fine_rollup_[a-f0-9]{32}")) admin.execute("DROP DATABASE " + database);
|
if (admin != null && database != null && database.matches("quality_fine_rollup_[a-f0-9]{32}")) cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ class SalesOrderTypesMySqlTest {
|
|||||||
assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
||||||
String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
||||||
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_sales_type_test_"+UUID.randomUUID().toString().replace("-","");
|
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_sales_type_test_"+UUID.randomUUID().toString().replace("-","");
|
||||||
admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4");
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);
|
||||||
var source=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(source);
|
var source=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(source);
|
||||||
try(var connection=source.getConnection()){ScriptUtils.executeSqlScript(connection,new ClassPathResource("db/migration/V39__sales_order_types.sql"));}
|
try(var connection=source.getConnection()){ScriptUtils.executeSqlScript(connection,new ClassPathResource("db/migration/V39__sales_order_types.sql"));}
|
||||||
jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64),label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)");
|
jdbc.execute("CREATE TABLE query_filter_field(field_key VARCHAR(64),label VARCHAR(80),black_column VARCHAR(160),bright_column VARCHAR(160),enabled BOOLEAN,allow_empty BOOLEAN,sort_order INT)");
|
||||||
@@ -41,7 +41,7 @@ class SalesOrderTypesMySqlTest {
|
|||||||
jdbc.update("INSERT INTO fact_black(id,build_id,source_revision_id,department,group_name,raw_values) VALUES(1,'black-build','black-rev','部门A','组A',JSON_OBJECT('销售订单号','0001')),(2,'black-build','black-rev','部门A','组A',JSON_OBJECT('销售订单号','absent')),(3,'black-build','black-rev','部门A','组A',JSON_OBJECT('销售订单号',NULL))");
|
jdbc.update("INSERT INTO fact_black(id,build_id,source_revision_id,department,group_name,raw_values) VALUES(1,'black-build','black-rev','部门A','组A',JSON_OBJECT('销售订单号','0001')),(2,'black-build','black-rev','部门A','组A',JSON_OBJECT('销售订单号','absent')),(3,'black-build','black-rev','部门A','组A',JSON_OBJECT('销售订单号',NULL))");
|
||||||
jdbc.update("INSERT INTO fact_bright(id,build_id,source_revision_id,department,group_name,raw_values) VALUES(4,'bright-build','bright-rev','部门A','组A',JSON_OBJECT('销售订单','0001'))");
|
jdbc.update("INSERT INTO fact_bright(id,build_id,source_revision_id,department,group_name,raw_values) VALUES(4,'bright-build','bright-rev','部门A','组A',JSON_OBJECT('销售订单','0001'))");
|
||||||
}
|
}
|
||||||
@AfterEach void cleanup(){RequestContextHolder.resetRequestAttributes();if(database!=null)admin.execute("DROP DATABASE "+database);}
|
@AfterEach void cleanup(){RequestContextHolder.resetRequestAttributes();if(database!=null)cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);}
|
||||||
Map<String,Object> load(String key,String type,String mode) throws Exception {
|
Map<String,Object> load(String key,String type,String mode) throws Exception {
|
||||||
var service=new SalesOrderTypeService(jdbc);var file=SalesOrderTypeServiceTest.workbook(new String[]{key,type});var preview=service.preview(file,mode);
|
var service=new SalesOrderTypeService(jdbc);var file=SalesOrderTypeServiceTest.workbook(new String[]{key,type});var preview=service.preview(file,mode);
|
||||||
return service.importFile(file,mode,(String)preview.get("expectedVersion"),(String)preview.get("sha256"),actor);
|
return service.importFile(file,mode,(String)preview.get("expectedVersion"),(String)preview.get("sha256"),actor);
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ class SourceColumnQueriesMySqlTest {
|
|||||||
String password = System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
String password = System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
||||||
admin = new JdbcTemplate(new DriverManagerDataSource(url, user, password));
|
admin = new JdbcTemplate(new DriverManagerDataSource(url, user, password));
|
||||||
database = "quality_source_column_test_" + UUID.randomUUID().toString().replace("-", "");
|
database = "quality_source_column_test_" + UUID.randomUUID().toString().replace("-", "");
|
||||||
admin.execute("CREATE DATABASE " + database + " CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);
|
||||||
created = true;
|
created = true;
|
||||||
var dataSource = new DriverManagerDataSource(url.replaceFirst("/\\?", "/" + database + "?"), user, password);
|
var dataSource = new DriverManagerDataSource(url.replaceFirst("/\\?", "/" + database + "?"), user, password);
|
||||||
jdbc = new JdbcTemplate(dataSource);
|
jdbc = new JdbcTemplate(dataSource);
|
||||||
@@ -57,7 +57,7 @@ class SourceColumnQueriesMySqlTest {
|
|||||||
|
|
||||||
@AfterEach void cleanup() {
|
@AfterEach void cleanup() {
|
||||||
if (created && database != null && database.matches("quality_source_column_test_[0-9a-f]{32}"))
|
if (created && database != null && database.matches("quality_source_column_test_[0-9a-f]{32}"))
|
||||||
admin.execute("DROP DATABASE " + database);
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test void catalogIsScopedAuthorizedAndSupportsThirdAndSingleStageColumns() {
|
@Test void catalogIsScopedAuthorizedAndSupportsThirdAndSingleStageColumns() {
|
||||||
|
|||||||
+2
-2
@@ -18,7 +18,7 @@ class ClassificationPreviewMySqlTest {
|
|||||||
@BeforeEach void create(){
|
@BeforeEach void create(){
|
||||||
String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
||||||
String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
||||||
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_classification_preview_"+UUID.randomUUID().toString().replace("-","");admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_classification_preview_"+UUID.randomUUID().toString().replace("-","");cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);
|
||||||
jdbc=new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password));
|
jdbc=new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password));
|
||||||
jdbc.execute("CREATE TABLE current_release(dataset_id VARCHAR(20),release_id VARCHAR(20))");jdbc.update("INSERT INTO current_release VALUES('quality','r')");
|
jdbc.execute("CREATE TABLE current_release(dataset_id VARCHAR(20),release_id VARCHAR(20))");jdbc.update("INSERT INTO current_release VALUES('quality','r')");
|
||||||
jdbc.execute("CREATE TABLE release_member(release_id VARCHAR(20),transform_build_id VARCHAR(20))");jdbc.update("INSERT INTO release_member VALUES('r','b'),('r','l')");
|
jdbc.execute("CREATE TABLE release_member(release_id VARCHAR(20),transform_build_id VARCHAR(20))");jdbc.update("INSERT INTO release_member VALUES('r','b'),('r','l')");
|
||||||
@@ -31,7 +31,7 @@ class ClassificationPreviewMySqlTest {
|
|||||||
seed("bright","l","A","R","小写区域","2026-04","0001","x",200);
|
seed("bright","l","A","R","小写区域","2026-04","0001","x",200);
|
||||||
service=new ClassificationPreviewService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),mock(CandidateAcceptanceService.class));
|
service=new ClassificationPreviewService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),mock(CandidateAcceptanceService.class));
|
||||||
}
|
}
|
||||||
@AfterEach void cleanup(){if(database!=null&&database.matches("quality_classification_preview_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);}
|
@AfterEach void cleanup(){if(database!=null&&database.matches("quality_classification_preview_[a-f0-9]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);}
|
||||||
private void seed(String stage,String build,String department,String product,String group,String month,String code,String region,int weight){
|
private void seed(String stage,String build,String department,String product,String group,String month,String code,String region,int weight){
|
||||||
Map<String,Object> components=new HashMap<>();components.put("sourcePdtCode",code);components.put("sourceSalesRegion",region);
|
Map<String,Object> components=new HashMap<>();components.put("sourcePdtCode",code);components.put("sourceSalesRegion",region);
|
||||||
jdbc.update("INSERT INTO fact_"+stage+" VALUES(?,?,?,?,?,?,CAST(? AS JSON))",build,department,product,group,month,weight,JsonMapper.builder().build().writeValueAsString(components));
|
jdbc.update("INSERT INTO fact_"+stage+" VALUES(?,?,?,?,?,?,CAST(? AS JSON))",build,department,product,group,month,weight,JsonMapper.builder().build().writeValueAsString(components));
|
||||||
|
|||||||
+2
-2
@@ -19,7 +19,7 @@ class ClassificationSourceMySqlTest {
|
|||||||
@BeforeEach void create(){
|
@BeforeEach void create(){
|
||||||
String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
||||||
String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
||||||
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_classification_source_"+UUID.randomUUID().toString().replace("-","");admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_classification_source_"+UUID.randomUUID().toString().replace("-","");cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);
|
||||||
jdbc=new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password));
|
jdbc=new JdbcTemplate(new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password));
|
||||||
jdbc.execute("CREATE TABLE current_release(dataset_id VARCHAR(20),release_id VARCHAR(20))");jdbc.update("INSERT INTO current_release VALUES('quality','r')");
|
jdbc.execute("CREATE TABLE current_release(dataset_id VARCHAR(20),release_id VARCHAR(20))");jdbc.update("INSERT INTO current_release VALUES('quality','r')");
|
||||||
jdbc.execute("CREATE TABLE data_release(id VARCHAR(20),status VARCHAR(20))");jdbc.update("INSERT INTO data_release VALUES('r','PUBLISHED'),('new','PUBLISHED')");
|
jdbc.execute("CREATE TABLE data_release(id VARCHAR(20),status VARCHAR(20))");jdbc.update("INSERT INTO data_release VALUES('r','PUBLISHED'),('new','PUBLISHED')");
|
||||||
@@ -32,7 +32,7 @@ class ClassificationSourceMySqlTest {
|
|||||||
access=mock(CandidateAcceptanceService.class);fields=mock(GovernanceAccessService.class);when(fields.allowedFields(actor)).thenReturn(Set.of("客户"));
|
access=mock(CandidateAcceptanceService.class);fields=mock(GovernanceAccessService.class);when(fields.allowedFields(actor)).thenReturn(Set.of("客户"));
|
||||||
service=new ClassificationSourceService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),access,fields);
|
service=new ClassificationSourceService(new GovernanceSupport(jdbc,JsonMapper.builder().build()),access,fields);
|
||||||
}
|
}
|
||||||
@AfterEach void cleanup(){if(database!=null&&database.matches("quality_classification_source_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);}
|
@AfterEach void cleanup(){if(database!=null&&database.matches("quality_classification_source_[a-f0-9]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);}
|
||||||
@Test void exactSourceRowsPaginateFifteenAndStayPinnedWithoutPreviewOrTarget(){
|
@Test void exactSourceRowsPaginateFifteenAndStayPinnedWithoutPreviewOrTarget(){
|
||||||
var first=service.list(new ClassificationSourceService.Request("black",conditions,1,15,null),actor);
|
var first=service.list(new ClassificationSourceService.Request("black",conditions,1,15,null),actor);
|
||||||
assertEquals(18,first.get("total"));var rows=(List<?>)first.get("rows");assertEquals(15,rows.size());var row=(Map<?,?>)rows.getFirst();assertEquals("原始数据.xlsx",row.get("sourceFile"));assertEquals(1,row.get("rowNumber"));assertEquals(Map.of("客户","原始客户"),row.get("raw"));
|
assertEquals(18,first.get("total"));var rows=(List<?>)first.get("rows");assertEquals(15,rows.size());var row=(Map<?,?>)rows.getFirst();assertEquals("原始数据.xlsx",row.get("sourceFile"));assertEquals(1,row.get("rowNumber"));assertEquals(Map.of("客户","原始客户"),row.get("raw"));
|
||||||
|
|||||||
+2
-2
@@ -21,7 +21,7 @@ class UnclassifiedMappingRebuildMySqlTest {
|
|||||||
Map<String,Object> pending(){return Map.of("id","classification-test","name","区域补分类","enabled",true,"onlyUnclassified",true,"priority",-9900,"stage","all","target","group","value","G","conditions",List.of(Map.of("field","@department","operator","EQUALS","value","A"),Map.of("field","销售片区","operator","EQUALS","value","X")));}
|
Map<String,Object> pending(){return Map.of("id","classification-test","name","区域补分类","enabled",true,"onlyUnclassified",true,"priority",-9900,"stage","all","target","group","value","G","conditions",List.of(Map.of("field","@department","operator","EQUALS","value","A"),Map.of("field","销售片区","operator","EQUALS","value","X")));}
|
||||||
@BeforeEach void setup()throws Exception{
|
@BeforeEach void setup()throws Exception{
|
||||||
String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL"),user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
String url=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_SERVER_URL"),user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");assertTrue(url.startsWith("jdbc:mysql://127.0.0.1:")&&url.contains("/?"));
|
||||||
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_unclassified_delta_"+UUID.randomUUID().toString().replace("-","");admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));database="quality_unclassified_delta_"+UUID.randomUUID().toString().replace("-","");cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);
|
||||||
dataSource=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(dataSource);
|
dataSource=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(dataSource);
|
||||||
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY,source_revision_id VARCHAR(100),build_key CHAR(64),parser_version VARCHAR(100),rule_version VARCHAR(100),mapping_version VARCHAR(100),status VARCHAR(32),row_count BIGINT,rejected_count BIGINT,component_summary JSON,error_summary JSON,started_at TIMESTAMP(6) NULL,completed_at TIMESTAMP(6) NULL,parser_config JSON,parser_config_hash VARCHAR(64))");
|
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY,source_revision_id VARCHAR(100),build_key CHAR(64),parser_version VARCHAR(100),rule_version VARCHAR(100),mapping_version VARCHAR(100),status VARCHAR(32),row_count BIGINT,rejected_count BIGINT,component_summary JSON,error_summary JSON,started_at TIMESTAMP(6) NULL,completed_at TIMESTAMP(6) NULL,parser_config JSON,parser_config_hash VARCHAR(64))");
|
||||||
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(190),status VARCHAR(32),parent_release_id CHAR(36),expected_current_release_id CHAR(36),rule_version VARCHAR(100),mapping_version VARCHAR(100),target_version_id VARCHAR(100),complete BOOLEAN,validation JSON,created_by CHAR(36))");
|
jdbc.execute("CREATE TABLE data_release(id CHAR(36) PRIMARY KEY,name VARCHAR(190),status VARCHAR(32),parent_release_id CHAR(36),expected_current_release_id CHAR(36),rule_version VARCHAR(100),mapping_version VARCHAR(100),target_version_id VARCHAR(100),complete BOOLEAN,validation JSON,created_by CHAR(36))");
|
||||||
@@ -47,7 +47,7 @@ class UnclassifiedMappingRebuildMySqlTest {
|
|||||||
jdbc.update("INSERT INTO defect_component(build_id,stage,sheet_name,physical_row_no,defect_name,weight,source_field) VALUES(?,?,'S',?,'料废',?,'原字段')",build,stage,row,scrap);
|
jdbc.update("INSERT INTO defect_component(build_id,stage,sheet_name,physical_row_no,defect_name,weight,source_field) VALUES(?,?,'S',?,'料废',?,'原字段')",build,stage,row,scrap);
|
||||||
}
|
}
|
||||||
UnclassifiedMappingRebuild.Candidate run(MetricsRollupService metrics){return new TransactionTemplate(new DataSourceTransactionManager(dataSource)).execute(status->new UnclassifiedMappingRebuild(jdbc,json,worker,metrics).create(UUID.randomUUID().toString(),parent,"new-map","rule",Map.of("matchRules",List.of(pending())),List.of(pending()),actor));}
|
UnclassifiedMappingRebuild.Candidate run(MetricsRollupService metrics){return new TransactionTemplate(new DataSourceTransactionManager(dataSource)).execute(status->new UnclassifiedMappingRebuild(jdbc,json,worker,metrics).create(UUID.randomUUID().toString(),parent,"new-map","rule",Map.of("matchRules",List.of(pending())),List.of(pending()),actor));}
|
||||||
@AfterEach void cleanup(){if(admin!=null&&database!=null&&database.matches("quality_unclassified_delta_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);}
|
@AfterEach void cleanup(){if(admin!=null&&database!=null&&database.matches("quality_unclassified_delta_[a-f0-9]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);}
|
||||||
@Test void onlyEmptyRowsAreMatchedAndOldFactsWeightsAndExistingClassificationsRemainUnchanged(){
|
@Test void onlyEmptyRowsAreMatchedAndOldFactsWeightsAndExistingClassificationsRemainUnchanged(){
|
||||||
var candidate=run(rollups);assertEquals(3,candidate.scannedRows());assertEquals(2,candidate.classifiedRows());
|
var candidate=run(rollups);assertEquals(3,candidate.scannedRows());assertEquals(2,candidate.classifiedRows());
|
||||||
assertEquals(2,jdbc.queryForObject("SELECT COUNT(*) FROM release_member m JOIN fact_black f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND f.group_name IS NULL",Integer.class,parent));
|
assertEquals(2,jdbc.queryForObject("SELECT COUNT(*) FROM release_member m JOIN fact_black f ON f.build_id=m.transform_build_id WHERE m.release_id=? AND f.group_name IS NULL",Integer.class,parent));
|
||||||
|
|||||||
+2
-2
@@ -30,7 +30,7 @@ class SalesOrderMetricRollupsMySqlTest {
|
|||||||
String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
String user=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_USER"),password=System.getenv("QUALITY_SOURCE_COLUMN_TEST_MYSQL_PASSWORD");
|
||||||
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));
|
admin=new JdbcTemplate(new DriverManagerDataSource(url,user,password));
|
||||||
database="quality_sales_metric_test_"+UUID.randomUUID().toString().replace("-","");
|
database="quality_sales_metric_test_"+UUID.randomUUID().toString().replace("-","");
|
||||||
admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.create(admin.getDataSource(),database);
|
||||||
var source=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(source);
|
var source=new DriverManagerDataSource(url.replaceFirst("/\\?","/"+database+"?"),user,password);jdbc=new JdbcTemplate(source);
|
||||||
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
|
jdbc.execute("CREATE TABLE transform_build(id CHAR(36) PRIMARY KEY)");
|
||||||
jdbc.execute("CREATE TABLE data_release(id CHAR(36),name VARCHAR(100),status VARCHAR(32),rule_version VARCHAR(100))");
|
jdbc.execute("CREATE TABLE data_release(id CHAR(36),name VARCHAR(100),status VARCHAR(32),rule_version VARCHAR(100))");
|
||||||
@@ -53,7 +53,7 @@ class SalesOrderMetricRollupsMySqlTest {
|
|||||||
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));when(actor.getDepartments()).thenReturn(List.of("*"));when(actor.getGroups()).thenReturn(List.of());
|
when(governance.allowedFields(actor)).thenReturn(Set.of("*"));when(actor.getDepartments()).thenReturn(List.of("*"));when(actor.getGroups()).thenReturn(List.of());
|
||||||
metrics=new MetricsService(jdbc,governance);rollups=new SalesOrderMetricRollups(jdbc);
|
metrics=new MetricsService(jdbc,governance);rollups=new SalesOrderMetricRollups(jdbc);
|
||||||
}
|
}
|
||||||
@AfterEach void cleanup(){if(database!=null)admin.execute("DROP DATABASE "+database);}
|
@AfterEach void cleanup(){if(database!=null)cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);}
|
||||||
Object normalized(Object value){return json.readValue(json.writeValueAsString(value),Object.class);}
|
Object normalized(Object value){return json.readValue(json.writeValueAsString(value),Object.class);}
|
||||||
MetricsService.MetricsQuery query(String view, boolean blanks, Map<String,List<String>> filters){return new MetricsService.MetricsQuery(2026,null,null,null,null,null,null,view,blanks,release,"month",Map.of(),filters);}
|
MetricsService.MetricsQuery query(String view, boolean blanks, Map<String,List<String>> filters){return new MetricsService.MetricsQuery(2026,null,null,null,null,null,null,view,blanks,release,"month",Map.of(),filters);}
|
||||||
|
|
||||||
|
|||||||
@@ -74,10 +74,11 @@ class SourceUploadAppendMySqlTest {
|
|||||||
String user=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_USER"),password=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_PASSWORD");
|
String user=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_USER"),password=System.getenv("QUALITY_MULTIFILE_TEST_MYSQL_PASSWORD");
|
||||||
admin=new JdbcTemplate(new DriverManagerDataSource(serverUrl,user,password));
|
admin=new JdbcTemplate(new DriverManagerDataSource(serverUrl,user,password));
|
||||||
database="quality_verify_multifile_"+UUID.randomUUID().toString().replace("-","");
|
database="quality_verify_multifile_"+UUID.randomUUID().toString().replace("-","");
|
||||||
admin.execute("CREATE DATABASE "+database+" CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci");
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.createEmpty(admin.getDataSource(),database);
|
||||||
dataSource=new DriverManagerDataSource(serverUrl.replaceFirst("/\\?","/"+database+"?"),user,password);
|
dataSource=new DriverManagerDataSource(serverUrl.replaceFirst("/\\?","/"+database+"?"),user,password);
|
||||||
Flyway.configure().dataSource(dataSource).locations("classpath:db/migration")
|
Flyway.configure().dataSource(dataSource).locations("classpath:db/migration")
|
||||||
.placeholders(Map.of("legacyTimestampOffsetSeconds","0")).load().migrate();
|
.placeholders(Map.of("legacyTimestampOffsetSeconds","0")).load().migrate();
|
||||||
|
cn.yunsheng.quality.testsupport.OwnedTestDatabase.seal(admin.getDataSource(),database);
|
||||||
jdbc=new JdbcTemplate(dataSource);tx=new TransactionTemplate(new DataSourceTransactionManager(dataSource));
|
jdbc=new JdbcTemplate(dataSource);tx=new TransactionTemplate(new DataSourceTransactionManager(dataSource));
|
||||||
archiveRoot=tempDir.resolve("archive");Files.createDirectories(archiveRoot);
|
archiveRoot=tempDir.resolve("archive");Files.createDirectories(archiveRoot);
|
||||||
adminId=UUID.randomUUID();
|
adminId=UUID.randomUUID();
|
||||||
@@ -101,7 +102,7 @@ class SourceUploadAppendMySqlTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@AfterEach void cleanup(){
|
@AfterEach void cleanup(){
|
||||||
if(admin!=null&&database!=null&&database.matches("quality_verify_multifile_[a-f0-9]{32}"))admin.execute("DROP DATABASE "+database);
|
if(admin!=null&&database!=null&&database.matches("quality_verify_multifile_[a-f0-9]{32}"))cn.yunsheng.quality.testsupport.OwnedTestDatabase.drop(admin.getDataSource(),database);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test void mainAndTwoAppendFilesImportPublishReplaceWithdrawAndKeepHistory() throws Exception {
|
@Test void mainAndTwoAppendFilesImportPublishReplaceWithdrawAndKeepHistory() throws Exception {
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
package cn.yunsheng.quality.testsupport;
|
||||||
|
|
||||||
|
import java.nio.file.*;
|
||||||
|
import java.nio.file.attribute.PosixFilePermissions;
|
||||||
|
import java.sql.*;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.UUID;
|
||||||
|
import javax.sql.DataSource;
|
||||||
|
|
||||||
|
/** Durable ownership for test schemas. Production classes never use this helper. */
|
||||||
|
public final class OwnedTestDatabase {
|
||||||
|
private static final String MARKER = "__quality_test_owner";
|
||||||
|
private OwnedTestDatabase() {}
|
||||||
|
|
||||||
|
private static Path root() throws Exception {
|
||||||
|
for (Path p = Path.of("").toAbsolutePath().normalize(); p != null; p = p.getParent()) {
|
||||||
|
if (Files.isRegularFile(p.resolve(".runtime/local-config.json"))) return p.toRealPath();
|
||||||
|
}
|
||||||
|
throw new IllegalStateException("Project test configuration not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void valid(String name) {
|
||||||
|
if (name == null || name.length() > 64 || !name.matches("quality_(?:classification_source|classification_preview|sales_type_test|unclassified_delta|sales_metric_test|source_column_test|protocol_test|fine_rollup|verify_multifile)_[a-f0-9]{32}"))
|
||||||
|
throw new IllegalArgumentException("Not a disposable test database");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void protect(Path root, String database) throws Exception {
|
||||||
|
var match = java.util.regex.Pattern.compile("\"mysqlDatabase\"\\s*:\\s*\"([^\"]+)\"")
|
||||||
|
.matcher(Files.readString(root.resolve(".runtime/local-config.json")));
|
||||||
|
if (!match.find() || database.equals(match.group(1)))
|
||||||
|
throw new IllegalArgumentException("Active database is protected");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String quote(String text) {
|
||||||
|
StringBuilder s = new StringBuilder("\"");
|
||||||
|
for (char c : text.toCharArray()) {
|
||||||
|
if (c == '"' || c == '\\') s.append('\\').append(c);
|
||||||
|
else if (c < 32) s.append(String.format("\\u%04x", (int)c));
|
||||||
|
else s.append(c);
|
||||||
|
}
|
||||||
|
return s.append('"').toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void write(Path path, String value) throws Exception {
|
||||||
|
Path temp = path.resolveSibling(path.getFileName() + "." + UUID.randomUUID() + ".tmp");
|
||||||
|
Files.writeString(temp, value, StandardOpenOption.CREATE_NEW);
|
||||||
|
if (Files.getFileStore(temp).supportsFileAttributeView("posix"))
|
||||||
|
Files.setPosixFilePermissions(temp, PosixFilePermissions.fromString("rw-------"));
|
||||||
|
try (var stream = java.nio.channels.FileChannel.open(temp, StandardOpenOption.WRITE)) { stream.force(true); }
|
||||||
|
Files.move(temp, path, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Path record(Path root, String database) throws Exception {
|
||||||
|
Path base = root.resolve(".runtime/test-resources");
|
||||||
|
if (Files.isSymbolicLink(root.resolve(".runtime")) || Files.isSymbolicLink(base))
|
||||||
|
throw new IllegalStateException("Unsafe test registry");
|
||||||
|
Files.createDirectories(base);
|
||||||
|
if (Files.getFileStore(base).supportsFileAttributeView("posix"))
|
||||||
|
Files.setPosixFilePermissions(base, PosixFilePermissions.fromString("rwx------"));
|
||||||
|
return base.resolve(database + ".json");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void create(DataSource source, String database) {
|
||||||
|
create(source, database, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void createEmpty(DataSource source, String database) {
|
||||||
|
create(source, database, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void create(DataSource source, String database, boolean empty) {
|
||||||
|
valid(database);
|
||||||
|
try {
|
||||||
|
Path root = root(), path = record(root, database);
|
||||||
|
protect(root, database);
|
||||||
|
String token = UUID.randomUUID().toString().replace("-", "");
|
||||||
|
double now = Instant.now().toEpochMilli() / 1000.0;
|
||||||
|
double started = ProcessHandle.current().info().startInstant().orElseThrow().toEpochMilli() / 1000.0;
|
||||||
|
try (Connection c = source.getConnection(); Statement s = c.createStatement()) {
|
||||||
|
String server;
|
||||||
|
try (ResultSet rs = s.executeQuery("SELECT @@server_uuid")) { rs.next(); server = rs.getString(1); }
|
||||||
|
if (!server.matches("[a-fA-F0-9-]{36}")) throw new IllegalStateException("Server identity unavailable");
|
||||||
|
String prefix = "{\"kind\":\"quality-owned-test-db\",\"formatVersion\":1,\"root\":" + quote(root.toString()) +
|
||||||
|
",\"database\":" + quote(database) + ",\"token\":" + quote(token) + ",\"serverUuid\":" + quote(server) +
|
||||||
|
",\"ownerPid\":" + ProcessHandle.current().pid() + ",\"ownerCreated\":" + started +
|
||||||
|
",\"createdAt\":" + now + ",\"expiresAt\":" + (now + 86400) + ",\"children\":[],\"users\":[]";
|
||||||
|
Files.createFile(path); write(path, prefix + ",\"created\":false}");
|
||||||
|
try { s.execute("CREATE DATABASE `" + database + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); }
|
||||||
|
catch (SQLException error) { Files.deleteIfExists(path); throw error; }
|
||||||
|
write(path, prefix + ",\"created\":true}");
|
||||||
|
if (!empty) {
|
||||||
|
s.execute("CREATE TABLE `" + database + "`.`" + MARKER + "` (token CHAR(32) PRIMARY KEY)");
|
||||||
|
s.execute("INSERT INTO `" + database + "`.`" + MARKER + "` VALUES ('" + token + "')");
|
||||||
|
write(path, prefix + ",\"created\":true,\"ready\":true}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception error) { throw new IllegalStateException("Owned test database creation failed", error); }
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void seal(DataSource source, String database) {
|
||||||
|
valid(database);
|
||||||
|
try {
|
||||||
|
Path root = root(); protect(root, database); Path path = record(root, database);
|
||||||
|
String value = Files.readString(path);
|
||||||
|
var match = java.util.regex.Pattern.compile("\"token\"\\s*:\\s*\"([a-f0-9]{32})\"").matcher(value);
|
||||||
|
if (!match.find()) throw new IllegalStateException("Test token unavailable");
|
||||||
|
try (Connection c = source.getConnection(); Statement s = c.createStatement()) {
|
||||||
|
s.execute("CREATE TABLE `" + database + "`.`" + MARKER + "` (token CHAR(32) PRIMARY KEY)");
|
||||||
|
s.execute("INSERT INTO `" + database + "`.`" + MARKER + "` VALUES ('" + match.group(1) + "')");
|
||||||
|
}
|
||||||
|
write(path, value.substring(0, value.length()-1) + ",\"ready\":true}");
|
||||||
|
} catch (Exception error) { throw new IllegalStateException("Test schema registration failed", error); }
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void drop(DataSource source, String database) {
|
||||||
|
valid(database);
|
||||||
|
try {
|
||||||
|
Path root = root(); protect(root, database);
|
||||||
|
Path path = record(root, database);
|
||||||
|
if (Files.isSymbolicLink(path)) throw new IllegalStateException("Unsafe test owner record");
|
||||||
|
String record = Files.readString(path);
|
||||||
|
var tokenMatch = java.util.regex.Pattern.compile("\"token\"\\s*:\\s*\"([a-f0-9]{32})\"").matcher(record);
|
||||||
|
var serverMatch = java.util.regex.Pattern.compile("\"serverUuid\"\\s*:\\s*\"([a-fA-F0-9-]{36})\"").matcher(record);
|
||||||
|
if (!tokenMatch.find() || !serverMatch.find()) throw new IllegalStateException("Test identity unavailable");
|
||||||
|
try (Connection c = source.getConnection(); Statement s = c.createStatement()) {
|
||||||
|
try (ResultSet rs = s.executeQuery("SELECT @@server_uuid")) {
|
||||||
|
rs.next(); if (!serverMatch.group(1).equals(rs.getString(1))) throw new IllegalStateException("Test server changed");
|
||||||
|
}
|
||||||
|
if (record.contains("\"ready\":true")) {
|
||||||
|
try (ResultSet rs = s.executeQuery("SELECT token FROM `" + database + "`.`" + MARKER + "`")) {
|
||||||
|
if (!rs.next() || !tokenMatch.group(1).equals(rs.getString(1))) throw new IllegalStateException("Test schema identity changed");
|
||||||
|
}
|
||||||
|
} else if (!record.contains("\"ownerPid\":" + ProcessHandle.current().pid() + ",")) {
|
||||||
|
throw new IllegalStateException("Unsealed test owner not confirmed");
|
||||||
|
}
|
||||||
|
s.execute("DROP DATABASE `" + database + "`");
|
||||||
|
Files.delete(path);
|
||||||
|
}
|
||||||
|
} catch (Exception error) { throw new IllegalStateException("Owned test database cleanup deferred", error); }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
# 此文件只放非敏感选项,真实密码由 manage.py init 生成。
|
||||||
|
HTTP_BIND=127.0.0.1
|
||||||
|
HTTP_PORT=5280
|
||||||
|
QUALITY_IMAGE_TAG=local
|
||||||
|
MYSQL_DATABASE=quality_platform
|
||||||
|
MYSQL_USER=quality_app
|
||||||
|
QUALITY_ADMIN_USERNAME=admin
|
||||||
|
JAVA_MAX_HEAP=2048m
|
||||||
|
# 从旧库迁移时必须核实旧版本会话的时区,不要猜测。
|
||||||
|
MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS=UNCONFIRMED
|
||||||
|
QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED=false
|
||||||
|
QUALITY_ANALYSIS_AUTONOMOUS_ENABLED=false
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# syntax=docker/dockerfile:1
|
||||||
|
FROM maven:3.9-eclipse-temurin-21 AS java-build
|
||||||
|
WORKDIR /build
|
||||||
|
COPY backend/pom.xml backend/pom.xml
|
||||||
|
RUN --mount=type=cache,target=/root/.m2 mvn -B -f backend/pom.xml dependency:go-offline
|
||||||
|
COPY backend/src backend/src
|
||||||
|
# 默认执行不需要私有工作簿的测试;不把生产库传入构建。
|
||||||
|
RUN --mount=type=cache,target=/root/.m2 mvn -B -f backend/pom.xml package
|
||||||
|
|
||||||
|
FROM python:3.11-slim-bookworm AS runtimes
|
||||||
|
ARG TARGETARCH
|
||||||
|
RUN test "$TARGETARCH" = amd64
|
||||||
|
WORKDIR /build
|
||||||
|
COPY deploy/docker/assets.lock.json deploy/docker/fetch_runtimes.py ./
|
||||||
|
RUN python fetch_runtimes.py
|
||||||
|
|
||||||
|
FROM debian:bookworm-slim AS app
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates tzdata tini procps libaio1 libnuma1 libncurses6 libtirpc3 libstdc++6 libgomp1 \
|
||||||
|
libatomic1 libcrypt1 libzstd1 liblz4-1 libx11-6 libxext6 libxi6 libxrender1 libxtst6 libasound2 fontconfig fonts-dejavu-core \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& groupadd -g 10001 quality && useradd -u 10001 -g quality -m quality
|
||||||
|
COPY --from=runtimes /runtimes/jre /opt/java
|
||||||
|
COPY --from=runtimes /runtimes/python /opt/python
|
||||||
|
COPY --from=runtimes /runtimes/mysql /opt/mysql
|
||||||
|
ENV JAVA_HOME=/opt/java PATH=/opt/nao/bin:/opt/java/bin:/opt/mysql/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||||
|
PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 QUALITY_CONTAINER=1 TZ=Asia/Shanghai
|
||||||
|
COPY deploy/docker/requirements-linux.lock deploy/macos/retention-requirements.txt /tmp/requirements/
|
||||||
|
RUN /opt/python/bin/python3 -m venv /opt/nao \
|
||||||
|
&& /opt/nao/bin/pip install --no-cache-dir --require-hashes -r /tmp/requirements/requirements-linux.lock -r /tmp/requirements/retention-requirements.txt \
|
||||||
|
&& /opt/nao/bin/python -c "import nao_core, pymysql, psutil; import importlib.metadata as m; assert m.version('nao-core') == '0.3.26'" \
|
||||||
|
&& /opt/java/bin/java -version && /opt/mysql/bin/mysqld --version \
|
||||||
|
&& rm -rf /tmp/requirements
|
||||||
|
WORKDIR /srv/quality
|
||||||
|
COPY .dockerignore runtime.lock.json requirements-nao.in requirements-nao.lock ./
|
||||||
|
COPY backend/pom.xml backend/pom.xml
|
||||||
|
COPY backend/src backend/src
|
||||||
|
COPY web/package.json web/pnpm-lock.yaml web/index.html web/tsconfig.json web/vite.config.ts web/pnpm-workspace.yaml web/
|
||||||
|
COPY web/src web/src
|
||||||
|
COPY web/public web/public
|
||||||
|
COPY scripts scripts
|
||||||
|
COPY ai ai
|
||||||
|
COPY deploy/macos deploy/macos
|
||||||
|
COPY deploy/docker deploy/docker
|
||||||
|
COPY --from=java-build /build/backend/target/quality-platform-backend-0.1.0-SNAPSHOT.jar app/backend.jar
|
||||||
|
RUN sha256sum app/backend.jar | cut -d ' ' -f 1 > app/backend.sha256 \
|
||||||
|
&& mkdir -p .runtime output/backups output/logs runtimedata/archive \
|
||||||
|
&& chown -R quality:quality .runtime output runtimedata
|
||||||
|
USER 10001:10001
|
||||||
|
EXPOSE 5281
|
||||||
|
ENTRYPOINT ["/usr/bin/tini", "--", "/opt/nao/bin/python", "deploy/docker/app.py"]
|
||||||
|
CMD ["serve"]
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# syntax=docker/dockerfile:1
|
||||||
|
FROM node:22-bookworm-slim AS build
|
||||||
|
RUN npm install --global pnpm@10.15.1
|
||||||
|
WORKDIR /build
|
||||||
|
COPY web/package.json web/pnpm-lock.yaml web/pnpm-workspace.yaml ./
|
||||||
|
RUN --mount=type=cache,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile
|
||||||
|
COPY web ./
|
||||||
|
RUN pnpm build
|
||||||
|
|
||||||
|
FROM nginx:stable-alpine
|
||||||
|
COPY deploy/docker/nginx.conf /etc/nginx/conf.d/default.conf
|
||||||
|
COPY --from=build /build/dist /usr/share/nginx/html
|
||||||
|
EXPOSE 8080
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
# Linux Docker 部署
|
||||||
|
|
||||||
|
本目录是项目源码的一部分,随 Gitea 仓库同步。它提供手动构建与运行入口,不需要 Gitea Actions、Runner 或自动发布流水线。构建需要联网获取公共依赖;已构建镜像在启动时不运行 Maven、pnpm、pip 或下载运行时。AI 调用仍需要可用的模型服务。
|
||||||
|
|
||||||
|
当前准备目标为 **Linux x86_64 / Docker Engine / Compose V2(建议 2.24 以上)**。应用运行时固定为 Java 21.0.13、Python 3.11.15、MySQL 8.4.13、nao-core 0.3.26。原生运行时校验和在 `assets.lock.json`,Python 依赖使用带哈希的锁文件。基础系统镜像按发行系列获取安全更新;正式发布时应记录构建提交和镜像 digest。ARM64 不在本版本支持范围内。
|
||||||
|
|
||||||
|
## 服务与数据
|
||||||
|
|
||||||
|
| 服务 | 内容 | 数据位置 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| mysql | MySQL,关闭 binary log,保留事务 redo/undo | `mysql_data` 命名卷 |
|
||||||
|
| app | 后端、AI、自动回收、备份验证、容量监测 | `state`、`archives`、`backups`、`logs` 命名卷 |
|
||||||
|
| web | 前端与 API 代理,支持流式响应 | 镜像内静态文件 |
|
||||||
|
|
||||||
|
只发布网页端口,默认 `127.0.0.1:5280`;MySQL 和后端没有宿主机端口映射。应用以 UID 10001 运行,镜像只读,不挂载 Docker socket。后端及所有维护子进程共享同一容器的进程空间,避免误判其他容器内的校验任务已经退出。不要同时启动第二个 app 副本访问同一组数据卷。
|
||||||
|
|
||||||
|
首次初始化后,密码存放于项目私有目录 `.runtime/docker/secrets/`,不是源码、镜像层或命令参数。Compose 从管理入口的环境变量读取密码,再作为 secret 文件交给容器。主机管理员仍能查看容器配置,需限制 Docker 管理权限。数据库初始化后不要直接重新生成密码;密码轮换必须同步修改数据库账户。
|
||||||
|
|
||||||
|
## 从仓库构建与启动
|
||||||
|
|
||||||
|
在项目根目录执行:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 deploy/docker/manage.py init
|
||||||
|
python3 deploy/docker/manage.py config
|
||||||
|
python3 deploy/docker/manage.py build
|
||||||
|
python3 deploy/docker/manage.py up
|
||||||
|
python3 deploy/docker/manage.py status
|
||||||
|
```
|
||||||
|
|
||||||
|
`init` 不覆盖已有设置或密码。非敏感设置位于 `.runtime/docker/config.env`;内网访问时可把 `HTTP_BIND` 改为服务器内网 IP,配合防火墙限制访问范围。管理员初始密码位于私有文件 `.runtime/docker/secrets/admin_password`,登录后及时更改。首次克隆为空系统,业务数据需单独授权导入。
|
||||||
|
|
||||||
|
启用服务器上的 Docker 开机启动后,已创建且未手动停止的容器通过 `restart: unless-stopped` 随 Docker 自动启动;MySQL 就绪后才启动应用,应用就绪后再启动网页。后端异常退出会使 app 退出并由 Docker 重启;维护进程退出由入口进程重新拉起。健康检查用于报告就绪状态,不会因一次查询超时强杀正在工作的数据库。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sudo systemctl enable --now docker
|
||||||
|
python3 deploy/docker/manage.py logs
|
||||||
|
python3 deploy/docker/manage.py stop
|
||||||
|
```
|
||||||
|
|
||||||
|
`stop` 保留所有数据卷,并表示主动停服;之后用 `up` 恢复。不要执行 `docker compose down -v`,它会删除数据库、归档和备份卷。操作系统关机时仍会停服;要摆脱测试 Mac 的关机和网络影响,必须将整套服务部署在持续开机联网的服务器上。
|
||||||
|
|
||||||
|
## 自动控制空间增长
|
||||||
|
|
||||||
|
复用 `deploy/macos/retention.py` 中的平台通用回收逻辑和 `scripts/` 内的保留模块;macOS 的 launchd 安装代码不会在 Linux 执行。
|
||||||
|
|
||||||
|
- 每 5 分钟检查旧版本、终态无引用构建、过期上传草稿和无引用旧来源修订。当前快照、候选基线、保存的报表及分析引用均保护;删除前重新确认当前版本与任务状态。
|
||||||
|
- 导入、分析、排队、重试或未知状态任务存在时延期,锁冲突也延期,不终止任务。无引用对象连续 30 分钟后才进入回收;上传草稿先经过 24 小时有效期。
|
||||||
|
- 发现新正式版本或距上次备份超过 24 小时,在空闲时生成候选备份。新候选通过独立、无网络 MySQL 实例恢复、表行数、业务汇总、来源归档及应用文件校验后,才替换上一份有效备份。失败时保留旧备份;创建期间会短暂同时存在新旧两份及恢复校验空间。
|
||||||
|
- 测试数据库按所有者、进程身份、服务器 UUID 和标记登记。正常完成即清理;异常遗留超过 24 小时且所有者及子进程已退出后再处理。不按名称前缀批量删除未知数据库。新增数据库集成测试应使用 `OwnedTestDatabase` 或 `managed_test_resources.py`,并在 finally 中释放。
|
||||||
|
- Java 日志按日期及每段 10MB 滚动,保留 7 天,归档总量上限 80MB,为活动文件留出空间。关闭的其他项目日志每类最多 100MiB、最长 7 天;活动文件受保护。Docker 每个容器日志使用 `local` 驱动、10m×10 段限制。MySQL 错误日志进入受限制的容器日志;binary log 关闭后不支持历史时间点恢复,恢复依赖最新有效快照。
|
||||||
|
- 容量每 5 分钟采样,保留 30 天日报。业务数据、测试/校验临时空间分别达到 30GiB 提醒、50GiB 严重提醒;磁盘余量低于 50/20GiB、一天增长超过 2GiB 且超过 20%、连续维护失败也告警。告警写入容器日志及 `state` 卷的 `operations/alerts.jsonl`,同类 6 小时内去重。阈值在 `state` 卷的 `operations/policy.json`,不应为压低统计而删除业务数据。
|
||||||
|
|
||||||
|
Linux 的 MySQL 用量来自 `information_schema` 表和索引估算,不包含系统表空间、redo/undo 和 Docker 镜像缓存;宿主机还应使用磁盘监测与 `docker system df` 监测这些空间。容器内报告的可用空间取各业务卷的最小值;MySQL 卷所在磁盘也要由主机监控。此版本不提供 Linux 桌面弹窗或外部通知推送,容器日志可接入主机现有告警系统。
|
||||||
|
|
||||||
|
手动生成并验证一份备份:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 deploy/docker/manage.py backup
|
||||||
|
```
|
||||||
|
|
||||||
|
维护记录分别在 `state` 卷的 `retention/state.json`、`operations/monitor.json`。运行包或数据库结构改变时,回收会暂停;检查新版本兼容性后才执行 `python3 deploy/docker/manage.py review-retention`。此命令只更新兼容性标记,不取消任务保护或备份校验。
|
||||||
|
|
||||||
|
## 数据迁移与公网入口
|
||||||
|
|
||||||
|
当前测试 Mac 不会因本目录更新而切换运行包、迁移数据库或改变公网入口。源码中也不包含测试库、来源表格、现有备份和 API 密钥。
|
||||||
|
|
||||||
|
正式迁移采用逻辑备份及来源归档,不把 Mac 的 MySQL 数据目录直接复制到 Linux。先做 `backup.py create-and-verify`,保留原机;在新服务器的新数据库和独立目录恢复并重绑定归档绝对路径,核对用户、当前发布版本、月度汇总、归档摘要、登录和 AI,再切换域名。`backup.py restore --help` 提供新库/新目录的防覆盖恢复入口;它不是自动覆盖容器数据卷的导入命令。旧库的时间迁移参数必须从原部署核实。
|
||||||
|
|
||||||
|
长期入口建议为:域名 → 服务器 HTTPS 反向代理 → 本服务器 web → app → MySQL。公网只开放 443(按需开放 80 做跳转/证书),不再依赖测试 Mac 的 SSH 隧道。另一条路径是服务器应用连接托管 MySQL,另配对象存储及异地备份;该方案需要单独评估数据库权限、备份方式及来源路径,不能直接套用本地卷配置。
|
||||||
|
|
||||||
|
## 验收边界
|
||||||
|
|
||||||
|
本目录当前已准备源码与测试;没有 Linux Docker 主机时,不把静态检查或 Mac 上的测试称作容器启动成功。上主机后至少完成:构建三项服务、空库初始化、登录/导入/AI、后端与 MySQL 异常退出重启、主机重启、有效备份恢复、新旧数据对账、进行中任务保护、日志滚动及告警、HTTPS 与断开测试 Mac 后访问。容量规划还应为新旧备份和独立恢复校验预留余量。
|
||||||
|
|
||||||
|
参考:[Compose 启动顺序](https://docs.docker.com/compose/how-tos/startup-order/)、[Compose 服务与重启策略](https://docs.docker.com/reference/compose-file/services/)、[MySQL 官方镜像](https://hub.docker.com/_/mysql)、[Logback 日志保留](https://logback.qos.ch/manual/appenders-rolling.html)。
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""容器入口:固定运行包、私有配置、子进程守护和安全停止。"""
|
||||||
|
import argparse
|
||||||
|
import fcntl
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')]
|
||||||
|
|
||||||
|
|
||||||
|
def secret(name):
|
||||||
|
path = Path('/run/secrets') / name
|
||||||
|
value = path.read_text().strip()
|
||||||
|
if not re.fullmatch(r'[A-Za-z0-9_-]{24,128}', value):
|
||||||
|
raise ValueError('Invalid secret: ' + name)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def identifier(value):
|
||||||
|
if not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,47}', value):
|
||||||
|
raise ValueError('Invalid database identifier')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def prepare(root=ROOT):
|
||||||
|
import backup_retention as br
|
||||||
|
root = Path(root)
|
||||||
|
for relative in ('.runtime/retention', '.runtime/operations', '.runtime/nao-users',
|
||||||
|
'.runtime/nao-trial', 'runtimedata/archive', 'output/backups', 'output/logs'):
|
||||||
|
(root / relative).mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
for relative, destination in (('.runtime/mysql', '/opt/mysql'), ('.runtime/nao-trial/venv', '/opt/nao')):
|
||||||
|
path = root / relative
|
||||||
|
if path.is_symlink():
|
||||||
|
if str(path.readlink()) != destination: raise ValueError('Unexpected runtime link')
|
||||||
|
elif path.exists(): raise ValueError('Runtime path already occupied')
|
||||||
|
else: path.symlink_to(destination, target_is_directory=True)
|
||||||
|
value = {'mysqlHost': 'mysql', 'mysqlPort': 3306,
|
||||||
|
'mysqlDatabase': identifier(os.environ.get('MYSQL_DATABASE', 'quality_platform')),
|
||||||
|
'mysqlUser': identifier(os.environ.get('MYSQL_USER', 'quality_app')),
|
||||||
|
'mysqlPassword': secret('mysql_password'), 'rootPassword': secret('mysql_root_password'),
|
||||||
|
'adminUsername': os.environ.get('QUALITY_ADMIN_USERNAME', 'admin'),
|
||||||
|
'adminPassword': secret('admin_password'), 'databaseReady': True, 'binaryLogEnabled': False,
|
||||||
|
'legacyTimestampOffsetSeconds': os.environ.get('MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS', 'UNCONFIRMED')}
|
||||||
|
config = root / '.runtime/local-config.json'
|
||||||
|
if config.exists():
|
||||||
|
old = json.loads(config.read_text())
|
||||||
|
# 更改持久数据库的密码需要专门轮换,不能在重启时默默写入不一致配置。
|
||||||
|
for key in ('mysqlDatabase', 'mysqlUser', 'mysqlPassword', 'rootPassword'):
|
||||||
|
if old.get(key) != value[key]: raise ValueError('Persisted database settings changed: ' + key)
|
||||||
|
br.atomic(config, value)
|
||||||
|
cnf = root / '.runtime/mysql-root.cnf'
|
||||||
|
cnf.write_text('[client]\nuser=root\npassword=' + value['rootPassword'] + '\nhost=mysql\nport=3306\nprotocol=TCP\n')
|
||||||
|
cnf.chmod(0o600)
|
||||||
|
import operations
|
||||||
|
policy = root / '.runtime/operations/policy.json'
|
||||||
|
if not policy.exists(): br.atomic(policy, operations.DEFAULTS)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def backend_environment(value):
|
||||||
|
env = os.environ.copy()
|
||||||
|
env.update({'MYSQL_URL': 'jdbc:mysql://mysql:3306/' + value['mysqlDatabase'] +
|
||||||
|
'?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true',
|
||||||
|
'MYSQL_USER': value['mysqlUser'], 'MYSQL_PASSWORD': value['mysqlPassword'],
|
||||||
|
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': value['adminUsername'],
|
||||||
|
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': value['adminPassword'],
|
||||||
|
'SERVER_ADDRESS': '0.0.0.0', 'QUALITY_PROJECT_ROOT': str(ROOT),
|
||||||
|
'QUALITY_ARCHIVE_ROOT': str(ROOT / 'runtimedata/archive'), 'QUALITY_ALLOWED_ROOTS': '',
|
||||||
|
'LOGGING_CONFIG': str(ROOT / 'deploy/macos/logback-quality.xml'),
|
||||||
|
'QUALITY_LOG_DIR': str(ROOT / 'output/logs'), 'QUALITY_CONTAINER': '1'})
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def ready():
|
||||||
|
try:
|
||||||
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||||
|
with opener.open('http://127.0.0.1:5281/actuator/health', timeout=3) as response:
|
||||||
|
return json.load(response).get('status') == 'UP'
|
||||||
|
except (OSError, ValueError): return False
|
||||||
|
|
||||||
|
|
||||||
|
def review_retention(first=False):
|
||||||
|
import backup_retention as br
|
||||||
|
from retention import Database
|
||||||
|
path = ROOT / '.runtime/retention/config.json'
|
||||||
|
if first and path.exists(): return
|
||||||
|
with br.lock(ROOT, timeout=0):
|
||||||
|
db = Database(ROOT)
|
||||||
|
try:
|
||||||
|
if db.busy(): raise RuntimeError('Application jobs are active; review deferred')
|
||||||
|
accepted = br.accepted_backend(ROOT)
|
||||||
|
if not accepted: raise RuntimeError('Pinned backend not found')
|
||||||
|
br.atomic(path, {'schemaSignature': db.schema_signature, 'acceptedJarSha256': accepted[1],
|
||||||
|
'retirementGraceSeconds': 1800, 'intervalSeconds': 300})
|
||||||
|
finally: db.conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def serve():
|
||||||
|
import backup_retention as br
|
||||||
|
(ROOT / '.runtime').mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
guard = (ROOT / '.runtime/container.lock').open('a')
|
||||||
|
fcntl.flock(guard, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
value = prepare()
|
||||||
|
br.accepted_backend(ROOT)
|
||||||
|
heap = os.environ.get('JAVA_MAX_HEAP', '2048m')
|
||||||
|
if not re.fullmatch(r'[1-9][0-9]*[mMgG]', heap): raise ValueError('Invalid Java heap size')
|
||||||
|
env = backend_environment(value)
|
||||||
|
children = {'backend': subprocess.Popen(['java', '-Xms128m', '-Xmx' + heap, '-jar', str(ROOT / 'app/backend.jar')],
|
||||||
|
env=env, start_new_session=True)}
|
||||||
|
stopping = False
|
||||||
|
def stop(signum, frame):
|
||||||
|
nonlocal stopping
|
||||||
|
stopping = True
|
||||||
|
signal.signal(signal.SIGTERM, stop); signal.signal(signal.SIGINT, stop)
|
||||||
|
try:
|
||||||
|
while not stopping:
|
||||||
|
if children['backend'].poll() is not None: return 1
|
||||||
|
if ready():
|
||||||
|
try: review_retention(first=True)
|
||||||
|
except Exception as error: print('retention-initialization: ' + type(error).__name__, flush=True)
|
||||||
|
for role in ('retention', 'monitor'):
|
||||||
|
if role not in children or children[role].poll() is not None:
|
||||||
|
children[role] = subprocess.Popen([sys.executable, str(ROOT / 'deploy/docker/worker.py'), role],
|
||||||
|
env=env, start_new_session=True)
|
||||||
|
time.sleep(1)
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
for process in children.values():
|
||||||
|
if process.poll() is None:
|
||||||
|
try: os.killpg(process.pid, signal.SIGTERM)
|
||||||
|
except ProcessLookupError: pass
|
||||||
|
deadline = time.monotonic() + 270
|
||||||
|
for process in children.values():
|
||||||
|
try: process.wait(timeout=max(.1, deadline - time.monotonic()))
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
os.killpg(process.pid, signal.SIGKILL); process.wait()
|
||||||
|
guard.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('action', choices=['serve', 'health', 'review-retention'], default='serve', nargs='?')
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.action == 'health': return 0 if ready() else 1
|
||||||
|
if args.action == 'review-retention': review_retention(); return 0
|
||||||
|
return serve()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
try: raise SystemExit(main())
|
||||||
|
except Exception as error:
|
||||||
|
print('Container startup failed: ' + type(error).__name__, file=sys.stderr)
|
||||||
|
raise SystemExit(1)
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "jre",
|
||||||
|
"version": "21.0.13+9.0.LTS",
|
||||||
|
"url": "https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.13%2B9/OpenJDK21U-jre_x64_linux_hotspot_21.0.13_9.tar.gz",
|
||||||
|
"sha256": "a535a1df9c6284d9618872984e026ae0352d1f7e9e0605425c0e6114f47351dd"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "python",
|
||||||
|
"version": "3.11.15",
|
||||||
|
"url": "https://github.com/astral-sh/python-build-standalone/releases/download/20260623/cpython-3.11.15%2B20260623-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz",
|
||||||
|
"sha256": "0604cd029b142dc223e131f17f5941c0c8d2d5074997c8178b515b19eea2a6c2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "mysql",
|
||||||
|
"version": "8.4.13",
|
||||||
|
"url": "https://cdn.mysql.com/Downloads/MySQL-8.4/mysql-8.4.13-linux-glibc2.28-x86_64-minimal.tar.xz",
|
||||||
|
"sha256": "5dfbb764f731df40ef9e4f5d69aed32bdac8ba1af20bd96b7d40a942d83d4ed2"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
name: quality-center
|
||||||
|
|
||||||
|
x-logging: &bounded-logging
|
||||||
|
driver: local
|
||||||
|
options:
|
||||||
|
max-size: "10m"
|
||||||
|
max-file: "10"
|
||||||
|
|
||||||
|
services:
|
||||||
|
mysql:
|
||||||
|
image: mysql:8.4.13
|
||||||
|
platform: linux/amd64
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
MYSQL_DATABASE: ${MYSQL_DATABASE:-quality_platform}
|
||||||
|
MYSQL_USER: ${MYSQL_USER:-quality_app}
|
||||||
|
MYSQL_ROOT_HOST: "%"
|
||||||
|
MYSQL_ROOT_PASSWORD_FILE: /run/secrets/mysql_root_password
|
||||||
|
MYSQL_PASSWORD_FILE: /run/secrets/mysql_password
|
||||||
|
TZ: Asia/Shanghai
|
||||||
|
secrets: [mysql_root_password, mysql_password]
|
||||||
|
command:
|
||||||
|
- --skip-log-bin
|
||||||
|
- --mysqlx=OFF
|
||||||
|
- --max-connections=60
|
||||||
|
- --innodb-buffer-pool-size=512M
|
||||||
|
- --innodb-redo-log-capacity=128M
|
||||||
|
volumes: [mysql_data:/var/lib/mysql]
|
||||||
|
networks: [database]
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD-SHELL, "MYSQL_PWD=$$(cat /run/secrets/mysql_root_password) mysql --user=root --batch --skip-column-names -e 'SELECT 1' >/dev/null"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 30
|
||||||
|
start_period: 60s
|
||||||
|
stop_grace_period: 2m
|
||||||
|
mem_limit: 2g
|
||||||
|
logging: *bounded-logging
|
||||||
|
|
||||||
|
app:
|
||||||
|
image: quality-center-app:${QUALITY_IMAGE_TAG:-local}
|
||||||
|
build:
|
||||||
|
context: ../..
|
||||||
|
dockerfile: deploy/docker/Dockerfile.app
|
||||||
|
platform: linux/amd64
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on:
|
||||||
|
mysql: {condition: service_healthy}
|
||||||
|
environment:
|
||||||
|
MYSQL_DATABASE: ${MYSQL_DATABASE:-quality_platform}
|
||||||
|
MYSQL_USER: ${MYSQL_USER:-quality_app}
|
||||||
|
QUALITY_ADMIN_USERNAME: ${QUALITY_ADMIN_USERNAME:-admin}
|
||||||
|
MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS: ${MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS:-UNCONFIRMED}
|
||||||
|
QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED: ${QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED:-false}
|
||||||
|
QUALITY_ANALYSIS_AUTONOMOUS_ENABLED: ${QUALITY_ANALYSIS_AUTONOMOUS_ENABLED:-false}
|
||||||
|
JAVA_MAX_HEAP: ${JAVA_MAX_HEAP:-2048m}
|
||||||
|
secrets: [mysql_root_password, mysql_password, admin_password]
|
||||||
|
volumes:
|
||||||
|
- state:/srv/quality/.runtime
|
||||||
|
- archives:/srv/quality/runtimedata/archive
|
||||||
|
- backups:/srv/quality/output/backups
|
||||||
|
- logs:/srv/quality/output/logs
|
||||||
|
networks: [database, frontend]
|
||||||
|
read_only: true
|
||||||
|
tmpfs: [/tmp:mode=1777,size=536870912]
|
||||||
|
cap_drop: [ALL]
|
||||||
|
security_opt: [no-new-privileges:true]
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD, /opt/nao/bin/python, deploy/docker/app.py, health]
|
||||||
|
interval: 15s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 6
|
||||||
|
start_period: 15m
|
||||||
|
stop_grace_period: 5m
|
||||||
|
mem_limit: 6g
|
||||||
|
logging: *bounded-logging
|
||||||
|
|
||||||
|
web:
|
||||||
|
image: quality-center-web:${QUALITY_IMAGE_TAG:-local}
|
||||||
|
build:
|
||||||
|
context: ../..
|
||||||
|
dockerfile: deploy/docker/Dockerfile.web
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on:
|
||||||
|
app: {condition: service_healthy}
|
||||||
|
ports: ["${HTTP_BIND:-127.0.0.1}:${HTTP_PORT:-5280}:8080"]
|
||||||
|
networks: [frontend]
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/healthz]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
mem_limit: 256m
|
||||||
|
logging: *bounded-logging
|
||||||
|
|
||||||
|
networks:
|
||||||
|
database: {internal: true}
|
||||||
|
frontend: {}
|
||||||
|
volumes:
|
||||||
|
mysql_data: {}
|
||||||
|
state: {}
|
||||||
|
archives: {}
|
||||||
|
backups: {}
|
||||||
|
logs: {}
|
||||||
|
secrets:
|
||||||
|
mysql_root_password: {environment: QUALITY_SECRET_MYSQL_ROOT}
|
||||||
|
mysql_password: {environment: QUALITY_SECRET_MYSQL_APP}
|
||||||
|
admin_password: {environment: QUALITY_SECRET_ADMIN}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
"""构建时下载并校验固定 Linux x86_64 运行时,运行阶段不再下载。"""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
def install(lock, destination):
|
||||||
|
destination = Path(destination)
|
||||||
|
for asset in json.loads(Path(lock).read_text()):
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
temporary = Path(temporary); archive = temporary / 'runtime.tar'
|
||||||
|
with urllib.request.urlopen(asset['url'], timeout=120) as source, archive.open('wb') as output:
|
||||||
|
shutil.copyfileobj(source, output)
|
||||||
|
with archive.open('rb') as source: digest = hashlib.file_digest(source, 'sha256').hexdigest()
|
||||||
|
if digest != asset['sha256']: raise ValueError('Runtime checksum mismatch: ' + asset['name'])
|
||||||
|
unpacked = temporary / 'unpacked'; unpacked.mkdir()
|
||||||
|
with tarfile.open(archive) as source: source.extractall(unpacked, filter='data')
|
||||||
|
children = list(unpacked.iterdir())
|
||||||
|
if len(children) != 1 or not children[0].is_dir(): raise ValueError('Unexpected archive layout')
|
||||||
|
shutil.copytree(children[0], destination / asset['name'], symlinks=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': install('/build/assets.lock.json', '/runtimes')
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""统一 Docker Compose 入口;密码只从本机私有文件加载,不进入源码或命令参数。"""
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SECRET_NAMES = {'mysql_root_password': 'QUALITY_SECRET_MYSQL_ROOT',
|
||||||
|
'mysql_password': 'QUALITY_SECRET_MYSQL_APP', 'admin_password': 'QUALITY_SECRET_ADMIN'}
|
||||||
|
|
||||||
|
|
||||||
|
def initialize(root=ROOT):
|
||||||
|
base = Path(root) / '.runtime/docker'
|
||||||
|
directory = base / 'secrets'
|
||||||
|
if base.is_symlink() or directory.is_symlink(): raise ValueError('Unsafe private settings path')
|
||||||
|
directory.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
base.chmod(0o700); directory.chmod(0o700)
|
||||||
|
for name in SECRET_NAMES:
|
||||||
|
path = directory / name
|
||||||
|
if path.is_symlink(): raise ValueError('Unsafe secret file')
|
||||||
|
if path.exists(): continue
|
||||||
|
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
||||||
|
with os.fdopen(fd, 'w') as stream: stream.write(secrets.token_urlsafe(32) + '\n')
|
||||||
|
config = base / 'config.env'
|
||||||
|
if not config.exists():
|
||||||
|
shutil.copyfile(Path(root) / 'deploy/docker/.env.example', config); config.chmod(0o600)
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def environment(root=ROOT):
|
||||||
|
env = os.environ.copy(); directory = Path(root) / '.runtime/docker/secrets'
|
||||||
|
for name, variable in SECRET_NAMES.items():
|
||||||
|
path = directory / name
|
||||||
|
if path.is_symlink() or not path.is_file() or path.stat().st_mode & 0o077:
|
||||||
|
raise ValueError('Missing or unsafe secret file; run init first')
|
||||||
|
value = path.read_text().strip()
|
||||||
|
if not re.fullmatch(r'[A-Za-z0-9_-]{24,128}', value): raise ValueError('Invalid secret file')
|
||||||
|
env[variable] = value
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def command(action):
|
||||||
|
actions = {'config': ['config', '--quiet'], 'build': ['build', '--pull'], 'up': ['up', '-d', '--wait', '--wait-timeout', '1200'],
|
||||||
|
'stop': ['stop'], 'status': ['ps'], 'logs': ['logs', '--tail', '100'],
|
||||||
|
'backup': ['exec', '-T', 'app', '/opt/nao/bin/python', 'scripts/backup.py', 'create-and-verify'],
|
||||||
|
'review-retention': ['exec', '-T', 'app', '/opt/nao/bin/python', 'deploy/docker/app.py', 'review-retention']}
|
||||||
|
return ['docker', 'compose', '-f', str(ROOT / 'deploy/docker/compose.yaml'),
|
||||||
|
'--env-file', str(ROOT / '.runtime/docker/config.env'), *actions[action]]
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('action', choices=['init', 'config', 'build', 'up', 'stop', 'status', 'logs', 'backup', 'review-retention'])
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.action == 'init':
|
||||||
|
base = initialize(); print('Private settings prepared: ' + str(base)); return
|
||||||
|
if not shutil.which('docker'): parser.error('Docker Engine and Compose V2 are required')
|
||||||
|
subprocess.run(command(args.action), cwd=ROOT, env=environment(), check=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': main()
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
server {
|
||||||
|
listen 8080;
|
||||||
|
server_name _;
|
||||||
|
root /usr/share/nginx/html;
|
||||||
|
client_max_body_size 260m;
|
||||||
|
resolver 127.0.0.11 valid=10s;
|
||||||
|
set $quality_backend http://app:5281;
|
||||||
|
location = /healthz { access_log off; proxy_pass $quality_backend/actuator/health; }
|
||||||
|
location /api/ {
|
||||||
|
proxy_pass $quality_backend$request_uri;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header Connection '';
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_read_timeout 1800s;
|
||||||
|
proxy_send_timeout 1800s;
|
||||||
|
}
|
||||||
|
location / { try_files $uri $uri/ /index.html; }
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,148 @@
|
|||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')]
|
||||||
|
|
||||||
|
|
||||||
|
def module(name, path):
|
||||||
|
spec = importlib.util.spec_from_file_location(name, path)
|
||||||
|
result = importlib.util.module_from_spec(spec); spec.loader.exec_module(result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
app = module('container_app', Path(__file__).with_name('app.py'))
|
||||||
|
manage = module('container_manage', Path(__file__).with_name('manage.py'))
|
||||||
|
import backup
|
||||||
|
import backup_retention as br
|
||||||
|
|
||||||
|
|
||||||
|
class ContainerOperationsTests(unittest.TestCase):
|
||||||
|
def test_retirement_rename_stays_on_the_backup_volume_and_resumes_cleanup(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root=Path(directory).resolve();folder=root/'output/backups/20261110T010000Z-12345678'
|
||||||
|
folder.mkdir(parents=True);(folder/'snapshot').write_bytes(b'old backup')
|
||||||
|
keep=folder.parent/'unrelated';keep.mkdir()
|
||||||
|
br.retire_folder(root,folder,'backup')
|
||||||
|
self.assertFalse(folder.exists())
|
||||||
|
record=next((root/'.runtime/retention/retired').glob('*.json'))
|
||||||
|
value=json.loads(record.read_text());self.assertEqual(value['storage'],'backup-volume')
|
||||||
|
self.assertTrue((root/'output/backups/.retired'/value['target']).is_dir())
|
||||||
|
self.assertEqual(br.clean_retired(root),1);self.assertTrue(keep.is_dir())
|
||||||
|
self.assertFalse(record.exists())
|
||||||
|
|
||||||
|
def test_secret_initialization_is_private_idempotent_and_not_in_command_arguments(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory); (root / 'deploy/docker').mkdir(parents=True)
|
||||||
|
(root / 'deploy/docker/.env.example').write_text('HTTP_BIND=127.0.0.1\n')
|
||||||
|
base = manage.initialize(root)
|
||||||
|
before = {p.name: p.read_bytes() for p in (base / 'secrets').iterdir()}
|
||||||
|
manage.initialize(root)
|
||||||
|
self.assertEqual(before, {p.name: p.read_bytes() for p in (base / 'secrets').iterdir()})
|
||||||
|
for path in (base / 'secrets').iterdir(): self.assertEqual(path.stat().st_mode & 0o077, 0)
|
||||||
|
env = manage.environment(root)
|
||||||
|
self.assertEqual(len(set(env[k] for k in manage.SECRET_NAMES.values())), 3)
|
||||||
|
args = manage.command('up')
|
||||||
|
for value in before.values(): self.assertNotIn(value.decode().strip(), str(args))
|
||||||
|
self.assertNotIn('-v', manage.command('stop'))
|
||||||
|
|
||||||
|
def test_config_uses_tcp_and_refuses_uncoordinated_secret_changes(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
with patch.object(app, 'secret', return_value='a' * 32), patch.dict(os.environ, {}, clear=True):
|
||||||
|
first = app.prepare(root)
|
||||||
|
self.assertEqual(first['mysqlHost'], 'mysql'); self.assertFalse(first['binaryLogEnabled'])
|
||||||
|
self.assertEqual((root / '.runtime/local-config.json').stat().st_mode & 0o077, 0)
|
||||||
|
env = app.backend_environment(first)
|
||||||
|
self.assertIn('jdbc:mysql://mysql:3306/', env['MYSQL_URL'])
|
||||||
|
self.assertEqual(env['SERVER_ADDRESS'], '0.0.0.0')
|
||||||
|
with patch.object(app, 'secret', return_value='b' * 32):
|
||||||
|
with self.assertRaises(ValueError): app.prepare(root)
|
||||||
|
|
||||||
|
def test_backend_exit_causes_container_failure_instead_of_leaving_a_dead_api(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory); (root / '.runtime').mkdir()
|
||||||
|
child = Mock(pid=777); child.poll.return_value = 2
|
||||||
|
with patch.object(app, 'ROOT', root), patch.object(app, 'prepare', return_value={}), \
|
||||||
|
patch.object(app, 'backend_environment', return_value={}), patch.object(br, 'accepted_backend'), \
|
||||||
|
patch.object(app.subprocess, 'Popen', return_value=child), patch.object(app.signal, 'signal'):
|
||||||
|
self.assertEqual(app.serve(), 1)
|
||||||
|
child.wait.assert_called_once()
|
||||||
|
|
||||||
|
def test_shutdown_signals_backend_and_workers_before_waiting(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory); (root / '.runtime').mkdir()
|
||||||
|
children = [Mock(pid=100 + i) for i in range(3)]
|
||||||
|
for child in children: child.poll.return_value = None
|
||||||
|
handlers = {}
|
||||||
|
def register(signum, handler): handlers[signum] = handler
|
||||||
|
def tick(_): handlers[app.signal.SIGTERM](app.signal.SIGTERM, None)
|
||||||
|
with patch.object(app, 'ROOT', root), patch.object(app, 'prepare', return_value={}), \
|
||||||
|
patch.object(app, 'backend_environment', return_value={}), patch.object(br, 'accepted_backend'), \
|
||||||
|
patch.object(app.subprocess, 'Popen', side_effect=children), patch.object(app.signal, 'signal', side_effect=register), \
|
||||||
|
patch.object(app, 'ready', return_value=True), patch.object(app, 'review_retention'), \
|
||||||
|
patch.object(app.time, 'sleep', side_effect=tick), patch.object(app.os, 'killpg') as kill:
|
||||||
|
self.assertEqual(app.serve(), 0)
|
||||||
|
self.assertEqual(kill.call_count, 3)
|
||||||
|
for child in children: child.wait.assert_called_once()
|
||||||
|
|
||||||
|
def test_container_pinned_jar_rejects_corruption(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {'QUALITY_CONTAINER': '1'}):
|
||||||
|
root = Path(directory); (root / 'app').mkdir()
|
||||||
|
jar = root / 'app/backend.jar'; jar.write_bytes(b'accepted')
|
||||||
|
(root / 'app/backend.sha256').write_text(br.sha(jar))
|
||||||
|
self.assertEqual(br.accepted_backend(root)[0], jar.resolve())
|
||||||
|
jar.write_bytes(b'changed')
|
||||||
|
with self.assertRaises(ValueError): br.accepted_backend(root)
|
||||||
|
|
||||||
|
def test_backup_uses_owned_private_instance_for_container_and_owns_temporary_files(self):
|
||||||
|
from contextlib import contextmanager
|
||||||
|
import backup_verifier
|
||||||
|
configuration = Path('/tmp/owned-verifier/client.cnf')
|
||||||
|
@contextmanager
|
||||||
|
def instance(root): yield configuration
|
||||||
|
seen = []
|
||||||
|
def verify(folder):
|
||||||
|
seen.append((backup.client_config(), backup.verification_temp_root()))
|
||||||
|
return 'verified'
|
||||||
|
with tempfile.TemporaryDirectory() as directory, patch.object(backup, 'ROOT', Path(directory)), \
|
||||||
|
patch.dict(os.environ, {'QUALITY_CONTAINER': '1'}), patch.object(backup_verifier, 'instance', instance), \
|
||||||
|
patch.object(backup, '_verify', side_effect=verify):
|
||||||
|
self.assertEqual(backup.verify('unused'), 'verified')
|
||||||
|
self.assertEqual(seen, [(configuration, configuration.parent)])
|
||||||
|
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
|
||||||
|
|
||||||
|
def test_compose_preserves_volumes_bounded_logs_secrets_and_private_networks(self):
|
||||||
|
try: import yaml
|
||||||
|
except ImportError: self.skipTest('Install PyYAML for Compose contract checks')
|
||||||
|
compose = yaml.safe_load((ROOT / 'deploy/docker/compose.yaml').read_text())
|
||||||
|
services = compose['services']
|
||||||
|
self.assertNotIn('ports', services['mysql']); self.assertNotIn('ports', services['app'])
|
||||||
|
self.assertEqual(services['mysql']['command'][0], '--skip-log-bin')
|
||||||
|
self.assertEqual(len(compose['volumes']), 5)
|
||||||
|
self.assertEqual(services['app']['depends_on']['mysql']['condition'], 'service_healthy')
|
||||||
|
for service in services.values():
|
||||||
|
self.assertEqual(service['restart'], 'unless-stopped')
|
||||||
|
self.assertEqual(service['logging']['options']['max-size'], '10m')
|
||||||
|
self.assertTrue(services['app']['read_only'])
|
||||||
|
self.assertNotIn('docker.sock', json.dumps(compose))
|
||||||
|
self.assertFalse((ROOT / '.gitea/workflows').exists())
|
||||||
|
|
||||||
|
def test_build_context_excludes_live_data_and_private_baseline(self):
|
||||||
|
ignore = (ROOT / '.dockerignore').read_text()
|
||||||
|
self.assertEqual(next(line for line in ignore.splitlines() if line and not line.startswith('#')), '**')
|
||||||
|
for allowed in ('!.runtime', '!runtimedata', '!output', '!质量数据', '!apikey.txt'):
|
||||||
|
self.assertNotIn(allowed, ignore)
|
||||||
|
self.assertIn('**/original-workbook-acceptance.json', ignore)
|
||||||
|
nginx = (ROOT / 'deploy/docker/nginx.conf').read_text()
|
||||||
|
self.assertIn('proxy_buffering off', nginx)
|
||||||
|
self.assertIn('resolver 127.0.0.11', nginx)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': unittest.main()
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""与后端共享 PID 命名空间的低优先级维护任务;不向容器暴露 Docker socket。"""
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'deploy/macos')]
|
||||||
|
import backup_retention as br
|
||||||
|
import operations
|
||||||
|
import retention
|
||||||
|
|
||||||
|
|
||||||
|
def console_notify(title, message):
|
||||||
|
print(json.dumps({'event': 'capacity-alert', 'title': title, 'message': message}, ensure_ascii=False), flush=True)
|
||||||
|
return {'submitted': True, 'channel': 'container-log'}
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot(root, support=None, now=None):
|
||||||
|
import backup
|
||||||
|
import managed_test_resources as resources
|
||||||
|
import log_retention
|
||||||
|
root = Path(root); now = time.time() if now is None else now
|
||||||
|
cfg = json.loads((root / '.runtime/local-config.json').read_text())
|
||||||
|
rows = backup.client('SELECT TABLE_SCHEMA,COALESCE(SUM(DATA_LENGTH+INDEX_LENGTH),0) FROM information_schema.tables GROUP BY TABLE_SCHEMA;')
|
||||||
|
estimates = {parts[0]: int(parts[1]) for line in rows.splitlines() if len(parts := line.split('\t')) == 2}
|
||||||
|
registered = 0; overdue = 0
|
||||||
|
for path in resources.registry(root).glob('*.json'):
|
||||||
|
item = resources.load(root, path.stem)
|
||||||
|
if not item or not item.get('created'): continue
|
||||||
|
registered += estimates.get(item['database'], 0)
|
||||||
|
if now >= item.get('expiresAt', now + 1): overdue += 1
|
||||||
|
private = sum(operations.allocated(p) for p in (root / '.runtime/retention').glob('verify-*') if p.is_dir() and not p.is_symlink())
|
||||||
|
mysql = estimates.get(cfg['mysqlDatabase'], 0)
|
||||||
|
archive = operations.allocated(root / 'runtimedata/archive'); backups = operations.allocated(root / 'output/backups')
|
||||||
|
native = operations.allocated(root / '.runtime/nao-users')
|
||||||
|
logs = sum(p.stat().st_blocks * 512 for p in log_retention.candidates(root, root / '.runtime/operations'))
|
||||||
|
core = mysql + archive + backups + native + logs
|
||||||
|
free = min(shutil.disk_usage(root / p).free for p in ('.runtime', 'output/backups', 'output/logs', 'runtimedata/archive'))
|
||||||
|
return {'at': now, 'mysqlBytes': mysql, 'mysqlMeasurement': 'information_schema_estimate_excludes_system_redo_undo',
|
||||||
|
'archiveBytes': archive, 'backupBytes': backups, 'logBytes': logs, 'nativeDataBytes': native,
|
||||||
|
'registeredTestBytes': registered, 'privateVerifierBytes': private, 'temporaryBytes': registered + private,
|
||||||
|
'coreBytes': core, 'artifactBytes': 0, 'permanentBytes': core, 'freeBytes': free,
|
||||||
|
'overdueTestResources': overdue}
|
||||||
|
|
||||||
|
|
||||||
|
def once(role):
|
||||||
|
if role == 'monitor':
|
||||||
|
operations.snapshot = snapshot
|
||||||
|
return operations.run(ROOT, support=ROOT / '.runtime/operations', notifier=console_notify)
|
||||||
|
return retention.run(ROOT)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
role = sys.argv[1]
|
||||||
|
if role not in ('retention', 'monitor'): raise ValueError('Unknown worker role')
|
||||||
|
try: os.nice(10)
|
||||||
|
except OSError: pass
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
||||||
|
stopping = False
|
||||||
|
def stop(signum, frame):
|
||||||
|
nonlocal stopping
|
||||||
|
stopping = True
|
||||||
|
signal.signal(signal.SIGTERM, stop); signal.signal(signal.SIGINT, stop)
|
||||||
|
while not stopping:
|
||||||
|
try:
|
||||||
|
once(role)
|
||||||
|
except (br.Busy, retention.Busy): pass
|
||||||
|
except Exception as error:
|
||||||
|
if role == 'monitor': operations.failed_run(ROOT, error, notifier=console_notify)
|
||||||
|
else:
|
||||||
|
path = ROOT / '.runtime/retention/state.json'
|
||||||
|
state = json.loads(path.read_text()) if path.exists() else {}
|
||||||
|
state['consecutiveFailures'] = min(1000, state.get('consecutiveFailures', 0) + 1)
|
||||||
|
state['lastError'] = {'time': time.time(), 'type': type(error).__name__}
|
||||||
|
br.atomic(path, state)
|
||||||
|
print(json.dumps({'event': 'maintenance-error', 'role': role, 'errorType': type(error).__name__}), flush=True)
|
||||||
|
for _ in range(300):
|
||||||
|
if stopping: break
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': main()
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
# 本机服务自动启动
|
||||||
|
|
||||||
|
本机使用 `com.lyunlab.quality-center-service` LaunchAgent:解锁并登录当前 macOS 用户后启动,持续守护现有 MySQL 和已验收的后端 JAR。SSH 隧道由已有的 `com.lyunlab.quality-center-tunnel` 独立守护。
|
||||||
|
|
||||||
|
启用 FileVault 的电脑在解锁登录前无法运行用户级守护。退出用户会话、电脑关机、休眠或断网期间,不能依靠这台 Mac 持续提供公网服务。需要全天候服务时,按 [Docker 部署说明](../docker/README.md) 将后台和数据库部署到常在线服务器。
|
||||||
|
|
||||||
|
## 管理
|
||||||
|
|
||||||
|
首次安装维护环境时运行 `python3 -m venv .runtime/retention-env`,再使用 `.runtime/retention-env/bin/pip install --require-hashes -r deploy/macos/retention-requirements.txt` 安装固定依赖。该步骤只安装维护工具,不初始化业务数据库。
|
||||||
|
|
||||||
|
在项目目录执行:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python3 deploy/macos/service.py status
|
||||||
|
python3 deploy/macos/service.py pause
|
||||||
|
python3 deploy/macos/service.py stop
|
||||||
|
python3 deploy/macos/service.py start
|
||||||
|
```
|
||||||
|
|
||||||
|
- `status`:显示守护、进程身份、数据库和接口就绪状态。
|
||||||
|
- `pause`:停用自动启动和守护,保留当前 MySQL / 后端进程,适合维护前使用。
|
||||||
|
- `stop`:停用守护,按后端、MySQL 的顺序正常停服;超时不会强制结束数据库。
|
||||||
|
- `start`:重新启用自动启动和守护,先确认数据库可查询,再启动后端。
|
||||||
|
|
||||||
|
维护期间先暂停守护,再操作服务。直接运行旧的停止脚本后,仍启用的守护会把进程重新拉起。
|
||||||
|
|
||||||
|
## 实现与边界
|
||||||
|
|
||||||
|
- 安装时登记已验收 JAR 的 SHA-256,从 `.runtime/backend-builds/` 启动对应的不可变运行包。
|
||||||
|
- 只使用 `.runtime/local-config.json` 指定的已初始化数据库;缺少数据标记或运行包摘要不符时停止启动,不初始化、不导入数据库。
|
||||||
|
- 每 3 秒检查进程;异常退出后等待 10 秒重启。数据库可查询后才启动后端。端口被未知进程占用或进程身份无法确认时保留现场。
|
||||||
|
- 守护自身由 launchd 的 `KeepAlive` 负责重启。重新启动守护时会核对启动时间、用户、程序和参数,接管仍存活的服务,避免重复启动。
|
||||||
|
- 只对已经退出的进程自动重启。健康检查暂时失败时不强行终止正在导入或查询的数据库。
|
||||||
|
- 守护脚本按摘要复制到用户的 Application Support 目录。项目源文件的后续编辑不会立即改变已安装脚本。
|
||||||
|
- MySQL 的 `binaryLogEnabled` 配置为 false 时,使用 `--skip-log-bin` 启动。事务重做日志和 undo 文件继续保留。历史时间点恢复不可用,恢复依赖最新完整快照。
|
||||||
|
|
||||||
|
启动配置:`~/Library/LaunchAgents/com.lyunlab.quality-center-service.plist`。
|
||||||
|
|
||||||
|
状态和滚动日志:`~/Library/Application Support/QualityCenterService/`。
|
||||||
|
|
||||||
|
进程业务日志:项目的 `output/logs/`。
|
||||||
|
|
||||||
|
重新安装或升级守护脚本时使用 `service.py install --root <项目绝对路径> --jar <已验收JAR绝对路径>`。部署配置中只记录路径与摘要,密码仍从原有私有配置读取。
|
||||||
|
|
||||||
|
## 验证
|
||||||
|
|
||||||
|
```sh
|
||||||
|
PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/macos -p 'test_service.py' -v
|
||||||
|
```
|
||||||
|
|
||||||
|
隔离进程检查覆盖 MySQL 与后端异常退出重启、数据库启动顺序、已有进程接管、未知进程保护和运行包/数据目录前置校验。实际重启电脑和 FileVault 解锁后的启动仍需单独验收。
|
||||||
|
|
||||||
|
## 自动数据保留
|
||||||
|
|
||||||
|
`com.lyunlab.quality-center-retention` 是独立的低优先级维护任务,解锁登录后运行,每5分钟检查一次。它使用当前已验收 JAR,独立运行维护脚本;不重新构建业务后端。
|
||||||
|
|
||||||
|
- **旧版本**:当前正式版本、候选版本及基线、待配置来源的基线、分析会话、已保存报表/快照/清单、巡检等业务引用均保护。其他已发布版本连续30分钟未被引用,且当前版本已有恢复验证通过的备份后,才进入回收。
|
||||||
|
- **旧构建**:只处理已无版本引用的终态构建。分批清理,每批最多1000行;先撤销其可复用状态和唯一构建键,避免新导入复用正在清理的半成品。仍有未知业务引用时保留。
|
||||||
|
- **旧来源修订**:每5分钟检查,已完成修订连续30分钟没有任何版本、构建、明细、来源清单、替换/撤销/恢复记录或保存内容引用后才回收。修订身份变化或重新出现引用会重置保护期;未知状态、缺失或校验异常的文件保留。SHA-256、大小、路径边界与文件身份全部通过后才删除修订记录,原件另行核对共享路径和保存内容引用。共享文件保留;文件删除意图先持久登记,意外退出后可续清,不扫描删除未登记的孤立文件。
|
||||||
|
- **上传草稿**:24小时过期后再留30分钟保护期。已入库文件先核对正式归档摘要,再重绑定来源路径、清理临时副本;待导入来源仍引用的文件保留。无数据库记录的上传临时文件超过24小时才检查回收。
|
||||||
|
- **备份**:检测到新正式版本或距离上次备份超过24小时,在任务空闲时生成新备份。新备份必须通过独立 MySQL 实例恢复、表行数、正式数据汇总、应用文件和来源归档校验,才原子更新最新备份入口并回收旧备份。失败时保留旧有效备份,失败重试间隔至少1小时。
|
||||||
|
- **任务保护**:导入、数据工作流、分析、报告、巡检、排队任务及待自动重试任务存在时,自动维护延期。每一批删除都重新检查任务状态和当前版本;短暂取得数据库排他访问失败时延期,不终止业务查询或任务。
|
||||||
|
- **互斥与失败恢复**:手动备份、恢复和自动维护共享 `.runtime/data-maintenance.lock`。恢复校验使用临时、仅本机私有 socket 可访问的独立 MySQL,完成后正常关闭并删除;异常遗留只根据所有者记录及进程身份处理,不按数据库名前缀批量删除。
|
||||||
|
- **维护暂停**:`service.py pause/stop` 同时写入维护暂停标记;`service.py start/install` 解除。执行中的备份允许安全完成,后续删除批次会停下。
|
||||||
|
- **升级保护**:当前 JAR 摘要或数据库结构发生变化时暂停自动清理,重新检查兼容性后执行维护安装命令。未知任务状态也按进行中处理。
|
||||||
|
- **空间余量**:开始新备份前预留至少三倍数据库表/索引容量、归档空间和额外安全余量;空间不足时保留现有有效备份并延期。失败的新候选只清理本次登记的文件,未知目录不会删除。
|
||||||
|
|
||||||
|
最新备份记录在 `.runtime/retention/latest.json`,实际目录在 `output/backups/`,界面“任务与监控”的备份列表可查看。
|
||||||
|
|
||||||
|
维护任务状态与滚动日志分别在 `.runtime/retention/state.json` 和 `~/Library/Application Support/QualityCenterService/logs/retention.log`。
|
||||||
|
|
||||||
|
来源修订回收计数为 `sourceRevisionsRemoved`、`sourceArchiveFilesRemoved` 和 `sourceArchiveBytesRemoved`,待解除文件引用的登记数为 `sourceFilesPending`。登记目录为 `.runtime/retention/source-files/`。来源清单中仍可重新导入的记录(包括已导入状态)属于有效引用,不能作为无引用修订删除。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
.runtime/retention-env/bin/python3 deploy/macos/retention.py status --root "$PWD"
|
||||||
|
.runtime/retention-env/bin/python3 deploy/macos/retention.py install --root "$PWD"
|
||||||
|
PYTHONDONTWRITEBYTECODE=1 .runtime/retention-env/bin/python3 -m unittest discover -s deploy/macos -p 'test_retention.py' -v
|
||||||
|
```
|
||||||
|
|
||||||
|
维护依赖 PyMySQL 固定版本及 SHA-256 在 `retention-requirements.txt` 中。私有 Python 环境与依赖不修改全局安装。日常删除释放的 InnoDB 表内空间由后续导入复用;必要的物理表空间整理另在低峰进行。
|
||||||
|
|
||||||
|
## 容量、测试资源与日志
|
||||||
|
|
||||||
|
`operations.py` 安装独立的 5 分钟检查任务。容量阈值为 30/50GiB,余量阈值为 50/20GiB;监测业务数据、交付产物、临时校验空间及约 24 小时增长,保留 30 天日报。告警通过 macOS 系统通知发送,并记录在 `.runtime/operations/`,相同告警 6 小时内去重。
|
||||||
|
|
||||||
|
注册过的测试数据库在正常退出时回收;异常遗留超过 24 小时,确认所有者和子进程退出、标记一致且无连接后才删除。未登记数据库继续保留。日志按 7 天、每类 100MiB 限制;Java 原生日志滚动配置在 `logback-quality.xml`,MySQL 通过重新打开错误日志完成滚动,不重启数据库,不截断活动文件。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
.runtime/retention-env/bin/python3 deploy/macos/operations.py install --root "$PWD"
|
||||||
|
.runtime/retention-env/bin/python3 deploy/macos/operations.py status --root "$PWD"
|
||||||
|
.runtime/retention-env/bin/python3 deploy/macos/operations.py notify-test --root "$PWD"
|
||||||
|
```
|
||||||
|
|
||||||
|
安装日志配置后,后端需在任务空闲时正常重启一次才能加载;不要为加载日志配置中断导入。Linux 复用同一回收逻辑,采用容器日志告警,见 Docker 说明。
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<configuration>
|
||||||
|
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||||
|
<property name="QUALITY_LOG_DIR" value="${QUALITY_LOG_DIR:-output/logs}"/>
|
||||||
|
<appender name="QUALITY_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||||
|
<file>${QUALITY_LOG_DIR}/backend.log</file>
|
||||||
|
<append>true</append>
|
||||||
|
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||||
|
<fileNamePattern>${QUALITY_LOG_DIR}/backend.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
|
||||||
|
<maxFileSize>10MB</maxFileSize>
|
||||||
|
<maxHistory>7</maxHistory>
|
||||||
|
<totalSizeCap>80MB</totalSizeCap>
|
||||||
|
<cleanHistoryOnStart>true</cleanHistoryOnStart>
|
||||||
|
</rollingPolicy>
|
||||||
|
<encoder><pattern>${FILE_LOG_PATTERN}</pattern><charset>UTF-8</charset></encoder>
|
||||||
|
</appender>
|
||||||
|
<logger name="org.springframework.security" level="WARN"/>
|
||||||
|
<root level="INFO"><appender-ref ref="QUALITY_FILE"/></root>
|
||||||
|
</configuration>
|
||||||
@@ -0,0 +1,276 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Independent capacity monitoring, registered-test cleanup and log retention."""
|
||||||
|
import argparse
|
||||||
|
import fcntl
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from logging.handlers import RotatingFileHandler
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import plistlib
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
|
LABEL = 'com.lyunlab.quality-center-operations'
|
||||||
|
SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
|
||||||
|
GIB = 1024**3
|
||||||
|
DEFAULTS = {'intervalSeconds': 300, 'warningBytes': 30 * GIB, 'criticalBytes': 50 * GIB,
|
||||||
|
'freeWarningBytes': 50 * GIB, 'freeCriticalBytes': 20 * GIB,
|
||||||
|
'growthBytes': 2 * GIB, 'growthFraction': .2, 'notificationCooldownSeconds': 21600,
|
||||||
|
'logMaxAgeSeconds': 604800, 'logClassMaxBytes': 100 * 1024**2,
|
||||||
|
'testMaxAgeSeconds': 86400}
|
||||||
|
|
||||||
|
|
||||||
|
def allocated(path):
|
||||||
|
path = Path(path)
|
||||||
|
if not path.exists() or path.is_symlink(): return 0
|
||||||
|
result = subprocess.run(['/usr/bin/du', '-sk', str(path)], capture_output=True, text=True, timeout=45)
|
||||||
|
lines = result.stdout.splitlines()
|
||||||
|
if result.returncode or not lines: raise RuntimeError('Capacity scan incomplete')
|
||||||
|
return int(lines[-1].split()[0]) * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot(root, support=SUPPORT, now=None):
|
||||||
|
import managed_test_resources as resources
|
||||||
|
import log_retention
|
||||||
|
root = Path(root).resolve(); now = time.time() if now is None else now
|
||||||
|
mysql = allocated(root / '.runtime/mysql-data')
|
||||||
|
archives = allocated(root / 'runtimedata/archive'); backups = allocated(root / 'output/backups')
|
||||||
|
runtime = allocated(root / '.runtime'); outputs = allocated(root / 'output')
|
||||||
|
logs = log_retention.candidates(root, support)
|
||||||
|
log_bytes = sum(p.stat().st_blocks * 512 for p in logs)
|
||||||
|
output_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / 'output'))
|
||||||
|
runtime_logs = sum(p.stat().st_blocks * 512 for p in logs if p.is_relative_to(root / '.runtime'))
|
||||||
|
native = allocated(root / '.runtime/nao-users')
|
||||||
|
registered = 0; overdue = 0
|
||||||
|
for p in resources.registry(root).glob('*.json'):
|
||||||
|
value = resources.load(root, p.stem)
|
||||||
|
if not value or not value.get('created'): continue
|
||||||
|
if now >= value.get('expiresAt', now + 1): overdue += 1
|
||||||
|
registered += allocated(root / '.runtime/mysql-data' / value['database'])
|
||||||
|
private = 0
|
||||||
|
for p in (root / '.runtime/retention').glob('verify-*'):
|
||||||
|
if p.is_symlink(): continue
|
||||||
|
try: owner = json.loads((p / 'owner.json').read_text())
|
||||||
|
except (OSError, ValueError): continue
|
||||||
|
if owner.get('kind') == 'private-quality-restore' and owner.get('directory') == str(p): private += allocated(p)
|
||||||
|
# Core service data is the same scope as the earlier 11.86GiB measurement.
|
||||||
|
# Software/build/delivery artifacts are visible separately and included in growth checks.
|
||||||
|
core = max(0, mysql - registered) + archives + backups + log_bytes + native
|
||||||
|
artifacts = max(0, runtime - mysql - private - native - runtime_logs) + max(0, outputs - backups - output_logs)
|
||||||
|
return {'at': now, 'mysqlBytes': mysql, 'archiveBytes': archives, 'backupBytes': backups,
|
||||||
|
'logBytes': log_bytes, 'nativeDataBytes': native, 'registeredTestBytes': registered,
|
||||||
|
'privateVerifierBytes': private, 'temporaryBytes': registered + private,
|
||||||
|
'coreBytes': core, 'artifactBytes': artifacts, 'permanentBytes': core + artifacts,
|
||||||
|
'freeBytes': shutil.disk_usage(root).free, 'overdueTestResources': overdue}
|
||||||
|
|
||||||
|
|
||||||
|
def conditions(sample, policy, maintenance, history):
|
||||||
|
alerts = {}
|
||||||
|
if sample['coreBytes'] >= policy['criticalBytes']:
|
||||||
|
alerts['capacity'] = {'severity': 'critical', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB,请检查增长来源。"}
|
||||||
|
elif sample['coreBytes'] >= policy['warningBytes']:
|
||||||
|
alerts['capacity'] = {'severity': 'warning', 'message': f"业务数据占用已达{sample['coreBytes']/GIB:.1f}GiB。"}
|
||||||
|
for key, label, size in [('temporary', '测试及校验临时空间', sample.get('temporaryBytes', 0)),
|
||||||
|
('artifacts', '交付及构建产物', sample.get('artifactBytes', max(0, sample['permanentBytes'] - sample['coreBytes'])))]:
|
||||||
|
if size >= policy['warningBytes']:
|
||||||
|
alerts[key] = {'severity': 'critical' if size >= policy['criticalBytes'] else 'warning',
|
||||||
|
'message': f'{label}已占用{size/GIB:.1f}GiB,请核对;正在运行的任务会继续保护。'}
|
||||||
|
if sample['freeBytes'] < policy['freeCriticalBytes']:
|
||||||
|
alerts['disk'] = {'severity': 'critical', 'message': f"磁盘仅余{sample['freeBytes']/GIB:.1f}GiB。"}
|
||||||
|
elif sample['freeBytes'] < policy['freeWarningBytes']:
|
||||||
|
alerts['disk'] = {'severity': 'warning', 'message': f"磁盘剩余{sample['freeBytes']/GIB:.1f}GiB,请预留备份空间。"}
|
||||||
|
if maintenance.get('consecutiveFailures', 0) >= 3:
|
||||||
|
alerts['maintenance'] = {'severity': 'critical', 'message': '数据维护已连续失败3次,请检查维护记录。'}
|
||||||
|
if sample['overdueTestResources']:
|
||||||
|
alerts['tests'] = {'severity': 'warning', 'message': f"有{sample['overdueTestResources']}个测试资源已超过24小时;正在运行或身份未确认的资源将继续保留。"}
|
||||||
|
old = [v for v in history if 21 * 3600 <= sample['at'] - v['at'] <= 27 * 3600]
|
||||||
|
if old:
|
||||||
|
baseline = min(old, key=lambda x: abs(sample['at'] - x['at'] - 86400))
|
||||||
|
changes = []
|
||||||
|
for label, key in [('业务数据', 'coreBytes'), ('交付及构建产物', 'artifactBytes')]:
|
||||||
|
before = baseline.get(key, baseline['permanentBytes'] - baseline['coreBytes'])
|
||||||
|
after = sample.get(key, sample['permanentBytes'] - sample['coreBytes'])
|
||||||
|
delta = after - before
|
||||||
|
if delta > max(policy['growthBytes'], before * policy['growthFraction']):
|
||||||
|
changes.append(f'{label}增长{delta/GIB:.1f}GiB')
|
||||||
|
if changes:
|
||||||
|
alerts['growth'] = {'severity': 'warning', 'message': '约24小时' + ','.join(changes) + ',请核对。'}
|
||||||
|
return alerts
|
||||||
|
|
||||||
|
|
||||||
|
def notify(title, message):
|
||||||
|
script = 'on run argv\ndisplay notification (item 2 of argv) with title (item 1 of argv)\nend run\n'
|
||||||
|
try:
|
||||||
|
p = subprocess.run(['/usr/bin/osascript', '-', title, message], input=script, text=True,
|
||||||
|
capture_output=True, timeout=10)
|
||||||
|
return {'submitted': p.returncode == 0, 'channel': 'macOS', 'exitCode': p.returncode}
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
return {'submitted': False, 'channel': 'macOS'}
|
||||||
|
|
||||||
|
|
||||||
|
def dispatch(alerts, previous, now, policy, notifier=notify):
|
||||||
|
updated = {}; events = []
|
||||||
|
for key, value in alerts.items():
|
||||||
|
prior = previous.get(key, {})
|
||||||
|
due = (not prior or not prior.get('lastSubmittedAt') or prior.get('severity') != value['severity'] or
|
||||||
|
now - prior.get('lastSubmittedAt', 0) >= policy['notificationCooldownSeconds'])
|
||||||
|
entry = {**value, 'since': prior.get('since', now), 'lastSubmittedAt': prior.get('lastSubmittedAt', 0)}
|
||||||
|
if due:
|
||||||
|
result = notifier('质量平台容量告警', value['message'])
|
||||||
|
entry['notification'] = result
|
||||||
|
# Failed submission retries on the next check; duplicates are limited after success.
|
||||||
|
if result.get('submitted'): entry['lastSubmittedAt'] = now
|
||||||
|
events.append({'kind': 'alert', 'key': key, **value, 'at': now, 'notification': result})
|
||||||
|
updated[key] = entry
|
||||||
|
for key in set(previous) - set(alerts):
|
||||||
|
events.append({'kind': 'resolved', 'key': key, 'at': now})
|
||||||
|
return updated, events
|
||||||
|
|
||||||
|
|
||||||
|
def janitor(root, support, policy):
|
||||||
|
import backup_retention as br
|
||||||
|
import backup_verifier
|
||||||
|
import managed_test_resources as resources
|
||||||
|
import log_retention
|
||||||
|
from retention import Database, Busy
|
||||||
|
root = Path(root).resolve()
|
||||||
|
if (root / '.runtime/retention/paused').exists(): return {'deferred': 'Maintenance is paused'}
|
||||||
|
try:
|
||||||
|
with br.lock(root, timeout=0):
|
||||||
|
db = Database(root)
|
||||||
|
try:
|
||||||
|
config = json.loads((root / '.runtime/retention/config.json').read_text())
|
||||||
|
accepted = br.accepted_backend(root)
|
||||||
|
if config['schemaSignature'] != db.schema_signature or not accepted or config['acceptedJarSha256'] != accepted[1]:
|
||||||
|
return {'deferred': 'Application version changed'}
|
||||||
|
db.current()
|
||||||
|
if db.busy(): return {'deferred': 'Application jobs are active'}
|
||||||
|
# Use a short-lived CLI connection so no observer holds a test schema open.
|
||||||
|
import backup
|
||||||
|
result = resources.cleanup(root, backup.client)
|
||||||
|
result['abandonedVerifiersRemoved'] = backup_verifier.clean_abandoned(root)
|
||||||
|
result['mysqlLogsRotated'] = log_retention.rotate_mysql(root, backup.client, br)
|
||||||
|
result.update(log_retention.retain(root, support,
|
||||||
|
max_bytes=policy['logClassMaxBytes'], max_age=policy['logMaxAgeSeconds']))
|
||||||
|
return result
|
||||||
|
finally: db.conn.close()
|
||||||
|
except (br.Busy, Busy): return {'deferred': 'Another maintenance or application operation is active'}
|
||||||
|
|
||||||
|
|
||||||
|
def run(root, support=SUPPORT, notifier=notify):
|
||||||
|
import backup_retention as br
|
||||||
|
root = Path(root).resolve(); base = root / '.runtime/operations'
|
||||||
|
if base.is_symlink() or (root / '.runtime').is_symlink(): raise ValueError('Unsafe operations state path')
|
||||||
|
base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
policy = {**DEFAULTS, **json.loads((base / 'policy.json').read_text())}
|
||||||
|
state_path = base / 'monitor.json'; previous = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||||
|
history_path = base / 'samples.json'; history = json.loads(history_path.read_text()) if history_path.exists() else []
|
||||||
|
now = time.time(); sample = snapshot(root, support, now)
|
||||||
|
maintenance_path = root / '.runtime/retention/state.json'
|
||||||
|
maintenance = json.loads(maintenance_path.read_text()) if maintenance_path.exists() else {}
|
||||||
|
alerts = conditions(sample, policy, maintenance, history)
|
||||||
|
cleanup = {}; failures = previous.get('cleanupFailures', 0)
|
||||||
|
try:
|
||||||
|
cleanup = janitor(root, support, policy)
|
||||||
|
if 'deferred' not in cleanup: failures = 0
|
||||||
|
except Exception as error:
|
||||||
|
failures += 1; cleanup = {'errorType': type(error).__name__}
|
||||||
|
if failures >= 3: alerts['cleanup'] = {'severity': 'warning', 'message': '日志或测试环境清理已连续失败3次,请检查维护记录。'}
|
||||||
|
active, events = dispatch(alerts, previous.get('alerts', {}), now, policy, notifier)
|
||||||
|
state = {'checkedAt': now, 'sample': sample, 'cleanup': cleanup, 'cleanupFailures': failures, 'alerts': active}
|
||||||
|
br.atomic(state_path, state)
|
||||||
|
(base / 'failure.json').unlink(missing_ok=True)
|
||||||
|
history = [v for v in history if now - v['at'] <= 27 * 3600]
|
||||||
|
history.append(sample); br.atomic(history_path, history[-330:])
|
||||||
|
daily_path = base / 'daily.json'; daily = json.loads(daily_path.read_text()) if daily_path.exists() else {}
|
||||||
|
day = datetime.fromtimestamp(now, ZoneInfo('Asia/Shanghai')).date().isoformat(); daily[day] = sample
|
||||||
|
br.atomic(daily_path, {k: daily[k] for k in sorted(daily)[-30:]})
|
||||||
|
if events:
|
||||||
|
handler = RotatingFileHandler(base / 'alerts.jsonl', maxBytes=1024**2, backupCount=2)
|
||||||
|
handler.setFormatter(logging.Formatter('%(message)s'))
|
||||||
|
logger = logging.getLogger('quality-operations-alerts'); logger.addHandler(handler); logger.setLevel(logging.INFO); logger.propagate = False
|
||||||
|
try:
|
||||||
|
for event in events: logger.info(json.dumps(event, ensure_ascii=False))
|
||||||
|
finally: logger.removeHandler(handler); handler.close()
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
def failed_run(root, error, notifier=notify):
|
||||||
|
import backup_retention as br
|
||||||
|
base = Path(root).resolve() / '.runtime/operations'
|
||||||
|
if base.is_symlink() or base.parent.is_symlink(): return
|
||||||
|
path = base / 'failure.json'; value = json.loads(path.read_text()) if path.exists() else {}
|
||||||
|
now = time.time(); value.update({'count': value.get('count', 0) + 1, 'time': now, 'type': type(error).__name__})
|
||||||
|
if value['count'] >= 3 and now - value.get('lastSubmittedAt', 0) >= DEFAULTS['notificationCooldownSeconds']:
|
||||||
|
result = notifier('质量平台监测异常', '容量监测已连续失败3次,请查看监测状态。')
|
||||||
|
value['notification'] = result
|
||||||
|
if result.get('submitted'): value['lastSubmittedAt'] = now
|
||||||
|
br.atomic(path, value)
|
||||||
|
|
||||||
|
|
||||||
|
def install(root):
|
||||||
|
import backup_retention as br
|
||||||
|
root = Path(root).resolve(); base = root / '.runtime/operations'
|
||||||
|
base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
policy_path = base / 'policy.json'
|
||||||
|
policy = {**DEFAULTS, **(json.loads(policy_path.read_text()) if policy_path.exists() else {})}
|
||||||
|
br.atomic(policy_path, policy)
|
||||||
|
files = [Path(__file__).resolve(), root / 'deploy/macos/retention.py'] + [root / 'scripts' / p for p in
|
||||||
|
('backup_retention.py', 'revision_retention.py', 'backup_verifier.py', 'managed_test_resources.py', 'log_retention.py')]
|
||||||
|
digest = hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest()
|
||||||
|
version = SUPPORT / 'operations' / digest; version.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
for p in files: shutil.copy2(p, version / p.name); (version / p.name).chmod(0o600)
|
||||||
|
log_config = root / 'deploy/macos/logback-quality.xml'
|
||||||
|
shutil.copy2(log_config, base / 'logback-quality.xml'); (base / 'logback-quality.xml').chmod(0o600)
|
||||||
|
logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
value = {'Label': LABEL, 'ProgramArguments': [str(root / '.runtime/retention-env/bin/python3'), str(version / 'operations.py'), 'run', '--root', str(root)],
|
||||||
|
'RunAtLoad': True, 'StartInterval': 300, 'WorkingDirectory': str(root), 'ProcessType': 'Background',
|
||||||
|
'Nice': 10, 'LowPriorityIO': True, 'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'},
|
||||||
|
'StandardOutPath': str(logs / 'operations-launchd.log'), 'StandardErrorPath': str(logs / 'operations-launchd-error.log')}
|
||||||
|
domain = 'gui/' + str(os.getuid()); plist = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist')
|
||||||
|
subprocess.run(['launchctl', 'bootout', domain + '/' + LABEL], capture_output=True)
|
||||||
|
with plist.open('wb') as f: plistlib.dump(value, f)
|
||||||
|
plist.chmod(0o600); subprocess.run(['launchctl', 'enable', domain + '/' + LABEL], check=True)
|
||||||
|
for _ in range(50):
|
||||||
|
p = subprocess.run(['launchctl', 'bootstrap', domain, str(plist)], capture_output=True)
|
||||||
|
if p.returncode == 0: break
|
||||||
|
time.sleep(.2)
|
||||||
|
else: raise RuntimeError('Operations job registration did not complete')
|
||||||
|
print('Installed independent five-minute capacity and housekeeping checks')
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('action', choices=['install', 'run', 'status', 'notify-test']); parser.add_argument('--root', required=True)
|
||||||
|
args = parser.parse_args(); root = Path(args.root).resolve(); sys.path.append(str(root / 'scripts'))
|
||||||
|
import backup_retention as br
|
||||||
|
if args.action == 'install':
|
||||||
|
with br.lock(root, timeout=0): install(root)
|
||||||
|
elif args.action == 'status':
|
||||||
|
print((root / '.runtime/operations/monitor.json').read_text())
|
||||||
|
elif args.action == 'notify-test':
|
||||||
|
print(json.dumps(notify('质量平台通知测试', '容量监测已启用。这是测试通知,当前没有容量告警。'), ensure_ascii=False))
|
||||||
|
else:
|
||||||
|
logs = SUPPORT / 'logs'; logs.mkdir(parents=True, exist_ok=True)
|
||||||
|
handler = RotatingFileHandler(logs / 'operations.log', maxBytes=2 * 1024**2, backupCount=2)
|
||||||
|
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
|
||||||
|
logging.getLogger().addHandler(handler); logging.getLogger().setLevel(logging.INFO)
|
||||||
|
with (root / '.runtime/operations/monitor.lock').open('a') as monitor_lock:
|
||||||
|
fcntl.flock(monitor_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
try:
|
||||||
|
state = run(root)
|
||||||
|
logging.info('operations complete coreGiB=%.3f temporaryGiB=%.3f artifactsGiB=%.3f cleanup=%s alerts=%s',
|
||||||
|
state['sample']['coreBytes'] / GIB, state['sample']['temporaryBytes'] / GIB,
|
||||||
|
state['sample']['artifactBytes'] / GIB, json.dumps(state['cleanup']), list(state['alerts']))
|
||||||
|
except Exception as error:
|
||||||
|
logging.error('operations failed safely: %s', type(error).__name__)
|
||||||
|
failed_run(root, error); raise SystemExit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': main()
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
PyMySQL==1.2.3 --hash=sha256:14f1c68e2ed859243ae5ca41ffbe677027fc46bc136a9f0be8a4e928e5e7415a
|
||||||
|
psutil==7.2.2 \
|
||||||
|
--hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \
|
||||||
|
--hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \
|
||||||
|
--hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \
|
||||||
|
--hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \
|
||||||
|
--hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \
|
||||||
|
--hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \
|
||||||
|
--hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \
|
||||||
|
--hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \
|
||||||
|
--hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \
|
||||||
|
--hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \
|
||||||
|
--hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \
|
||||||
|
--hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \
|
||||||
|
--hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \
|
||||||
|
--hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \
|
||||||
|
--hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \
|
||||||
|
--hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \
|
||||||
|
--hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \
|
||||||
|
--hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \
|
||||||
|
--hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \
|
||||||
|
--hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \
|
||||||
|
--hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8
|
||||||
@@ -0,0 +1,525 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Conservative, online retention for this Mac's accepted Quality Center runtime."""
|
||||||
|
import argparse
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import importlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from logging.handlers import RotatingFileHandler
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import plistlib
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
import pymysql
|
||||||
|
from pymysql.cursors import DictCursor
|
||||||
|
|
||||||
|
LABEL = 'com.lyunlab.quality-center-retention'
|
||||||
|
SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
|
||||||
|
TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED')
|
||||||
|
BUILD_CHILDREN = ('defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup',
|
||||||
|
'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build',
|
||||||
|
'fact_black', 'fact_bright')
|
||||||
|
OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion',
|
||||||
|
'release_action', 'candidate_reconciliation')
|
||||||
|
MUTABLE_RELEASE_REFS = ('data_release', 'import_run', 'release_action', 'candidate_reconciliation')
|
||||||
|
|
||||||
|
|
||||||
|
def ident(value):
|
||||||
|
if not re.fullmatch(r'[A-Za-z0-9_]+', value):
|
||||||
|
raise ValueError('Invalid SQL identifier')
|
||||||
|
return '`' + value + '`'
|
||||||
|
|
||||||
|
|
||||||
|
class Busy(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def safe_file(path, boundary):
|
||||||
|
path, boundary = Path(path), Path(boundary).absolute()
|
||||||
|
if not path.is_absolute():
|
||||||
|
return False
|
||||||
|
if not path.is_relative_to(boundary) or not path.resolve().is_relative_to(boundary.resolve()):
|
||||||
|
return False
|
||||||
|
cursor = path
|
||||||
|
while cursor != boundary:
|
||||||
|
if cursor.is_symlink():
|
||||||
|
return False
|
||||||
|
cursor = cursor.parent
|
||||||
|
return not boundary.is_symlink() and path.is_file()
|
||||||
|
|
||||||
|
|
||||||
|
def fingerprint(path):
|
||||||
|
s = Path(path).stat()
|
||||||
|
return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns)
|
||||||
|
|
||||||
|
|
||||||
|
class Database:
|
||||||
|
def __init__(self, root, database=None):
|
||||||
|
self.root = Path(root).resolve()
|
||||||
|
self.config = json.loads((self.root / '.runtime/local-config.json').read_text())
|
||||||
|
self.name = database or self.config['mysqlDatabase']
|
||||||
|
ident(self.name)
|
||||||
|
endpoint = ({'host': self.config['mysqlHost'], 'port': int(self.config['mysqlPort'])}
|
||||||
|
if self.config.get('mysqlHost') else {'unix_socket': str(self.root / '.runtime/mysql.sock')})
|
||||||
|
self.conn = pymysql.connect(**endpoint, user='root',
|
||||||
|
password=self.config['rootPassword'], database=self.name, charset='utf8mb4',
|
||||||
|
autocommit=True, cursorclass=DictCursor, connect_timeout=3, read_timeout=10, write_timeout=5)
|
||||||
|
self.execute("SET SESSION time_zone='+00:00'")
|
||||||
|
self.execute('SET SESSION lock_wait_timeout=1')
|
||||||
|
self.execute('SET SESSION innodb_lock_wait_timeout=1')
|
||||||
|
rows = self.rows('SELECT TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE FROM information_schema.columns WHERE TABLE_SCHEMA=%s', (self.name,))
|
||||||
|
self.columns = {}
|
||||||
|
self.column_types = {}
|
||||||
|
for row in rows:
|
||||||
|
self.columns.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['IS_NULLABLE'] == 'YES'
|
||||||
|
self.column_types.setdefault(row['TABLE_NAME'], {})[row['COLUMN_NAME']] = row['DATA_TYPE']
|
||||||
|
self.tables = sorted(self.columns)
|
||||||
|
self.fks = self.rows('SELECT TABLE_NAME,COLUMN_NAME,REFERENCED_TABLE_NAME,REFERENCED_COLUMN_NAME FROM information_schema.key_column_usage WHERE TABLE_SCHEMA=%s AND REFERENCED_TABLE_NAME IS NOT NULL', (self.name,))
|
||||||
|
self.schema_signature=hashlib.sha256(json.dumps({'columns':self.columns,'fks':sorted(self.fks,key=lambda r:json.dumps(r,sort_keys=True))},sort_keys=True).encode()).hexdigest()
|
||||||
|
self.longest_exclusion=0.0
|
||||||
|
if not {'current_release','data_release','release_member','transform_build','source_upload_draft'}.issubset(self.columns):
|
||||||
|
raise ValueError('Unexpected application schema')
|
||||||
|
|
||||||
|
def rows(self, sql, args=()):
|
||||||
|
with self.conn.cursor() as cursor:
|
||||||
|
cursor.execute(sql, args)
|
||||||
|
return list(cursor.fetchall())
|
||||||
|
|
||||||
|
def execute(self, sql, args=()):
|
||||||
|
with self.conn.cursor() as cursor:
|
||||||
|
return cursor.execute(sql, args)
|
||||||
|
|
||||||
|
def current(self):
|
||||||
|
rows = self.rows("SELECT release_id FROM current_release WHERE dataset_id='quality'")
|
||||||
|
if len(rows) != 1 or not rows[0]['release_id']:
|
||||||
|
raise Busy('Current release not confirmed')
|
||||||
|
fields='status,complete' if 'complete' in self.columns['data_release'] else 'status'
|
||||||
|
release=self.rows('SELECT '+fields+' FROM data_release WHERE id=%s',(rows[0]['release_id'],))
|
||||||
|
if len(release)!=1 or release[0]['status']!='PUBLISHED' or release[0].get('complete',True) not in (True,1):
|
||||||
|
raise Busy('Current release is not a complete publication')
|
||||||
|
return rows[0]['release_id']
|
||||||
|
|
||||||
|
def busy(self):
|
||||||
|
for table in ('task','import_run','data_workflow_operation','analysis_turn','analysis_protocol_execution','quality_inspection_run'):
|
||||||
|
if table not in self.columns or 'status' not in self.columns[table]:
|
||||||
|
continue
|
||||||
|
sql = 'SELECT 1 FROM ' + ident(table) + ' WHERE (status IS NULL OR status NOT IN (' + ','.join(['%s']*len(TERMINAL)) + '))'
|
||||||
|
if table == 'task' and {'next_retry_at','retry_count','max_retries'}.issubset(self.columns[table]):
|
||||||
|
sql += " OR (status='FAILED' AND next_retry_at IS NOT NULL AND retry_count<max_retries)"
|
||||||
|
if self.rows(sql + ' LIMIT 1', TERMINAL):
|
||||||
|
return True
|
||||||
|
if 'analysis_report' in self.columns:
|
||||||
|
# Report rows keep QUEUED after handing work to analysis_turn. The
|
||||||
|
# linked turn, not this stored label, is the execution state.
|
||||||
|
for report in self.rows('SELECT status,turn_id FROM analysis_report'):
|
||||||
|
if report['turn_id']:
|
||||||
|
turns=self.rows('SELECT status FROM analysis_turn WHERE id=%s',(report['turn_id'],))
|
||||||
|
if not turns or turns[0]['status'] not in TERMINAL:return True
|
||||||
|
elif report['status'] not in (*TERMINAL,'BLOCKED'):
|
||||||
|
return True
|
||||||
|
if 'source_candidate' in self.columns:
|
||||||
|
return bool(self.rows("SELECT 1 FROM source_candidate WHERE status IN ('QUEUED','RUNNING','IMPORTING') LIMIT 1"))
|
||||||
|
return False
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def frozen(self, expected):
|
||||||
|
"""Short exclusion against writers; never START TRANSACTION after LOCK TABLES.
|
||||||
|
|
||||||
|
All aliases are intentionally avoided while these MySQL locks are held.
|
||||||
|
Busy jobs and a changed release cause rollback without deleting anything.
|
||||||
|
"""
|
||||||
|
if (self.root/'.runtime/retention/paused').exists():raise Busy('Service maintenance is paused')
|
||||||
|
if self.busy():
|
||||||
|
raise Busy('An application job is active')
|
||||||
|
self.conn.autocommit(False)
|
||||||
|
locked = False
|
||||||
|
acquired=None
|
||||||
|
try:
|
||||||
|
self.execute('LOCK TABLES ' + ','.join(ident(t) + ' WRITE' for t in self.tables))
|
||||||
|
locked = True
|
||||||
|
acquired=time.monotonic()
|
||||||
|
if sorted(next(iter(r.values())) for r in self.rows('SHOW TABLES'))!=self.tables:
|
||||||
|
raise Busy('Application schema changed')
|
||||||
|
if self.current() != expected or self.busy():
|
||||||
|
raise Busy('The publication or job state changed')
|
||||||
|
yield
|
||||||
|
self.conn.commit()
|
||||||
|
except BaseException as error:
|
||||||
|
self.conn.rollback()
|
||||||
|
if isinstance(error,pymysql.err.OperationalError) and error.args[0] in (1205,3572):
|
||||||
|
raise Busy('Application tables are in use') from error
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if locked:
|
||||||
|
self.execute('UNLOCK TABLES')
|
||||||
|
self.longest_exclusion=max(self.longest_exclusion,time.monotonic()-acquired)
|
||||||
|
self.conn.autocommit(True)
|
||||||
|
|
||||||
|
def protected_releases(self, current):
|
||||||
|
releases = self.rows('SELECT id,status,parent_release_id,expected_current_release_id FROM data_release')
|
||||||
|
protected = {current}
|
||||||
|
pending = {r['id'] for r in releases if r['status'] != 'PUBLISHED'}
|
||||||
|
protected.update(pending)
|
||||||
|
# Keep candidate baselines and all their ancestors: review/retry must work.
|
||||||
|
while True:
|
||||||
|
more = {r[c] for r in releases if r['id'] in pending for c in ('parent_release_id','expected_current_release_id') if r[c]}
|
||||||
|
if more.issubset(pending):
|
||||||
|
break
|
||||||
|
pending.update(more); protected.update(more)
|
||||||
|
for fk in self.fks:
|
||||||
|
t, c = fk['TABLE_NAME'], fk['COLUMN_NAME']
|
||||||
|
if fk['REFERENCED_TABLE_NAME'] != 'data_release' or t in MUTABLE_RELEASE_REFS or t in OWNED_RELEASE:
|
||||||
|
continue
|
||||||
|
protected.update(r[c] for r in self.rows('SELECT DISTINCT ' + ident(c) + ' FROM ' + ident(t) + ' WHERE ' + ident(c) + ' IS NOT NULL'))
|
||||||
|
# This field is intentionally not a database FK, but is a real user reference.
|
||||||
|
if 'analysis_session' in self.columns:
|
||||||
|
protected.update(r['release_id'] for r in self.rows('SELECT DISTINCT release_id FROM analysis_session'))
|
||||||
|
if 'source_candidate' in self.columns and 'expected_current_release_id' in self.columns['source_candidate']:
|
||||||
|
protected.update(r['expected_current_release_id'] for r in self.rows("SELECT DISTINCT expected_current_release_id FROM source_candidate WHERE status IN ('DISCOVERED','QUEUED','RUNNING') AND expected_current_release_id IS NOT NULL"))
|
||||||
|
if 'source_change_request' in self.columns:
|
||||||
|
cols = [c for c in ('requested_from_release_id','result_release_id') if c in self.columns['source_change_request']]
|
||||||
|
for row in self.rows('SELECT ' + ','.join(map(ident, cols)) + " FROM source_change_request WHERE status IN ('REQUESTED','APPROVED')") if cols else []:
|
||||||
|
protected.update(v for v in row.values() if v)
|
||||||
|
if 'candidate_reconciliation' in self.columns:
|
||||||
|
for row in self.rows('SELECT release_id,baseline_release_id FROM candidate_reconciliation'):
|
||||||
|
if row['release_id'] in pending and row['baseline_release_id']:
|
||||||
|
protected.add(row['baseline_release_id'])
|
||||||
|
return releases, protected
|
||||||
|
|
||||||
|
def release_step(self, current, state, grace, now):
|
||||||
|
releases, protected = self.protected_releases(current)
|
||||||
|
seen = state.setdefault('releases', {})
|
||||||
|
eligible = []
|
||||||
|
ids = {r['id'] for r in releases}
|
||||||
|
for stale in set(seen) - ids:
|
||||||
|
seen.pop(stale)
|
||||||
|
for row in releases:
|
||||||
|
rid = row['id']
|
||||||
|
if rid in protected:
|
||||||
|
seen.pop(rid, None)
|
||||||
|
elif row['status'] == 'PUBLISHED':
|
||||||
|
seen.setdefault(rid, now)
|
||||||
|
if now - seen[rid] >= grace:
|
||||||
|
eligible.append(rid)
|
||||||
|
if not eligible:
|
||||||
|
return 0
|
||||||
|
rid = sorted(eligible, key=lambda x: seen[x])[0]
|
||||||
|
# Clear optional historical links only. Unknown/business references were
|
||||||
|
# included in protected above and are never cascaded away.
|
||||||
|
for fk in self.fks:
|
||||||
|
t,c = fk['TABLE_NAME'],fk['COLUMN_NAME']
|
||||||
|
if fk['REFERENCED_TABLE_NAME'] == 'data_release' and t in MUTABLE_RELEASE_REFS and self.columns[t][c]:
|
||||||
|
self.execute('UPDATE ' + ident(t) + ' SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,))
|
||||||
|
if 'data_workflow_operation' in self.columns:
|
||||||
|
for c in ('release_id','expected_current_release_id'):
|
||||||
|
if c in self.columns['data_workflow_operation']:
|
||||||
|
self.execute('UPDATE data_workflow_operation SET ' + ident(c) + '=NULL WHERE ' + ident(c) + '=%s', (rid,))
|
||||||
|
for t in OWNED_RELEASE:
|
||||||
|
if t in self.columns:
|
||||||
|
self.execute('DELETE FROM ' + ident(t) + ' WHERE release_id=%s', (rid,))
|
||||||
|
self.execute('DELETE FROM data_release WHERE id=%s', (rid,))
|
||||||
|
seen.pop(rid, None)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
def build_step(self, state, grace, now, batch):
|
||||||
|
referenced = {r['transform_build_id'] for r in self.rows('SELECT DISTINCT transform_build_id FROM release_member')}
|
||||||
|
builds = self.rows('SELECT id,status,build_key FROM transform_build')
|
||||||
|
seen = state.setdefault('builds', {})
|
||||||
|
ids = {r['id'] for r in builds}
|
||||||
|
for stale in set(seen) - ids:
|
||||||
|
seen.pop(stale)
|
||||||
|
for row in builds:
|
||||||
|
bid = row['id']
|
||||||
|
unknown_reference = any(self.rows('SELECT 1 FROM ' + ident(f['TABLE_NAME']) + ' WHERE ' + ident(f['COLUMN_NAME']) + '=%s LIMIT 1', (bid,))
|
||||||
|
for f in self.fks if f['REFERENCED_TABLE_NAME'] == 'transform_build' and f['TABLE_NAME'] not in (*BUILD_CHILDREN, 'release_member'))
|
||||||
|
if bid in referenced or unknown_reference or row['status'] not in ('COMPLETED','FAILED','CANCELLED','RETIRED'):
|
||||||
|
seen.pop(bid, None)
|
||||||
|
continue
|
||||||
|
seen.setdefault(bid, now)
|
||||||
|
if row['status'] != 'RETIRED' and now - seen[bid] < grace:
|
||||||
|
continue
|
||||||
|
if row['status'] != 'RETIRED':
|
||||||
|
# Free the unique reuse key before deleting chunks. A new import
|
||||||
|
# can build a fresh result, never reuse this partially emptied one.
|
||||||
|
tombstone = hashlib.sha256(('retired:'+bid+':'+row['build_key']).encode()).hexdigest()
|
||||||
|
self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid))
|
||||||
|
for t in BUILD_CHILDREN:
|
||||||
|
if t in self.columns:
|
||||||
|
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
|
||||||
|
if count:
|
||||||
|
return count, 0
|
||||||
|
self.execute('DELETE FROM transform_build WHERE id=%s', (bid,))
|
||||||
|
seen.pop(bid, None)
|
||||||
|
return 0, 1
|
||||||
|
return 0, 0
|
||||||
|
|
||||||
|
def file_referenced(self, path):
|
||||||
|
for table, cols in self.columns.items():
|
||||||
|
for col in cols:
|
||||||
|
if col.endswith('_path') and self.rows('SELECT 1 FROM ' + ident(table) + ' WHERE ' + ident(col) + '=%s LIMIT 1', (str(path),)):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def expired(self, limit=10):
|
||||||
|
return self.rows('SELECT * FROM source_upload_draft WHERE expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6)) ORDER BY expires_at LIMIT ' + str(int(limit)))
|
||||||
|
|
||||||
|
def promotion(self, draft):
|
||||||
|
if not draft.get('candidate_id') or not {'source_revision','source_document','source_candidate'}.issubset(self.columns):
|
||||||
|
return None
|
||||||
|
rows = self.rows("SELECT * FROM source_candidate WHERE id=%s AND status='IMPORTED'", (draft['candidate_id'],))
|
||||||
|
if not rows:
|
||||||
|
return None
|
||||||
|
c = rows[0]
|
||||||
|
docs = {d['id']:d for d in self.rows('SELECT id,source_slot FROM source_document')}
|
||||||
|
for rev in self.rows('SELECT source_document_id,archive_path,sha256,parser_config_hash,file_size FROM source_revision WHERE sha256=%s AND parser_config_hash=%s', (c['sha256'],c['parser_config_hash'])):
|
||||||
|
if docs.get(rev['source_document_id'],{}).get('source_slot') == c['source_slot']:
|
||||||
|
p=Path(rev['archive_path'])
|
||||||
|
if safe_file(p,self.root/'runtimedata/archive/source-revisions') and p.stat().st_size==rev['file_size']:
|
||||||
|
return c,rev,p
|
||||||
|
return None
|
||||||
|
|
||||||
|
def clean_owned_scratch(self, br):
|
||||||
|
import psutil
|
||||||
|
server=self.rows('SELECT @@server_uuid AS value')[0]['value']
|
||||||
|
directory=self.root/'.runtime/retention/scratch'
|
||||||
|
removed=0
|
||||||
|
for path in directory.glob('*.json'):
|
||||||
|
if path.is_symlink():continue
|
||||||
|
value=json.loads(path.read_text());name=value.get('database','')
|
||||||
|
if value.get('kind')!='quality-restore-check' or value.get('server')!=server or not value.get('created') or name==self.name or not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',name):continue
|
||||||
|
if time.time()-value['createdAt']<300:continue
|
||||||
|
alive=False
|
||||||
|
for pid_key,created_key in [('ownerPid','ownerCreated'),('childPid','childCreated')]:
|
||||||
|
if pid_key not in value:continue
|
||||||
|
try:
|
||||||
|
process=psutil.Process(value[pid_key])
|
||||||
|
if abs(process.create_time()-value[created_key])<.01 and process.is_running() and process.status()!=psutil.STATUS_ZOMBIE:alive=True
|
||||||
|
except psutil.NoSuchProcess:pass
|
||||||
|
except psutil.Error:alive=True
|
||||||
|
if alive or self.rows('SELECT 1 FROM information_schema.processlist WHERE DB=%s LIMIT 1',(name,)):continue
|
||||||
|
self.execute('DROP DATABASE IF EXISTS '+ident(name));path.unlink();removed+=1
|
||||||
|
return removed
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_drafts(db, current, state, now, br):
|
||||||
|
removed=0; bytes_removed=0
|
||||||
|
# Checksum expensive files outside the brief database exclusion window.
|
||||||
|
for draft in db.expired():
|
||||||
|
path=Path(draft['source_path']); boundary=db.root/'runtimedata/archive/upload-drafts'
|
||||||
|
info=None
|
||||||
|
if safe_file(path,boundary):
|
||||||
|
info=fingerprint(path)
|
||||||
|
if info[2]!=draft['file_size'] or br.sha(path)!=draft['sha256']:
|
||||||
|
continue
|
||||||
|
promotion=db.promotion(draft)
|
||||||
|
if promotion:
|
||||||
|
c,rev,target=promotion
|
||||||
|
target_stamp=fingerprint(target)
|
||||||
|
if br.sha(target)!=rev['sha256']:
|
||||||
|
continue
|
||||||
|
with db.frozen(current):
|
||||||
|
live=db.rows('SELECT * FROM source_upload_draft WHERE id=%s AND expires_at<=TIMESTAMPADD(MINUTE,-30,CURRENT_TIMESTAMP(6))',(draft['id'],))
|
||||||
|
if not live or live[0]!=draft:
|
||||||
|
continue
|
||||||
|
if info and (not safe_file(path,boundary) or fingerprint(path)!=info):
|
||||||
|
continue
|
||||||
|
if promotion and safe_file(target,db.root/'runtimedata/archive/source-revisions') and fingerprint(target)==target_stamp:
|
||||||
|
# The WHERE identity prevents overwriting a source reconfigured
|
||||||
|
# between preflight hashing and acquiring the database exclusion.
|
||||||
|
db.execute("UPDATE source_candidate SET source_path=%s,archive_path=%s WHERE id=%s AND status='IMPORTED' AND source_path=%s AND sha256=%s AND parser_config_hash=%s",
|
||||||
|
(str(target),str(target),c['id'],c['source_path'],c['sha256'],c['parser_config_hash']))
|
||||||
|
db.execute('DELETE FROM source_upload_draft WHERE id=%s',(draft['id'],))
|
||||||
|
if info:
|
||||||
|
with db.frozen(current):
|
||||||
|
if safe_file(path,boundary) and fingerprint(path)==info and not db.file_referenced(path):
|
||||||
|
bytes_removed+=path.stat().st_size
|
||||||
|
path.unlink()
|
||||||
|
removed+=1
|
||||||
|
# Interrupted uploads can leave no database row at all. Their age and every
|
||||||
|
# direct path reference are rechecked under the same exclusion before unlink.
|
||||||
|
boundary=db.root/'runtimedata/archive/upload-drafts'
|
||||||
|
for path in list(boundary.glob('upload-*.xlsx'))[:100]:
|
||||||
|
if not safe_file(path,boundary) or now-path.stat().st_mtime<86400:
|
||||||
|
continue
|
||||||
|
info=fingerprint(path)
|
||||||
|
with db.frozen(current):
|
||||||
|
if fingerprint(path)==info and not db.file_referenced(path):
|
||||||
|
size=path.stat().st_size;path.unlink();bytes_removed+=size
|
||||||
|
return removed,bytes_removed
|
||||||
|
|
||||||
|
|
||||||
|
def run(root, max_batches=30, grace=1800):
|
||||||
|
root=Path(root).resolve();sys.path.append(str(root/'scripts'))
|
||||||
|
import backup_retention as br
|
||||||
|
import backup
|
||||||
|
import backup_verifier
|
||||||
|
import revision_retention
|
||||||
|
if (root/'.runtime/retention/paused').exists():raise Busy('Service maintenance is paused')
|
||||||
|
readiness=urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||||
|
try:
|
||||||
|
with readiness.open('http://127.0.0.1:5281/actuator/health',timeout=2) as response:
|
||||||
|
if json.load(response).get('status')!='UP':raise Busy('Backend is not ready')
|
||||||
|
except (OSError,ValueError) as error:raise Busy('Backend is not ready') from error
|
||||||
|
state_path=root/'.runtime/retention/state.json'
|
||||||
|
state=json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||||
|
result={'checkedAt':time.time(),'releasesRemoved':0,'buildsRemoved':0,'rowsRemoved':0,'draftsRemoved':0,'bytesRemoved':0}
|
||||||
|
with br.lock(root,timeout=0):
|
||||||
|
result['retiredBackupFoldersRemoved']=br.clean_retired(root)
|
||||||
|
result['privateVerifiersRemoved']=backup_verifier.clean_abandoned(root)
|
||||||
|
db=Database(root)
|
||||||
|
try:
|
||||||
|
policy_path=root/'.runtime/retention/config.json'
|
||||||
|
if policy_path.exists():
|
||||||
|
policy=json.loads(policy_path.read_text());accepted=br.accepted_backend(root)
|
||||||
|
if policy['schemaSignature']!=db.schema_signature or not accepted or policy['acceptedJarSha256']!=accepted[1]:
|
||||||
|
raise Busy('Application version changed; retention compatibility review required')
|
||||||
|
result['scratchRemoved']=db.clean_owned_scratch(br)
|
||||||
|
current=db.current()
|
||||||
|
if db.busy():
|
||||||
|
raise Busy('An application job is active')
|
||||||
|
latest=br.latest(root)
|
||||||
|
# Recover a restore-verified candidate left between verification and
|
||||||
|
# pointer publication by a previous crash, without creating a duplicate.
|
||||||
|
if not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current:
|
||||||
|
candidates=[]
|
||||||
|
for folder in (root/'output/backups').iterdir() if (root/'output/backups').exists() else []:
|
||||||
|
candidate=br.verified(folder,checksums=False)
|
||||||
|
if candidate and candidate.get('queryEvidence',{}).get('releaseId')==current:candidates.append((candidate.get('createdAt',''),folder))
|
||||||
|
if candidates:
|
||||||
|
selected=max(candidates)[1]
|
||||||
|
if br.verified(selected):br.rotate(root,selected);latest=br.latest(root)
|
||||||
|
if latest:
|
||||||
|
state['lastBackupAt']=max(state.get('lastBackupAt',0),datetime.fromisoformat(latest[1]['createdAt']).timestamp())
|
||||||
|
due=not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current
|
||||||
|
due=due or time.time()-state.get('lastBackupAt',0)>86400
|
||||||
|
due=due or (state.get('dirtySince') and not state.get('builds') and time.time()-state['dirtySince']>300)
|
||||||
|
if due:
|
||||||
|
database_bytes=int(db.rows('SELECT COALESCE(SUM(DATA_LENGTH+INDEX_LENGTH),0) AS size FROM information_schema.tables WHERE TABLE_SCHEMA=%s',(db.name,))[0]['size'])
|
||||||
|
archive_bytes=sum(p.stat().st_size for p in (root/'runtimedata/archive').rglob('*') if p.is_file())
|
||||||
|
required_free=max(10*1024**3,3*database_bytes+2*archive_bytes+5*1024**3)
|
||||||
|
reserves = [root / '.runtime', root / 'output/backups']
|
||||||
|
for reserve in reserves: reserve.mkdir(parents=True, exist_ok=True)
|
||||||
|
if min(shutil.disk_usage(p).free for p in reserves)<required_free:raise Busy('Insufficient reserve for a verified backup; existing backup preserved')
|
||||||
|
if state.get('lastBackupAttemptRelease')==current and time.time()-state.get('lastBackupAttempt',0)<3600:
|
||||||
|
raise Busy('Waiting before retrying a failed backup')
|
||||||
|
state['lastBackupAttempt']=time.time();state['lastBackupAttemptRelease']=current;br.atomic(state_path,state)
|
||||||
|
logging.info('creating consistent backup; previous verified backup remains available')
|
||||||
|
before=br.backup_ids(root)
|
||||||
|
try:
|
||||||
|
folder=backup.create();backup.verify(folder)
|
||||||
|
except BaseException:
|
||||||
|
br.discard_failed_new(root,before)
|
||||||
|
raise
|
||||||
|
if db.current()!=current:
|
||||||
|
raise Busy('Publication changed while backing up; cleanup deferred')
|
||||||
|
latest=br.latest(root) # rotate is shared with manual/UI backup operations
|
||||||
|
if latest is None or latest[0]!=folder:
|
||||||
|
br.rotate(root,folder);latest=br.latest(root)
|
||||||
|
state['lastBackupAt']=time.time();state.pop('dirtySince',None)
|
||||||
|
br.atomic(state_path,state)
|
||||||
|
logging.info('isolated restore verified; latest backup published')
|
||||||
|
if not latest or latest[1].get('queryEvidence',{}).get('releaseId')!=current:
|
||||||
|
raise Busy('No verified backup for the current publication')
|
||||||
|
# Every data-deletion batch rechecks both job state and current pointer.
|
||||||
|
deadline=time.monotonic()+20
|
||||||
|
for _ in range(max_batches):
|
||||||
|
with db.frozen(current):
|
||||||
|
releases=db.release_step(current,state,grace,time.time())
|
||||||
|
rows,builds=db.build_step(state,grace,time.time(),1000)
|
||||||
|
result['releasesRemoved']+=releases;result['rowsRemoved']+=rows;result['buildsRemoved']+=builds
|
||||||
|
if not (releases or rows or builds) or time.monotonic()>deadline or db.longest_exclusion>.5:
|
||||||
|
break
|
||||||
|
time.sleep(.05)
|
||||||
|
drafts,size=cleanup_drafts(db,current,state,time.time(),br)
|
||||||
|
result['draftsRemoved']=drafts;result['bytesRemoved']=size
|
||||||
|
sources=revision_retention.Cleaner(db,br,safe_file,fingerprint,Busy)
|
||||||
|
result.update(sources.collect(current,state,time.time(),grace))
|
||||||
|
result['maxExclusiveSeconds']=round(db.longest_exclusion,4)
|
||||||
|
if any(result[k] for k in ('releasesRemoved','rowsRemoved','buildsRemoved','draftsRemoved','bytesRemoved','sourceRevisionsRemoved','sourceArchiveBytesRemoved')):
|
||||||
|
state.setdefault('dirtySince',time.time())
|
||||||
|
state['lastResult']=result;state.pop('lastError',None)
|
||||||
|
state['consecutiveFailures']=0
|
||||||
|
br.atomic(state_path,state)
|
||||||
|
return result
|
||||||
|
except Busy as error:
|
||||||
|
state['lastDeferred']={'time':time.time(),'reason':str(error)}
|
||||||
|
state['lastResult']=result;br.atomic(state_path,state)
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
db.conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def install(root):
|
||||||
|
root=Path(root).resolve();sys.path.insert(0,str(root/'scripts'));import backup_retention as br
|
||||||
|
python=root/'.runtime/retention-env/bin/python3'
|
||||||
|
if not python.is_file():
|
||||||
|
raise ValueError('Private maintenance runtime missing')
|
||||||
|
files=[Path(__file__).resolve(),root/'scripts/backup_retention.py',root/'scripts/revision_retention.py']
|
||||||
|
db=Database(root)
|
||||||
|
try:
|
||||||
|
accepted=br.accepted_backend(root)
|
||||||
|
if not accepted:raise ValueError('Accepted backend runtime is not registered')
|
||||||
|
br.atomic(root/'.runtime/retention/config.json',{'schemaSignature':db.schema_signature,
|
||||||
|
'acceptedJarSha256':accepted[1],'retirementGraceSeconds':1800,'intervalSeconds':300,
|
||||||
|
'backupIntervalSeconds':86400,'draftExpiryGraceSeconds':1800,'sourceRevisionGraceSeconds':1800})
|
||||||
|
finally:db.conn.close()
|
||||||
|
digest=hashlib.sha256(b''.join(p.read_bytes() for p in files)).hexdigest()
|
||||||
|
version=SUPPORT/'retention'/digest;version.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||||
|
for p in files:
|
||||||
|
shutil.copy2(p,version/p.name);(version/p.name).chmod(0o600)
|
||||||
|
legacy=root/'.runtime/current-snapshot-20261109'
|
||||||
|
if legacy.is_dir() and not legacy.is_symlink() and (legacy/'manifest.json').is_file():
|
||||||
|
br.atomic(root/'.runtime/retention/legacy.json',{'path':'.runtime/current-snapshot-20261109','manifestSha256':br.sha(legacy/'manifest.json')})
|
||||||
|
logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||||
|
plist=Path.home()/'Library/LaunchAgents'/(LABEL+'.plist')
|
||||||
|
value={'Label':LABEL,'ProgramArguments':[str(python),str(version/'retention.py'),'run','--root',str(root)],
|
||||||
|
'WorkingDirectory':str(root),'RunAtLoad':True,'StartInterval':300,'ProcessType':'Background','Nice':10,
|
||||||
|
'LowPriorityIO':True,'ExitTimeOut':300,'EnvironmentVariables':{'PYTHONDONTWRITEBYTECODE':'1'},
|
||||||
|
'StandardOutPath':str(logs/'retention-launchd.log'),'StandardErrorPath':str(logs/'retention-launchd-error.log')}
|
||||||
|
subprocess.run(['/bin/launchctl','bootout','gui/'+str(os.getuid())+'/'+LABEL],capture_output=True)
|
||||||
|
with plist.open('wb') as out:plistlib.dump(value,out)
|
||||||
|
plist.chmod(0o600)
|
||||||
|
subprocess.run(['/bin/launchctl','enable','gui/'+str(os.getuid())+'/'+LABEL],check=True)
|
||||||
|
for attempt in range(50):
|
||||||
|
loaded=subprocess.run(['/bin/launchctl','bootstrap','gui/'+str(os.getuid()),str(plist)],capture_output=True)
|
||||||
|
if loaded.returncode==0:break
|
||||||
|
time.sleep(.2)
|
||||||
|
else:raise RuntimeError('Maintenance registration did not complete')
|
||||||
|
print('Installed five-minute idle maintenance and daily/publication backup checks')
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser=argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('action',choices=['run','install','status'])
|
||||||
|
parser.add_argument('--root',required=True)
|
||||||
|
args=parser.parse_args();root=Path(args.root).resolve();sys.path.append(str(root/'scripts'));import backup_retention as br
|
||||||
|
if args.action=='install':
|
||||||
|
with br.lock(root,timeout=0):install(root)
|
||||||
|
return
|
||||||
|
if args.action=='status':
|
||||||
|
p=root/'.runtime/retention/state.json';print(p.read_text() if p.exists() else '{}');return
|
||||||
|
logs=SUPPORT/'logs';logs.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||||
|
handler=RotatingFileHandler(logs/'retention.log',maxBytes=2*1024*1024,backupCount=2)
|
||||||
|
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
|
||||||
|
logging.getLogger().addHandler(handler);logging.getLogger().setLevel(logging.INFO)
|
||||||
|
try:
|
||||||
|
result=run(root);logging.info('maintenance complete %s',json.dumps(result))
|
||||||
|
except (Busy,br.Busy) as error:
|
||||||
|
logging.info('maintenance deferred: %s',str(error))
|
||||||
|
except Exception as error:
|
||||||
|
p=root/'.runtime/retention/state.json';state=json.loads(p.read_text()) if p.exists() else {}
|
||||||
|
state['lastError']={'time':time.time(),'type':type(error).__name__};br.atomic(p,state)
|
||||||
|
state['consecutiveFailures']=min(1000,state.get('consecutiveFailures',0)+1);br.atomic(p,state)
|
||||||
|
logging.error('maintenance failed safely: %s',type(error).__name__)
|
||||||
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__=='__main__':main()
|
||||||
@@ -0,0 +1,351 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Pinned macOS service guardian. Never initializes or imports a database."""
|
||||||
|
import argparse
|
||||||
|
import fcntl
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from logging.handlers import RotatingFileHandler
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import plistlib
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
import psutil
|
||||||
|
|
||||||
|
LABEL = 'com.lyunlab.quality-center-service'
|
||||||
|
HOME = Path.home() / 'Library/Application Support/QualityCenterService'
|
||||||
|
CONFIG = HOME / 'service.json'
|
||||||
|
PLIST = Path.home() / 'Library/LaunchAgents' / (LABEL + '.plist')
|
||||||
|
DOMAIN = 'gui/' + str(os.getuid())
|
||||||
|
|
||||||
|
|
||||||
|
def atomic(path, value):
|
||||||
|
path = Path(path)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
temporary = path.with_name(path.name + '.tmp-' + str(os.getpid()))
|
||||||
|
with temporary.open('w') as stream:
|
||||||
|
os.chmod(temporary, 0o600)
|
||||||
|
json.dump(value, stream, ensure_ascii=False, indent=2)
|
||||||
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def stamp(pid):
|
||||||
|
return subprocess.run(['/bin/ps', '-p', str(pid), '-o', 'lstart='],
|
||||||
|
capture_output=True, text=True).stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def matches(record, command=None):
|
||||||
|
"""True=confirmed identity, False=gone/reused, None=unknown identity."""
|
||||||
|
if not record:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
p = psutil.Process(record['pid'])
|
||||||
|
if not p.is_running() or p.status() == psutil.STATUS_ZOMBIE:
|
||||||
|
return False
|
||||||
|
if 'created' in record and abs(p.create_time() - record['created']) > .01:
|
||||||
|
return False
|
||||||
|
if 'started' in record and stamp(p.pid) != record['started']:
|
||||||
|
return False
|
||||||
|
expected = command or record['command']
|
||||||
|
if p.uids().real != os.getuid():
|
||||||
|
return None
|
||||||
|
if Path(p.exe()).resolve() != Path(expected[0]).resolve() or p.cmdline() != expected:
|
||||||
|
return None
|
||||||
|
return True
|
||||||
|
except psutil.NoSuchProcess:
|
||||||
|
return False
|
||||||
|
except (psutil.Error, KeyError, ValueError, OSError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def record(pid, command):
|
||||||
|
return {'pid': pid, 'created': psutil.Process(pid).create_time(),
|
||||||
|
'command': command, 'started': stamp(pid)}
|
||||||
|
|
||||||
|
|
||||||
|
class Guardian:
|
||||||
|
def __init__(self, config, home=HOME):
|
||||||
|
self.config = config
|
||||||
|
self.home = Path(home)
|
||||||
|
self.root = Path(config['root']).resolve()
|
||||||
|
self.runtime = self.root / '.runtime'
|
||||||
|
self.records = {}
|
||||||
|
self.children = {}
|
||||||
|
self.next_start = {'mysql': 0, 'backend': 0}
|
||||||
|
self.restarts = {'mysql': 0, 'backend': 0}
|
||||||
|
self.stopping = False
|
||||||
|
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||||
|
|
||||||
|
def preflight(self):
|
||||||
|
self.value = json.loads((self.runtime / 'local-config.json').read_text())
|
||||||
|
if not self.value.get('databaseReady') or not (self.runtime / 'mysql-data/auto.cnf').is_file():
|
||||||
|
raise RuntimeError('Existing initialized database not confirmed; refusing bootstrap')
|
||||||
|
self.jar = self.runtime / 'backend-builds' / (self.config['jarSha256'] + '.jar')
|
||||||
|
if hashlib.sha256(self.jar.read_bytes()).hexdigest() != self.config['jarSha256']:
|
||||||
|
raise RuntimeError('Accepted backend digest mismatch')
|
||||||
|
for binary in [self.runtime / 'mysql/bin/mysqld', self.runtime / 'jdk/Contents/Home/bin/java']:
|
||||||
|
if not os.access(binary, os.X_OK):
|
||||||
|
raise RuntimeError('Required runtime missing')
|
||||||
|
|
||||||
|
def command(self, role):
|
||||||
|
r = self.runtime
|
||||||
|
if role == 'backend':
|
||||||
|
return [str(r / 'jdk/Contents/Home/bin/java'), '-Xms128m', '-Xmx2048m', '-jar', str(self.jar)]
|
||||||
|
return [str(r / 'mysql/bin/mysqld'), '--no-defaults', '--basedir=' + str(r / 'mysql'),
|
||||||
|
'--datadir=' + str(r / 'mysql-data'), '--port=' + str(self.value['mysqlPort']),
|
||||||
|
'--bind-address=127.0.0.1', '--socket=' + str(r / 'mysql.sock'),
|
||||||
|
'--pid-file=' + str(r / 'mysql-server.pid'),
|
||||||
|
'--log-error=' + str(self.root / 'output/logs/mysql-server.log'), '--mysqlx=OFF',
|
||||||
|
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M',
|
||||||
|
*(['--skip-log-bin'] if self.value.get('binaryLogEnabled', True) is False else [])]
|
||||||
|
|
||||||
|
def environment(self):
|
||||||
|
v, r = self.value, self.runtime
|
||||||
|
env = os.environ.copy()
|
||||||
|
env.update({
|
||||||
|
'JAVA_HOME': str(r / 'jdk/Contents/Home'),
|
||||||
|
'PATH': str(r / 'jdk/Contents/Home/bin') + ':' + self.config['path'],
|
||||||
|
'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{v["mysqlPort"]}/{v["mysqlDatabase"]}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true',
|
||||||
|
'MYSQL_USER': v['mysqlUser'], 'MYSQL_PASSWORD': v['mysqlPassword'],
|
||||||
|
'MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS': str(v.get('legacyTimestampOffsetSeconds', 'UNCONFIRMED')),
|
||||||
|
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': v['adminUsername'],
|
||||||
|
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': v['adminPassword'],
|
||||||
|
'QUALITY_ARCHIVE_ROOT': str(self.root / 'runtimedata/archive'),
|
||||||
|
'QUALITY_ALLOWED_ROOTS': str(self.root / '质量数据/系统原始数据示例'),
|
||||||
|
'QUALITY_DATA_WORKFLOW_TEST_MODE': 'true',
|
||||||
|
'QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED': str(v.get('analysisProtocolV1Enabled', False)).lower(),
|
||||||
|
'QUALITY_ANALYSIS_AUTONOMOUS_ENABLED': str(v.get('analysisAutonomousEnabled', False)).lower(),
|
||||||
|
'QUALITY_PROJECT_ROOT': str(self.root), 'QUALITY_DEFAULT_SOURCE_YEAR': '2026',
|
||||||
|
'SERVER_ADDRESS': '127.0.0.1', 'PORT': '5281', 'PYTHONDONTWRITEBYTECODE': '1',
|
||||||
|
})
|
||||||
|
log_config = r / 'operations/logback-quality.xml'
|
||||||
|
if log_config.is_file() and not log_config.is_symlink():
|
||||||
|
env['LOGGING_CONFIG'] = str(log_config)
|
||||||
|
env['QUALITY_LOG_DIR'] = str(self.root / 'output/logs')
|
||||||
|
return env
|
||||||
|
|
||||||
|
def port(self, role):
|
||||||
|
return self.value['mysqlPort'] if role == 'mysql' else 5281
|
||||||
|
|
||||||
|
def occupied(self, role):
|
||||||
|
try:
|
||||||
|
with socket.create_connection(('127.0.0.1', self.port(role)), timeout=.5):
|
||||||
|
return True
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def db_ready(self):
|
||||||
|
result = subprocess.run([str(self.runtime / 'mysql/bin/mysql'),
|
||||||
|
'--defaults-file=' + str(self.runtime / 'mysql-root.cnf'), '--connect-timeout=2',
|
||||||
|
'--batch', '--skip-column-names', '-e', 'SELECT 1;'],
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4)
|
||||||
|
return result.returncode == 0 and result.stdout.strip() == b'1'
|
||||||
|
|
||||||
|
def backend_ready(self):
|
||||||
|
try:
|
||||||
|
with self.opener.open('http://127.0.0.1:5281/actuator/health', timeout=2) as response:
|
||||||
|
return json.load(response).get('status') == 'UP'
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def adopt(self, role):
|
||||||
|
path = self.runtime / (role + '.process.json')
|
||||||
|
if not path.exists():
|
||||||
|
return False
|
||||||
|
info = json.loads(path.read_text())
|
||||||
|
identity = matches(info, self.command(role))
|
||||||
|
if identity is None:
|
||||||
|
raise RuntimeError(role + ': existing process identity cannot be confirmed')
|
||||||
|
if identity:
|
||||||
|
self.records[role] = record(info['pid'], self.command(role))
|
||||||
|
logging.info('%s adopted pid=%s', role, info['pid'])
|
||||||
|
return bool(identity)
|
||||||
|
|
||||||
|
def ensure(self, role):
|
||||||
|
child = self.children.get(role)
|
||||||
|
if child is not None:
|
||||||
|
child.poll() # Reap exited children before checking PID identity.
|
||||||
|
info = self.records.get(role)
|
||||||
|
state = matches(info, self.command(role))
|
||||||
|
if state is None:
|
||||||
|
raise RuntimeError(role + ': process identity is unknown; no replacement started')
|
||||||
|
if state:
|
||||||
|
return True
|
||||||
|
if info:
|
||||||
|
self.records.pop(role)
|
||||||
|
self.next_start[role] = time.monotonic() + 10
|
||||||
|
logging.warning('%s exited; restart scheduled in 10 seconds', role)
|
||||||
|
if time.monotonic() < self.next_start[role]:
|
||||||
|
return False
|
||||||
|
if self.adopt(role):
|
||||||
|
return True
|
||||||
|
if self.occupied(role):
|
||||||
|
raise RuntimeError(role + ': port occupied by an unconfirmed process')
|
||||||
|
command = self.command(role)
|
||||||
|
logs = self.root / 'output/logs'
|
||||||
|
logs.mkdir(parents=True, exist_ok=True)
|
||||||
|
log_name = ('backend-console.log' if role == 'backend' and
|
||||||
|
(self.runtime / 'operations/logback-quality.xml').is_file() else role + '.log')
|
||||||
|
with (logs / log_name).open('ab') as log:
|
||||||
|
child = subprocess.Popen(command, cwd=self.root, env=self.environment(),
|
||||||
|
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
|
||||||
|
self.children[role] = child
|
||||||
|
self.records[role] = info = record(child.pid, command)
|
||||||
|
(self.runtime / (role + '.pid')).write_text(str(child.pid))
|
||||||
|
atomic(self.runtime / (role + '.process.json'), {'pid': child.pid, 'started': info['started']})
|
||||||
|
self.restarts[role] += 1
|
||||||
|
logging.info('%s started pid=%s', role, child.pid)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def tick(self):
|
||||||
|
mysql_alive = self.ensure('mysql')
|
||||||
|
ready = mysql_alive and self.db_ready()
|
||||||
|
if ready:
|
||||||
|
self.ensure('backend')
|
||||||
|
atomic(self.home / 'state.json', {'updatedAt': time.time(), 'supervisorPid': os.getpid(),
|
||||||
|
'processes': self.records, 'starts': self.restarts,
|
||||||
|
'databaseReady': bool(ready), 'backendReady': self.backend_ready()})
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
self.preflight()
|
||||||
|
with (self.runtime / 'quality-service.lock').open('a') as lock:
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||||
|
signal.signal(sig, lambda *_: setattr(self, 'stopping', True))
|
||||||
|
logging.info('guardian started; accepted backend=%s', self.config['jarSha256'])
|
||||||
|
while not self.stopping:
|
||||||
|
try:
|
||||||
|
self.tick()
|
||||||
|
except Exception as error:
|
||||||
|
# Never log exception arguments: database credentials may be embedded.
|
||||||
|
detail = str(error) if type(error) is RuntimeError else type(error).__name__
|
||||||
|
logging.error('check failed: %s; leaving existing processes intact', detail)
|
||||||
|
atomic(self.home / 'error.json', {'time': time.time(), 'type': type(error).__name__})
|
||||||
|
for _ in range(30):
|
||||||
|
if self.stopping:
|
||||||
|
break
|
||||||
|
time.sleep(.1)
|
||||||
|
logging.info('guardian stopped; service processes remain available for maintenance')
|
||||||
|
|
||||||
|
|
||||||
|
def ctl(*args, check=True):
|
||||||
|
return subprocess.run(['/bin/launchctl', *args], check=check, capture_output=True, text=True)
|
||||||
|
|
||||||
|
|
||||||
|
def install(root, jar, clear_maintenance_pause=True):
|
||||||
|
root, jar = Path(root).resolve(), Path(jar).resolve()
|
||||||
|
cfg = {'root': str(root), 'jarSha256': hashlib.sha256(jar.read_bytes()).hexdigest(),
|
||||||
|
'path': '/opt/homebrew/bin:/usr/local/bin:/Library/Frameworks/Python.framework/Versions/3.13/bin:/usr/bin:/bin:/usr/sbin:/sbin'}
|
||||||
|
Guardian(cfg).preflight()
|
||||||
|
HOME.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
HOME.chmod(0o700)
|
||||||
|
script = Path(__file__).resolve()
|
||||||
|
digest = hashlib.sha256(script.read_bytes()).hexdigest()
|
||||||
|
target = HOME / 'versions' / digest / 'service.py'
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
shutil.copyfile(script, target)
|
||||||
|
target.chmod(0o600)
|
||||||
|
# Remove only the previous guardian; adopted MySQL/backend processes keep running.
|
||||||
|
ctl('bootout', DOMAIN + '/' + LABEL, check=False)
|
||||||
|
atomic(CONFIG, cfg)
|
||||||
|
logs = HOME / 'logs'
|
||||||
|
logs.mkdir(mode=0o700, exist_ok=True)
|
||||||
|
plist = {'Label': LABEL, 'ProgramArguments': [str(Path(sys.executable).resolve()), str(target), 'run'],
|
||||||
|
'WorkingDirectory': str(HOME), 'RunAtLoad': True, 'KeepAlive': True,
|
||||||
|
'ThrottleInterval': 10, 'ProcessType': 'Background', 'ExitTimeOut': 30,
|
||||||
|
'AbandonProcessGroup': True,
|
||||||
|
'EnvironmentVariables': {'PYTHONDONTWRITEBYTECODE': '1'},
|
||||||
|
'StandardOutPath': str(logs / 'launchd.log'), 'StandardErrorPath': str(logs / 'launchd-error.log')}
|
||||||
|
PLIST.parent.mkdir(exist_ok=True)
|
||||||
|
with PLIST.open('wb') as stream:
|
||||||
|
plistlib.dump(plist, stream)
|
||||||
|
PLIST.chmod(0o600)
|
||||||
|
ctl('enable', DOMAIN + '/' + LABEL)
|
||||||
|
for attempt in range(50):
|
||||||
|
loaded=ctl('bootstrap', DOMAIN, str(PLIST), check=False)
|
||||||
|
if loaded.returncode==0:break
|
||||||
|
time.sleep(.2)
|
||||||
|
else:raise RuntimeError('Service registration did not complete; existing processes preserved')
|
||||||
|
if clear_maintenance_pause:
|
||||||
|
(root/'.runtime/retention/paused').unlink(missing_ok=True)
|
||||||
|
print('Installed login startup and continuous guardian:', PLIST)
|
||||||
|
|
||||||
|
|
||||||
|
def status():
|
||||||
|
loaded = ctl('print', DOMAIN + '/' + LABEL, check=False)
|
||||||
|
state_path = HOME / 'state.json'
|
||||||
|
state = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||||
|
output = {'launchdLoaded': loaded.returncode == 0, 'stateFresh': time.time() - state.get('updatedAt', 0) < 20,
|
||||||
|
'databaseReady': state.get('databaseReady'), 'backendReady': state.get('backendReady'),
|
||||||
|
'supervisorPid': state.get('supervisorPid'), 'starts': state.get('starts')}
|
||||||
|
output['processes'] = {k: {'pid': v['pid'], 'confirmed': matches(v)} for k, v in state.get('processes', {}).items()}
|
||||||
|
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
def stop_services():
|
||||||
|
cfg = json.loads(CONFIG.read_text())
|
||||||
|
guardian = Guardian(cfg)
|
||||||
|
guardian.preflight()
|
||||||
|
for role in ('backend', 'mysql'):
|
||||||
|
if not guardian.adopt(role):
|
||||||
|
continue
|
||||||
|
info = guardian.records[role]
|
||||||
|
if matches(info) is not True:
|
||||||
|
raise RuntimeError('Cannot confirm process identity')
|
||||||
|
psutil.Process(info['pid']).terminate()
|
||||||
|
deadline = time.monotonic() + 300
|
||||||
|
while matches(info) is True and time.monotonic() < deadline:
|
||||||
|
time.sleep(.2)
|
||||||
|
if matches(info) is not False:
|
||||||
|
raise RuntimeError('Graceful shutdown incomplete; no forced termination performed')
|
||||||
|
print(role + ' stopped gracefully')
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('action', choices=['install', 'run', 'status', 'start', 'pause', 'stop'])
|
||||||
|
parser.add_argument('--root')
|
||||||
|
parser.add_argument('--jar')
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.action == 'install':
|
||||||
|
if not args.root or not args.jar:
|
||||||
|
parser.error('install requires --root and --jar')
|
||||||
|
install(args.root, args.jar)
|
||||||
|
elif args.action == 'status':
|
||||||
|
status()
|
||||||
|
elif args.action in ('pause', 'stop'):
|
||||||
|
cfg = json.loads(CONFIG.read_text())
|
||||||
|
atomic(Path(cfg['root'])/'.runtime/retention/paused', {'since':time.time()})
|
||||||
|
ctl('disable', DOMAIN + '/' + LABEL)
|
||||||
|
ctl('bootout', DOMAIN + '/' + LABEL, check=False)
|
||||||
|
if args.action == 'stop':
|
||||||
|
stop_services()
|
||||||
|
print('Automatic restart disabled until the start command is used')
|
||||||
|
elif args.action == 'start':
|
||||||
|
cfg = json.loads(CONFIG.read_text())
|
||||||
|
(Path(cfg['root'])/'.runtime/retention/paused').unlink(missing_ok=True)
|
||||||
|
ctl('enable', DOMAIN + '/' + LABEL)
|
||||||
|
if ctl('print', DOMAIN + '/' + LABEL, check=False).returncode:
|
||||||
|
ctl('bootstrap', DOMAIN, str(PLIST))
|
||||||
|
else:
|
||||||
|
ctl('kickstart', DOMAIN + '/' + LABEL)
|
||||||
|
status()
|
||||||
|
else:
|
||||||
|
logs = HOME / 'logs'
|
||||||
|
logs.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
handler = RotatingFileHandler(logs / 'guardian.log', maxBytes=5*1024*1024, backupCount=4)
|
||||||
|
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
|
||||||
|
logging.getLogger().addHandler(handler)
|
||||||
|
logging.getLogger().setLevel(logging.INFO)
|
||||||
|
Guardian(json.loads(CONFIG.read_text())).run()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
"""Verify the actual JDK, accepted logging libraries, JDBC and cross-language registry."""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0,str(ROOT/'scripts'))
|
||||||
|
import backup
|
||||||
|
import backup_retention as br
|
||||||
|
import managed_test_resources as resources
|
||||||
|
|
||||||
|
|
||||||
|
HARNESS=r'''
|
||||||
|
import java.sql.*;
|
||||||
|
import javax.sql.DataSource;
|
||||||
|
import cn.yunsheng.quality.testsupport.OwnedTestDatabase;
|
||||||
|
public class TestDbProbe {
|
||||||
|
static class DS implements DataSource {
|
||||||
|
public Connection getConnection() throws SQLException {return DriverManager.getConnection(System.getenv("TEST_DB_URL"),"root",System.getenv("TEST_DB_PASSWORD"));}
|
||||||
|
public Connection getConnection(String u,String p)throws SQLException{return DriverManager.getConnection(System.getenv("TEST_DB_URL"),u,p);}
|
||||||
|
public java.io.PrintWriter getLogWriter(){return null;} public void setLogWriter(java.io.PrintWriter w){}
|
||||||
|
public void setLoginTimeout(int t){} public int getLoginTimeout(){return 5;}
|
||||||
|
public java.util.logging.Logger getParentLogger(){return java.util.logging.Logger.getGlobal();}
|
||||||
|
public <T>T unwrap(Class<T> c)throws SQLException{throw new SQLException();} public boolean isWrapperFor(Class<?> c){return false;}
|
||||||
|
}
|
||||||
|
public static void main(String[] a)throws Exception{
|
||||||
|
Class.forName("com.mysql.cj.jdbc.Driver");DS ds=new DS();String name=a[1];
|
||||||
|
if(a[0].equals("normal")){OwnedTestDatabase.create(ds,name);OwnedTestDatabase.drop(ds,name);}
|
||||||
|
else if(a[0].equals("empty")){
|
||||||
|
OwnedTestDatabase.createEmpty(ds,name);
|
||||||
|
try(Connection c=ds.getConnection();Statement s=c.createStatement();ResultSet rs=s.executeQuery("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='"+name+"'")){
|
||||||
|
rs.next();if(rs.getInt(1)!=0)throw new AssertionError("Fresh schema is not empty");}
|
||||||
|
OwnedTestDatabase.seal(ds,name);OwnedTestDatabase.drop(ds,name);
|
||||||
|
}else OwnedTestDatabase.create(ds,name);
|
||||||
|
System.out.println("PROBE_OK");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'''
|
||||||
|
|
||||||
|
LOG_PROBE=r'''
|
||||||
|
import ch.qos.logback.classic.LoggerContext;
|
||||||
|
import ch.qos.logback.classic.joran.JoranConfigurator;
|
||||||
|
import ch.qos.logback.core.status.Status;
|
||||||
|
import java.nio.file.*;
|
||||||
|
public class LogProbe {
|
||||||
|
public static void main(String[] args)throws Exception{
|
||||||
|
LoggerContext c=new LoggerContext();c.setMDCAdapter(new ch.qos.logback.classic.util.LogbackMDCAdapter());c.putProperty("QUALITY_LOG_DIR",args[1]);
|
||||||
|
JoranConfigurator config=new JoranConfigurator();config.setContext(c);config.doConfigure(args[0]);
|
||||||
|
for(Status s:c.getStatusManager().getCopyOfStatusList())if(s.getLevel()>=Status.ERROR)throw new AssertionError(s.toString());
|
||||||
|
var logger=c.getLogger("probe");
|
||||||
|
for(int i=0;i<12000;i++)logger.info("{} {}",java.util.UUID.randomUUID(),java.util.UUID.randomUUID());
|
||||||
|
Thread.sleep(1500);
|
||||||
|
for(Status s:c.getStatusManager().getCopyOfStatusList())if(s.getLevel()>=Status.ERROR)throw new AssertionError(s.toString());
|
||||||
|
c.stop();System.out.println("LOG_PROBE_OK");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
class NativeOperationsTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Accepted local JDK/MySQL runtime is not configured')
|
||||||
|
cls.temp=tempfile.TemporaryDirectory();cls.build=Path(cls.temp.name).resolve();cls.libs=cls.build/'lib';cls.libs.mkdir()
|
||||||
|
cfg=json.loads((Path.home()/'Library/Application Support/QualityCenterService/service.json').read_text())
|
||||||
|
jar=ROOT/'.runtime/backend-builds'/(cfg['jarSha256']+'.jar')
|
||||||
|
with zipfile.ZipFile(jar) as z:
|
||||||
|
for n in z.namelist():
|
||||||
|
if n.startswith('BOOT-INF/lib/') and any(x in Path(n).name for x in ['logback-','slf4j-api-','spring-boot-4.','spring-core-','mysql-connector-']):
|
||||||
|
(cls.libs/Path(n).name).write_bytes(z.read(n))
|
||||||
|
(cls.build/'TestDbProbe.java').write_text(HARNESS);(cls.build/'LogProbe.java').write_text(LOG_PROBE)
|
||||||
|
cls.java=ROOT/'.runtime/jdk/Contents/Home/bin/java';cls.javac=cls.java.with_name('javac')
|
||||||
|
cls.classes=cls.build/'classes';cls.classes.mkdir()
|
||||||
|
helper=ROOT/'backend/src/test/java/cn/yunsheng/quality/testsupport/OwnedTestDatabase.java'
|
||||||
|
p=subprocess.run([str(cls.javac),'-cp',str(cls.libs/'*'),'-d',str(cls.classes),str(helper),str(cls.build/'TestDbProbe.java'),str(cls.build/'LogProbe.java')],text=True,capture_output=True)
|
||||||
|
if p.returncode:raise RuntimeError(p.stderr)
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):cls.temp.cleanup()
|
||||||
|
|
||||||
|
def test_java_registration_normal_drop_fresh_schema_and_expired_orphan_cleanup(self):
|
||||||
|
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root=Path(directory).resolve();br.atomic(root/'.runtime/local-config.json',{'mysqlDatabase':cfg['mysqlDatabase']})
|
||||||
|
env=os.environ.copy();env.update({'TEST_DB_URL':f"jdbc:mysql://127.0.0.1:{cfg['mysqlPort']}/?useSSL=false&allowPublicKeyRetrieval=true",'TEST_DB_PASSWORD':cfg['rootPassword']})
|
||||||
|
names=[]
|
||||||
|
try:
|
||||||
|
for mode in ['normal','empty','orphan']:
|
||||||
|
name='quality_sales_type_test_'+uuid.uuid4().hex;names.append(name)
|
||||||
|
p=subprocess.run([str(self.java),'-cp',str(self.classes)+os.pathsep+str(self.libs/'*'),'TestDbProbe',mode,name],cwd=root,env=env,text=True,capture_output=True,timeout=30)
|
||||||
|
self.assertEqual(p.returncode,0,p.stderr);self.assertIn('PROBE_OK',p.stdout)
|
||||||
|
if mode!='orphan':self.assertIsNone(resources.load(root,name))
|
||||||
|
else:
|
||||||
|
record=resources.load(root,name);self.assertTrue(record['ready'])
|
||||||
|
self.assertFalse(resources.active_process(record['ownerPid'],record['ownerCreated']))
|
||||||
|
result=resources.cleanup(root,backup.client,record['expiresAt']+1)
|
||||||
|
self.assertEqual(result['testDatabasesRemoved'],1);self.assertIsNone(resources.load(root,name))
|
||||||
|
finally:
|
||||||
|
for name in names:backup.client('DROP DATABASE IF EXISTS `'+name+'`;')
|
||||||
|
|
||||||
|
def test_actual_logback_configuration_rolls_and_bounds_files(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root=Path(directory);xml=(ROOT/'deploy/macos/logback-quality.xml').read_text()
|
||||||
|
# Exercise identical policy at a smaller scale, retaining asynchronous-roll headroom.
|
||||||
|
self.assertIn('<maxFileSize>10MB</maxFileSize>',xml);self.assertIn('<totalSizeCap>80MB</totalSizeCap>',xml)
|
||||||
|
config=root/'logback.xml';config.write_text(xml.replace('10MB','10KB').replace('80MB','80KB'))
|
||||||
|
p=subprocess.run([str(self.java),'-cp',str(self.classes)+os.pathsep+str(self.libs/'*'),'LogProbe',str(config),str(root)],text=True,capture_output=True,timeout=30)
|
||||||
|
self.assertEqual(p.returncode,0,p.stderr+p.stdout);self.assertIn('LOG_PROBE_OK',p.stdout)
|
||||||
|
archives=list(root.glob('backend.*.log.gz'));self.assertTrue(archives)
|
||||||
|
self.assertLessEqual(sum(f.stat().st_size for f in archives),90*1024)
|
||||||
|
self.assertLessEqual(sum(f.stat().st_size for f in archives)+(root/'backend.log').stat().st_size,100*1024)
|
||||||
|
self.assertLessEqual((root/'backend.log').stat().st_size,12*1024)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__=='__main__':unittest.main()
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||||
|
import backup
|
||||||
|
import backup_retention as br
|
||||||
|
import log_retention as logs
|
||||||
|
import managed_test_resources as resources
|
||||||
|
spec = importlib.util.spec_from_file_location('quality_operations', Path(__file__).with_name('operations.py'))
|
||||||
|
ops = importlib.util.module_from_spec(spec); spec.loader.exec_module(ops)
|
||||||
|
|
||||||
|
|
||||||
|
class OperationsTests(unittest.TestCase):
|
||||||
|
def sample(self, **extra):
|
||||||
|
return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB,
|
||||||
|
'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra}
|
||||||
|
|
||||||
|
def test_capacity_levels_free_space_failures_and_growth_are_separate(self):
|
||||||
|
self.assertEqual(ops.conditions(self.sample(), ops.DEFAULTS, {}, []), {})
|
||||||
|
self.assertEqual(ops.conditions(self.sample(coreBytes=30*ops.GIB), ops.DEFAULTS, {}, [])['capacity']['severity'], 'warning')
|
||||||
|
self.assertIn('temporary',ops.conditions(self.sample(temporaryBytes=30*ops.GIB),ops.DEFAULTS,{},[]))
|
||||||
|
self.assertIn('artifacts',ops.conditions(self.sample(artifactBytes=50*ops.GIB),ops.DEFAULTS,{},[]))
|
||||||
|
found = ops.conditions(self.sample(coreBytes=50*ops.GIB, freeBytes=19*ops.GIB, overdueTestResources=1),
|
||||||
|
ops.DEFAULTS, {'consecutiveFailures': 3}, [])
|
||||||
|
self.assertEqual(set(found), {'capacity', 'disk', 'maintenance', 'tests'})
|
||||||
|
old = self.sample(at=200000-86400, permanentBytes=30*ops.GIB)
|
||||||
|
self.assertIn('growth', ops.conditions(self.sample(), ops.DEFAULTS, {}, [old]))
|
||||||
|
self.assertNotIn('maintenance', ops.conditions(self.sample(), ops.DEFAULTS, {'lastDeferred': {'reason': 'active'}}, []))
|
||||||
|
old = self.sample(at=200000-86400)
|
||||||
|
self.assertIn('growth',ops.conditions(self.sample(coreBytes=15*ops.GIB,permanentBytes=43*ops.GIB),ops.DEFAULTS,{},[old]))
|
||||||
|
|
||||||
|
def test_monitor_failure_notifies_after_three_attempts_without_repeated_noise(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
root=Path(d).resolve();sent=[]
|
||||||
|
def notify(*_):sent.append(True);return {'submitted':True}
|
||||||
|
for _ in range(4):ops.failed_run(root,RuntimeError('not logged'),notify)
|
||||||
|
self.assertEqual(len(sent),1)
|
||||||
|
self.assertEqual(json.loads((root/'.runtime/operations/failure.json').read_text())['count'],4)
|
||||||
|
|
||||||
|
def test_alert_deduplication_escalation_recovery_and_failed_delivery_retry(self):
|
||||||
|
sent = []
|
||||||
|
def notify(title, message): sent.append(message); return {'submitted': True}
|
||||||
|
warning = {'capacity': {'severity': 'warning', 'message': 'warning'}}
|
||||||
|
first, events = ops.dispatch(warning, {}, 100, ops.DEFAULTS, notify)
|
||||||
|
second, events = ops.dispatch(warning, first, 200, ops.DEFAULTS, notify)
|
||||||
|
self.assertEqual(len(sent), 1); self.assertFalse(events)
|
||||||
|
critical = {'capacity': {'severity': 'critical', 'message': 'critical'}}
|
||||||
|
third, _ = ops.dispatch(critical, second, 300, ops.DEFAULTS, notify)
|
||||||
|
self.assertEqual(len(sent), 2)
|
||||||
|
_, events = ops.dispatch({}, third, 400, ops.DEFAULTS, notify)
|
||||||
|
self.assertEqual(events[0]['kind'], 'resolved')
|
||||||
|
failed, _ = ops.dispatch(warning, {}, 500, ops.DEFAULTS, lambda *_: {'submitted': False})
|
||||||
|
ops.dispatch(warning, failed, 600, ops.DEFAULTS, notify)
|
||||||
|
self.assertEqual(len(sent), 3)
|
||||||
|
|
||||||
|
def test_registered_and_private_temporary_storage_is_not_permanent_growth(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
root = Path(d).resolve(); support = root / 'support'
|
||||||
|
name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
|
||||||
|
for path in [root/'.runtime/mysql-data/current/data.ibd', root/'.runtime/mysql-data'/name/'sample.ibd',
|
||||||
|
root/'output/backups/current/file.gz', root/'runtimedata/archive/source.xlsx']:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(b'x'*8192)
|
||||||
|
br.atomic(resources.registry(root)/(name+'.json'), {'kind':'quality-owned-test-db','root':str(root),
|
||||||
|
'database':name,'token':'1'*32,'created':True,'expiresAt':time.time()+86400})
|
||||||
|
private=root/'.runtime/retention/verify-fixture';private.mkdir(parents=True)
|
||||||
|
(private/'data').write_bytes(b'x'*16384)
|
||||||
|
br.atomic(private/'owner.json',{'kind':'private-quality-restore','directory':str(private)})
|
||||||
|
sample=ops.snapshot(root,support)
|
||||||
|
self.assertGreater(sample['registeredTestBytes'],0); self.assertGreater(sample['privateVerifierBytes'],0)
|
||||||
|
self.assertLess(sample['coreBytes'],sample['mysqlBytes']+sample['backupBytes']+sample['archiveBytes'])
|
||||||
|
|
||||||
|
def test_monitor_still_collects_when_housekeeping_is_deferred_and_history_is_bounded(self):
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
root=Path(d).resolve();base=root/'.runtime/operations';base.mkdir(parents=True)
|
||||||
|
br.atomic(base/'policy.json',ops.DEFAULTS)
|
||||||
|
history=[self.sample(at=time.time()-i*100) for i in range(450)]
|
||||||
|
br.atomic(base/'samples.json',history)
|
||||||
|
with patch.object(ops,'snapshot',return_value=self.sample(at=time.time())), \
|
||||||
|
patch.object(ops,'janitor',return_value={'deferred':'Application jobs are active'}):
|
||||||
|
result=ops.run(root,root/'support',lambda *_:{'submitted':True})
|
||||||
|
self.assertIn('sample',result);self.assertIn('deferred',result['cleanup'])
|
||||||
|
self.assertLessEqual(len(json.loads((base/'samples.json').read_text())),330)
|
||||||
|
|
||||||
|
|
||||||
|
class LogRetentionTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve();self.support=self.root/'support'
|
||||||
|
def tearDown(self):self.temp.cleanup()
|
||||||
|
def file(self,name,age=0,size=100):
|
||||||
|
p=self.root/'output/logs'/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(b'line\n'*(size//5))
|
||||||
|
os.utime(p,(time.time()-age,time.time()-age));return p
|
||||||
|
|
||||||
|
def test_old_closed_logs_removed_but_active_symlinks_and_business_files_preserved(self):
|
||||||
|
old=self.file('old.log',age=8*86400);active=self.file('backend.log',age=8*86400)
|
||||||
|
other=self.file('worker.log',age=8*86400);outside=self.root/'important.xlsx';outside.write_bytes(b'keep')
|
||||||
|
link=old.with_name('link.log');link.symlink_to(outside)
|
||||||
|
result=logs.retain(self.root,self.support,open_paths={str(other)})
|
||||||
|
self.assertEqual(result['logsRemoved'],1);self.assertFalse(old.exists())
|
||||||
|
self.assertTrue(active.exists());self.assertTrue(other.exists());self.assertTrue(outside.exists());self.assertTrue(link.is_symlink())
|
||||||
|
|
||||||
|
def test_per_class_cap_prunes_old_archives_and_bounds_a_closed_console_tail(self):
|
||||||
|
old=self.file('worker.log.1',age=100,size=100);new=self.file('worker.log',age=0,size=100)
|
||||||
|
logs.retain(self.root,self.support,open_paths=set(),max_bytes=150)
|
||||||
|
self.assertFalse(old.exists());self.assertTrue(new.exists())
|
||||||
|
big=self.file('huge.log',size=1000)
|
||||||
|
logs.retain(self.root,self.support,open_paths=set(),max_bytes=100)
|
||||||
|
self.assertLessEqual(big.stat().st_size,100)
|
||||||
|
|
||||||
|
def test_mysql_rotation_uses_reopen_and_recovers_after_interruption(self):
|
||||||
|
source=self.file('mysql-server.log',age=86400,size=100);original=source.read_bytes()
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
logs.rotate_mysql(self.root,lambda *_:(_ for _ in ()).throw(RuntimeError('interrupted')),br)
|
||||||
|
self.assertFalse(source.exists());self.assertTrue((self.root/'.runtime/operations/mysql-log-rotation.json').exists())
|
||||||
|
calls=[]
|
||||||
|
def reopen(sql,_):calls.append(sql);source.write_bytes(b'new log')
|
||||||
|
self.assertEqual(logs.rotate_mysql(self.root,reopen,br),1)
|
||||||
|
self.assertEqual(calls,['FLUSH ERROR LOGS;'])
|
||||||
|
archived=list(source.parent.glob('mysql-server.roll-*.log'));self.assertEqual(archived[0].read_bytes(),original)
|
||||||
|
self.assertEqual(source.read_bytes(),b'new log')
|
||||||
|
|
||||||
|
|
||||||
|
class ManagedResourceTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured')
|
||||||
|
def setUp(self):
|
||||||
|
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve()
|
||||||
|
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
|
||||||
|
br.atomic(self.root/'.runtime/local-config.json',{'mysqlDatabase':cfg['mysqlDatabase']})
|
||||||
|
self.names=[];self.users=[]
|
||||||
|
def tearDown(self):
|
||||||
|
for n in self.names:backup.client('DROP DATABASE IF EXISTS `'+n+'`;')
|
||||||
|
for u in self.users:backup.client("DROP USER IF EXISTS '"+u+"'@'127.0.0.1';")
|
||||||
|
self.temp.cleanup()
|
||||||
|
def create(self,empty=False):
|
||||||
|
name='quality_retention_test_'+uuid.uuid4().hex[:12];self.names.append(name)
|
||||||
|
resources.create(self.root,name,backup.client,empty=empty);return name
|
||||||
|
|
||||||
|
def test_successful_create_and_drop_unregisters_only_its_schema(self):
|
||||||
|
name=self.create();self.assertTrue(resources.load(self.root,name)['ready'])
|
||||||
|
resources.drop(self.root,name,backup.client);self.assertIsNone(resources.load(self.root,name))
|
||||||
|
|
||||||
|
def test_expired_dead_owner_is_cleaned_but_live_owner_is_protected(self):
|
||||||
|
name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1
|
||||||
|
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0)
|
||||||
|
with patch.object(resources,'active_process',return_value=False):
|
||||||
|
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],1)
|
||||||
|
self.assertIsNone(resources.load(self.root,name))
|
||||||
|
|
||||||
|
def test_unregistered_schema_wrong_server_and_wrong_marker_are_never_dropped(self):
|
||||||
|
name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1
|
||||||
|
value['serverUuid']='different';br.atomic(resources.registry(self.root)/(name+'.json'),value)
|
||||||
|
with patch.object(resources,'active_process',return_value=False):
|
||||||
|
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0)
|
||||||
|
with self.assertRaises(ValueError):resources.drop(self.root,name,backup.client)
|
||||||
|
value['serverUuid']=backup.client('SELECT @@server_uuid;').strip();br.atomic(resources.registry(self.root)/(name+'.json'),value)
|
||||||
|
backup.client('UPDATE `'+name+'`.`'+resources.MARKER+"` SET token='different';")
|
||||||
|
with patch.object(resources,'active_process',return_value=False):
|
||||||
|
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0)
|
||||||
|
with self.assertRaises(ValueError):resources.create(self.root,'quality_platform',backup.client)
|
||||||
|
|
||||||
|
def test_empty_schema_stays_empty_until_migrations_finish_then_seals(self):
|
||||||
|
name=self.create(empty=True)
|
||||||
|
self.assertEqual(backup.client('SHOW TABLES;',name).strip(),'')
|
||||||
|
backup.client('CREATE TABLE business_data(id INT);',name)
|
||||||
|
resources.seal(self.root,name,backup.client)
|
||||||
|
self.assertTrue(resources.marker_matches(resources.load(self.root,name),backup.client))
|
||||||
|
resources.drop(self.root,name,backup.client)
|
||||||
|
|
||||||
|
def test_empty_failed_schema_can_be_dropped_by_its_current_owner(self):
|
||||||
|
name=self.create(empty=True);resources.drop(self.root,name,backup.client)
|
||||||
|
self.assertIsNone(resources.load(self.root,name))
|
||||||
|
|
||||||
|
def test_registered_test_account_is_cleaned_after_expired_schema(self):
|
||||||
|
name=self.create();user='qverify_'+uuid.uuid4().hex[:12];self.users.append(user)
|
||||||
|
backup.client("CREATE USER '"+user+"'@'127.0.0.1' IDENTIFIED BY 'disposable-test-account';")
|
||||||
|
backup.client('GRANT ALL ON `'+name+"`.* TO '"+user+"'@'127.0.0.1';")
|
||||||
|
resources.attach_user(self.root,name,user)
|
||||||
|
with patch.object(resources,'active_process',return_value=False):
|
||||||
|
result=resources.cleanup(self.root,backup.client,resources.load(self.root,name)['expiresAt']+1)
|
||||||
|
self.assertEqual(result['testDatabasesRemoved'],1);self.assertEqual(result['testAccountsRemoved'],1)
|
||||||
|
self.assertIsNone(resources.load(self.root,name))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__=='__main__':unittest.main()
|
||||||
@@ -0,0 +1,305 @@
|
|||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import tarfile
|
||||||
|
import gzip
|
||||||
|
from contextlib import contextmanager
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0,str(ROOT/'scripts'))
|
||||||
|
import backup_retention as br
|
||||||
|
spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py'))
|
||||||
|
ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret)
|
||||||
|
|
||||||
|
|
||||||
|
def good_backup(root, suffix, created='2026-11-09T01:00:00+00:00'):
|
||||||
|
folder=root/'output/backups'/('20261109T010000Z-'+suffix)
|
||||||
|
folder.mkdir(parents=True)
|
||||||
|
m={'formatVersion':3,'status':'ISOLATED_RESTORE_VERIFIED','createdAt':created,
|
||||||
|
'restoreVerifiedAt':created,'queryEvidence':{'releaseId':'current'},
|
||||||
|
'verification':{k:True for k in br.CHECKS}}
|
||||||
|
for name,key in br.ARTIFACTS.items():
|
||||||
|
(folder/name).write_bytes(name.encode());m[key]=br.sha(folder/name)
|
||||||
|
br.atomic(folder/'manifest.json',m)
|
||||||
|
return folder.resolve()
|
||||||
|
|
||||||
|
|
||||||
|
class BackupRotationTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
|
||||||
|
def tearDown(self):self.temp.cleanup()
|
||||||
|
|
||||||
|
def test_failed_or_corrupted_new_backup_keeps_previous(self):
|
||||||
|
old=good_backup(self.root,'11111111');br.rotate(self.root,old)
|
||||||
|
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00')
|
||||||
|
(new/'database.sql.gz').write_bytes(b'corrupted')
|
||||||
|
with self.assertRaises(ValueError):br.rotate(self.root,new)
|
||||||
|
self.assertTrue(old.is_dir());self.assertEqual(br.latest(self.root)[0],old)
|
||||||
|
|
||||||
|
def test_rotation_keeps_one_and_verifying_older_never_rolls_back_latest(self):
|
||||||
|
old=good_backup(self.root,'11111111');br.rotate(self.root,old)
|
||||||
|
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00');br.rotate(self.root,new)
|
||||||
|
self.assertFalse(old.exists());self.assertEqual(br.latest(self.root)[0],new)
|
||||||
|
older=good_backup(self.root,'33333333');br.rotate(self.root,older)
|
||||||
|
self.assertEqual(br.latest(self.root)[0],new);self.assertFalse(older.exists())
|
||||||
|
|
||||||
|
def test_unknown_and_active_incomplete_folders_are_preserved(self):
|
||||||
|
old=good_backup(self.root,'11111111');unknown=old.parent/'unrelated';unknown.mkdir()
|
||||||
|
incomplete=old.parent/'20261109T010000Z-abcdefab';incomplete.mkdir()
|
||||||
|
br.atomic(incomplete/'.owner.json',{'kind':'quality-backup','root':str(self.root),'id':incomplete.name})
|
||||||
|
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00');br.rotate(self.root,new)
|
||||||
|
self.assertTrue(unknown.exists());self.assertTrue(incomplete.exists())
|
||||||
|
|
||||||
|
def test_shared_lock_excludes_another_process(self):
|
||||||
|
code='import sys;from pathlib import Path;sys.path.insert(0,sys.argv[1]);import backup_retention as b\ntry:\n with b.lock(Path(sys.argv[2]),timeout=0):pass\nexcept b.Busy:sys.exit(17)'
|
||||||
|
with br.lock(self.root):
|
||||||
|
child=subprocess.run([sys.executable,'-c',code,str(ROOT/'scripts'),str(self.root)],capture_output=True)
|
||||||
|
self.assertEqual(child.returncode,17)
|
||||||
|
|
||||||
|
def test_managed_backup_root_cannot_be_redirected_outside_project(self):
|
||||||
|
with tempfile.TemporaryDirectory() as external:
|
||||||
|
folder=good_backup(Path(external),'11111111')
|
||||||
|
(self.root/'output').mkdir();(self.root/'output/backups').symlink_to(folder.parent,target_is_directory=True)
|
||||||
|
with self.assertRaises(ValueError):br.rotate(self.root,folder)
|
||||||
|
self.assertTrue(folder.is_dir())
|
||||||
|
|
||||||
|
def test_interrupted_old_backup_removal_can_resume_without_touching_latest(self):
|
||||||
|
old=good_backup(self.root,'11111111');br.rotate(self.root,old)
|
||||||
|
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00')
|
||||||
|
with patch.object(br.shutil,'rmtree',side_effect=OSError('simulated removal failure')):
|
||||||
|
br.rotate(self.root,new)
|
||||||
|
self.assertEqual(br.latest(self.root)[0],new)
|
||||||
|
self.assertEqual(len(list((self.root/'.runtime/retention/retired').glob('*.json'))),1)
|
||||||
|
br.clean_retired(self.root)
|
||||||
|
self.assertFalse(list((self.root/'.runtime/retention/retired').glob('*.json')))
|
||||||
|
self.assertTrue(new.is_dir())
|
||||||
|
|
||||||
|
|
||||||
|
class DatabaseRetentionTests(unittest.TestCase):
|
||||||
|
"""Real MySQL locking/FK tests in a newly owned, tiny disposable database."""
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured')
|
||||||
|
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
|
||||||
|
cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True)
|
||||||
|
cls.name='quality_retention_test_'+uuid.uuid4().hex[:12]
|
||||||
|
with cls.admin.cursor() as c:
|
||||||
|
c.execute('CREATE DATABASE '+ret.ident(cls.name))
|
||||||
|
c.execute('USE '+ret.ident(cls.name))
|
||||||
|
ddl=[
|
||||||
|
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
|
||||||
|
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
|
||||||
|
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)',
|
||||||
|
'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))',
|
||||||
|
'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||||
|
'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||||
|
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)',
|
||||||
|
'CREATE TABLE import_run(id int PRIMARY KEY,status varchar(24),expected_current_release_id varchar(36),result_release_id varchar(36),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id),FOREIGN KEY(result_release_id) REFERENCES data_release(id))',
|
||||||
|
'CREATE TABLE release_action(id int PRIMARY KEY,release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
|
||||||
|
'CREATE TABLE candidate_reconciliation(id int PRIMARY KEY,release_id varchar(36),baseline_release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(baseline_release_id) REFERENCES data_release(id))',
|
||||||
|
'CREATE TABLE analysis_session(id int PRIMARY KEY,release_id varchar(36))',
|
||||||
|
'CREATE TABLE saved_business_object(id int PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
|
||||||
|
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
|
||||||
|
'CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),parser_config_hash varchar(64),file_size bigint)',
|
||||||
|
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),source_slot varchar(100),source_path varchar(1000),archive_path varchar(1000),sha256 char(64),parser_config_hash varchar(64),expected_current_release_id varchar(36))',
|
||||||
|
'CREATE TABLE source_upload_draft(id varchar(36) PRIMARY KEY,source_file varchar(100),source_path varchar(1000),sha256 char(64),file_size bigint,candidate_id varchar(36),created_at datetime,expires_at datetime)']
|
||||||
|
for statement in ddl:c.execute(statement+' ENGINE=InnoDB')
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name))
|
||||||
|
cls.admin.close()
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
|
||||||
|
self.db=ret.Database(ROOT,self.name);self.db.root=self.root
|
||||||
|
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
|
||||||
|
for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table))
|
||||||
|
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
|
||||||
|
self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')")
|
||||||
|
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
|
||||||
|
self.db.execute("INSERT INTO transform_build VALUES('shared','COMPLETED',%s),('obsolete','COMPLETED',%s)",('1'*64,'2'*64))
|
||||||
|
self.db.execute("INSERT INTO release_member VALUES('current','shared'),('old','shared'),('old','obsolete')")
|
||||||
|
self.db.execute("INSERT INTO fact_black VALUES(1,'shared','current data'),(2,'obsolete','old data'),(3,'obsolete','old data')")
|
||||||
|
|
||||||
|
def tearDown(self):self.db.conn.close();self.temp.cleanup()
|
||||||
|
|
||||||
|
def retire_old(self):
|
||||||
|
state={'releases':{'old':1}}
|
||||||
|
with self.db.frozen('current'):self.assertEqual(self.db.release_step('current',state,1800,4000),1)
|
||||||
|
|
||||||
|
def test_running_queued_retry_and_unknown_tasks_prevent_cleanup(self):
|
||||||
|
for status,retry in [('RUNNING',False),('QUEUED',False),('FAILED',True),('FUTURE_STATE',False)]:
|
||||||
|
self.db.execute('DELETE FROM task')
|
||||||
|
self.db.execute('INSERT INTO task VALUES(1,%s,%s,0,3)',(status,datetime_from_future() if retry else None))
|
||||||
|
with self.assertRaises(ret.Busy):
|
||||||
|
with self.db.frozen('current'):self.fail('busy job was ignored')
|
||||||
|
self.assertEqual(len(self.db.rows('SELECT * FROM data_release')),2)
|
||||||
|
|
||||||
|
def test_pending_candidate_and_saved_business_references_are_protected(self):
|
||||||
|
self.db.execute("INSERT INTO data_release VALUES('candidate','CANDIDATE','old','old')")
|
||||||
|
with self.db.frozen('current'):
|
||||||
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
|
||||||
|
self.db.execute("DELETE FROM data_release WHERE id='candidate'")
|
||||||
|
self.db.execute("INSERT INTO saved_business_object VALUES(1,'old')")
|
||||||
|
with self.db.frozen('current'):
|
||||||
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
|
||||||
|
|
||||||
|
def test_analysis_session_reference_is_protected_without_database_fk(self):
|
||||||
|
self.db.execute("INSERT INTO analysis_session VALUES(1,'old')")
|
||||||
|
with self.db.frozen('current'):
|
||||||
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
|
||||||
|
|
||||||
|
def test_pending_source_configuration_preserves_its_old_baseline(self):
|
||||||
|
self.db.execute("INSERT INTO source_candidate VALUES('pending','DISCOVERED','slot','path',NULL,%s,'legacy','old')",('3'*64,))
|
||||||
|
with self.db.frozen('current'):
|
||||||
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
|
||||||
|
|
||||||
|
def test_unknown_business_build_reference_is_preserved(self):
|
||||||
|
self.retire_old();self.db.execute("INSERT INTO unknown_build_use VALUES(1,'obsolete')")
|
||||||
|
with self.db.frozen('current'):
|
||||||
|
self.assertEqual(self.db.build_step({'builds':{'obsolete':1}},1800,4000,1),(0,0))
|
||||||
|
self.assertEqual(self.db.rows("SELECT status FROM transform_build WHERE id='obsolete'")[0]['status'],'COMPLETED')
|
||||||
|
|
||||||
|
def test_changed_publication_and_grace_period_prevent_cleanup(self):
|
||||||
|
with self.assertRaises(ret.Busy):
|
||||||
|
with self.db.frozen('old'):pass
|
||||||
|
with self.db.frozen('current'):
|
||||||
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':3900}},1800,4000),0)
|
||||||
|
|
||||||
|
def test_old_release_removed_but_current_membership_and_rows_preserved(self):
|
||||||
|
self.db.execute("INSERT INTO import_run VALUES(1,'COMPLETED','old','old')")
|
||||||
|
self.db.execute("INSERT INTO release_action VALUES(1,'old',NULL)")
|
||||||
|
self.db.execute("INSERT INTO candidate_reconciliation VALUES(1,'current','old')")
|
||||||
|
self.retire_old()
|
||||||
|
self.assertEqual(self.db.rows("SELECT payload FROM fact_black WHERE build_id='shared'")[0]['payload'],'current data')
|
||||||
|
self.assertEqual(self.db.rows("SELECT * FROM release_member"),[{'release_id':'current','transform_build_id':'shared'}])
|
||||||
|
self.assertIsNone(self.db.rows('SELECT * FROM import_run')[0]['expected_current_release_id'])
|
||||||
|
self.assertIsNone(self.db.rows("SELECT * FROM data_release WHERE id='current'")[0]['parent_release_id'])
|
||||||
|
|
||||||
|
def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self):
|
||||||
|
self.retire_old();state={'builds':{'obsolete':1}}
|
||||||
|
with self.db.frozen('current'):
|
||||||
|
self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0))
|
||||||
|
old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0]
|
||||||
|
self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64)
|
||||||
|
self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,))
|
||||||
|
for _ in range(4):
|
||||||
|
with self.db.frozen('current'):self.db.build_step(state,1800,4000,1)
|
||||||
|
self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'"))
|
||||||
|
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'"))
|
||||||
|
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'"))
|
||||||
|
|
||||||
|
def add_draft(self,candidate=None,where=None,age=25):
|
||||||
|
path=where or self.root/'runtimedata/archive/upload-drafts/upload-fixture.xlsx'
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'workbook bytes')
|
||||||
|
self.db.execute('INSERT INTO source_upload_draft VALUES(%s,%s,%s,%s,%s,%s,TIMESTAMPADD(HOUR,-30,CURRENT_TIMESTAMP),TIMESTAMPADD(HOUR,%s,CURRENT_TIMESTAMP))',('draft','input.xlsx',str(path),br.sha(path),path.stat().st_size,candidate,-age))
|
||||||
|
return path
|
||||||
|
|
||||||
|
def test_expired_unbound_upload_removed_but_fresh_preview_is_kept(self):
|
||||||
|
path=self.add_draft()
|
||||||
|
count,size=ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
||||||
|
self.assertEqual(count,1);self.assertGreater(size,0);self.assertFalse(path.exists())
|
||||||
|
self.add_draft(age=0)
|
||||||
|
self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br),(0,0))
|
||||||
|
|
||||||
|
def test_upload_still_referenced_by_pending_candidate_is_kept(self):
|
||||||
|
path=self.add_draft(candidate='pending')
|
||||||
|
self.db.execute("INSERT INTO source_candidate VALUES('pending','DISCOVERED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path)))
|
||||||
|
self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br)[0],1)
|
||||||
|
self.assertTrue(path.exists())
|
||||||
|
|
||||||
|
def test_imported_upload_is_rebound_to_verified_archive_before_cleanup(self):
|
||||||
|
path=self.add_draft(candidate='imported');archive=self.root/'runtimedata/archive/source-revisions/formal.xlsx';archive.parent.mkdir();archive.write_bytes(path.read_bytes())
|
||||||
|
self.db.execute("INSERT INTO source_document VALUES('doc','slot')")
|
||||||
|
self.db.execute("INSERT INTO source_revision VALUES('revision','doc',%s,%s,'legacy',%s)",(str(archive),br.sha(archive),archive.stat().st_size))
|
||||||
|
self.db.execute("INSERT INTO source_candidate VALUES('imported','IMPORTED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path)))
|
||||||
|
self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br)[0],1)
|
||||||
|
self.assertFalse(path.exists());self.assertTrue(archive.exists())
|
||||||
|
self.assertEqual(self.db.rows('SELECT source_path FROM source_candidate')[0]['source_path'],str(archive))
|
||||||
|
|
||||||
|
def test_outside_file_is_never_deleted(self):
|
||||||
|
path=self.add_draft(where=self.root/'unmanaged/private.xlsx')
|
||||||
|
ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
||||||
|
self.assertTrue(path.exists())
|
||||||
|
|
||||||
|
def test_symlink_is_not_followed_or_deleted(self):
|
||||||
|
outside=self.root/'important.xlsx';outside.write_bytes(b'keep this')
|
||||||
|
link=self.root/'runtimedata/archive/upload-drafts/upload-link.xlsx';link.parent.mkdir(parents=True);link.symlink_to(outside)
|
||||||
|
self.db.execute('INSERT INTO source_upload_draft VALUES(%s,%s,%s,%s,%s,NULL,TIMESTAMPADD(HOUR,-30,CURRENT_TIMESTAMP),TIMESTAMPADD(HOUR,-25,CURRENT_TIMESTAMP))',('draft','link.xlsx',str(link),br.sha(outside),outside.stat().st_size))
|
||||||
|
ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
||||||
|
self.assertTrue(outside.exists());self.assertTrue(link.is_symlink())
|
||||||
|
|
||||||
|
def test_file_changed_after_checksum_is_not_deleted(self):
|
||||||
|
path=self.add_draft();original=ret.fingerprint;calls=0
|
||||||
|
def changed(p):
|
||||||
|
nonlocal calls
|
||||||
|
calls+=1
|
||||||
|
if calls==2:path.write_bytes(b'new file contents')
|
||||||
|
return original(p)
|
||||||
|
with patch.object(ret,'fingerprint',side_effect=changed):
|
||||||
|
count,size=ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
||||||
|
self.assertEqual((count,size),(0,0));self.assertTrue(path.exists())
|
||||||
|
self.assertTrue(self.db.rows('SELECT * FROM source_upload_draft'))
|
||||||
|
|
||||||
|
def test_new_reference_before_file_unlink_is_rechecked(self):
|
||||||
|
path=self.add_draft();freeze=self.db.frozen;calls=0
|
||||||
|
@contextmanager
|
||||||
|
def insert_reference(expected):
|
||||||
|
nonlocal calls
|
||||||
|
calls+=1
|
||||||
|
with freeze(expected):
|
||||||
|
if calls==2:self.db.execute("INSERT INTO source_candidate VALUES('new','DISCOVERED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path)))
|
||||||
|
yield
|
||||||
|
with patch.object(self.db,'frozen',side_effect=insert_reference):
|
||||||
|
ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
||||||
|
self.assertTrue(path.exists());self.assertFalse(self.db.rows('SELECT * FROM source_upload_draft'))
|
||||||
|
|
||||||
|
def test_long_running_sql_is_not_interrupted_for_cleanup(self):
|
||||||
|
other=ret.Database(ROOT,self.name)
|
||||||
|
try:
|
||||||
|
other.conn.autocommit(False);other.rows('SELECT * FROM fact_black WHERE id=1 FOR UPDATE')
|
||||||
|
with self.assertRaises(ret.Busy):
|
||||||
|
with self.db.frozen('current'):pass
|
||||||
|
self.assertTrue(other.rows('SELECT 1'));other.conn.rollback()
|
||||||
|
finally:other.conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def datetime_from_future():
|
||||||
|
from datetime import datetime,timedelta,timezone
|
||||||
|
return datetime.now(timezone.utc).replace(tzinfo=None)+timedelta(hours=1)
|
||||||
|
|
||||||
|
|
||||||
|
class PrivateVerifierTests(unittest.TestCase):
|
||||||
|
def test_small_backup_restores_on_private_server_and_cleans_up(self):
|
||||||
|
import backup
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
root=Path(temporary).resolve();(root/'.runtime').mkdir()
|
||||||
|
(root/'.runtime/mysql').symlink_to(ROOT/'.runtime/mysql',target_is_directory=True)
|
||||||
|
base=root/'output/backups';folder=base/'20261109T020000Z-12345678';folder.mkdir(parents=True)
|
||||||
|
source=root/'source';source.mkdir();(source/'config.json').write_text('{}')
|
||||||
|
with tarfile.open(folder/'application.tar.gz','w:gz') as tar:tar.add(source/'config.json',arcname='config.json')
|
||||||
|
with tarfile.open(folder/'archive.tar.gz','w:gz'):pass
|
||||||
|
sql="CREATE TABLE current_release(dataset_id varchar(20),release_id varchar(36)); INSERT INTO current_release VALUES('quality',NULL); CREATE TABLE source_revision(id varchar(36),archive_path varchar(1024),sha256 char(64),file_size bigint);"
|
||||||
|
with gzip.open(folder/'database.sql.gz','wt') as out:out.write(sql)
|
||||||
|
manifest={'formatVersion':3,'createdAt':'2026-11-09T02:00:00+00:00','files':{},'archiveSourceRoot':str(root/'runtimedata/archive'),
|
||||||
|
'applicationFiles':{'config.json':br.sha(source/'config.json')},'requiredApplicationFiles':['config.json'],
|
||||||
|
'tableCounts':{'current_release':1,'source_revision':0},'queryEvidence':{'releaseId':None,'months':[]}}
|
||||||
|
for name,key in br.ARTIFACTS.items():manifest[key]=br.sha(folder/name)
|
||||||
|
br.atomic(folder/'manifest.json',manifest)
|
||||||
|
with patch.object(backup,'ROOT',root),patch.object(backup,'BACKUP_ROOT',base),patch.object(backup,'REQUIRED_APPLICATION_FILES',('config.json',)):
|
||||||
|
result=backup.verify(folder)
|
||||||
|
self.assertEqual(result['status'],'ISOLATED_RESTORE_VERIFIED')
|
||||||
|
self.assertFalse(list((root/'.runtime/retention').glob('verify-*')))
|
||||||
|
self.assertFalse(list((root/'.runtime/retention/scratch').glob('*.json')))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__=='__main__':unittest.main()
|
||||||
@@ -0,0 +1,237 @@
|
|||||||
|
"""Exercise source retirement against an owned disposable MySQL schema and files."""
|
||||||
|
from contextlib import contextmanager
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||||
|
import backup_retention as br
|
||||||
|
import revision_retention as revisions
|
||||||
|
spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py'))
|
||||||
|
ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret)
|
||||||
|
|
||||||
|
|
||||||
|
class SourceRevisionTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured')
|
||||||
|
cfg = json.loads((ROOT / '.runtime/local-config.json').read_text())
|
||||||
|
cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root',
|
||||||
|
password=cfg['rootPassword'], autocommit=True)
|
||||||
|
cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
|
||||||
|
ddl = [
|
||||||
|
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))',
|
||||||
|
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))',
|
||||||
|
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
|
||||||
|
"CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)",
|
||||||
|
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||||
|
'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||||
|
'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||||
|
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))',
|
||||||
|
'CREATE TABLE source_upload_draft(id varchar(36) PRIMARY KEY,source_path varchar(1000))',
|
||||||
|
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),payload json)',
|
||||||
|
'CREATE TABLE source_change_request(id int PRIMARY KEY,original_revision_id varchar(36),status varchar(24))',
|
||||||
|
'CREATE TABLE saved_context(id int PRIMARY KEY,context_json json,attachment_path varchar(1000))',
|
||||||
|
'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))']
|
||||||
|
try:
|
||||||
|
with cls.admin.cursor() as c:
|
||||||
|
c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name))
|
||||||
|
for statement in ddl:
|
||||||
|
c.execute(statement + ' ENGINE=InnoDB')
|
||||||
|
except BaseException:
|
||||||
|
with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name))
|
||||||
|
cls.admin.close(); raise
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name))
|
||||||
|
cls.admin.close()
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve()
|
||||||
|
self.db = ret.Database(ROOT, self.name); self.db.root = self.root
|
||||||
|
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
|
||||||
|
for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
|
||||||
|
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
|
||||||
|
self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')")
|
||||||
|
self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')")
|
||||||
|
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
|
||||||
|
self.current_file = self.add_revision('current-rev', b'current workbook')
|
||||||
|
self.db.execute("INSERT INTO release_member VALUES('current','current-rev')")
|
||||||
|
self.db.execute("INSERT INTO transform_build VALUES('current-build','current-rev')")
|
||||||
|
self.cleaner = revisions.Cleaner(self.db, br, ret.safe_file, ret.fingerprint, ret.Busy)
|
||||||
|
self.state = {}
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.db.conn.close(); self.temp.cleanup()
|
||||||
|
|
||||||
|
def add_revision(self, rid='old-rev-unique', content=b'old workbook'):
|
||||||
|
digest = __import__('hashlib').sha256(content).hexdigest()
|
||||||
|
path = self.root / 'runtimedata/archive/source-revisions/black/2026/01' / (digest + '.xlsx')
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(content)
|
||||||
|
self.db.execute('INSERT INTO source_revision(id,source_document_id,archive_path,sha256,file_size) VALUES(%s,%s,%s,%s,%s)',
|
||||||
|
(rid, 'doc', str(path), digest, len(content)))
|
||||||
|
return path
|
||||||
|
|
||||||
|
def collect(self, now):
|
||||||
|
return self.cleaner.collect('current', self.state, now)
|
||||||
|
|
||||||
|
def present(self, rid='old-rev-unique'):
|
||||||
|
return bool(self.db.rows('SELECT id FROM source_revision WHERE id=%s', (rid,)))
|
||||||
|
|
||||||
|
def test_unreferenced_revision_waits_grace_then_removes_only_its_owned_file(self):
|
||||||
|
path = self.add_revision()
|
||||||
|
self.assertEqual(self.collect(100)['sourceRevisionsRemoved'], 0)
|
||||||
|
result = self.collect(2000)
|
||||||
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||||
|
self.assertEqual(result['sourceArchiveFilesRemoved'], 1)
|
||||||
|
self.assertEqual(result['sourceArchiveBytesRemoved'], len(b'old workbook'))
|
||||||
|
self.assertFalse(self.present()); self.assertFalse(path.exists())
|
||||||
|
self.assertTrue(self.present('current-rev')); self.assertTrue(self.current_file.exists())
|
||||||
|
self.assertTrue(self.db.rows('SELECT * FROM source_document'))
|
||||||
|
|
||||||
|
def test_any_release_build_fact_or_unknown_foreign_key_protects_revision(self):
|
||||||
|
for rid, statement in [
|
||||||
|
('candidate', "INSERT INTO release_member VALUES('candidate','candidate')"),
|
||||||
|
('build', "INSERT INTO transform_build VALUES('old-build','build')"),
|
||||||
|
('fact', "INSERT INTO fact_black VALUES(1,'fact',JSON_OBJECT(),JSON_OBJECT())"),
|
||||||
|
('future', "INSERT INTO unknown_reference VALUES(1,'future')")]:
|
||||||
|
self.add_revision(rid, rid.encode()); self.db.execute(statement)
|
||||||
|
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
||||||
|
for rid in ('candidate', 'build', 'fact', 'future'): self.assertTrue(self.present(rid))
|
||||||
|
|
||||||
|
def test_replacement_withdrawal_and_saved_json_ids_without_fks_are_protected(self):
|
||||||
|
for rid in ('replacement', 'withdrawal', 'lineage', 'saved'): self.add_revision(rid, rid.encode())
|
||||||
|
self.db.execute("INSERT INTO source_candidate VALUES('candidate','DISCOVERED',NULL,NULL,NULL,'replacement')")
|
||||||
|
self.db.execute("INSERT INTO source_change_request VALUES(1,'withdrawal','APPROVED')")
|
||||||
|
self.db.execute("UPDATE source_revision SET replaces_revision_id='lineage' WHERE id='current-rev'")
|
||||||
|
self.db.execute("INSERT INTO saved_context VALUES(1,JSON_OBJECT('sourceRevisionId','saved'),NULL)")
|
||||||
|
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
||||||
|
|
||||||
|
def test_catalog_hash_protects_original_even_with_external_path(self):
|
||||||
|
path = self.add_revision(); digest = br.sha(path)
|
||||||
|
self.db.execute("INSERT INTO source_candidate VALUES('catalog','IMPORTED',%s,'/external/original.xlsx',NULL,NULL)", (digest,))
|
||||||
|
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
||||||
|
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
||||||
|
|
||||||
|
def test_new_reference_resets_continuous_grace(self):
|
||||||
|
self.add_revision(); self.collect(100)
|
||||||
|
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500)
|
||||||
|
self.db.execute('DELETE FROM unknown_reference')
|
||||||
|
self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
||||||
|
self.assertEqual(self.collect(3801)['sourceRevisionsRemoved'], 1)
|
||||||
|
|
||||||
|
def test_revision_metadata_change_restarts_grace(self):
|
||||||
|
self.add_revision(); self.collect(100)
|
||||||
|
self.db.execute("UPDATE source_revision SET parser_config=JSON_OBJECT('changed',TRUE) WHERE id='old-rev-unique'")
|
||||||
|
self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
||||||
|
self.assertEqual(self.collect(3801)['sourceRevisionsRemoved'], 1)
|
||||||
|
|
||||||
|
def test_shared_archive_stays_until_its_other_revision_is_unreferenced(self):
|
||||||
|
path = self.add_revision(content=b'current workbook')
|
||||||
|
self.collect(100); result = self.collect(2000)
|
||||||
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||||
|
self.assertEqual(result['sourceArchiveFilesRemoved'], 0)
|
||||||
|
self.assertTrue(path.exists()); self.assertTrue(self.present('current-rev'))
|
||||||
|
|
||||||
|
def test_saved_file_path_protects_file_after_metadata_removal(self):
|
||||||
|
path = self.add_revision()
|
||||||
|
self.db.execute('INSERT INTO saved_context VALUES(1,JSON_OBJECT(\'archivePath\',%s),NULL)', (str(path),))
|
||||||
|
self.collect(100); result = self.collect(2000)
|
||||||
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||||
|
self.assertEqual(result['sourceFilesPending'], 1); self.assertTrue(path.exists())
|
||||||
|
self.db.execute('DELETE FROM saved_context')
|
||||||
|
self.assertEqual(self.cleaner.drain('current')[0], 1); self.assertFalse(path.exists())
|
||||||
|
|
||||||
|
def test_external_symlink_mismatched_or_incomplete_files_are_kept(self):
|
||||||
|
for rid in ('outside', 'symlink', 'damaged', 'incomplete'): self.add_revision(rid, rid.encode())
|
||||||
|
external = self.root / 'important.xlsx'; external.write_bytes(b'outside')
|
||||||
|
self.db.execute("UPDATE source_revision SET archive_path=%s WHERE id='outside'", (str(external),))
|
||||||
|
row = self.db.rows("SELECT archive_path FROM source_revision WHERE id='symlink'")[0]
|
||||||
|
link = Path(row['archive_path']); link.unlink(); link.symlink_to(external)
|
||||||
|
row = self.db.rows("SELECT archive_path FROM source_revision WHERE id='damaged'")[0]
|
||||||
|
Path(row['archive_path']).write_bytes(b'changed')
|
||||||
|
self.db.execute("UPDATE source_revision SET status='RECEIVED' WHERE id='incomplete'")
|
||||||
|
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
||||||
|
for rid in ('outside', 'symlink', 'damaged', 'incomplete'): self.assertTrue(self.present(rid))
|
||||||
|
self.assertTrue(external.exists()); self.assertTrue(link.is_symlink())
|
||||||
|
|
||||||
|
def test_reference_added_before_database_delete_is_rechecked(self):
|
||||||
|
path = self.add_revision(); self.collect(100); original = self.db.frozen
|
||||||
|
@contextmanager
|
||||||
|
def add_reference(current):
|
||||||
|
with original(current):
|
||||||
|
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')")
|
||||||
|
yield
|
||||||
|
with patch.object(self.db, 'frozen', side_effect=add_reference): result = self.collect(2000)
|
||||||
|
self.assertEqual(result['sourceRevisionsRemoved'], 0)
|
||||||
|
self.assertTrue(path.exists()); self.assertTrue(self.present())
|
||||||
|
|
||||||
|
def test_file_changed_after_hash_is_kept(self):
|
||||||
|
path = self.add_revision(); self.collect(100); original = self.db.frozen
|
||||||
|
@contextmanager
|
||||||
|
def change_file(current):
|
||||||
|
path.write_bytes(b'file edited during preflight')
|
||||||
|
with original(current): yield
|
||||||
|
with patch.object(self.db, 'frozen', side_effect=change_file): result = self.collect(2000)
|
||||||
|
self.assertEqual(result['sourceRevisionsRemoved'], 0)
|
||||||
|
self.assertTrue(path.exists()); self.assertTrue(self.present())
|
||||||
|
|
||||||
|
def test_crash_after_database_commit_resumes_registered_file_removal(self):
|
||||||
|
path = self.add_revision(); self.collect(100)
|
||||||
|
with patch.object(self.cleaner, 'drain', side_effect=RuntimeError('interrupted')):
|
||||||
|
with self.assertRaises(RuntimeError): self.collect(2000)
|
||||||
|
self.assertFalse(self.present()); self.assertTrue(path.exists())
|
||||||
|
self.assertEqual(self.cleaner.drain('current')[0], 1)
|
||||||
|
self.assertFalse(path.exists()); self.assertTrue(self.current_file.exists())
|
||||||
|
|
||||||
|
def test_database_rollback_keeps_file_and_discards_only_journal(self):
|
||||||
|
path = self.add_revision(); self.collect(100); original = self.db.execute
|
||||||
|
def rollback(sql, args=()):
|
||||||
|
result = original(sql, args)
|
||||||
|
if sql.startswith('DELETE FROM source_revision'): raise RuntimeError('rollback')
|
||||||
|
return result
|
||||||
|
with patch.object(self.db, 'execute', side_effect=rollback):
|
||||||
|
with self.assertRaises(RuntimeError): self.collect(2000)
|
||||||
|
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
||||||
|
self.assertEqual(self.cleaner.drain('current'), (0, 0))
|
||||||
|
self.assertFalse(list(self.cleaner.journals.glob('*.json')))
|
||||||
|
|
||||||
|
def test_new_path_reference_before_unlink_protects_file(self):
|
||||||
|
path = self.add_revision(); self.collect(100); original = self.cleaner.drain
|
||||||
|
def add_reference(current, limit=5):
|
||||||
|
self.db.execute('INSERT INTO source_upload_draft VALUES(\'new\',%s)', (str(path),))
|
||||||
|
return original(current, limit)
|
||||||
|
with patch.object(self.cleaner, 'drain', side_effect=add_reference): result = self.collect(2000)
|
||||||
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||||
|
self.assertEqual(result['sourceArchiveFilesRemoved'], 0); self.assertTrue(path.exists())
|
||||||
|
|
||||||
|
def test_running_task_or_changed_publication_defers_without_deleting(self):
|
||||||
|
path = self.add_revision(); self.collect(100)
|
||||||
|
self.db.execute("INSERT INTO task VALUES(1,'RUNNING',NULL)")
|
||||||
|
with self.assertRaises(ret.Busy): self.collect(2000)
|
||||||
|
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
||||||
|
self.db.execute('DELETE FROM task')
|
||||||
|
self.db.execute("UPDATE current_release SET release_id='different'")
|
||||||
|
with self.assertRaises(ret.Busy): self.collect(2000)
|
||||||
|
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
||||||
|
|
||||||
|
def test_unsafe_registry_and_unknown_stray_files_are_not_deleted(self):
|
||||||
|
path = self.add_revision(); self.collect(100)
|
||||||
|
self.cleaner.journals.mkdir(parents=True)
|
||||||
|
br.atomic(self.cleaner.journals / ('1' * 32 + '.json'), {'kind': 'quality-source-retirement'})
|
||||||
|
unknown = path.with_name('unregistered.xlsx'); unknown.write_bytes(b'keep')
|
||||||
|
self.cleaner.drain('current'); self.assertTrue(unknown.exists()); self.assertTrue(path.exists())
|
||||||
|
with self.cleaner.quick_checks():
|
||||||
|
self.cleaner.reference_deadline = time.monotonic() - 1
|
||||||
|
with self.assertRaises(ret.Busy): self.cleaner.exists('source_revision', 'id=%s', ('old-rev-unique',))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': unittest.main()
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
spec = importlib.util.spec_from_file_location('quality_macos_service', Path(__file__).with_name('service.py'))
|
||||||
|
service = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(service)
|
||||||
|
|
||||||
|
|
||||||
|
class Fixture(service.Guardian):
|
||||||
|
def command(self, role):
|
||||||
|
return ['/bin/sleep', '90' if role == 'mysql' else '91']
|
||||||
|
|
||||||
|
def occupied(self, role):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def environment(self):
|
||||||
|
return os.environ.copy()
|
||||||
|
|
||||||
|
def db_ready(self):
|
||||||
|
return self.ready
|
||||||
|
|
||||||
|
def backend_ready(self):
|
||||||
|
return service.matches(self.records.get('backend')) is True
|
||||||
|
|
||||||
|
|
||||||
|
class GuardianTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temp = tempfile.TemporaryDirectory()
|
||||||
|
self.root = Path(self.temp.name)
|
||||||
|
(self.root / '.runtime').mkdir()
|
||||||
|
self.g = Fixture({'root': str(self.root)}, self.root / 'guardian')
|
||||||
|
self.g.value = {'mysqlPort': 59998}
|
||||||
|
self.g.ready = False
|
||||||
|
self.external = []
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
for p in list(self.g.children.values()) + self.external:
|
||||||
|
if p.poll() is None:
|
||||||
|
p.terminate()
|
||||||
|
p.wait(timeout=10)
|
||||||
|
self.temp.cleanup()
|
||||||
|
|
||||||
|
def test_database_must_be_ready_before_backend_starts(self):
|
||||||
|
self.g.tick()
|
||||||
|
self.assertIn('mysql', self.g.records)
|
||||||
|
self.assertNotIn('backend', self.g.records)
|
||||||
|
self.g.ready = True
|
||||||
|
self.g.tick()
|
||||||
|
self.assertTrue(service.matches(self.g.records['backend']))
|
||||||
|
|
||||||
|
def test_abnormal_child_exit_is_restarted_without_duplicate_mysql(self):
|
||||||
|
self.g.ready = True
|
||||||
|
self.g.tick()
|
||||||
|
mysql = self.g.records['mysql']['pid']
|
||||||
|
old = self.g.children['backend']
|
||||||
|
old.kill()
|
||||||
|
old.wait()
|
||||||
|
self.g.tick()
|
||||||
|
self.assertNotIn('backend', self.g.records)
|
||||||
|
self.assertEqual(self.g.records['mysql']['pid'], mysql)
|
||||||
|
self.g.next_start['backend'] = 0
|
||||||
|
self.g.tick()
|
||||||
|
self.assertNotEqual(self.g.records['backend']['pid'], old.pid)
|
||||||
|
self.assertEqual(self.g.restarts, {'mysql': 1, 'backend': 2})
|
||||||
|
|
||||||
|
def test_database_exit_is_restarted_before_missing_backend(self):
|
||||||
|
self.g.tick()
|
||||||
|
old = self.g.children['mysql']
|
||||||
|
old.kill()
|
||||||
|
old.wait()
|
||||||
|
self.g.tick()
|
||||||
|
self.assertNotIn('mysql', self.g.records)
|
||||||
|
self.g.next_start['mysql'] = 0
|
||||||
|
self.g.tick()
|
||||||
|
self.assertNotEqual(self.g.records['mysql']['pid'], old.pid)
|
||||||
|
self.assertNotIn('backend', self.g.records)
|
||||||
|
|
||||||
|
def test_existing_process_is_adopted_and_unknown_process_is_not_replaced(self):
|
||||||
|
command = self.g.command('mysql')
|
||||||
|
p = subprocess.Popen(command, start_new_session=True)
|
||||||
|
self.external.append(p)
|
||||||
|
info = service.record(p.pid, command)
|
||||||
|
service.atomic(self.root / '.runtime/mysql.process.json', {'pid': p.pid, 'started': info['started']})
|
||||||
|
self.assertTrue(self.g.ensure('mysql'))
|
||||||
|
self.assertEqual(self.g.restarts['mysql'], 0)
|
||||||
|
self.assertIsNone(service.matches(info, self.g.command('backend')))
|
||||||
|
self.assertIs(p.poll(), None)
|
||||||
|
|
||||||
|
def test_missing_data_marker_never_initializes_database(self):
|
||||||
|
service.atomic(self.root / '.runtime/local-config.json', {'databaseReady': True})
|
||||||
|
real = service.Guardian({'root': str(self.root), 'jarSha256': '0'*64}, self.root / 'guardian')
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
real.preflight()
|
||||||
|
self.assertFalse((self.root / '.runtime/mysql-data').exists())
|
||||||
|
|
||||||
|
def test_pinned_jar_mismatch_is_rejected(self):
|
||||||
|
r = self.root / '.runtime'
|
||||||
|
(r / 'mysql-data').mkdir()
|
||||||
|
(r / 'mysql-data/auto.cnf').write_text('existing marker')
|
||||||
|
(r / 'backend-builds').mkdir()
|
||||||
|
(r / 'backend-builds' / ('0'*64+'.jar')).write_bytes(b'unaccepted')
|
||||||
|
service.atomic(r / 'local-config.json', {'databaseReady': True})
|
||||||
|
real = service.Guardian({'root': str(self.root), 'jarSha256': '0'*64}, self.root / 'guardian')
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
real.preflight()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
+84
-12
@@ -1,6 +1,8 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Create and verify a local backup using an isolated database and archive tree."""
|
"""Create and verify a local backup using an isolated database and archive tree."""
|
||||||
import argparse
|
import argparse
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from contextvars import ContextVar
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
import gzip
|
import gzip
|
||||||
import hashlib
|
import hashlib
|
||||||
@@ -10,11 +12,13 @@ from pathlib import Path, PurePosixPath
|
|||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import sys
|
||||||
import tarfile
|
import tarfile
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
import project
|
import project
|
||||||
|
import backup_retention
|
||||||
|
|
||||||
ROOT = project.ROOT
|
ROOT = project.ROOT
|
||||||
BACKUP_ROOT = ROOT / 'output/backups'
|
BACKUP_ROOT = ROOT / 'output/backups'
|
||||||
@@ -55,6 +59,17 @@ SAFE_FAILURES = {
|
|||||||
('CLEANUP', 'SCRATCH_CLEANUP_FAILED'): '隔离恢复完成后,临时数据库清理失败。',
|
('CLEANUP', 'SCRATCH_CLEANUP_FAILED'): '隔离恢复完成后,临时数据库清理失败。',
|
||||||
('PROTOCOL', 'BACKUP_INTERNAL_ERROR'): '备份操作未完成,请检查本机运维状态。',
|
('PROTOCOL', 'BACKUP_INTERNAL_ERROR'): '备份操作未完成,请检查本机运维状态。',
|
||||||
}
|
}
|
||||||
|
CLIENT_CONFIG_OVERRIDE = ContextVar('quality_backup_client_config', default=None)
|
||||||
|
|
||||||
|
|
||||||
|
def client_config():
|
||||||
|
return CLIENT_CONFIG_OVERRIDE.get() or project.RUNTIME / 'mysql-root.cnf'
|
||||||
|
|
||||||
|
|
||||||
|
def verification_temp_root():
|
||||||
|
# 校验临时文件归属于独立 MySQL 的所有权目录;异常退出后可一并安全回收。
|
||||||
|
configuration = CLIENT_CONFIG_OVERRIDE.get()
|
||||||
|
return configuration.parent if configuration is not None else None
|
||||||
|
|
||||||
|
|
||||||
class BackupFailure(RuntimeError):
|
class BackupFailure(RuntimeError):
|
||||||
@@ -101,14 +116,14 @@ def application_files():
|
|||||||
if missing:
|
if missing:
|
||||||
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
|
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
|
||||||
paths = set()
|
paths = set()
|
||||||
for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist'):
|
for relative in (*REQUIRED_APPLICATION_DIRECTORIES, 'web/dist', 'deploy/macos', 'deploy/docker'):
|
||||||
folder = ROOT / relative
|
folder = ROOT / relative
|
||||||
for path in folder.rglob('*'):
|
for path in folder.rglob('*'):
|
||||||
if path.is_symlink():
|
if path.is_symlink():
|
||||||
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
||||||
if path.is_file() and '__pycache__' not in path.parts:
|
if path.is_file() and '__pycache__' not in path.parts:
|
||||||
paths.add(path)
|
paths.add(path)
|
||||||
for relative in (*REQUIRED_APPLICATION_FILES, '启动项目.command', '停止项目.command'):
|
for relative in (*REQUIRED_APPLICATION_FILES, '.dockerignore', 'app/backend.sha256', '启动项目.command', '停止项目.command'):
|
||||||
path = ROOT / relative
|
path = ROOT / relative
|
||||||
if path.exists():
|
if path.exists():
|
||||||
if path.is_symlink() or not path.is_file():
|
if path.is_symlink() or not path.is_file():
|
||||||
@@ -124,6 +139,10 @@ def application_files():
|
|||||||
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
||||||
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
||||||
paths.add(path)
|
paths.add(path)
|
||||||
|
accepted = backup_retention.accepted_backend(ROOT)
|
||||||
|
if accepted:
|
||||||
|
paths.add(accepted[0])
|
||||||
|
paths.update(backup_retention.native_files(ROOT))
|
||||||
if not set(required_paths).issubset(paths):
|
if not set(required_paths).issubset(paths):
|
||||||
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
|
fail('INPUTS', 'APPLICATION_INPUT_MISSING')
|
||||||
return sorted(paths)
|
return sorted(paths)
|
||||||
@@ -139,6 +158,13 @@ def archive_application(folder):
|
|||||||
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
||||||
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
fail('INPUTS', 'APPLICATION_INPUT_UNSAFE')
|
||||||
relative = path.relative_to(ROOT).as_posix()
|
relative = path.relative_to(ROOT).as_posix()
|
||||||
|
if path.name == 'nao.sqlite':
|
||||||
|
with tempfile.TemporaryDirectory(prefix='quality-native-snapshot-') as temporary:
|
||||||
|
copied = Path(temporary) / 'nao.sqlite'
|
||||||
|
backup_retention.snapshot_sqlite(path, copied)
|
||||||
|
files[relative] = file_sha256(copied)
|
||||||
|
tar.add(copied, arcname=relative, recursive=False)
|
||||||
|
continue
|
||||||
before = file_sha256(path)
|
before = file_sha256(path)
|
||||||
tar.add(path, arcname=relative, recursive=False)
|
tar.add(path, arcname=relative, recursive=False)
|
||||||
if before != file_sha256(path):
|
if before != file_sha256(path):
|
||||||
@@ -153,7 +179,7 @@ def archive_application(folder):
|
|||||||
|
|
||||||
|
|
||||||
def client(sql, database=None):
|
def client(sql, database=None):
|
||||||
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()),
|
||||||
'--batch', '--skip-column-names']
|
'--batch', '--skip-column-names']
|
||||||
if database:
|
if database:
|
||||||
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', database):
|
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', database):
|
||||||
@@ -181,7 +207,7 @@ def counts(database):
|
|||||||
def query_evidence(database):
|
def query_evidence(database):
|
||||||
"""Reproduce release membership and monthly formal input components in SQL."""
|
"""Reproduce release membership and monthly formal input components in SQL."""
|
||||||
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
|
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
|
||||||
if not current:
|
if not current or current == 'NULL':
|
||||||
return {'releaseId': None, 'months': []}
|
return {'releaseId': None, 'months': []}
|
||||||
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
|
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
|
||||||
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
|
fail('DATABASE_EXPORT', 'DATABASE_READ_FAILED')
|
||||||
@@ -332,25 +358,29 @@ def _snapshot_counts(sql):
|
|||||||
return snapshot_counts
|
return snapshot_counts
|
||||||
|
|
||||||
|
|
||||||
|
@backup_retention.serialized
|
||||||
def create():
|
def create():
|
||||||
# Validate the full, fixed application input set before creating any backup files.
|
# Validate the full, fixed application input set before creating any backup files.
|
||||||
selected_application = application_files()
|
selected_application = application_files()
|
||||||
config = database_config()
|
config = database_config()
|
||||||
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive', ROOT / 'runtimedata/archive')
|
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive', ROOT / 'runtimedata/archive')
|
||||||
before_queries = query_evidence(config['mysqlDatabase'])
|
before_queries = query_evidence(config['mysqlDatabase'])
|
||||||
|
before_counts = counts(config['mysqlDatabase'])
|
||||||
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
|
folder = BACKUP_ROOT / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
|
||||||
folder.mkdir(parents=True, exist_ok=False)
|
folder.mkdir(parents=True, exist_ok=False)
|
||||||
folder.chmod(0o700)
|
folder.chmod(0o700)
|
||||||
|
backup_retention.atomic(folder / '.owner.json', {'kind': 'quality-backup', 'root': str(ROOT.resolve()),
|
||||||
|
'id': folder.name, 'createdAt': time.time(), 'ownerPid': os.getpid()})
|
||||||
sql = folder / 'database.sql.gz'
|
sql = folder / 'database.sql.gz'
|
||||||
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
||||||
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
|
'--single-transaction', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
|
||||||
'--skip-extended-insert', config['mysqlDatabase']]
|
'--extended-insert', '--net-buffer-length=1048576', config['mysqlDatabase']]
|
||||||
process = None
|
process = None
|
||||||
try:
|
try:
|
||||||
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
|
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
|
||||||
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
|
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
|
||||||
try:
|
try:
|
||||||
with gzip.GzipFile(fileobj=output, mode='wb') as compressed:
|
with gzip.GzipFile(fileobj=output, mode='wb', compresslevel=1) as compressed:
|
||||||
shutil.copyfileobj(process.stdout, compressed)
|
shutil.copyfileobj(process.stdout, compressed)
|
||||||
process.stdout.close()
|
process.stdout.close()
|
||||||
if process.wait() != 0:
|
if process.wait() != 0:
|
||||||
@@ -373,7 +403,11 @@ def create():
|
|||||||
sql.chmod(0o600)
|
sql.chmod(0o600)
|
||||||
if query_evidence(config['mysqlDatabase']) != before_queries:
|
if query_evidence(config['mysqlDatabase']) != before_queries:
|
||||||
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
|
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
|
||||||
table_counts = _snapshot_counts(sql)
|
# The dump is a single consistent transaction. Restore verification below
|
||||||
|
# independently checks these counts, including a net-zero concurrent race.
|
||||||
|
if counts(config['mysqlDatabase']) != before_counts:
|
||||||
|
fail('DATABASE_EXPORT', 'DATABASE_CHANGED')
|
||||||
|
table_counts = before_counts
|
||||||
archive = folder / 'archive.tar.gz'
|
archive = folder / 'archive.tar.gz'
|
||||||
source = ROOT / 'runtimedata/archive'
|
source = ROOT / 'runtimedata/archive'
|
||||||
files = {}
|
files = {}
|
||||||
@@ -413,6 +447,9 @@ def create():
|
|||||||
'archiveReferencesVerified': False, 'applicationStartupVerified': False},
|
'archiveReferencesVerified': False, 'applicationStartupVerified': False},
|
||||||
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
|
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
|
||||||
'tableCounts': table_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
'tableCounts': table_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
||||||
|
accepted = backup_retention.accepted_backend(ROOT)
|
||||||
|
if accepted:
|
||||||
|
manifest['acceptedBackend'] = {'path': accepted[0].relative_to(ROOT).as_posix(), 'sha256': accepted[1]}
|
||||||
try:
|
try:
|
||||||
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -420,7 +457,7 @@ def create():
|
|||||||
return folder
|
return folder
|
||||||
|
|
||||||
|
|
||||||
def verify(folder):
|
def _verify(folder):
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
folder = Path(folder)
|
folder = Path(folder)
|
||||||
if folder.is_symlink():
|
if folder.is_symlink():
|
||||||
@@ -451,7 +488,7 @@ def verify(folder):
|
|||||||
try:
|
try:
|
||||||
if file_sha256(app_archive) != manifest['applicationSha256']:
|
if file_sha256(app_archive) != manifest['applicationSha256']:
|
||||||
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
fail('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH')
|
||||||
with tempfile.TemporaryDirectory(prefix='quality-restore-application-') as temp:
|
with tempfile.TemporaryDirectory(prefix='quality-restore-application-', dir=verification_temp_root()) as temp:
|
||||||
Path(temp).chmod(0o700)
|
Path(temp).chmod(0o700)
|
||||||
actual_app = extract_verified_tar(app_archive, Path(temp) / 'files', expected_app)
|
actual_app = extract_verified_tar(app_archive, Path(temp) / 'files', expected_app)
|
||||||
application_verified = actual_app == expected_app
|
application_verified = actual_app == expected_app
|
||||||
@@ -465,14 +502,17 @@ def verify(folder):
|
|||||||
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
fail('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
||||||
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
|
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
|
||||||
created_scratch = False
|
created_scratch = False
|
||||||
|
scratch_registration = backup_retention.scratch_register(ROOT, scratch, client('SELECT @@server_uuid;').strip())
|
||||||
try:
|
try:
|
||||||
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
||||||
created_scratch = True
|
created_scratch = True
|
||||||
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--database=' + scratch, '--init-command=SET SESSION autocommit=0']
|
backup_retention.scratch_created(scratch_registration)
|
||||||
|
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config()), '--database=' + scratch, '--init-command=SET SESSION autocommit=0']
|
||||||
process = None
|
process = None
|
||||||
try:
|
try:
|
||||||
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
|
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
|
||||||
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
|
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
|
||||||
|
backup_retention.scratch_child(scratch_registration, process.pid)
|
||||||
try:
|
try:
|
||||||
shutil.copyfileobj(data, process.stdin)
|
shutil.copyfileobj(data, process.stdin)
|
||||||
process.stdin.write(b'\nCOMMIT;\n')
|
process.stdin.write(b'\nCOMMIT;\n')
|
||||||
@@ -494,7 +534,7 @@ def verify(folder):
|
|||||||
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
||||||
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
|
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
|
||||||
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
fail('DATABASE_RESTORE', 'DATABASE_CONTENT_MISMATCH')
|
||||||
with tempfile.TemporaryDirectory(prefix='quality-restore-archive-') as temp:
|
with tempfile.TemporaryDirectory(prefix='quality-restore-archive-', dir=verification_temp_root()) as temp:
|
||||||
relocated_root = Path(temp) / 'restored' / 'runtimedata' / 'archive'
|
relocated_root = Path(temp) / 'restored' / 'runtimedata' / 'archive'
|
||||||
relocated_root.parent.mkdir(parents=True, exist_ok=True)
|
relocated_root.parent.mkdir(parents=True, exist_ok=True)
|
||||||
actual_archives = _safe_extract_tree(archive, relocated_root, manifest.get('files', {}))
|
actual_archives = _safe_extract_tree(archive, relocated_root, manifest.get('files', {}))
|
||||||
@@ -534,10 +574,27 @@ def verify(folder):
|
|||||||
if created_scratch:
|
if created_scratch:
|
||||||
try:
|
try:
|
||||||
client('DROP DATABASE `' + scratch + '`;')
|
client('DROP DATABASE `' + scratch + '`;')
|
||||||
|
scratch_registration.unlink(missing_ok=True)
|
||||||
except Exception:
|
except Exception:
|
||||||
fail('CLEANUP', 'SCRATCH_CLEANUP_FAILED')
|
fail('CLEANUP', 'SCRATCH_CLEANUP_FAILED')
|
||||||
|
else:
|
||||||
|
scratch_registration.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
@backup_retention.serialized
|
||||||
|
def verify(folder):
|
||||||
|
if sys.platform == 'darwin' or os.environ.get('QUALITY_CONTAINER') == '1':
|
||||||
|
import backup_verifier
|
||||||
|
with backup_verifier.instance(ROOT) as configuration:
|
||||||
|
token = CLIENT_CONFIG_OVERRIDE.set(configuration)
|
||||||
|
try:
|
||||||
|
return _verify(folder)
|
||||||
|
finally:
|
||||||
|
CLIENT_CONFIG_OVERRIDE.reset(token)
|
||||||
|
return _verify(folder)
|
||||||
|
|
||||||
|
|
||||||
|
@backup_retention.serialized
|
||||||
def restore(folder, target, database):
|
def restore(folder, target, database):
|
||||||
"""Restore a verified format-v3 backup to a new project directory and new database.
|
"""Restore a verified format-v3 backup to a new project directory and new database.
|
||||||
|
|
||||||
@@ -709,13 +766,14 @@ def restore(folder, target, database):
|
|||||||
fail('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED')
|
fail('CLEANUP', 'RESTORE_STAGE_CLEANUP_FAILED')
|
||||||
|
|
||||||
|
|
||||||
def main(argv=None):
|
def _main(argv=None):
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify', 'restore'])
|
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify', 'restore'])
|
||||||
parser.add_argument('--directory', type=Path)
|
parser.add_argument('--directory', type=Path)
|
||||||
parser.add_argument('--target', type=Path)
|
parser.add_argument('--target', type=Path)
|
||||||
parser.add_argument('--database')
|
parser.add_argument('--database')
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
|
previous_backups = backup_retention.backup_ids(ROOT)
|
||||||
try:
|
try:
|
||||||
if args.action == 'verify':
|
if args.action == 'verify':
|
||||||
if not args.directory:
|
if not args.directory:
|
||||||
@@ -730,17 +788,31 @@ def main(argv=None):
|
|||||||
result = {'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
result = {'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
||||||
if args.action == 'create-and-verify':
|
if args.action == 'create-and-verify':
|
||||||
result = verify(folder)
|
result = verify(folder)
|
||||||
|
if args.action in ('verify', 'create-and-verify'):
|
||||||
|
selected = args.directory if args.action == 'verify' else folder
|
||||||
|
backup_retention.rotate(ROOT, selected)
|
||||||
print(json.dumps({'event': 'completed', 'action': args.action, **result}, ensure_ascii=False,
|
print(json.dumps({'event': 'completed', 'action': args.action, **result}, ensure_ascii=False,
|
||||||
separators=(',', ':')), flush=True)
|
separators=(',', ':')), flush=True)
|
||||||
return 0
|
return 0
|
||||||
except BackupFailure as error:
|
except BackupFailure as error:
|
||||||
|
backup_retention.discard_failed_new(ROOT, previous_backups)
|
||||||
emit_failure(error)
|
emit_failure(error)
|
||||||
return 20 + sorted(SAFE_FAILURES).index((error.stage, error.code))
|
return 20 + sorted(SAFE_FAILURES).index((error.stage, error.code))
|
||||||
except Exception:
|
except Exception:
|
||||||
|
backup_retention.discard_failed_new(ROOT, previous_backups)
|
||||||
error = BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR')
|
error = BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR')
|
||||||
emit_failure(error)
|
emit_failure(error)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
try:
|
||||||
|
with backup_retention.lock(ROOT):
|
||||||
|
return _main(argv)
|
||||||
|
except backup_retention.Busy:
|
||||||
|
emit_failure(BackupFailure('PROTOCOL', 'BACKUP_INTERNAL_ERROR'))
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
raise SystemExit(main())
|
raise SystemExit(main())
|
||||||
|
|||||||
@@ -0,0 +1,381 @@
|
|||||||
|
"""Shared backup/maintenance exclusion and verified-only backup rotation."""
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from functools import wraps
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
_thread_lock = threading.RLock()
|
||||||
|
_held = {}
|
||||||
|
ID = re.compile(r'^\d{8}T\d{6}Z-[a-f0-9]{8}$')
|
||||||
|
CHECKS = ('databaseRestoreVerified', 'applicationFilesVerified', 'requiredApplicationInputsVerified',
|
||||||
|
'archiveFilesVerified', 'archiveReferencesVerified')
|
||||||
|
ARTIFACTS = {'database.sql.gz': 'databaseDumpSha256', 'archive.tar.gz': 'archiveSha256',
|
||||||
|
'application.tar.gz': 'applicationSha256'}
|
||||||
|
|
||||||
|
|
||||||
|
class Busy(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def sha(path):
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with Path(path).open('rb') as stream:
|
||||||
|
for block in iter(lambda: stream.read(1024 * 1024), b''):
|
||||||
|
digest.update(block)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def atomic(path, value):
|
||||||
|
path = Path(path)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
tmp = path.with_name(path.name + '.' + str(os.getpid()) + '.tmp')
|
||||||
|
with tmp.open('w') as output:
|
||||||
|
os.chmod(tmp, 0o600)
|
||||||
|
json.dump(value, output, ensure_ascii=False, indent=2)
|
||||||
|
output.flush()
|
||||||
|
os.fsync(output.fileno())
|
||||||
|
tmp.replace(path)
|
||||||
|
if os.name != 'nt':
|
||||||
|
fd = os.open(path.parent, os.O_RDONLY)
|
||||||
|
try:
|
||||||
|
os.fsync(fd)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def lock(root, timeout=1800):
|
||||||
|
"""One lock for UI backups, restores, scheduled backups and garbage collection."""
|
||||||
|
root = Path(root).resolve()
|
||||||
|
key = str(root)
|
||||||
|
with _thread_lock:
|
||||||
|
if key in _held:
|
||||||
|
yield
|
||||||
|
return
|
||||||
|
path = root / '.runtime/data-maintenance.lock'
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if path.parent.is_symlink() or path.is_symlink():
|
||||||
|
raise ValueError('Unsafe maintenance lock path')
|
||||||
|
stream = path.open('a+b')
|
||||||
|
os.chmod(path, 0o600)
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
if os.name == 'nt':
|
||||||
|
import msvcrt
|
||||||
|
stream.seek(0)
|
||||||
|
if not path.stat().st_size:
|
||||||
|
stream.write(b'0'); stream.flush()
|
||||||
|
stream.seek(0)
|
||||||
|
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
||||||
|
else:
|
||||||
|
import fcntl
|
||||||
|
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
break
|
||||||
|
except OSError:
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise Busy('Another backup or maintenance operation is active')
|
||||||
|
time.sleep(.2)
|
||||||
|
_held[key] = stream
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
_held.pop(key, None)
|
||||||
|
if os.name == 'nt':
|
||||||
|
import msvcrt
|
||||||
|
stream.seek(0); msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
||||||
|
else:
|
||||||
|
import fcntl
|
||||||
|
fcntl.flock(stream, fcntl.LOCK_UN)
|
||||||
|
finally:
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
|
||||||
|
def serialized(function):
|
||||||
|
@wraps(function)
|
||||||
|
def wrapped(*args, **kwargs):
|
||||||
|
with lock(function.__globals__['ROOT']):
|
||||||
|
return function(*args, **kwargs)
|
||||||
|
return wrapped
|
||||||
|
|
||||||
|
|
||||||
|
def verified(folder, checksums=True):
|
||||||
|
folder = Path(folder)
|
||||||
|
if folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
if (folder / 'manifest.json').is_symlink():
|
||||||
|
return None
|
||||||
|
manifest = json.loads((folder / 'manifest.json').read_text())
|
||||||
|
if manifest.get('formatVersion') != 3 or manifest.get('status') != 'ISOLATED_RESTORE_VERIFIED':
|
||||||
|
return None
|
||||||
|
if not all(manifest.get('verification', {}).get(c) is True for c in CHECKS):
|
||||||
|
return None
|
||||||
|
for name, field in ARTIFACTS.items():
|
||||||
|
p = folder / name
|
||||||
|
if p.is_symlink() or not p.is_file() or not manifest.get(field):
|
||||||
|
return None
|
||||||
|
if checksums and sha(p) != manifest[field]:
|
||||||
|
return None
|
||||||
|
return manifest
|
||||||
|
except (OSError, ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _rotate(root, selected):
|
||||||
|
"""Publish the new verified backup first. A failed candidate never deletes old backups."""
|
||||||
|
root = Path(root).resolve()
|
||||||
|
selected = Path(selected)
|
||||||
|
if selected.is_symlink():
|
||||||
|
raise ValueError('Backup directory cannot be a symbolic link')
|
||||||
|
selected = selected.resolve()
|
||||||
|
base = root / 'output/backups'
|
||||||
|
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):
|
||||||
|
raise ValueError('Unsafe managed backup root')
|
||||||
|
if selected.parent.resolve() != base.resolve():
|
||||||
|
raise ValueError('Backup is outside the managed backup directory')
|
||||||
|
manifest = verified(selected)
|
||||||
|
if manifest is None:
|
||||||
|
raise ValueError('Backup has not passed restore and artifact verification')
|
||||||
|
previous = latest(root)
|
||||||
|
if previous and previous[0] != selected and previous[1].get('createdAt', '') > manifest.get('createdAt', ''):
|
||||||
|
selected, manifest = previous
|
||||||
|
for name in (*ARTIFACTS, 'manifest.json'):
|
||||||
|
with (selected / name).open('r+b') as stream:
|
||||||
|
stream.flush(); os.fsync(stream.fileno())
|
||||||
|
if os.name!='nt':
|
||||||
|
for directory in (selected,base,base.parent):
|
||||||
|
fd=os.open(directory,os.O_RDONLY)
|
||||||
|
try:os.fsync(fd)
|
||||||
|
finally:os.close(fd)
|
||||||
|
pointer = {'id': selected.name, 'manifestSha256': sha(selected / 'manifest.json'),
|
||||||
|
'releaseId': manifest.get('queryEvidence', {}).get('releaseId'),
|
||||||
|
'verifiedAt': manifest.get('restoreVerifiedAt'), 'publishedAt': time.time()}
|
||||||
|
atomic(root / '.runtime/retention/latest.json', pointer)
|
||||||
|
removed = []
|
||||||
|
for folder in base.iterdir():
|
||||||
|
if folder == selected or folder.is_symlink() or not folder.is_dir() or not ID.fullmatch(folder.name):
|
||||||
|
continue
|
||||||
|
# A restore-verified older folder is safe to retire under the shared lock.
|
||||||
|
# Incomplete candidates are kept for 24h and never treated as good backups.
|
||||||
|
older = verified(folder, checksums=False)
|
||||||
|
try:
|
||||||
|
owner = json.loads((folder / '.owner.json').read_text())
|
||||||
|
owned = owner.get('kind') == 'quality-backup' and owner.get('root') == str(root) and owner.get('id') == folder.name
|
||||||
|
except (OSError, ValueError):
|
||||||
|
owned = False
|
||||||
|
stale = owned and time.time() - folder.stat().st_mtime > 86400
|
||||||
|
if older is not None or stale:
|
||||||
|
contents = list(folder.rglob('*'))
|
||||||
|
if any(p.is_symlink() for p in contents):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
retire_folder(root,folder,'backup')
|
||||||
|
removed.append(folder.name)
|
||||||
|
except OSError:
|
||||||
|
pass # Latest stays usable; a later maintenance pass retries retirement.
|
||||||
|
# A one-off snapshot from the previous manual cleanup is retired only when
|
||||||
|
# explicitly registered and its complete original file set still matches.
|
||||||
|
registration = root / '.runtime/retention/legacy.json'
|
||||||
|
if registration.is_file():
|
||||||
|
legacy_info = json.loads(registration.read_text())
|
||||||
|
if not legacy_info.get('retired') and legacy_info.get('path') == '.runtime/current-snapshot-20261109' and manifest.get('acceptedBackend'):
|
||||||
|
legacy = root / legacy_info['path']
|
||||||
|
if legacy.is_dir() and not legacy.is_symlink() and sha(legacy/'manifest.json') == legacy_info['manifestSha256']:
|
||||||
|
old = json.loads((legacy/'manifest.json').read_text())
|
||||||
|
expected = old.get('filesSha256', {})
|
||||||
|
actual = {p.relative_to(legacy).as_posix() for p in legacy.rglob('*') if p.is_file()}
|
||||||
|
if actual == set(expected)|{'manifest.json'} and all(not (legacy/k).is_symlink() and sha(legacy/k)==v for k,v in expected.items()):
|
||||||
|
evidence = root/'.runtime/retention/setup-evidence'
|
||||||
|
evidence.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||||
|
for name in ('cleanup-report.json','guardian-verification.json','api-verification.json'):
|
||||||
|
if (legacy/name).is_file():
|
||||||
|
shutil.copy2(legacy/name,evidence/name)
|
||||||
|
retire_folder(root,legacy,'legacy')
|
||||||
|
legacy_info['retired']=True
|
||||||
|
atomic(registration,legacy_info)
|
||||||
|
clean_retired(root)
|
||||||
|
return pointer, removed
|
||||||
|
|
||||||
|
|
||||||
|
def retire_folder(root, folder, kind):
|
||||||
|
"""Atomic rename makes a crash during removal safely resumable."""
|
||||||
|
root=Path(root).resolve();folder=Path(folder).resolve()
|
||||||
|
if kind=='backup':
|
||||||
|
if folder.parent!=root/'output/backups' or not ID.fullmatch(folder.name):raise ValueError('Unsafe retirement source')
|
||||||
|
elif kind=='legacy':
|
||||||
|
if folder!=root/'.runtime/current-snapshot-20261109':raise ValueError('Unsafe legacy source')
|
||||||
|
else:raise ValueError('Unknown retirement kind')
|
||||||
|
area=root/'.runtime/retention/retired';area.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||||
|
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):
|
||||||
|
raise ValueError('Unsafe backup retirement directory')
|
||||||
|
token=uuid.uuid4().hex
|
||||||
|
# Docker 的状态卷与备份卷可能是不同挂载点,重命名必须留在备份所在文件系统。
|
||||||
|
storage = root/'output/backups/.retired' if kind=='backup' else area
|
||||||
|
if storage.is_symlink() or not storage.resolve().is_relative_to(root):raise ValueError('Unsafe retirement storage')
|
||||||
|
storage.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||||
|
atomic(area/(token+'.json'),{'kind':'quality-retired-backup','root':str(root),'target':token,
|
||||||
|
'origin':str(folder.relative_to(root)),'storage':'backup-volume' if kind=='backup' else 'state-volume'})
|
||||||
|
folder.rename(storage/token)
|
||||||
|
if os.name!='nt':
|
||||||
|
fd=os.open(storage,os.O_RDONLY)
|
||||||
|
try:os.fsync(fd)
|
||||||
|
finally:os.close(fd)
|
||||||
|
|
||||||
|
|
||||||
|
def clean_retired(root):
|
||||||
|
root=Path(root).resolve();area=root/'.runtime/retention/retired';count=0
|
||||||
|
if area.is_symlink() or not area.resolve().is_relative_to(root) or any(p.is_symlink() for p in (root/'.runtime',root/'.runtime/retention')):return 0
|
||||||
|
for record in area.glob('*.json'):
|
||||||
|
try:
|
||||||
|
if record.is_symlink():continue
|
||||||
|
value=json.loads(record.read_text());token=value.get('target','')
|
||||||
|
if value.get('kind')!='quality-retired-backup' or value.get('root')!=str(root) or not re.fullmatch(r'[a-f0-9]{32}',token):continue
|
||||||
|
storage=value.get('storage','state-volume')
|
||||||
|
if storage not in ('state-volume','backup-volume'):continue
|
||||||
|
base=root/'output/backups/.retired' if storage=='backup-volume' else area
|
||||||
|
if any(p.is_symlink() for p in (base,base.parent,base.parent.parent)) or not base.resolve().is_relative_to(root):continue
|
||||||
|
folder=base/token
|
||||||
|
if folder.is_symlink():continue
|
||||||
|
if folder.exists():
|
||||||
|
if any(p.is_symlink() for p in folder.rglob('*')):continue
|
||||||
|
shutil.rmtree(folder);count+=1
|
||||||
|
record.unlink()
|
||||||
|
except (OSError,ValueError):continue
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
def rotate(root, selected):
|
||||||
|
with lock(root):
|
||||||
|
return _rotate(root, selected)
|
||||||
|
|
||||||
|
|
||||||
|
def latest(root, checksums=True):
|
||||||
|
root = Path(root).resolve()
|
||||||
|
try:
|
||||||
|
base=root/'output/backups'
|
||||||
|
if base.is_symlink() or base.parent.is_symlink() or not base.resolve().is_relative_to(root):return None
|
||||||
|
pointer = json.loads((root / '.runtime/retention/latest.json').read_text())
|
||||||
|
if not ID.fullmatch(pointer['id']):
|
||||||
|
return None
|
||||||
|
folder = root / 'output/backups' / pointer['id']
|
||||||
|
if sha(folder / 'manifest.json') != pointer['manifestSha256']:
|
||||||
|
return None
|
||||||
|
manifest = verified(folder, checksums=checksums)
|
||||||
|
return (folder, manifest) if manifest else None
|
||||||
|
except (OSError, ValueError, KeyError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def backup_ids(root):
|
||||||
|
base=Path(root)/'output/backups'
|
||||||
|
return {p.name for p in base.iterdir()} if base.exists() else set()
|
||||||
|
|
||||||
|
|
||||||
|
def discard_failed_new(root, before):
|
||||||
|
"""Delete only this call's registered, incomplete candidate after it has stopped."""
|
||||||
|
root=Path(root).resolve();base=root/'output/backups'
|
||||||
|
for folder in base.iterdir() if base.exists() else []:
|
||||||
|
if folder.name in before or not ID.fullmatch(folder.name) or folder.is_symlink() or not folder.is_dir():continue
|
||||||
|
try:
|
||||||
|
owner=json.loads((folder/'.owner.json').read_text())
|
||||||
|
if owner.get('root')!=str(root) or owner.get('ownerPid')!=os.getpid() or owner.get('id')!=folder.name:continue
|
||||||
|
if verified(folder,checksums=False) is not None:continue
|
||||||
|
if any(p.is_symlink() for p in folder.rglob('*')):continue
|
||||||
|
shutil.rmtree(folder)
|
||||||
|
except (OSError,ValueError):continue
|
||||||
|
|
||||||
|
|
||||||
|
def accepted_backend(root):
|
||||||
|
"""The running pinned jar is part of a valid application backup on this Mac."""
|
||||||
|
root = Path(root).resolve()
|
||||||
|
# 容器的只读镜像内保存运行包及构建时摘要,不依赖 macOS 服务注册表。
|
||||||
|
if os.environ.get('QUALITY_CONTAINER') == '1':
|
||||||
|
path = root / 'app/backend.jar'
|
||||||
|
digest = (root / 'app/backend.sha256').read_text().strip()
|
||||||
|
if path.is_symlink() or not re.fullmatch(r'[a-f0-9]{64}', digest) or sha(path) != digest:
|
||||||
|
raise ValueError('Container backend digest mismatch')
|
||||||
|
return path, digest
|
||||||
|
config = Path.home() / 'Library/Application Support/QualityCenterService/service.json'
|
||||||
|
if not config.exists():
|
||||||
|
return None
|
||||||
|
value = json.loads(config.read_text())
|
||||||
|
if Path(value['root']).resolve() != root:
|
||||||
|
return None
|
||||||
|
digest = value['jarSha256']
|
||||||
|
if not re.fullmatch(r'[a-f0-9]{64}', digest):
|
||||||
|
raise ValueError('Invalid accepted backend digest')
|
||||||
|
path = root / '.runtime/backend-builds' / (digest + '.jar')
|
||||||
|
if path.is_symlink() or sha(path) != digest:
|
||||||
|
raise ValueError('Accepted backend digest mismatch')
|
||||||
|
return path, digest
|
||||||
|
|
||||||
|
|
||||||
|
def native_files(root):
|
||||||
|
"""Retain restorable native settings/conversations; exclude live access tokens."""
|
||||||
|
for relative in ('.runtime/operations/policy.json','.runtime/operations/logback-quality.xml'):
|
||||||
|
path=Path(root)/relative
|
||||||
|
if path.is_symlink():raise ValueError('Unsafe operations configuration link')
|
||||||
|
if path.is_file():yield path
|
||||||
|
base=Path(root)/'.runtime/nao-users'
|
||||||
|
ignored={'logs','backups','__pycache__'}
|
||||||
|
ephemeral={'access.json','cookies.txt','sessions.json','current-session.json','instance.json'}
|
||||||
|
for path in base.rglob('*') if base.exists() else []:
|
||||||
|
if path.is_symlink():
|
||||||
|
raise ValueError('Unsafe native workspace link')
|
||||||
|
if not path.is_file() or any(p in ignored for p in path.relative_to(base).parts):
|
||||||
|
continue
|
||||||
|
if path.name in ephemeral or path.suffix in ('.lock','.pid') or path.name.endswith(('-wal','-shm')):
|
||||||
|
continue
|
||||||
|
yield path
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_sqlite(source, target):
|
||||||
|
original=sqlite3.connect(Path(source).resolve().as_uri()+'?mode=ro',uri=True)
|
||||||
|
copied=sqlite3.connect(target)
|
||||||
|
try:
|
||||||
|
original.backup(copied)
|
||||||
|
copied.execute('PRAGMA foreign_keys=OFF')
|
||||||
|
copied.execute('PRAGMA secure_delete=ON')
|
||||||
|
tables={r[0] for r in copied.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||||
|
for name in ('session','verification','api_key','jwks','oauth_access_token','oauth_refresh_token',
|
||||||
|
'oauth_consent','oauth_client','mcp_user_token','mcp_oauth_client','sandbox_secret','log','mcp_call_log'):
|
||||||
|
if name in tables:
|
||||||
|
copied.execute('DELETE FROM "'+name+'"')
|
||||||
|
copied.commit();copied.execute('VACUUM')
|
||||||
|
if copied.execute('PRAGMA integrity_check').fetchone()[0]!='ok':
|
||||||
|
raise ValueError('Native SQLite backup integrity failed')
|
||||||
|
finally:
|
||||||
|
original.close();copied.close()
|
||||||
|
|
||||||
|
|
||||||
|
def scratch_register(root, database, server):
|
||||||
|
import psutil
|
||||||
|
if not re.fullmatch(r'quality_restore_\d{14}_[a-f0-9]{8}',database):
|
||||||
|
raise ValueError('Unexpected verification database name')
|
||||||
|
path=Path(root)/'.runtime/retention/scratch'/(database+'.json')
|
||||||
|
value={'kind':'quality-restore-check','database':database,'server':server,
|
||||||
|
'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time(),'created':False}
|
||||||
|
atomic(path,value)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def scratch_created(path):
|
||||||
|
value=json.loads(Path(path).read_text());value['created']=True;atomic(path,value)
|
||||||
|
|
||||||
|
|
||||||
|
def scratch_child(path, pid):
|
||||||
|
import psutil
|
||||||
|
value=json.loads(Path(path).read_text())
|
||||||
|
value.update({'childPid':pid,'childCreated':psutil.Process(pid).create_time()})
|
||||||
|
atomic(path,value)
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
"""Private, socket-only MySQL instance for low-priority restore verification."""
|
||||||
|
from contextlib import contextmanager
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import psutil
|
||||||
|
import backup_retention as br
|
||||||
|
|
||||||
|
|
||||||
|
def stop_owned(value):
|
||||||
|
if value.get('initPid'):
|
||||||
|
try:
|
||||||
|
init=psutil.Process(value['initPid'])
|
||||||
|
if abs(init.create_time()-value['initCreated'])<.01 and init.is_running() and init.status()!=psutil.STATUS_ZOMBIE:return False
|
||||||
|
except psutil.NoSuchProcess:pass
|
||||||
|
except psutil.Error:return False
|
||||||
|
pid=value.get('serverPid')
|
||||||
|
if not pid:
|
||||||
|
p=Path(value['directory'])/'server.pid'
|
||||||
|
if not p.exists():return True
|
||||||
|
pid=int(p.read_text())
|
||||||
|
try:
|
||||||
|
process=psutil.Process(pid)
|
||||||
|
if value.get('serverCreated') and abs(process.create_time()-value['serverCreated'])>.01:return True
|
||||||
|
command=process.cmdline()
|
||||||
|
if process.uids().real!=os.getuid() or Path(process.exe()).resolve()!=Path(value['binary']).resolve() or '--datadir='+value['directory']+'/data' not in command or '--skip-networking' not in command:
|
||||||
|
return False
|
||||||
|
process.terminate()
|
||||||
|
try:process.wait(timeout=180)
|
||||||
|
except psutil.TimeoutExpired:return False
|
||||||
|
return True
|
||||||
|
except psutil.NoSuchProcess:return True
|
||||||
|
except psutil.Error:return False
|
||||||
|
|
||||||
|
|
||||||
|
def remove_owned(root, directory, value):
|
||||||
|
directory=Path(directory);base=Path(root)/'.runtime/retention'
|
||||||
|
if directory.parent!=base or not directory.name.startswith('verify-') or directory.is_symlink():
|
||||||
|
return False
|
||||||
|
if not stop_owned(value):return False
|
||||||
|
socket=Path(value['socketDirectory'])
|
||||||
|
if socket.parent.resolve()!=Path('/tmp').resolve() or not socket.name.startswith('qret-') or socket.is_symlink():return False
|
||||||
|
shutil.rmtree(socket,ignore_errors=True)
|
||||||
|
shutil.rmtree(directory)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def clean_abandoned(root):
|
||||||
|
base=Path(root)/'.runtime/retention';removed=0
|
||||||
|
for directory in base.glob('verify-*'):
|
||||||
|
path=directory/'owner.json'
|
||||||
|
if directory.is_symlink() or not path.is_file() or path.is_symlink():continue
|
||||||
|
value=json.loads(path.read_text())
|
||||||
|
if value.get('kind')!='private-quality-restore' or value.get('directory')!=str(directory):continue
|
||||||
|
try:
|
||||||
|
p=psutil.Process(value['ownerPid'])
|
||||||
|
if abs(p.create_time()-value['ownerCreated'])<.01 and p.is_running() and p.status()!=psutil.STATUS_ZOMBIE:continue
|
||||||
|
except psutil.NoSuchProcess:pass
|
||||||
|
except psutil.Error:continue
|
||||||
|
if time.time()-value['createdAt']<300:continue
|
||||||
|
if remove_owned(root,directory,value):removed+=1
|
||||||
|
return removed
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def instance(root):
|
||||||
|
root=Path(root).resolve();base=root/'.runtime/retention';base.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||||
|
directory=base/('verify-'+uuid.uuid4().hex);directory.mkdir(mode=0o700)
|
||||||
|
socket=Path(tempfile.mkdtemp(prefix='qret-',dir='/tmp')).resolve();socket.chmod(0o700)
|
||||||
|
binary=root/'.runtime/mysql/bin/mysqld'
|
||||||
|
value={'kind':'private-quality-restore','directory':str(directory),'socketDirectory':str(socket),
|
||||||
|
'binary':str(binary),'ownerPid':os.getpid(),'ownerCreated':psutil.Process().create_time(),'createdAt':time.time()}
|
||||||
|
br.atomic(directory/'owner.json',value)
|
||||||
|
process=None
|
||||||
|
try:
|
||||||
|
with (directory/'initialize.log').open('ab') as output:
|
||||||
|
init=subprocess.Popen([str(binary),'--no-defaults','--initialize-insecure','--basedir='+str(binary.parent.parent),
|
||||||
|
'--datadir='+str(directory/'data'),'--lower-case-table-names='+('2' if sys.platform == 'darwin' else '0'),'--skip-log-bin'],stdout=output,stderr=output)
|
||||||
|
value.update({'initPid':init.pid,'initCreated':psutil.Process(init.pid).create_time()});br.atomic(directory/'owner.json',value)
|
||||||
|
try:code=init.wait(timeout=180)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
init.kill();init.wait();raise
|
||||||
|
if code:raise RuntimeError('Private restore server initialization failed')
|
||||||
|
args=[str(binary),'--no-defaults','--basedir='+str(binary.parent.parent),'--datadir='+str(directory/'data'),
|
||||||
|
'--socket='+str(socket/'mysql.sock'),'--pid-file='+str(directory/'server.pid'),
|
||||||
|
'--log-error='+str(directory/'server.log'),'--skip-networking','--mysqlx=OFF','--skip-log-bin',
|
||||||
|
'--lower-case-table-names='+('2' if sys.platform == 'darwin' else '0'),'--innodb-buffer-pool-size=128M','--innodb-redo-log-capacity=64M','--max-connections=8']
|
||||||
|
with (directory/'stdout.log').open('ab') as output:
|
||||||
|
process=subprocess.Popen(args,stdout=output,stderr=output,stdin=subprocess.DEVNULL)
|
||||||
|
value.update({'serverPid':process.pid,'serverCreated':psutil.Process(process.pid).create_time()});br.atomic(directory/'owner.json',value)
|
||||||
|
config=directory/'client.cnf'
|
||||||
|
config.write_text('[client]\nuser=root\nsocket='+str(socket/'mysql.sock')+'\n');config.chmod(0o600)
|
||||||
|
deadline=time.monotonic()+90
|
||||||
|
while time.monotonic()<deadline:
|
||||||
|
if process.poll() is not None:raise RuntimeError('Private restore server exited')
|
||||||
|
ready=subprocess.run([str(binary.parent/'mysql'),'--defaults-file='+str(config),'--batch','--skip-column-names','-e','SELECT 1'],capture_output=True)
|
||||||
|
if ready.returncode==0:break
|
||||||
|
time.sleep(.2)
|
||||||
|
else:raise RuntimeError('Private restore server not ready')
|
||||||
|
yield config
|
||||||
|
finally:
|
||||||
|
if not remove_owned(root,directory,value):
|
||||||
|
raise RuntimeError('Private restore server cleanup requires inspection')
|
||||||
|
if process is not None:process.wait(timeout=5)
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
"""Conservative retention of application logs, with native MySQL reopening."""
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import psutil
|
||||||
|
|
||||||
|
MAX_BYTES = 100 * 1024**2
|
||||||
|
MAX_AGE = 7 * 86400
|
||||||
|
|
||||||
|
|
||||||
|
def candidates(root, support):
|
||||||
|
root, support = Path(root).resolve(), Path(support).resolve()
|
||||||
|
paths = set()
|
||||||
|
for base, depth in ((root / 'output', 3), (root / '.runtime', 2), (support / 'logs', 1)):
|
||||||
|
if not base.is_dir() or base.is_symlink():
|
||||||
|
continue
|
||||||
|
for directory, dirs, files in os.walk(base, followlinks=False):
|
||||||
|
parent = Path(directory)
|
||||||
|
if len(parent.relative_to(base).parts) >= depth - 1:
|
||||||
|
dirs[:] = []
|
||||||
|
else:
|
||||||
|
dirs[:] = [d for d in dirs if not (parent / d).is_symlink() and d not in
|
||||||
|
('backups', 'mysql-data', 'mysql', 'jdk', 'm2', 'backend-builds', 'node_modules')]
|
||||||
|
for name in files:
|
||||||
|
if re.fullmatch(r'.+\.(?:log|out|err)(?:\.[a-zA-Z0-9_-]+)*(?:\.gz)?', name):
|
||||||
|
p = parent / name
|
||||||
|
if not p.is_symlink() and p.is_file() and p.stat().st_nlink == 1:
|
||||||
|
paths.add(p)
|
||||||
|
return sorted(paths)
|
||||||
|
|
||||||
|
|
||||||
|
def opened_files():
|
||||||
|
"""Unknown access is a reason to defer deletion, never a reason to assume closed."""
|
||||||
|
paths = set()
|
||||||
|
for process in psutil.process_iter(['uids']):
|
||||||
|
try:
|
||||||
|
if process.info['uids'].real != os.getuid():
|
||||||
|
continue
|
||||||
|
for f in process.open_files():
|
||||||
|
paths.add(str(Path(f.path).absolute()))
|
||||||
|
except psutil.NoSuchProcess:
|
||||||
|
continue
|
||||||
|
except psutil.Error as error:
|
||||||
|
# macOS restricts system/GUI agents unrelated to these application log
|
||||||
|
# directories. A denied project runtime/interpreter still defers cleanup.
|
||||||
|
try:
|
||||||
|
executable = Path(process.exe()).resolve()
|
||||||
|
interpreter = re.fullmatch(r'(?:python.*|Python|java|node|bash|zsh|sh|uv|mvn)', executable.name)
|
||||||
|
if executable.is_relative_to(Path.cwd().resolve()) or interpreter:
|
||||||
|
raise RuntimeError('Project writer identity check unavailable') from error
|
||||||
|
except psutil.NoSuchProcess:
|
||||||
|
continue
|
||||||
|
except psutil.Error:
|
||||||
|
raise RuntimeError('Writer identity check unavailable') from error
|
||||||
|
return paths
|
||||||
|
|
||||||
|
|
||||||
|
def fingerprint(path):
|
||||||
|
s = path.stat()
|
||||||
|
return s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns
|
||||||
|
|
||||||
|
|
||||||
|
def category(path):
|
||||||
|
name = path.name.lstrip('.')
|
||||||
|
name = re.sub(r'\.\d{4}-\d{2}-\d{2}.*', '', name)
|
||||||
|
name = re.sub(r'\.roll-.*', '', name)
|
||||||
|
name = re.sub(r'\.(?:log|out|err)(?:\..*)?$', '', name)
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
def rotate_mysql(root, execute, br, now=None):
|
||||||
|
root = Path(root).resolve(); now = time.time() if now is None else now
|
||||||
|
source = root / 'output/logs/mysql-server.log'
|
||||||
|
journal = root / '.runtime/operations/mysql-log-rotation.json'
|
||||||
|
if any(p.is_symlink() for p in (root / 'output', source.parent, source, root / '.runtime', journal.parent, journal)):
|
||||||
|
raise ValueError('Unsafe MySQL log path')
|
||||||
|
pending = None
|
||||||
|
if journal.exists():
|
||||||
|
pending = json.loads(journal.read_text())
|
||||||
|
if (pending.get('root') != str(root) or pending.get('kind') != 'quality-mysql-log-rotation' or
|
||||||
|
not re.fullmatch(r'mysql-server\.roll-\d+-[a-f0-9]{8}\.log', pending.get('archive', ''))):
|
||||||
|
raise ValueError('Unknown MySQL log rotation intent')
|
||||||
|
elif source.is_file() and source.stat().st_size:
|
||||||
|
old_day = time.localtime(source.stat().st_mtime)[:3]
|
||||||
|
if source.stat().st_size < 10 * 1024**2 and old_day == time.localtime(now)[:3]:
|
||||||
|
return 0
|
||||||
|
pending = {'kind': 'quality-mysql-log-rotation', 'root': str(root),
|
||||||
|
'archive': 'mysql-server.roll-' + str(int(now)) + '-' + uuid.uuid4().hex[:8] + '.log',
|
||||||
|
'fingerprint': list(fingerprint(source))}
|
||||||
|
br.atomic(journal, pending)
|
||||||
|
if pending is None:
|
||||||
|
return 0
|
||||||
|
target = source.with_name(pending['archive'])
|
||||||
|
if target.is_symlink():
|
||||||
|
raise ValueError('Unsafe MySQL rotated log')
|
||||||
|
if not target.exists():
|
||||||
|
if not source.is_file() or list(fingerprint(source)) != pending['fingerprint']:
|
||||||
|
journal.unlink(); return 0
|
||||||
|
source.rename(target)
|
||||||
|
# MySQL closes the old descriptor and reopens its configured path. No truncation.
|
||||||
|
execute('FLUSH ERROR LOGS;', None)
|
||||||
|
if not source.is_file() or source.is_symlink() or source.stat().st_ino == target.stat().st_ino:
|
||||||
|
raise RuntimeError('MySQL error log reopening not confirmed')
|
||||||
|
source.chmod(0o600)
|
||||||
|
target.chmod(0o600)
|
||||||
|
journal.unlink()
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def retain(root, support, now=None, open_paths=None, max_bytes=MAX_BYTES, max_age=MAX_AGE):
|
||||||
|
now = time.time() if now is None else now
|
||||||
|
opened = opened_files() if open_paths is None else set(open_paths)
|
||||||
|
groups = {}; result = {'logsRemoved': 0, 'logBytesRemoved': 0, 'logsDeferred': 0}
|
||||||
|
for p in candidates(root, support):
|
||||||
|
groups.setdefault(category(p), []).append(p)
|
||||||
|
for files in groups.values():
|
||||||
|
files.sort(key=lambda p: p.stat().st_mtime)
|
||||||
|
total = sum(p.stat().st_size for p in files)
|
||||||
|
for p in files:
|
||||||
|
s = p.stat(); stamp = fingerprint(p)
|
||||||
|
if now - s.st_mtime <= max_age and total <= max_bytes:
|
||||||
|
continue
|
||||||
|
# Native appenders own these live files. The monitor handles archives;
|
||||||
|
# MySQL is reopened by its dedicated rotation function above.
|
||||||
|
if ((p.parent == Path(root) / 'output/logs' and p.name in ('backend.log', 'mysql-server.log', 'backend-console.log')) or
|
||||||
|
(p.parent == Path(support) / 'logs' and p.name in ('guardian.log', 'retention.log', 'operations.log'))):
|
||||||
|
result['logsDeferred'] += 1; continue
|
||||||
|
if str(p.absolute()) in opened:
|
||||||
|
result['logsDeferred'] += 1; continue
|
||||||
|
# Recheck descriptors immediately before mutation. Data/config/archives
|
||||||
|
# never enter this allowlist, and active writers are never truncated.
|
||||||
|
if p.is_symlink() or fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened):
|
||||||
|
result['logsDeferred'] += 1; continue
|
||||||
|
if s.st_size > max_bytes and now - s.st_mtime <= max_age and len(files) == 1:
|
||||||
|
# Keep the newest bounded tail of a closed oversized console log.
|
||||||
|
temp = p.with_name(p.name + '.' + uuid.uuid4().hex + '.tmp')
|
||||||
|
try:
|
||||||
|
with p.open('rb') as source, temp.open('xb') as output:
|
||||||
|
temp.chmod(0o600); source.seek(max(0, s.st_size - max_bytes))
|
||||||
|
source.readline() # Drop a possible partial leading line.
|
||||||
|
while block := source.read(1024 * 1024): output.write(block)
|
||||||
|
output.flush(); os.fsync(output.fileno())
|
||||||
|
if fingerprint(p) != stamp or str(p.absolute()) in (opened_files() if open_paths is None else opened):
|
||||||
|
result['logsDeferred'] += 1; continue
|
||||||
|
os.utime(temp, ns=(s.st_atime_ns, s.st_mtime_ns)); temp.replace(p)
|
||||||
|
removed = s.st_size - p.stat().st_size
|
||||||
|
total -= removed; result['logBytesRemoved'] += removed
|
||||||
|
finally:
|
||||||
|
temp.unlink(missing_ok=True)
|
||||||
|
else:
|
||||||
|
p.unlink(); total -= s.st_size
|
||||||
|
result['logsRemoved'] += 1; result['logBytesRemoved'] += s.st_size
|
||||||
|
return result
|
||||||
@@ -0,0 +1,207 @@
|
|||||||
|
"""Ownership records for disposable test schemas; never infer ownership from a prefix."""
|
||||||
|
from pathlib import Path
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import psutil
|
||||||
|
import backup_retention as br
|
||||||
|
|
||||||
|
PREFIXES = ('quality_verify_', 'quality_scale_', 'quality_retention_test_',
|
||||||
|
'quality_classification_source_', 'quality_classification_preview_',
|
||||||
|
'quality_sales_type_test_', 'quality_unclassified_delta_', 'quality_sales_metric_test_',
|
||||||
|
'quality_source_column_test_', 'quality_protocol_test_', 'quality_fine_rollup_')
|
||||||
|
MARKER = '__quality_test_owner'
|
||||||
|
|
||||||
|
|
||||||
|
def valid_name(name):
|
||||||
|
return (isinstance(name, str) and len(name) <= 64 and
|
||||||
|
re.fullmatch(r'[a-z0-9_]+_[a-f0-9]{8,32}', name) is not None and
|
||||||
|
name.startswith(PREFIXES))
|
||||||
|
|
||||||
|
|
||||||
|
def registry(root):
|
||||||
|
root = Path(root).resolve(); base = root / '.runtime/test-resources'
|
||||||
|
for path in (root / '.runtime', base):
|
||||||
|
if path.is_symlink() or not path.resolve().is_relative_to(root):
|
||||||
|
raise ValueError('Unsafe test registry path')
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def protect(root, name):
|
||||||
|
cfg = json.loads((Path(root) / '.runtime/local-config.json').read_text())
|
||||||
|
if not valid_name(name) or name == cfg['mysqlDatabase']:
|
||||||
|
raise ValueError('Not a disposable test database')
|
||||||
|
|
||||||
|
|
||||||
|
def active_process(pid, created):
|
||||||
|
try:
|
||||||
|
p = psutil.Process(int(pid))
|
||||||
|
return (abs(p.create_time() - float(created)) < .02 and p.is_running() and
|
||||||
|
p.status() != psutil.STATUS_ZOMBIE)
|
||||||
|
except psutil.NoSuchProcess:
|
||||||
|
return False
|
||||||
|
except (psutil.Error, TypeError, ValueError):
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def load(root, name):
|
||||||
|
path = registry(root) / (name + '.json')
|
||||||
|
if not path.is_file() or path.is_symlink():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text())
|
||||||
|
if (not isinstance(value, dict) or value.get('kind') != 'quality-owned-test-db' or
|
||||||
|
value.get('root') != str(Path(root).resolve()) or value.get('database') != name or
|
||||||
|
not valid_name(name) or not re.fullmatch(r'[a-f0-9]{32}', value.get('token', ''))):
|
||||||
|
return None
|
||||||
|
return value
|
||||||
|
except (OSError, ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def create(root, name, execute, ddl=None, empty=False):
|
||||||
|
root = Path(root).resolve(); protect(root, name)
|
||||||
|
base = registry(root); base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
path = base / (name + '.json')
|
||||||
|
# Exclusive registration also prevents a second process claiming an existing name.
|
||||||
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600); os.close(fd)
|
||||||
|
value = {'kind': 'quality-owned-test-db', 'formatVersion': 1, 'root': str(root),
|
||||||
|
'database': name, 'token': uuid.uuid4().hex, 'createdAt': time.time(),
|
||||||
|
'expiresAt': time.time() + 86400, 'ownerPid': os.getpid(),
|
||||||
|
'ownerCreated': psutil.Process().create_time(), 'children': [], 'users': [],
|
||||||
|
'serverUuid': execute('SELECT @@server_uuid;', None).strip(), 'created': False}
|
||||||
|
br.atomic(path, value)
|
||||||
|
try:
|
||||||
|
result = execute(ddl or 'CREATE DATABASE `' + name + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;', None)
|
||||||
|
except BaseException:
|
||||||
|
path.unlink(missing_ok=True); raise
|
||||||
|
value['created'] = True; br.atomic(path, value)
|
||||||
|
directory = root / '.runtime/mysql-data' / name
|
||||||
|
if directory.is_dir() and not directory.is_symlink():
|
||||||
|
stat = directory.stat()
|
||||||
|
value['directoryIdentity'] = [stat.st_dev, stat.st_ino]
|
||||||
|
br.atomic(path, value)
|
||||||
|
if not empty:
|
||||||
|
seal(root, name, execute)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def seal(root, name, execute):
|
||||||
|
value = load(root, name)
|
||||||
|
if value is None or value['ownerPid'] != os.getpid():
|
||||||
|
raise ValueError('Test resource owner not confirmed')
|
||||||
|
execute('CREATE TABLE `' + name + '`.`' + MARKER + '` (token CHAR(32) PRIMARY KEY);'
|
||||||
|
'INSERT INTO `' + name + '`.`' + MARKER + "` VALUES ('" + value['token'] + "');", None)
|
||||||
|
value['ready'] = True; br.atomic(registry(root) / (name + '.json'), value)
|
||||||
|
|
||||||
|
|
||||||
|
def same_directory(root, value):
|
||||||
|
p = Path(root) / '.runtime/mysql-data' / value['database']
|
||||||
|
if p.is_symlink() or not p.is_dir(): return False
|
||||||
|
stat = p.stat()
|
||||||
|
return value.get('directoryIdentity') == [stat.st_dev, stat.st_ino]
|
||||||
|
|
||||||
|
|
||||||
|
def attach_child(root, name, pid):
|
||||||
|
value = load(root, name)
|
||||||
|
if value is None or value['ownerPid'] != os.getpid():
|
||||||
|
raise ValueError('Test resource owner not confirmed')
|
||||||
|
value.setdefault('children', []).append({'pid': pid, 'created': psutil.Process(pid).create_time()})
|
||||||
|
br.atomic(registry(root) / (name + '.json'), value)
|
||||||
|
|
||||||
|
|
||||||
|
def attach_user(root, name, user):
|
||||||
|
if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user):
|
||||||
|
raise ValueError('Unsafe test account')
|
||||||
|
value = load(root, name)
|
||||||
|
if value is None or value['ownerPid'] != os.getpid():
|
||||||
|
raise ValueError('Test resource owner not confirmed')
|
||||||
|
value.setdefault('users', []).append(user)
|
||||||
|
br.atomic(registry(root) / (name + '.json'), value)
|
||||||
|
|
||||||
|
|
||||||
|
def marker_matches(value, execute):
|
||||||
|
if not value.get('ready') or execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']:
|
||||||
|
return False
|
||||||
|
name = value['database']
|
||||||
|
found = execute("SELECT COUNT(*) FROM information_schema.tables WHERE TABLE_SCHEMA='" + name +
|
||||||
|
"' AND TABLE_NAME='" + MARKER + "';", None).strip()
|
||||||
|
return found == '1' and execute('SELECT token FROM `' + name + '`.`' + MARKER + '`;', None).strip() == value['token']
|
||||||
|
|
||||||
|
|
||||||
|
def drop(root, name, execute):
|
||||||
|
protect(root, name); value = load(root, name)
|
||||||
|
if value is None:
|
||||||
|
raise ValueError('Registered test schema identity not confirmed')
|
||||||
|
same_server = execute('SELECT @@server_uuid;', None).strip() == value['serverUuid']
|
||||||
|
own_process = (value['ownerPid'] == os.getpid() and active_process(value['ownerPid'], value['ownerCreated']))
|
||||||
|
if not same_server or not (marker_matches(value, execute) or
|
||||||
|
(value.get('created') and not value.get('ready') and (own_process or same_directory(root, value)))):
|
||||||
|
raise ValueError('Registered test schema identity not confirmed')
|
||||||
|
result = execute('DROP DATABASE `' + name + '`;', None)
|
||||||
|
value['databaseDropped'] = True
|
||||||
|
path = registry(root) / (name + '.json')
|
||||||
|
if value.get('users'):
|
||||||
|
br.atomic(path, value)
|
||||||
|
else:
|
||||||
|
path.unlink()
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def user_dropped(root, user):
|
||||||
|
for path in registry(root).glob('*.json'):
|
||||||
|
value = load(root, path.stem)
|
||||||
|
if value is None or value['ownerPid'] != os.getpid() or user not in value.get('users', []):
|
||||||
|
continue
|
||||||
|
value['users'].remove(user)
|
||||||
|
if value.get('databaseDropped') and not value['users']:
|
||||||
|
path.unlink()
|
||||||
|
else:
|
||||||
|
br.atomic(path, value)
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup(root, execute, now=None, limit=3):
|
||||||
|
"""Called while maintenance owns its shared lock and the business application is idle."""
|
||||||
|
now = time.time() if now is None else now
|
||||||
|
result = {'testDatabasesRemoved': 0, 'testAccountsRemoved': 0, 'testResourcesDeferred': 0}
|
||||||
|
for path in sorted(registry(root).glob('*.json')):
|
||||||
|
if result['testDatabasesRemoved'] >= limit:
|
||||||
|
break
|
||||||
|
value = load(root, path.stem)
|
||||||
|
if value is None or not value.get('created') or now < value.get('expiresAt', now + 1):
|
||||||
|
continue
|
||||||
|
protect(root, value['database'])
|
||||||
|
if (active_process(value.get('ownerPid'), value.get('ownerCreated')) or
|
||||||
|
any(active_process(p.get('pid'), p.get('created')) for p in value.get('children', []))):
|
||||||
|
result['testResourcesDeferred'] += 1; continue
|
||||||
|
if execute('SELECT @@server_uuid;', None).strip() != value['serverUuid']:
|
||||||
|
result['testResourcesDeferred'] += 1; continue
|
||||||
|
name = value['database']
|
||||||
|
if execute("SELECT COUNT(*) FROM information_schema.processlist WHERE DB='" + name + "';", None).strip() != '0':
|
||||||
|
result['testResourcesDeferred'] += 1; continue
|
||||||
|
if not value.get('databaseDropped'):
|
||||||
|
if not marker_matches(value, execute) and not (not value.get('ready') and same_directory(root, value)):
|
||||||
|
result['testResourcesDeferred'] += 1; continue
|
||||||
|
drop(root, name, execute); result['testDatabasesRemoved'] += 1
|
||||||
|
value['databaseDropped'] = True
|
||||||
|
for user in list(value.get('users', [])):
|
||||||
|
if not re.fullmatch(r'q(?:verify|scale)_[a-f0-9]{12}', user):
|
||||||
|
continue
|
||||||
|
grantee = "'" + user + "'@'127.0.0.1'"
|
||||||
|
literal = "CONVERT(0x" + grantee.encode().hex() + " USING utf8mb4)"
|
||||||
|
unsafe = execute("SELECT (SELECT COUNT(*) FROM information_schema.USER_PRIVILEGES WHERE GRANTEE=" + literal +
|
||||||
|
" AND PRIVILEGE_TYPE<>'USAGE')+(SELECT COUNT(*) FROM information_schema.SCHEMA_PRIVILEGES WHERE GRANTEE=" + literal +
|
||||||
|
" AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.TABLE_PRIVILEGES WHERE GRANTEE=" + literal +
|
||||||
|
" AND TABLE_SCHEMA<>'" + name + "')+(SELECT COUNT(*) FROM information_schema.processlist WHERE USER='" + user + "');", None).strip()
|
||||||
|
if unsafe != '0':
|
||||||
|
result['testResourcesDeferred'] += 1; continue
|
||||||
|
execute("DROP USER IF EXISTS '" + user + "'@'127.0.0.1';", None)
|
||||||
|
value['users'].remove(user); result['testAccountsRemoved'] += 1
|
||||||
|
if not value.get('users'):
|
||||||
|
path.unlink(missing_ok=True)
|
||||||
|
else:
|
||||||
|
br.atomic(path, value)
|
||||||
|
return result
|
||||||
+2
-1
@@ -145,7 +145,8 @@ def start_mysql():
|
|||||||
'--datadir=' + str(data), '--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1',
|
'--datadir=' + str(data), '--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1',
|
||||||
'--socket=' + str(RUNTIME / 'mysql.sock'), '--pid-file=' + str(RUNTIME / 'mysql-server.pid'),
|
'--socket=' + str(RUNTIME / 'mysql.sock'), '--pid-file=' + str(RUNTIME / 'mysql-server.pid'),
|
||||||
'--log-error=' + str(OUTPUT / 'logs/mysql-server.log'), '--mysqlx=OFF',
|
'--log-error=' + str(OUTPUT / 'logs/mysql-server.log'), '--mysqlx=OFF',
|
||||||
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M'])
|
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M',
|
||||||
|
*(['--skip-log-bin'] if value.get('binaryLogEnabled', True) is False else [])])
|
||||||
client_config = RUNTIME / 'mysql-root.cnf'
|
client_config = RUNTIME / 'mysql-root.cnf'
|
||||||
save_private(client_config, '[client]\nuser=root\npassword=' + value['rootPassword'] + '\nsocket=' + str(RUNTIME / 'mysql.sock') + '\n')
|
save_private(client_config, '[client]\nuser=root\npassword=' + value['rootPassword'] + '\nsocket=' + str(RUNTIME / 'mysql.sock') + '\n')
|
||||||
for _ in range(120):
|
for _ in range(120):
|
||||||
|
|||||||
@@ -0,0 +1,213 @@
|
|||||||
|
"""Reclaim only unreferenced, completed source revisions and their owned archives."""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
|
||||||
|
# These are measured business values, not documents that refer to source IDs.
|
||||||
|
VALUE_JSON = {('fact_black', 'raw_values'), ('fact_black', 'components'),
|
||||||
|
('fact_bright', 'raw_values'), ('fact_bright', 'components'),
|
||||||
|
('metric_rollup', 'components'), ('sales_order_metric_rollup', 'components')}
|
||||||
|
TEXT_TYPES = {'json', 'text', 'mediumtext', 'longtext', 'tinytext'}
|
||||||
|
|
||||||
|
|
||||||
|
class Cleaner:
|
||||||
|
def __init__(self, db, br, safe_file, fingerprint, defer):
|
||||||
|
self.db, self.br, self.safe_file, self.fingerprint, self.defer = db, br, safe_file, fingerprint, defer
|
||||||
|
self.root = db.root.resolve()
|
||||||
|
self.boundary = self.root / 'runtimedata/archive/source-revisions'
|
||||||
|
self.journals = self.root / '.runtime/retention/source-files'
|
||||||
|
self.reference_deadline = None
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def quick_checks(self):
|
||||||
|
self.reference_deadline = time.monotonic() + .25
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
self.reference_deadline = None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def ident(value):
|
||||||
|
if not re.fullmatch(r'[A-Za-z0-9_]+', value):
|
||||||
|
raise ValueError('Invalid SQL identifier')
|
||||||
|
return '`' + value + '`'
|
||||||
|
|
||||||
|
def exists(self, table, condition, args):
|
||||||
|
if self.reference_deadline and time.monotonic() > self.reference_deadline:
|
||||||
|
raise self.defer('Source reference checks exceeded the brief exclusion budget')
|
||||||
|
try:
|
||||||
|
result = bool(self.db.rows('SELECT /*+ MAX_EXECUTION_TIME(100) */ 1 FROM ' +
|
||||||
|
self.ident(table) + ' WHERE ' + condition + ' LIMIT 1', args))
|
||||||
|
if self.reference_deadline and time.monotonic() > self.reference_deadline:
|
||||||
|
raise self.defer('Source reference checks exceeded the brief exclusion budget')
|
||||||
|
return result
|
||||||
|
except Exception as error:
|
||||||
|
if error.args and error.args[0] in (1317, 3024):
|
||||||
|
raise self.defer('Source reference check exceeded its short query budget') from error
|
||||||
|
raise
|
||||||
|
|
||||||
|
def metadata_mentions(self, needles, excluded=None):
|
||||||
|
for table, columns in self.db.column_types.items():
|
||||||
|
for column, kind in columns.items():
|
||||||
|
if kind not in TEXT_TYPES or (table, column) in VALUE_JSON:
|
||||||
|
continue
|
||||||
|
conditions = ['LOCATE(%s,CAST(' + self.ident(column) + ' AS CHAR))>0' for _ in needles]
|
||||||
|
args = list(needles)
|
||||||
|
if excluded and table == excluded[0]:
|
||||||
|
conditions = ['(' + ' OR '.join(conditions) + ')', self.ident('id') + '<>%s']
|
||||||
|
args.append(excluded[1])
|
||||||
|
condition = ' AND '.join(conditions)
|
||||||
|
else:
|
||||||
|
condition = ' OR '.join(conditions)
|
||||||
|
if self.exists(table, condition, args):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def referenced(self, revision):
|
||||||
|
rid = revision['id']
|
||||||
|
for fk in self.db.fks:
|
||||||
|
if fk['REFERENCED_TABLE_NAME'] == 'source_revision' and self.exists(
|
||||||
|
fk['TABLE_NAME'], self.ident(fk['COLUMN_NAME']) + '=%s', (rid,)):
|
||||||
|
return True
|
||||||
|
# Replacement, withdrawal and restore records include IDs without FKs.
|
||||||
|
for table, columns in self.db.columns.items():
|
||||||
|
for column in columns:
|
||||||
|
if column.endswith('revision_id'):
|
||||||
|
condition = self.ident(column) + '=%s'
|
||||||
|
args = [rid]
|
||||||
|
if table == 'source_revision':
|
||||||
|
condition += ' AND id<>%s'; args.append(rid)
|
||||||
|
if self.exists(table, condition, args):
|
||||||
|
return True
|
||||||
|
# A catalog source can be imported again. Retain its original even when
|
||||||
|
# its current source_path points to an external workbook rather than archive.
|
||||||
|
if 'source_candidate' in self.db.columns and 'sha256' in self.db.columns['source_candidate']:
|
||||||
|
if self.exists('source_candidate', 'sha256=%s', (revision['sha256'],)):
|
||||||
|
return True
|
||||||
|
return self.metadata_mentions([rid], ('source_revision', rid))
|
||||||
|
|
||||||
|
def file_referenced(self, path, exclude_revision=None):
|
||||||
|
for table, columns in self.db.columns.items():
|
||||||
|
for column in columns:
|
||||||
|
if column.endswith('_path'):
|
||||||
|
condition = self.ident(column) + '=%s'; args = [str(path)]
|
||||||
|
if table == 'source_revision' and exclude_revision:
|
||||||
|
condition += ' AND id<>%s'; args.append(exclude_revision)
|
||||||
|
if self.exists(table, condition, args):
|
||||||
|
return True
|
||||||
|
# Search both raw and JSON-escaped spelling, including saved snapshots.
|
||||||
|
needles = list(dict.fromkeys([str(path), json.dumps(str(path), ensure_ascii=False)[1:-1]]))
|
||||||
|
return self.metadata_mentions(needles, ('source_revision', exclude_revision) if exclude_revision else None)
|
||||||
|
|
||||||
|
def owned_file(self, revision):
|
||||||
|
path = Path(revision['archive_path'])
|
||||||
|
if not self.safe_file(path, self.boundary):
|
||||||
|
return None
|
||||||
|
relative = path.relative_to(self.boundary).parts
|
||||||
|
if (len(relative) != 4 or relative[0] not in ('black', 'bright') or
|
||||||
|
not re.fullmatch(r'\d{4}', relative[1]) or
|
||||||
|
not re.fullmatch(r'0[1-9]|1[0-2]', relative[2]) or
|
||||||
|
not re.fullmatch(r'[a-f0-9]{64}', revision['sha256']) or
|
||||||
|
relative[3] != revision['sha256'] + '.xlsx'):
|
||||||
|
return None
|
||||||
|
if path.stat().st_size != revision['file_size'] or path.stat().st_nlink != 1:
|
||||||
|
return None
|
||||||
|
return path
|
||||||
|
|
||||||
|
def safe_journals(self):
|
||||||
|
for p in (self.root / '.runtime', self.root / '.runtime/retention', self.journals):
|
||||||
|
if p.is_symlink() or not p.resolve().is_relative_to(self.root):
|
||||||
|
raise ValueError('Unsafe source file journal directory')
|
||||||
|
|
||||||
|
def drain(self, current, limit=5):
|
||||||
|
"""Resume only registered deletions after database commit; never scan stray files."""
|
||||||
|
self.safe_journals()
|
||||||
|
removed = size = 0
|
||||||
|
for record in sorted(self.journals.glob('*.json'))[:limit]:
|
||||||
|
if record.is_symlink():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
value = json.loads(record.read_text())
|
||||||
|
except (OSError, ValueError):
|
||||||
|
continue
|
||||||
|
token = value.get('token', '')
|
||||||
|
if (value.get('kind') != 'quality-source-retirement' or value.get('root') != str(self.root) or
|
||||||
|
value.get('database') != self.db.name or not isinstance(token, str) or
|
||||||
|
not re.fullmatch(r'[a-f0-9]{32}', token) or token + '.json' != record.name):
|
||||||
|
continue
|
||||||
|
revision = value.get('revision')
|
||||||
|
if (not isinstance(revision, dict) or not all(isinstance(revision.get(k), str) for k in ('id', 'archive_path', 'sha256')) or
|
||||||
|
not isinstance(revision.get('file_size'), int) or not isinstance(value.get('fingerprint'), list)):
|
||||||
|
continue
|
||||||
|
path = Path(revision['archive_path'])
|
||||||
|
if self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)):
|
||||||
|
record.unlink() # An interrupted/rolled-back delete still owns its file.
|
||||||
|
continue
|
||||||
|
if not path.exists() and not path.is_symlink():
|
||||||
|
record.unlink(); continue
|
||||||
|
if self.owned_file(revision) != path or list(self.fingerprint(path)) != value['fingerprint']:
|
||||||
|
continue
|
||||||
|
if self.br.sha(path) != revision['sha256']:
|
||||||
|
continue
|
||||||
|
with self.db.frozen(current), self.quick_checks():
|
||||||
|
if (self.db.rows('SELECT id FROM source_revision WHERE id=%s', (revision['id'],)) or
|
||||||
|
self.owned_file(revision) != path or
|
||||||
|
list(self.fingerprint(path)) != value['fingerprint'] or self.file_referenced(path)):
|
||||||
|
continue
|
||||||
|
size += path.stat().st_size
|
||||||
|
path.unlink(); removed += 1
|
||||||
|
record.unlink()
|
||||||
|
return removed, size
|
||||||
|
|
||||||
|
def collect(self, current, state, now, grace=1800, limit=3):
|
||||||
|
self.safe_journals()
|
||||||
|
seen = state.setdefault('sourceRevisions', {})
|
||||||
|
result = {'sourceRevisionsRemoved': 0, 'sourceArchiveFilesRemoved': 0, 'sourceArchiveBytesRemoved': 0}
|
||||||
|
# All registered/current revisions remain readable until both ownership and
|
||||||
|
# dependency checks pass. Unknown states and missing/damaged files are kept.
|
||||||
|
revisions = self.db.rows('SELECT * FROM source_revision')
|
||||||
|
present = {row['id'] for row in revisions}
|
||||||
|
for stale in set(seen) - present:
|
||||||
|
seen.pop(stale, None)
|
||||||
|
deadline = time.monotonic() + 5
|
||||||
|
for revision in revisions:
|
||||||
|
if time.monotonic() > deadline or result['sourceRevisionsRemoved'] >= limit:
|
||||||
|
break
|
||||||
|
rid = revision['id']
|
||||||
|
if revision.get('status') != 'TRANSFORMED' or self.referenced(revision):
|
||||||
|
seen.pop(rid, None); continue
|
||||||
|
identity = hashlib.sha256(json.dumps(revision, sort_keys=True, default=str).encode()).hexdigest()
|
||||||
|
if seen.get(rid, {}).get('identity') != identity:
|
||||||
|
seen[rid] = {'identity': identity, 'since': now}
|
||||||
|
if now - seen[rid]['since'] < grace:
|
||||||
|
continue
|
||||||
|
path = self.owned_file(revision)
|
||||||
|
if path is None:
|
||||||
|
continue
|
||||||
|
stamp = self.fingerprint(path)
|
||||||
|
if self.br.sha(path) != revision['sha256'] or self.fingerprint(path) != stamp:
|
||||||
|
continue
|
||||||
|
# Durable intent first: a crash after row deletion must not leave an
|
||||||
|
# unregistered orphan. The journal is harmless if the row still exists.
|
||||||
|
self.journals.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||||
|
token = uuid.uuid4().hex; record = self.journals / (token + '.json')
|
||||||
|
self.br.atomic(record, {'kind': 'quality-source-retirement', 'root': str(self.root),
|
||||||
|
'database': self.db.name, 'token': token, 'createdAt': now,
|
||||||
|
'revision': {k: revision[k] for k in ('id', 'archive_path', 'sha256', 'file_size')},
|
||||||
|
'fingerprint': stamp})
|
||||||
|
with self.db.frozen(current), self.quick_checks():
|
||||||
|
live = self.db.rows('SELECT * FROM source_revision WHERE id=%s', (rid,))
|
||||||
|
if (not live or live[0] != revision or self.referenced(revision) or
|
||||||
|
self.owned_file(revision) != path or self.fingerprint(path) != stamp):
|
||||||
|
record.unlink(); seen.pop(rid, None); continue
|
||||||
|
self.db.execute('DELETE FROM source_revision WHERE id=%s', (rid,))
|
||||||
|
result['sourceRevisionsRemoved'] += 1; seen.pop(rid, None)
|
||||||
|
files, size = self.drain(current)
|
||||||
|
result['sourceArchiveFilesRemoved'] = files; result['sourceArchiveBytesRemoved'] = size
|
||||||
|
result['sourceFilesPending'] = sum(1 for _ in self.journals.glob('*.json'))
|
||||||
|
return result
|
||||||
Reference in New Issue
Block a user