新增:Linux Docker 部署与自动回收、备份及容量监测

This commit is contained in:
2026-11-10 14:53:07 +08:00
parent 3962521a5c
commit dc5b9b76ec
45 changed files with 5591 additions and 35 deletions
+305
View File
@@ -0,0 +1,305 @@
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
import tarfile
import gzip
from contextlib import contextmanager
import unittest
import uuid
from unittest.mock import patch
ROOT=Path(__file__).resolve().parents[2]
sys.path.insert(0,str(ROOT/'scripts'))
import backup_retention as br
spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py'))
ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret)
def good_backup(root, suffix, created='2026-11-09T01:00:00+00:00'):
folder=root/'output/backups'/('20261109T010000Z-'+suffix)
folder.mkdir(parents=True)
m={'formatVersion':3,'status':'ISOLATED_RESTORE_VERIFIED','createdAt':created,
'restoreVerifiedAt':created,'queryEvidence':{'releaseId':'current'},
'verification':{k:True for k in br.CHECKS}}
for name,key in br.ARTIFACTS.items():
(folder/name).write_bytes(name.encode());m[key]=br.sha(folder/name)
br.atomic(folder/'manifest.json',m)
return folder.resolve()
class BackupRotationTests(unittest.TestCase):
def setUp(self):
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
def tearDown(self):self.temp.cleanup()
def test_failed_or_corrupted_new_backup_keeps_previous(self):
old=good_backup(self.root,'11111111');br.rotate(self.root,old)
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00')
(new/'database.sql.gz').write_bytes(b'corrupted')
with self.assertRaises(ValueError):br.rotate(self.root,new)
self.assertTrue(old.is_dir());self.assertEqual(br.latest(self.root)[0],old)
def test_rotation_keeps_one_and_verifying_older_never_rolls_back_latest(self):
old=good_backup(self.root,'11111111');br.rotate(self.root,old)
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00');br.rotate(self.root,new)
self.assertFalse(old.exists());self.assertEqual(br.latest(self.root)[0],new)
older=good_backup(self.root,'33333333');br.rotate(self.root,older)
self.assertEqual(br.latest(self.root)[0],new);self.assertFalse(older.exists())
def test_unknown_and_active_incomplete_folders_are_preserved(self):
old=good_backup(self.root,'11111111');unknown=old.parent/'unrelated';unknown.mkdir()
incomplete=old.parent/'20261109T010000Z-abcdefab';incomplete.mkdir()
br.atomic(incomplete/'.owner.json',{'kind':'quality-backup','root':str(self.root),'id':incomplete.name})
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00');br.rotate(self.root,new)
self.assertTrue(unknown.exists());self.assertTrue(incomplete.exists())
def test_shared_lock_excludes_another_process(self):
code='import sys;from pathlib import Path;sys.path.insert(0,sys.argv[1]);import backup_retention as b\ntry:\n with b.lock(Path(sys.argv[2]),timeout=0):pass\nexcept b.Busy:sys.exit(17)'
with br.lock(self.root):
child=subprocess.run([sys.executable,'-c',code,str(ROOT/'scripts'),str(self.root)],capture_output=True)
self.assertEqual(child.returncode,17)
def test_managed_backup_root_cannot_be_redirected_outside_project(self):
with tempfile.TemporaryDirectory() as external:
folder=good_backup(Path(external),'11111111')
(self.root/'output').mkdir();(self.root/'output/backups').symlink_to(folder.parent,target_is_directory=True)
with self.assertRaises(ValueError):br.rotate(self.root,folder)
self.assertTrue(folder.is_dir())
def test_interrupted_old_backup_removal_can_resume_without_touching_latest(self):
old=good_backup(self.root,'11111111');br.rotate(self.root,old)
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00')
with patch.object(br.shutil,'rmtree',side_effect=OSError('simulated removal failure')):
br.rotate(self.root,new)
self.assertEqual(br.latest(self.root)[0],new)
self.assertEqual(len(list((self.root/'.runtime/retention/retired').glob('*.json'))),1)
br.clean_retired(self.root)
self.assertFalse(list((self.root/'.runtime/retention/retired').glob('*.json')))
self.assertTrue(new.is_dir())
class DatabaseRetentionTests(unittest.TestCase):
"""Real MySQL locking/FK tests in a newly owned, tiny disposable database."""
@classmethod
def setUpClass(cls):
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured')
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True)
cls.name='quality_retention_test_'+uuid.uuid4().hex[:12]
with cls.admin.cursor() as c:
c.execute('CREATE DATABASE '+ret.ident(cls.name))
c.execute('USE '+ret.ident(cls.name))
ddl=[
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)',
'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))',
'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)',
'CREATE TABLE import_run(id int PRIMARY KEY,status varchar(24),expected_current_release_id varchar(36),result_release_id varchar(36),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id),FOREIGN KEY(result_release_id) REFERENCES data_release(id))',
'CREATE TABLE release_action(id int PRIMARY KEY,release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
'CREATE TABLE candidate_reconciliation(id int PRIMARY KEY,release_id varchar(36),baseline_release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(baseline_release_id) REFERENCES data_release(id))',
'CREATE TABLE analysis_session(id int PRIMARY KEY,release_id varchar(36))',
'CREATE TABLE saved_business_object(id int PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
'CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),parser_config_hash varchar(64),file_size bigint)',
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),source_slot varchar(100),source_path varchar(1000),archive_path varchar(1000),sha256 char(64),parser_config_hash varchar(64),expected_current_release_id varchar(36))',
'CREATE TABLE source_upload_draft(id varchar(36) PRIMARY KEY,source_file varchar(100),source_path varchar(1000),sha256 char(64),file_size bigint,candidate_id varchar(36),created_at datetime,expires_at datetime)']
for statement in ddl:c.execute(statement+' ENGINE=InnoDB')
@classmethod
def tearDownClass(cls):
with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name))
cls.admin.close()
def setUp(self):
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
self.db=ret.Database(ROOT,self.name);self.db.root=self.root
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table))
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')")
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
self.db.execute("INSERT INTO transform_build VALUES('shared','COMPLETED',%s),('obsolete','COMPLETED',%s)",('1'*64,'2'*64))
self.db.execute("INSERT INTO release_member VALUES('current','shared'),('old','shared'),('old','obsolete')")
self.db.execute("INSERT INTO fact_black VALUES(1,'shared','current data'),(2,'obsolete','old data'),(3,'obsolete','old data')")
def tearDown(self):self.db.conn.close();self.temp.cleanup()
def retire_old(self):
state={'releases':{'old':1}}
with self.db.frozen('current'):self.assertEqual(self.db.release_step('current',state,1800,4000),1)
def test_running_queued_retry_and_unknown_tasks_prevent_cleanup(self):
for status,retry in [('RUNNING',False),('QUEUED',False),('FAILED',True),('FUTURE_STATE',False)]:
self.db.execute('DELETE FROM task')
self.db.execute('INSERT INTO task VALUES(1,%s,%s,0,3)',(status,datetime_from_future() if retry else None))
with self.assertRaises(ret.Busy):
with self.db.frozen('current'):self.fail('busy job was ignored')
self.assertEqual(len(self.db.rows('SELECT * FROM data_release')),2)
def test_pending_candidate_and_saved_business_references_are_protected(self):
self.db.execute("INSERT INTO data_release VALUES('candidate','CANDIDATE','old','old')")
with self.db.frozen('current'):
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
self.db.execute("DELETE FROM data_release WHERE id='candidate'")
self.db.execute("INSERT INTO saved_business_object VALUES(1,'old')")
with self.db.frozen('current'):
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
def test_analysis_session_reference_is_protected_without_database_fk(self):
self.db.execute("INSERT INTO analysis_session VALUES(1,'old')")
with self.db.frozen('current'):
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
def test_pending_source_configuration_preserves_its_old_baseline(self):
self.db.execute("INSERT INTO source_candidate VALUES('pending','DISCOVERED','slot','path',NULL,%s,'legacy','old')",('3'*64,))
with self.db.frozen('current'):
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
def test_unknown_business_build_reference_is_preserved(self):
self.retire_old();self.db.execute("INSERT INTO unknown_build_use VALUES(1,'obsolete')")
with self.db.frozen('current'):
self.assertEqual(self.db.build_step({'builds':{'obsolete':1}},1800,4000,1),(0,0))
self.assertEqual(self.db.rows("SELECT status FROM transform_build WHERE id='obsolete'")[0]['status'],'COMPLETED')
def test_changed_publication_and_grace_period_prevent_cleanup(self):
with self.assertRaises(ret.Busy):
with self.db.frozen('old'):pass
with self.db.frozen('current'):
self.assertEqual(self.db.release_step('current',{'releases':{'old':3900}},1800,4000),0)
def test_old_release_removed_but_current_membership_and_rows_preserved(self):
self.db.execute("INSERT INTO import_run VALUES(1,'COMPLETED','old','old')")
self.db.execute("INSERT INTO release_action VALUES(1,'old',NULL)")
self.db.execute("INSERT INTO candidate_reconciliation VALUES(1,'current','old')")
self.retire_old()
self.assertEqual(self.db.rows("SELECT payload FROM fact_black WHERE build_id='shared'")[0]['payload'],'current data')
self.assertEqual(self.db.rows("SELECT * FROM release_member"),[{'release_id':'current','transform_build_id':'shared'}])
self.assertIsNone(self.db.rows('SELECT * FROM import_run')[0]['expected_current_release_id'])
self.assertIsNone(self.db.rows("SELECT * FROM data_release WHERE id='current'")[0]['parent_release_id'])
def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self):
self.retire_old();state={'builds':{'obsolete':1}}
with self.db.frozen('current'):
self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0))
old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0]
self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64)
self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,))
for _ in range(4):
with self.db.frozen('current'):self.db.build_step(state,1800,4000,1)
self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'"))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'"))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'"))
def add_draft(self,candidate=None,where=None,age=25):
path=where or self.root/'runtimedata/archive/upload-drafts/upload-fixture.xlsx'
path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'workbook bytes')
self.db.execute('INSERT INTO source_upload_draft VALUES(%s,%s,%s,%s,%s,%s,TIMESTAMPADD(HOUR,-30,CURRENT_TIMESTAMP),TIMESTAMPADD(HOUR,%s,CURRENT_TIMESTAMP))',('draft','input.xlsx',str(path),br.sha(path),path.stat().st_size,candidate,-age))
return path
def test_expired_unbound_upload_removed_but_fresh_preview_is_kept(self):
path=self.add_draft()
count,size=ret.cleanup_drafts(self.db,'current',{},time.time(),br)
self.assertEqual(count,1);self.assertGreater(size,0);self.assertFalse(path.exists())
self.add_draft(age=0)
self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br),(0,0))
def test_upload_still_referenced_by_pending_candidate_is_kept(self):
path=self.add_draft(candidate='pending')
self.db.execute("INSERT INTO source_candidate VALUES('pending','DISCOVERED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path)))
self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br)[0],1)
self.assertTrue(path.exists())
def test_imported_upload_is_rebound_to_verified_archive_before_cleanup(self):
path=self.add_draft(candidate='imported');archive=self.root/'runtimedata/archive/source-revisions/formal.xlsx';archive.parent.mkdir();archive.write_bytes(path.read_bytes())
self.db.execute("INSERT INTO source_document VALUES('doc','slot')")
self.db.execute("INSERT INTO source_revision VALUES('revision','doc',%s,%s,'legacy',%s)",(str(archive),br.sha(archive),archive.stat().st_size))
self.db.execute("INSERT INTO source_candidate VALUES('imported','IMPORTED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path)))
self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br)[0],1)
self.assertFalse(path.exists());self.assertTrue(archive.exists())
self.assertEqual(self.db.rows('SELECT source_path FROM source_candidate')[0]['source_path'],str(archive))
def test_outside_file_is_never_deleted(self):
path=self.add_draft(where=self.root/'unmanaged/private.xlsx')
ret.cleanup_drafts(self.db,'current',{},time.time(),br)
self.assertTrue(path.exists())
def test_symlink_is_not_followed_or_deleted(self):
outside=self.root/'important.xlsx';outside.write_bytes(b'keep this')
link=self.root/'runtimedata/archive/upload-drafts/upload-link.xlsx';link.parent.mkdir(parents=True);link.symlink_to(outside)
self.db.execute('INSERT INTO source_upload_draft VALUES(%s,%s,%s,%s,%s,NULL,TIMESTAMPADD(HOUR,-30,CURRENT_TIMESTAMP),TIMESTAMPADD(HOUR,-25,CURRENT_TIMESTAMP))',('draft','link.xlsx',str(link),br.sha(outside),outside.stat().st_size))
ret.cleanup_drafts(self.db,'current',{},time.time(),br)
self.assertTrue(outside.exists());self.assertTrue(link.is_symlink())
def test_file_changed_after_checksum_is_not_deleted(self):
path=self.add_draft();original=ret.fingerprint;calls=0
def changed(p):
nonlocal calls
calls+=1
if calls==2:path.write_bytes(b'new file contents')
return original(p)
with patch.object(ret,'fingerprint',side_effect=changed):
count,size=ret.cleanup_drafts(self.db,'current',{},time.time(),br)
self.assertEqual((count,size),(0,0));self.assertTrue(path.exists())
self.assertTrue(self.db.rows('SELECT * FROM source_upload_draft'))
def test_new_reference_before_file_unlink_is_rechecked(self):
path=self.add_draft();freeze=self.db.frozen;calls=0
@contextmanager
def insert_reference(expected):
nonlocal calls
calls+=1
with freeze(expected):
if calls==2:self.db.execute("INSERT INTO source_candidate VALUES('new','DISCOVERED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path)))
yield
with patch.object(self.db,'frozen',side_effect=insert_reference):
ret.cleanup_drafts(self.db,'current',{},time.time(),br)
self.assertTrue(path.exists());self.assertFalse(self.db.rows('SELECT * FROM source_upload_draft'))
def test_long_running_sql_is_not_interrupted_for_cleanup(self):
other=ret.Database(ROOT,self.name)
try:
other.conn.autocommit(False);other.rows('SELECT * FROM fact_black WHERE id=1 FOR UPDATE')
with self.assertRaises(ret.Busy):
with self.db.frozen('current'):pass
self.assertTrue(other.rows('SELECT 1'));other.conn.rollback()
finally:other.conn.close()
def datetime_from_future():
from datetime import datetime,timedelta,timezone
return datetime.now(timezone.utc).replace(tzinfo=None)+timedelta(hours=1)
class PrivateVerifierTests(unittest.TestCase):
def test_small_backup_restores_on_private_server_and_cleans_up(self):
import backup
with tempfile.TemporaryDirectory() as temporary:
root=Path(temporary).resolve();(root/'.runtime').mkdir()
(root/'.runtime/mysql').symlink_to(ROOT/'.runtime/mysql',target_is_directory=True)
base=root/'output/backups';folder=base/'20261109T020000Z-12345678';folder.mkdir(parents=True)
source=root/'source';source.mkdir();(source/'config.json').write_text('{}')
with tarfile.open(folder/'application.tar.gz','w:gz') as tar:tar.add(source/'config.json',arcname='config.json')
with tarfile.open(folder/'archive.tar.gz','w:gz'):pass
sql="CREATE TABLE current_release(dataset_id varchar(20),release_id varchar(36)); INSERT INTO current_release VALUES('quality',NULL); CREATE TABLE source_revision(id varchar(36),archive_path varchar(1024),sha256 char(64),file_size bigint);"
with gzip.open(folder/'database.sql.gz','wt') as out:out.write(sql)
manifest={'formatVersion':3,'createdAt':'2026-11-09T02:00:00+00:00','files':{},'archiveSourceRoot':str(root/'runtimedata/archive'),
'applicationFiles':{'config.json':br.sha(source/'config.json')},'requiredApplicationFiles':['config.json'],
'tableCounts':{'current_release':1,'source_revision':0},'queryEvidence':{'releaseId':None,'months':[]}}
for name,key in br.ARTIFACTS.items():manifest[key]=br.sha(folder/name)
br.atomic(folder/'manifest.json',manifest)
with patch.object(backup,'ROOT',root),patch.object(backup,'BACKUP_ROOT',base),patch.object(backup,'REQUIRED_APPLICATION_FILES',('config.json',)):
result=backup.verify(folder)
self.assertEqual(result['status'],'ISOLATED_RESTORE_VERIFIED')
self.assertFalse(list((root/'.runtime/retention').glob('verify-*')))
self.assertFalse(list((root/'.runtime/retention/scratch').glob('*.json')))
if __name__=='__main__':unittest.main()