332 lines
23 KiB
Python
332 lines
23 KiB
Python
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import tarfile
|
|
import gzip
|
|
from contextlib import contextmanager
|
|
import unittest
|
|
import uuid
|
|
from unittest.mock import patch
|
|
|
|
ROOT=Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0,str(ROOT/'scripts'))
|
|
import backup_retention as br
|
|
import managed_test_resources
|
|
spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py'))
|
|
ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret)
|
|
|
|
|
|
def good_backup(root, suffix, created='2026-11-09T01:00:00+00:00'):
|
|
folder=root/'output/backups'/('20261109T010000Z-'+suffix)
|
|
folder.mkdir(parents=True)
|
|
m={'formatVersion':3,'status':'ISOLATED_RESTORE_VERIFIED','createdAt':created,
|
|
'restoreVerifiedAt':created,'queryEvidence':{'releaseId':'current'},
|
|
'verification':{k:True for k in br.CHECKS}}
|
|
for name,key in br.ARTIFACTS.items():
|
|
(folder/name).write_bytes(name.encode());m[key]=br.sha(folder/name)
|
|
br.atomic(folder/'manifest.json',m)
|
|
return folder.resolve()
|
|
|
|
|
|
class BackupRotationTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
|
|
def tearDown(self):self.temp.cleanup()
|
|
|
|
def test_failed_or_corrupted_new_backup_keeps_previous(self):
|
|
old=good_backup(self.root,'11111111');br.rotate(self.root,old)
|
|
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00')
|
|
(new/'database.sql.gz').write_bytes(b'corrupted')
|
|
with self.assertRaises(ValueError):br.rotate(self.root,new)
|
|
self.assertTrue(old.is_dir());self.assertEqual(br.latest(self.root)[0],old)
|
|
|
|
def test_rotation_keeps_one_and_verifying_older_never_rolls_back_latest(self):
|
|
old=good_backup(self.root,'11111111');br.rotate(self.root,old)
|
|
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00');br.rotate(self.root,new)
|
|
self.assertFalse(old.exists());self.assertEqual(br.latest(self.root)[0],new)
|
|
older=good_backup(self.root,'33333333');br.rotate(self.root,older)
|
|
self.assertEqual(br.latest(self.root)[0],new);self.assertFalse(older.exists())
|
|
|
|
def test_unknown_and_active_incomplete_folders_are_preserved(self):
|
|
old=good_backup(self.root,'11111111');unknown=old.parent/'unrelated';unknown.mkdir()
|
|
incomplete=old.parent/'20261109T010000Z-abcdefab';incomplete.mkdir()
|
|
br.atomic(incomplete/'.owner.json',{'kind':'quality-backup','root':str(self.root),'id':incomplete.name})
|
|
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00');br.rotate(self.root,new)
|
|
self.assertTrue(unknown.exists());self.assertTrue(incomplete.exists())
|
|
|
|
def test_shared_lock_excludes_another_process(self):
|
|
code='import sys;from pathlib import Path;sys.path.insert(0,sys.argv[1]);import backup_retention as b\ntry:\n with b.lock(Path(sys.argv[2]),timeout=0):pass\nexcept b.Busy:sys.exit(17)'
|
|
with br.lock(self.root):
|
|
child=subprocess.run([sys.executable,'-c',code,str(ROOT/'scripts'),str(self.root)],capture_output=True)
|
|
self.assertEqual(child.returncode,17)
|
|
|
|
def test_managed_backup_root_cannot_be_redirected_outside_project(self):
|
|
with tempfile.TemporaryDirectory() as external:
|
|
folder=good_backup(Path(external),'11111111')
|
|
(self.root/'output').mkdir();(self.root/'output/backups').symlink_to(folder.parent,target_is_directory=True)
|
|
with self.assertRaises(ValueError):br.rotate(self.root,folder)
|
|
self.assertTrue(folder.is_dir())
|
|
|
|
def test_interrupted_old_backup_removal_can_resume_without_touching_latest(self):
|
|
old=good_backup(self.root,'11111111');br.rotate(self.root,old)
|
|
new=good_backup(self.root,'22222222','2026-11-09T02:00:00+00:00')
|
|
with patch.object(br.shutil,'rmtree',side_effect=OSError('simulated removal failure')):
|
|
br.rotate(self.root,new)
|
|
self.assertEqual(br.latest(self.root)[0],new)
|
|
self.assertEqual(len(list((self.root/'.runtime/retention/retired').glob('*.json'))),1)
|
|
br.clean_retired(self.root)
|
|
self.assertFalse(list((self.root/'.runtime/retention/retired').glob('*.json')))
|
|
self.assertTrue(new.is_dir())
|
|
|
|
|
|
class DatabaseRetentionTests(unittest.TestCase):
|
|
"""Real MySQL locking/FK tests in a newly owned, tiny disposable database."""
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured')
|
|
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
|
|
cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True)
|
|
cls.name='quality_retention_test_'+uuid.uuid4().hex[:12]
|
|
managed_test_resources.create(ROOT,cls.name,cls.execute_admin)
|
|
with cls.admin.cursor() as c:
|
|
c.execute('USE '+ret.ident(cls.name))
|
|
ddl=[
|
|
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
|
|
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
|
|
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)',
|
|
'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))',
|
|
'CREATE TABLE source_filter_build(build_id varchar(36) PRIMARY KEY,format_version int,FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
|
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
|
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
|
|
'CREATE TABLE source_filter_member(field_id bigint NOT NULL,fact_id bigint NOT NULL,value_id bigint NOT NULL,PRIMARY KEY(field_id,fact_id),KEY(value_id,fact_id),FOREIGN KEY(field_id) REFERENCES source_filter_field(id),FOREIGN KEY(value_id) REFERENCES source_filter_value(id))',
|
|
'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
|
'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
|
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)',
|
|
'CREATE TABLE import_run(id int PRIMARY KEY,status varchar(24),expected_current_release_id varchar(36),result_release_id varchar(36),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id),FOREIGN KEY(result_release_id) REFERENCES data_release(id))',
|
|
'CREATE TABLE release_action(id int PRIMARY KEY,release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
|
|
'CREATE TABLE candidate_reconciliation(id int PRIMARY KEY,release_id varchar(36),baseline_release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(baseline_release_id) REFERENCES data_release(id))',
|
|
'CREATE TABLE analysis_session(id int PRIMARY KEY,release_id varchar(36))',
|
|
'CREATE TABLE saved_business_object(id int PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
|
|
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
|
|
'CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),parser_config_hash varchar(64),file_size bigint)',
|
|
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),source_slot varchar(100),source_path varchar(1000),archive_path varchar(1000),sha256 char(64),parser_config_hash varchar(64),expected_current_release_id varchar(36))',
|
|
'CREATE TABLE source_upload_draft(id varchar(36) PRIMARY KEY,source_file varchar(100),source_path varchar(1000),sha256 char(64),file_size bigint,candidate_id varchar(36),created_at datetime,expires_at datetime)']
|
|
for statement in ddl:c.execute(statement+' ENGINE=InnoDB')
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
managed_test_resources.drop(ROOT,cls.name,cls.execute_admin)
|
|
cls.admin.close()
|
|
|
|
@classmethod
|
|
def execute_admin(cls,sql,args=None):
|
|
result=''
|
|
statements=[part.strip() for part in sql.split(';') if part.strip()]
|
|
with cls.admin.cursor() as cursor:
|
|
for statement in statements:
|
|
cursor.execute(statement,args if len(statements)==1 else None)
|
|
result='\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
|
|
return result
|
|
|
|
def setUp(self):
|
|
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
|
|
self.db=ret.Database(ROOT,self.name);self.db.root=self.root
|
|
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
|
|
for table in self.db.tables:
|
|
if table!='__quality_test_owner':self.db.execute('TRUNCATE TABLE '+ret.ident(table))
|
|
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
|
|
self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')")
|
|
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
|
|
self.db.execute("INSERT INTO transform_build VALUES('shared','COMPLETED',%s),('obsolete','COMPLETED',%s)",('1'*64,'2'*64))
|
|
self.db.execute("INSERT INTO release_member VALUES('current','shared'),('old','shared'),('old','obsolete')")
|
|
self.db.execute("INSERT INTO fact_black VALUES(1,'shared','current data'),(2,'obsolete','old data'),(3,'obsolete','old data')")
|
|
|
|
def tearDown(self):self.db.conn.close();self.temp.cleanup()
|
|
|
|
def retire_old(self):
|
|
state={'releases':{'old':1}}
|
|
with self.db.frozen('current'):self.assertEqual(self.db.release_step('current',state,1800,4000),1)
|
|
|
|
def test_running_queued_retry_and_unknown_tasks_prevent_cleanup(self):
|
|
for status,retry in [('RUNNING',False),('QUEUED',False),('FAILED',True),('FUTURE_STATE',False)]:
|
|
self.db.execute('DELETE FROM task')
|
|
self.db.execute('INSERT INTO task VALUES(1,%s,%s,0,3)',(status,datetime_from_future() if retry else None))
|
|
with self.assertRaises(ret.Busy):
|
|
with self.db.frozen('current'):self.fail('busy job was ignored')
|
|
self.assertEqual(len(self.db.rows('SELECT * FROM data_release')),2)
|
|
|
|
def test_pending_candidate_and_saved_business_references_are_protected(self):
|
|
self.db.execute("INSERT INTO data_release VALUES('candidate','CANDIDATE','old','old')")
|
|
with self.db.frozen('current'):
|
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
|
|
self.db.execute("DELETE FROM data_release WHERE id='candidate'")
|
|
self.db.execute("INSERT INTO saved_business_object VALUES(1,'old')")
|
|
with self.db.frozen('current'):
|
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
|
|
|
|
def test_analysis_session_reference_is_protected_without_database_fk(self):
|
|
self.db.execute("INSERT INTO analysis_session VALUES(1,'old')")
|
|
with self.db.frozen('current'):
|
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
|
|
|
|
def test_pending_source_configuration_preserves_its_old_baseline(self):
|
|
self.db.execute("INSERT INTO source_candidate VALUES('pending','DISCOVERED','slot','path',NULL,%s,'legacy','old')",('3'*64,))
|
|
with self.db.frozen('current'):
|
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':1}},1800,4000),0)
|
|
|
|
def test_unknown_business_build_reference_is_preserved(self):
|
|
self.retire_old();self.db.execute("INSERT INTO unknown_build_use VALUES(1,'obsolete')")
|
|
with self.db.frozen('current'):
|
|
self.assertEqual(self.db.build_step({'builds':{'obsolete':1}},1800,4000,1),(0,0))
|
|
self.assertEqual(self.db.rows("SELECT status FROM transform_build WHERE id='obsolete'")[0]['status'],'COMPLETED')
|
|
|
|
def test_changed_publication_and_grace_period_prevent_cleanup(self):
|
|
with self.assertRaises(ret.Busy):
|
|
with self.db.frozen('old'):pass
|
|
with self.db.frozen('current'):
|
|
self.assertEqual(self.db.release_step('current',{'releases':{'old':3900}},1800,4000),0)
|
|
|
|
def test_old_release_removed_but_current_membership_and_rows_preserved(self):
|
|
self.db.execute("INSERT INTO import_run VALUES(1,'COMPLETED','old','old')")
|
|
self.db.execute("INSERT INTO release_action VALUES(1,'old',NULL)")
|
|
self.db.execute("INSERT INTO candidate_reconciliation VALUES(1,'current','old')")
|
|
self.retire_old()
|
|
self.assertEqual(self.db.rows("SELECT payload FROM fact_black WHERE build_id='shared'")[0]['payload'],'current data')
|
|
self.assertEqual(self.db.rows("SELECT * FROM release_member"),[{'release_id':'current','transform_build_id':'shared'}])
|
|
self.assertIsNone(self.db.rows('SELECT * FROM import_run')[0]['expected_current_release_id'])
|
|
self.assertIsNone(self.db.rows("SELECT * FROM data_release WHERE id='current'")[0]['parent_release_id'])
|
|
|
|
def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self):
|
|
self.retire_old();state={'builds':{'obsolete':1}}
|
|
self.db.execute("INSERT INTO source_filter_build VALUES('obsolete',1)")
|
|
self.db.execute("INSERT INTO source_filter_field VALUES(9,'obsolete','black','空字段',%s),(10,'obsolete','black','客户',%s),(11,'obsolete','black','工单',%s)", (b'0'*32,b'1'*32,b'2'*32))
|
|
self.db.execute("INSERT INTO source_filter_value VALUES(100,10,%s,'客户A'),(101,10,%s,'客户B'),(102,11,%s,'工单A')", (b'1'*32,b'2'*32,b'3'*32))
|
|
self.db.execute("INSERT INTO source_filter_member VALUES(10,1,100),(10,2,101),(11,3,102)")
|
|
with self.db.frozen('current'):
|
|
self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0))
|
|
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_member')[0]['n'],2)
|
|
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_value')[0]['n'],3)
|
|
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_field')[0]['n'],3)
|
|
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_build')[0]['n'],1)
|
|
old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0]
|
|
self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64)
|
|
self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,))
|
|
for _ in range(20):
|
|
with self.db.frozen('current'):self.db.build_step(state,1800,4000,1)
|
|
self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'"))
|
|
for table in ('source_filter_member','source_filter_value','source_filter_field','source_filter_build'):
|
|
self.assertFalse(self.db.rows('SELECT * FROM '+ret.ident(table)))
|
|
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'"))
|
|
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'"))
|
|
|
|
def add_draft(self,candidate=None,where=None,age=25):
|
|
path=where or self.root/'runtimedata/archive/upload-drafts/upload-fixture.xlsx'
|
|
path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'workbook bytes')
|
|
self.db.execute('INSERT INTO source_upload_draft VALUES(%s,%s,%s,%s,%s,%s,TIMESTAMPADD(HOUR,-30,CURRENT_TIMESTAMP),TIMESTAMPADD(HOUR,%s,CURRENT_TIMESTAMP))',('draft','input.xlsx',str(path),br.sha(path),path.stat().st_size,candidate,-age))
|
|
return path
|
|
|
|
def test_expired_unbound_upload_removed_but_fresh_preview_is_kept(self):
|
|
path=self.add_draft()
|
|
count,size=ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
|
self.assertEqual(count,1);self.assertGreater(size,0);self.assertFalse(path.exists())
|
|
self.add_draft(age=0)
|
|
self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br),(0,0))
|
|
|
|
def test_upload_still_referenced_by_pending_candidate_is_kept(self):
|
|
path=self.add_draft(candidate='pending')
|
|
self.db.execute("INSERT INTO source_candidate VALUES('pending','DISCOVERED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path)))
|
|
self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br)[0],1)
|
|
self.assertTrue(path.exists())
|
|
|
|
def test_imported_upload_is_rebound_to_verified_archive_before_cleanup(self):
|
|
path=self.add_draft(candidate='imported');archive=self.root/'runtimedata/archive/source-revisions/formal.xlsx';archive.parent.mkdir();archive.write_bytes(path.read_bytes())
|
|
self.db.execute("INSERT INTO source_document VALUES('doc','slot')")
|
|
self.db.execute("INSERT INTO source_revision VALUES('revision','doc',%s,%s,'legacy',%s)",(str(archive),br.sha(archive),archive.stat().st_size))
|
|
self.db.execute("INSERT INTO source_candidate VALUES('imported','IMPORTED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path)))
|
|
self.assertEqual(ret.cleanup_drafts(self.db,'current',{},time.time(),br)[0],1)
|
|
self.assertFalse(path.exists());self.assertTrue(archive.exists())
|
|
self.assertEqual(self.db.rows('SELECT source_path FROM source_candidate')[0]['source_path'],str(archive))
|
|
|
|
def test_outside_file_is_never_deleted(self):
|
|
path=self.add_draft(where=self.root/'unmanaged/private.xlsx')
|
|
ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
|
self.assertTrue(path.exists())
|
|
|
|
def test_symlink_is_not_followed_or_deleted(self):
|
|
outside=self.root/'important.xlsx';outside.write_bytes(b'keep this')
|
|
link=self.root/'runtimedata/archive/upload-drafts/upload-link.xlsx';link.parent.mkdir(parents=True);link.symlink_to(outside)
|
|
self.db.execute('INSERT INTO source_upload_draft VALUES(%s,%s,%s,%s,%s,NULL,TIMESTAMPADD(HOUR,-30,CURRENT_TIMESTAMP),TIMESTAMPADD(HOUR,-25,CURRENT_TIMESTAMP))',('draft','link.xlsx',str(link),br.sha(outside),outside.stat().st_size))
|
|
ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
|
self.assertTrue(outside.exists());self.assertTrue(link.is_symlink())
|
|
|
|
def test_file_changed_after_checksum_is_not_deleted(self):
|
|
path=self.add_draft();original=ret.fingerprint;calls=0
|
|
def changed(p):
|
|
nonlocal calls
|
|
calls+=1
|
|
if calls==2:path.write_bytes(b'new file contents')
|
|
return original(p)
|
|
with patch.object(ret,'fingerprint',side_effect=changed):
|
|
count,size=ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
|
self.assertEqual((count,size),(0,0));self.assertTrue(path.exists())
|
|
self.assertTrue(self.db.rows('SELECT * FROM source_upload_draft'))
|
|
|
|
def test_new_reference_before_file_unlink_is_rechecked(self):
|
|
path=self.add_draft();freeze=self.db.frozen;calls=0
|
|
@contextmanager
|
|
def insert_reference(expected):
|
|
nonlocal calls
|
|
calls+=1
|
|
with freeze(expected):
|
|
if calls==2:self.db.execute("INSERT INTO source_candidate VALUES('new','DISCOVERED','slot',%s,NULL,%s,'legacy',NULL)",(str(path),br.sha(path)))
|
|
yield
|
|
with patch.object(self.db,'frozen',side_effect=insert_reference):
|
|
ret.cleanup_drafts(self.db,'current',{},time.time(),br)
|
|
self.assertTrue(path.exists());self.assertFalse(self.db.rows('SELECT * FROM source_upload_draft'))
|
|
|
|
def test_long_running_sql_is_not_interrupted_for_cleanup(self):
|
|
other=ret.Database(ROOT,self.name)
|
|
try:
|
|
other.conn.autocommit(False);other.rows('SELECT * FROM fact_black WHERE id=1 FOR UPDATE')
|
|
with self.assertRaises(ret.Busy):
|
|
with self.db.frozen('current'):pass
|
|
self.assertTrue(other.rows('SELECT 1'));other.conn.rollback()
|
|
finally:other.conn.close()
|
|
|
|
|
|
def datetime_from_future():
|
|
from datetime import datetime,timedelta,timezone
|
|
return datetime.now(timezone.utc).replace(tzinfo=None)+timedelta(hours=1)
|
|
|
|
|
|
class PrivateVerifierTests(unittest.TestCase):
|
|
def test_small_backup_restores_on_private_server_and_cleans_up(self):
|
|
import backup
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root=Path(temporary).resolve();(root/'.runtime').mkdir()
|
|
(root/'.runtime/mysql').symlink_to(ROOT/'.runtime/mysql',target_is_directory=True)
|
|
base=root/'output/backups';folder=base/'20261109T020000Z-12345678';folder.mkdir(parents=True)
|
|
source=root/'source';source.mkdir();(source/'config.json').write_text('{}')
|
|
with tarfile.open(folder/'application.tar.gz','w:gz') as tar:tar.add(source/'config.json',arcname='config.json')
|
|
with tarfile.open(folder/'archive.tar.gz','w:gz'):pass
|
|
sql="CREATE TABLE current_release(dataset_id varchar(20),release_id varchar(36)); INSERT INTO current_release VALUES('quality',NULL); CREATE TABLE source_revision(id varchar(36),archive_path varchar(1024),sha256 char(64),file_size bigint);"
|
|
with gzip.open(folder/'database.sql.gz','wt') as out:out.write(sql)
|
|
manifest={'formatVersion':3,'createdAt':'2026-11-09T02:00:00+00:00','files':{},'archiveSourceRoot':str(root/'runtimedata/archive'),
|
|
'applicationFiles':{'config.json':br.sha(source/'config.json')},'requiredApplicationFiles':['config.json'],
|
|
'tableCounts':{'current_release':1,'source_revision':0},'queryEvidence':{'releaseId':None,'months':[]}}
|
|
for name,key in br.ARTIFACTS.items():manifest[key]=br.sha(folder/name)
|
|
br.atomic(folder/'manifest.json',manifest)
|
|
with patch.object(backup,'ROOT',root),patch.object(backup,'BACKUP_ROOT',base),patch.object(backup,'REQUIRED_APPLICATION_FILES',('config.json',)):
|
|
result=backup.verify(folder)
|
|
self.assertEqual(result['status'],'ISOLATED_RESTORE_VERIFIED')
|
|
self.assertFalse(list((root/'.runtime/retention').glob('verify-*')))
|
|
self.assertFalse(list((root/'.runtime/retention/scratch').glob('*.json')))
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|