Files
quality_analyse/deploy/macos/test_revision_retention.py
T

268 lines
17 KiB
Python

"""Exercise source retirement against an owned disposable MySQL schema and files."""
from contextlib import contextmanager
import importlib.util
import json
from pathlib import Path
import sys
import tempfile
import time
import unittest
import uuid
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import backup_retention as br
import managed_test_resources
import revision_retention as revisions
spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py'))
ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret)
class SourceRevisionTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured')
cfg = json.loads((ROOT / '.runtime/local-config.json').read_text())
cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root',
password=cfg['rootPassword'], autocommit=True)
cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
managed_test_resources.create(ROOT, cls.name, cls.execute_admin)
ddl = [
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))',
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))',
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
"CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)",
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))',
'CREATE TABLE source_upload_draft(id varchar(36) PRIMARY KEY,source_path varchar(1000))',
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),payload json)',
'CREATE TABLE source_change_request(id int PRIMARY KEY,original_revision_id varchar(36),status varchar(24))',
'CREATE TABLE saved_context(id int PRIMARY KEY,context_json json,attachment_path varchar(1000))',
'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))']
try:
with cls.admin.cursor() as c:
c.execute('USE ' + ret.ident(cls.name))
for statement in ddl:
c.execute(statement + ' ENGINE=InnoDB')
except BaseException:
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
cls.admin.close(); raise
@classmethod
def tearDownClass(cls):
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
cls.admin.close()
@classmethod
def execute_admin(cls, sql, args=None):
result = ''
statements = [part.strip() for part in sql.split(';') if part.strip()]
with cls.admin.cursor() as cursor:
for statement in statements:
cursor.execute(statement, args if len(statements) == 1 else None)
result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
return result
def setUp(self):
self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve()
self.db = ret.Database(ROOT, self.name); self.db.root = self.root
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
for table in self.db.tables:
if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')")
self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')")
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
self.current_file = self.add_revision('current-rev', b'current workbook')
self.db.execute("INSERT INTO release_member VALUES('current','current-rev')")
self.db.execute("INSERT INTO transform_build VALUES('current-build','current-rev')")
self.cleaner = revisions.Cleaner(self.db, br, ret.safe_file, ret.fingerprint, ret.Busy)
self.state = {}
def tearDown(self):
self.db.conn.close(); self.temp.cleanup()
def add_revision(self, rid='old-rev-unique', content=b'old workbook'):
digest = __import__('hashlib').sha256(content).hexdigest()
path = self.root / 'runtimedata/archive/source-revisions/black/2026/01' / (digest + '.xlsx')
path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(content)
self.db.execute('INSERT INTO source_revision(id,source_document_id,archive_path,sha256,file_size) VALUES(%s,%s,%s,%s,%s)',
(rid, 'doc', str(path), digest, len(content)))
return path
def collect(self, now):
return self.cleaner.collect('current', self.state, now)
def present(self, rid='old-rev-unique'):
return bool(self.db.rows('SELECT id FROM source_revision WHERE id=%s', (rid,)))
def test_unreferenced_revision_waits_grace_then_removes_only_its_owned_file(self):
path = self.add_revision()
self.assertEqual(self.collect(100)['sourceRevisionsRemoved'], 0)
result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertEqual(result['sourceArchiveFilesRemoved'], 1)
self.assertEqual(result['sourceArchiveBytesRemoved'], len(b'old workbook'))
self.assertFalse(self.present()); self.assertFalse(path.exists())
self.assertTrue(self.present('current-rev')); self.assertTrue(self.current_file.exists())
self.assertTrue(self.db.rows('SELECT * FROM source_document'))
def test_any_release_build_fact_or_unknown_foreign_key_protects_revision(self):
for rid, statement in [
('candidate', "INSERT INTO release_member VALUES('candidate','candidate')"),
('build', "INSERT INTO transform_build VALUES('old-build','build')"),
('fact', "INSERT INTO fact_black VALUES(1,'fact',JSON_OBJECT(),JSON_OBJECT())"),
('future', "INSERT INTO unknown_reference VALUES(1,'future')")]:
self.add_revision(rid, rid.encode()); self.db.execute(statement)
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
for rid in ('candidate', 'build', 'fact', 'future'): self.assertTrue(self.present(rid))
def test_replacement_withdrawal_and_saved_json_ids_without_fks_are_protected(self):
for rid in ('replacement', 'withdrawal', 'lineage', 'saved'): self.add_revision(rid, rid.encode())
self.db.execute("INSERT INTO source_candidate VALUES('candidate','DISCOVERED',NULL,NULL,NULL,'replacement')")
self.db.execute("INSERT INTO source_change_request VALUES(1,'withdrawal','APPROVED')")
self.db.execute("UPDATE source_revision SET replaces_revision_id='lineage' WHERE id='current-rev'")
self.db.execute("INSERT INTO saved_context VALUES(1,JSON_OBJECT('sourceRevisionId','saved'),NULL)")
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
def test_catalog_hash_protects_original_even_with_external_path(self):
path = self.add_revision(); digest = br.sha(path)
self.db.execute("INSERT INTO source_candidate VALUES('catalog','IMPORTED',%s,'/external/original.xlsx',NULL,NULL)", (digest,))
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
self.assertTrue(self.present()); self.assertTrue(path.exists())
def test_indexed_business_values_do_not_count_as_revision_metadata(self):
path = self.add_revision('indexed-only', b'indexed-only workbook')
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,))
self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,))
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists())
def test_indexed_field_names_do_not_count_as_revision_metadata(self):
path = self.add_revision('indexed-field-only', b'field-only workbook')
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,))
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists())
def test_new_reference_resets_continuous_grace(self):
self.add_revision(); self.collect(100)
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500)
self.db.execute('DELETE FROM unknown_reference')
self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
self.assertEqual(self.collect(3801)['sourceRevisionsRemoved'], 1)
def test_revision_metadata_change_restarts_grace(self):
self.add_revision(); self.collect(100)
self.db.execute("UPDATE source_revision SET parser_config=JSON_OBJECT('changed',TRUE) WHERE id='old-rev-unique'")
self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
self.assertEqual(self.collect(3801)['sourceRevisionsRemoved'], 1)
def test_shared_archive_stays_until_its_other_revision_is_unreferenced(self):
path = self.add_revision(content=b'current workbook')
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertEqual(result['sourceArchiveFilesRemoved'], 0)
self.assertTrue(path.exists()); self.assertTrue(self.present('current-rev'))
def test_saved_file_path_protects_file_after_metadata_removal(self):
path = self.add_revision()
self.db.execute('INSERT INTO saved_context VALUES(1,JSON_OBJECT(\'archivePath\',%s),NULL)', (str(path),))
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertEqual(result['sourceFilesPending'], 1); self.assertTrue(path.exists())
self.db.execute('DELETE FROM saved_context')
self.assertEqual(self.cleaner.drain('current')[0], 1); self.assertFalse(path.exists())
def test_external_symlink_mismatched_or_incomplete_files_are_kept(self):
for rid in ('outside', 'symlink', 'damaged', 'incomplete'): self.add_revision(rid, rid.encode())
external = self.root / 'important.xlsx'; external.write_bytes(b'outside')
self.db.execute("UPDATE source_revision SET archive_path=%s WHERE id='outside'", (str(external),))
row = self.db.rows("SELECT archive_path FROM source_revision WHERE id='symlink'")[0]
link = Path(row['archive_path']); link.unlink(); link.symlink_to(external)
row = self.db.rows("SELECT archive_path FROM source_revision WHERE id='damaged'")[0]
Path(row['archive_path']).write_bytes(b'changed')
self.db.execute("UPDATE source_revision SET status='RECEIVED' WHERE id='incomplete'")
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
for rid in ('outside', 'symlink', 'damaged', 'incomplete'): self.assertTrue(self.present(rid))
self.assertTrue(external.exists()); self.assertTrue(link.is_symlink())
def test_reference_added_before_database_delete_is_rechecked(self):
path = self.add_revision(); self.collect(100); original = self.db.frozen
@contextmanager
def add_reference(current):
with original(current):
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')")
yield
with patch.object(self.db, 'frozen', side_effect=add_reference): result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 0)
self.assertTrue(path.exists()); self.assertTrue(self.present())
def test_file_changed_after_hash_is_kept(self):
path = self.add_revision(); self.collect(100); original = self.db.frozen
@contextmanager
def change_file(current):
path.write_bytes(b'file edited during preflight')
with original(current): yield
with patch.object(self.db, 'frozen', side_effect=change_file): result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 0)
self.assertTrue(path.exists()); self.assertTrue(self.present())
def test_crash_after_database_commit_resumes_registered_file_removal(self):
path = self.add_revision(); self.collect(100)
with patch.object(self.cleaner, 'drain', side_effect=RuntimeError('interrupted')):
with self.assertRaises(RuntimeError): self.collect(2000)
self.assertFalse(self.present()); self.assertTrue(path.exists())
self.assertEqual(self.cleaner.drain('current')[0], 1)
self.assertFalse(path.exists()); self.assertTrue(self.current_file.exists())
def test_database_rollback_keeps_file_and_discards_only_journal(self):
path = self.add_revision(); self.collect(100); original = self.db.execute
def rollback(sql, args=()):
result = original(sql, args)
if sql.startswith('DELETE FROM source_revision'): raise RuntimeError('rollback')
return result
with patch.object(self.db, 'execute', side_effect=rollback):
with self.assertRaises(RuntimeError): self.collect(2000)
self.assertTrue(self.present()); self.assertTrue(path.exists())
self.assertEqual(self.cleaner.drain('current'), (0, 0))
self.assertFalse(list(self.cleaner.journals.glob('*.json')))
def test_new_path_reference_before_unlink_protects_file(self):
path = self.add_revision(); self.collect(100); original = self.cleaner.drain
def add_reference(current, limit=5):
self.db.execute('INSERT INTO source_upload_draft VALUES(\'new\',%s)', (str(path),))
return original(current, limit)
with patch.object(self.cleaner, 'drain', side_effect=add_reference): result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertEqual(result['sourceArchiveFilesRemoved'], 0); self.assertTrue(path.exists())
def test_running_task_or_changed_publication_defers_without_deleting(self):
path = self.add_revision(); self.collect(100)
self.db.execute("INSERT INTO task VALUES(1,'RUNNING',NULL)")
with self.assertRaises(ret.Busy): self.collect(2000)
self.assertTrue(self.present()); self.assertTrue(path.exists())
self.db.execute('DELETE FROM task')
self.db.execute("UPDATE current_release SET release_id='different'")
with self.assertRaises(ret.Busy): self.collect(2000)
self.assertTrue(self.present()); self.assertTrue(path.exists())
def test_unsafe_registry_and_unknown_stray_files_are_not_deleted(self):
path = self.add_revision(); self.collect(100)
self.cleaner.journals.mkdir(parents=True)
br.atomic(self.cleaner.journals / ('1' * 32 + '.json'), {'kind': 'quality-source-retirement'})
unknown = path.with_name('unregistered.xlsx'); unknown.write_bytes(b'keep')
self.cleaner.drain('current'); self.assertTrue(unknown.exists()); self.assertTrue(path.exists())
with self.cleaner.quick_checks():
self.cleaner.reference_deadline = time.monotonic() - 1
with self.assertRaises(ret.Busy): self.cleaner.exists('source_revision', 'id=%s', ('old-rev-unique',))
if __name__ == '__main__': unittest.main()