268 lines
17 KiB
Python
268 lines
17 KiB
Python
"""Exercise source retirement against an owned disposable MySQL schema and files."""
|
|
from contextlib import contextmanager
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
import uuid
|
|
from unittest.mock import patch
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
import backup_retention as br
|
|
import managed_test_resources
|
|
import revision_retention as revisions
|
|
spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py'))
|
|
ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret)
|
|
|
|
|
|
class SourceRevisionTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured')
|
|
cfg = json.loads((ROOT / '.runtime/local-config.json').read_text())
|
|
cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root',
|
|
password=cfg['rootPassword'], autocommit=True)
|
|
cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
|
|
managed_test_resources.create(ROOT, cls.name, cls.execute_admin)
|
|
ddl = [
|
|
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))',
|
|
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))',
|
|
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
|
|
"CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)",
|
|
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
|
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
|
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
|
|
'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
|
'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
|
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))',
|
|
'CREATE TABLE source_upload_draft(id varchar(36) PRIMARY KEY,source_path varchar(1000))',
|
|
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),payload json)',
|
|
'CREATE TABLE source_change_request(id int PRIMARY KEY,original_revision_id varchar(36),status varchar(24))',
|
|
'CREATE TABLE saved_context(id int PRIMARY KEY,context_json json,attachment_path varchar(1000))',
|
|
'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))']
|
|
try:
|
|
with cls.admin.cursor() as c:
|
|
c.execute('USE ' + ret.ident(cls.name))
|
|
for statement in ddl:
|
|
c.execute(statement + ' ENGINE=InnoDB')
|
|
except BaseException:
|
|
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
|
|
cls.admin.close(); raise
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
|
|
cls.admin.close()
|
|
|
|
@classmethod
|
|
def execute_admin(cls, sql, args=None):
|
|
result = ''
|
|
statements = [part.strip() for part in sql.split(';') if part.strip()]
|
|
with cls.admin.cursor() as cursor:
|
|
for statement in statements:
|
|
cursor.execute(statement, args if len(statements) == 1 else None)
|
|
result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
|
|
return result
|
|
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve()
|
|
self.db = ret.Database(ROOT, self.name); self.db.root = self.root
|
|
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
|
|
for table in self.db.tables:
|
|
if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
|
|
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
|
|
self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')")
|
|
self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')")
|
|
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
|
|
self.current_file = self.add_revision('current-rev', b'current workbook')
|
|
self.db.execute("INSERT INTO release_member VALUES('current','current-rev')")
|
|
self.db.execute("INSERT INTO transform_build VALUES('current-build','current-rev')")
|
|
self.cleaner = revisions.Cleaner(self.db, br, ret.safe_file, ret.fingerprint, ret.Busy)
|
|
self.state = {}
|
|
|
|
def tearDown(self):
|
|
self.db.conn.close(); self.temp.cleanup()
|
|
|
|
def add_revision(self, rid='old-rev-unique', content=b'old workbook'):
|
|
digest = __import__('hashlib').sha256(content).hexdigest()
|
|
path = self.root / 'runtimedata/archive/source-revisions/black/2026/01' / (digest + '.xlsx')
|
|
path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(content)
|
|
self.db.execute('INSERT INTO source_revision(id,source_document_id,archive_path,sha256,file_size) VALUES(%s,%s,%s,%s,%s)',
|
|
(rid, 'doc', str(path), digest, len(content)))
|
|
return path
|
|
|
|
def collect(self, now):
|
|
return self.cleaner.collect('current', self.state, now)
|
|
|
|
def present(self, rid='old-rev-unique'):
|
|
return bool(self.db.rows('SELECT id FROM source_revision WHERE id=%s', (rid,)))
|
|
|
|
def test_unreferenced_revision_waits_grace_then_removes_only_its_owned_file(self):
|
|
path = self.add_revision()
|
|
self.assertEqual(self.collect(100)['sourceRevisionsRemoved'], 0)
|
|
result = self.collect(2000)
|
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
|
self.assertEqual(result['sourceArchiveFilesRemoved'], 1)
|
|
self.assertEqual(result['sourceArchiveBytesRemoved'], len(b'old workbook'))
|
|
self.assertFalse(self.present()); self.assertFalse(path.exists())
|
|
self.assertTrue(self.present('current-rev')); self.assertTrue(self.current_file.exists())
|
|
self.assertTrue(self.db.rows('SELECT * FROM source_document'))
|
|
|
|
def test_any_release_build_fact_or_unknown_foreign_key_protects_revision(self):
|
|
for rid, statement in [
|
|
('candidate', "INSERT INTO release_member VALUES('candidate','candidate')"),
|
|
('build', "INSERT INTO transform_build VALUES('old-build','build')"),
|
|
('fact', "INSERT INTO fact_black VALUES(1,'fact',JSON_OBJECT(),JSON_OBJECT())"),
|
|
('future', "INSERT INTO unknown_reference VALUES(1,'future')")]:
|
|
self.add_revision(rid, rid.encode()); self.db.execute(statement)
|
|
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
|
for rid in ('candidate', 'build', 'fact', 'future'): self.assertTrue(self.present(rid))
|
|
|
|
def test_replacement_withdrawal_and_saved_json_ids_without_fks_are_protected(self):
|
|
for rid in ('replacement', 'withdrawal', 'lineage', 'saved'): self.add_revision(rid, rid.encode())
|
|
self.db.execute("INSERT INTO source_candidate VALUES('candidate','DISCOVERED',NULL,NULL,NULL,'replacement')")
|
|
self.db.execute("INSERT INTO source_change_request VALUES(1,'withdrawal','APPROVED')")
|
|
self.db.execute("UPDATE source_revision SET replaces_revision_id='lineage' WHERE id='current-rev'")
|
|
self.db.execute("INSERT INTO saved_context VALUES(1,JSON_OBJECT('sourceRevisionId','saved'),NULL)")
|
|
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
|
|
|
def test_catalog_hash_protects_original_even_with_external_path(self):
|
|
path = self.add_revision(); digest = br.sha(path)
|
|
self.db.execute("INSERT INTO source_candidate VALUES('catalog','IMPORTED',%s,'/external/original.xlsx',NULL,NULL)", (digest,))
|
|
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
|
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
|
|
|
def test_indexed_business_values_do_not_count_as_revision_metadata(self):
|
|
path = self.add_revision('indexed-only', b'indexed-only workbook')
|
|
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,))
|
|
self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,))
|
|
self.collect(100); result = self.collect(2000)
|
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
|
self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists())
|
|
|
|
def test_indexed_field_names_do_not_count_as_revision_metadata(self):
|
|
path = self.add_revision('indexed-field-only', b'field-only workbook')
|
|
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,))
|
|
self.collect(100); result = self.collect(2000)
|
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
|
self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists())
|
|
|
|
def test_new_reference_resets_continuous_grace(self):
|
|
self.add_revision(); self.collect(100)
|
|
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500)
|
|
self.db.execute('DELETE FROM unknown_reference')
|
|
self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
|
self.assertEqual(self.collect(3801)['sourceRevisionsRemoved'], 1)
|
|
|
|
def test_revision_metadata_change_restarts_grace(self):
|
|
self.add_revision(); self.collect(100)
|
|
self.db.execute("UPDATE source_revision SET parser_config=JSON_OBJECT('changed',TRUE) WHERE id='old-rev-unique'")
|
|
self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
|
self.assertEqual(self.collect(3801)['sourceRevisionsRemoved'], 1)
|
|
|
|
def test_shared_archive_stays_until_its_other_revision_is_unreferenced(self):
|
|
path = self.add_revision(content=b'current workbook')
|
|
self.collect(100); result = self.collect(2000)
|
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
|
self.assertEqual(result['sourceArchiveFilesRemoved'], 0)
|
|
self.assertTrue(path.exists()); self.assertTrue(self.present('current-rev'))
|
|
|
|
def test_saved_file_path_protects_file_after_metadata_removal(self):
|
|
path = self.add_revision()
|
|
self.db.execute('INSERT INTO saved_context VALUES(1,JSON_OBJECT(\'archivePath\',%s),NULL)', (str(path),))
|
|
self.collect(100); result = self.collect(2000)
|
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
|
self.assertEqual(result['sourceFilesPending'], 1); self.assertTrue(path.exists())
|
|
self.db.execute('DELETE FROM saved_context')
|
|
self.assertEqual(self.cleaner.drain('current')[0], 1); self.assertFalse(path.exists())
|
|
|
|
def test_external_symlink_mismatched_or_incomplete_files_are_kept(self):
|
|
for rid in ('outside', 'symlink', 'damaged', 'incomplete'): self.add_revision(rid, rid.encode())
|
|
external = self.root / 'important.xlsx'; external.write_bytes(b'outside')
|
|
self.db.execute("UPDATE source_revision SET archive_path=%s WHERE id='outside'", (str(external),))
|
|
row = self.db.rows("SELECT archive_path FROM source_revision WHERE id='symlink'")[0]
|
|
link = Path(row['archive_path']); link.unlink(); link.symlink_to(external)
|
|
row = self.db.rows("SELECT archive_path FROM source_revision WHERE id='damaged'")[0]
|
|
Path(row['archive_path']).write_bytes(b'changed')
|
|
self.db.execute("UPDATE source_revision SET status='RECEIVED' WHERE id='incomplete'")
|
|
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
|
for rid in ('outside', 'symlink', 'damaged', 'incomplete'): self.assertTrue(self.present(rid))
|
|
self.assertTrue(external.exists()); self.assertTrue(link.is_symlink())
|
|
|
|
def test_reference_added_before_database_delete_is_rechecked(self):
|
|
path = self.add_revision(); self.collect(100); original = self.db.frozen
|
|
@contextmanager
|
|
def add_reference(current):
|
|
with original(current):
|
|
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')")
|
|
yield
|
|
with patch.object(self.db, 'frozen', side_effect=add_reference): result = self.collect(2000)
|
|
self.assertEqual(result['sourceRevisionsRemoved'], 0)
|
|
self.assertTrue(path.exists()); self.assertTrue(self.present())
|
|
|
|
def test_file_changed_after_hash_is_kept(self):
|
|
path = self.add_revision(); self.collect(100); original = self.db.frozen
|
|
@contextmanager
|
|
def change_file(current):
|
|
path.write_bytes(b'file edited during preflight')
|
|
with original(current): yield
|
|
with patch.object(self.db, 'frozen', side_effect=change_file): result = self.collect(2000)
|
|
self.assertEqual(result['sourceRevisionsRemoved'], 0)
|
|
self.assertTrue(path.exists()); self.assertTrue(self.present())
|
|
|
|
def test_crash_after_database_commit_resumes_registered_file_removal(self):
|
|
path = self.add_revision(); self.collect(100)
|
|
with patch.object(self.cleaner, 'drain', side_effect=RuntimeError('interrupted')):
|
|
with self.assertRaises(RuntimeError): self.collect(2000)
|
|
self.assertFalse(self.present()); self.assertTrue(path.exists())
|
|
self.assertEqual(self.cleaner.drain('current')[0], 1)
|
|
self.assertFalse(path.exists()); self.assertTrue(self.current_file.exists())
|
|
|
|
def test_database_rollback_keeps_file_and_discards_only_journal(self):
|
|
path = self.add_revision(); self.collect(100); original = self.db.execute
|
|
def rollback(sql, args=()):
|
|
result = original(sql, args)
|
|
if sql.startswith('DELETE FROM source_revision'): raise RuntimeError('rollback')
|
|
return result
|
|
with patch.object(self.db, 'execute', side_effect=rollback):
|
|
with self.assertRaises(RuntimeError): self.collect(2000)
|
|
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
|
self.assertEqual(self.cleaner.drain('current'), (0, 0))
|
|
self.assertFalse(list(self.cleaner.journals.glob('*.json')))
|
|
|
|
def test_new_path_reference_before_unlink_protects_file(self):
|
|
path = self.add_revision(); self.collect(100); original = self.cleaner.drain
|
|
def add_reference(current, limit=5):
|
|
self.db.execute('INSERT INTO source_upload_draft VALUES(\'new\',%s)', (str(path),))
|
|
return original(current, limit)
|
|
with patch.object(self.cleaner, 'drain', side_effect=add_reference): result = self.collect(2000)
|
|
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
|
self.assertEqual(result['sourceArchiveFilesRemoved'], 0); self.assertTrue(path.exists())
|
|
|
|
def test_running_task_or_changed_publication_defers_without_deleting(self):
|
|
path = self.add_revision(); self.collect(100)
|
|
self.db.execute("INSERT INTO task VALUES(1,'RUNNING',NULL)")
|
|
with self.assertRaises(ret.Busy): self.collect(2000)
|
|
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
|
self.db.execute('DELETE FROM task')
|
|
self.db.execute("UPDATE current_release SET release_id='different'")
|
|
with self.assertRaises(ret.Busy): self.collect(2000)
|
|
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
|
|
|
def test_unsafe_registry_and_unknown_stray_files_are_not_deleted(self):
|
|
path = self.add_revision(); self.collect(100)
|
|
self.cleaner.journals.mkdir(parents=True)
|
|
br.atomic(self.cleaner.journals / ('1' * 32 + '.json'), {'kind': 'quality-source-retirement'})
|
|
unknown = path.with_name('unregistered.xlsx'); unknown.write_bytes(b'keep')
|
|
self.cleaner.drain('current'); self.assertTrue(unknown.exists()); self.assertTrue(path.exists())
|
|
with self.cleaner.quick_checks():
|
|
self.cleaner.reference_deadline = time.monotonic() - 1
|
|
with self.assertRaises(ret.Busy): self.cleaner.exists('source_revision', 'id=%s', ('old-rev-unique',))
|
|
|
|
|
|
if __name__ == '__main__': unittest.main()
|