import importlib.util import json import os from pathlib import Path import subprocess import sys import tempfile import time import unittest import uuid from unittest.mock import patch ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'scripts')) import backup import backup_retention as br import log_retention as logs import managed_test_resources as resources spec = importlib.util.spec_from_file_location('quality_operations', Path(__file__).with_name('operations.py')) ops = importlib.util.module_from_spec(spec); spec.loader.exec_module(ops) class OperationsTests(unittest.TestCase): def test_native_logs_are_counted_once_and_do_not_hide_other_runtime_artifacts(self): with tempfile.TemporaryDirectory() as d: root=Path(d).resolve();native=root/'.runtime/nao-users/actor' for relative in ['logs/server.log','private/nao.sqlite']: path=native/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'x'*8192) (root/'.runtime/tool.bin').write_bytes(b'x'*16384) sample=ops.snapshot(root,root/'support') self.assertEqual(sample['coreBytes'],sum(sample[k] for k in ['mysqlBytes','archiveBytes','backupBytes','logBytes','nativeDataBytes'])) self.assertEqual(sample['artifactBytes'],ops.allocated(root/'.runtime')-ops.allocated(root/'.runtime/nao-users')) def sample(self, **extra): return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB, 'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra} def test_capacity_levels_free_space_failures_and_growth_are_separate(self): self.assertEqual(ops.conditions(self.sample(), ops.DEFAULTS, {}, []), {}) self.assertEqual(ops.conditions(self.sample(coreBytes=30*ops.GIB), ops.DEFAULTS, {}, [])['capacity']['severity'], 'warning') self.assertIn('temporary',ops.conditions(self.sample(temporaryBytes=30*ops.GIB),ops.DEFAULTS,{},[])) self.assertIn('artifacts',ops.conditions(self.sample(artifactBytes=50*ops.GIB),ops.DEFAULTS,{},[])) found = ops.conditions(self.sample(coreBytes=50*ops.GIB, freeBytes=19*ops.GIB, overdueTestResources=1), ops.DEFAULTS, {'consecutiveFailures': 3}, []) self.assertEqual(set(found), {'capacity', 'disk', 'maintenance', 'tests'}) old = self.sample(at=200000-86400, permanentBytes=30*ops.GIB) self.assertIn('growth', ops.conditions(self.sample(), ops.DEFAULTS, {}, [old])) self.assertNotIn('maintenance', ops.conditions(self.sample(), ops.DEFAULTS, {'lastDeferred': {'reason': 'active'}}, [])) old = self.sample(at=200000-86400) self.assertIn('growth',ops.conditions(self.sample(coreBytes=15*ops.GIB,permanentBytes=43*ops.GIB),ops.DEFAULTS,{},[old])) def test_monitor_failure_notifies_after_three_attempts_without_repeated_noise(self): with tempfile.TemporaryDirectory() as d: root=Path(d).resolve();sent=[] def notify(*_):sent.append(True);return {'submitted':True} for _ in range(4):ops.failed_run(root,RuntimeError('not logged'),notify) self.assertEqual(len(sent),1) self.assertEqual(json.loads((root/'.runtime/operations/failure.json').read_text())['count'],4) def test_alert_deduplication_escalation_recovery_and_failed_delivery_retry(self): sent = [] def notify(title, message): sent.append(message); return {'submitted': True} warning = {'capacity': {'severity': 'warning', 'message': 'warning'}} first, events = ops.dispatch(warning, {}, 100, ops.DEFAULTS, notify) second, events = ops.dispatch(warning, first, 200, ops.DEFAULTS, notify) self.assertEqual(len(sent), 1); self.assertFalse(events) critical = {'capacity': {'severity': 'critical', 'message': 'critical'}} third, _ = ops.dispatch(critical, second, 300, ops.DEFAULTS, notify) self.assertEqual(len(sent), 2) _, events = ops.dispatch({}, third, 400, ops.DEFAULTS, notify) self.assertEqual(events[0]['kind'], 'resolved') failed, _ = ops.dispatch(warning, {}, 500, ops.DEFAULTS, lambda *_: {'submitted': False}) ops.dispatch(warning, failed, 600, ops.DEFAULTS, notify) self.assertEqual(len(sent), 3) def test_registered_and_private_temporary_storage_is_not_permanent_growth(self): with tempfile.TemporaryDirectory() as d: root = Path(d).resolve(); support = root / 'support' name = 'quality_retention_test_' + uuid.uuid4().hex[:12] for path in [root/'.runtime/mysql-data/current/data.ibd', root/'.runtime/mysql-data'/name/'sample.ibd', root/'output/backups/current/file.gz', root/'runtimedata/archive/source.xlsx']: path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(b'x'*8192) br.atomic(resources.registry(root)/(name+'.json'), {'kind':'quality-owned-test-db','root':str(root), 'database':name,'token':'1'*32,'created':True,'expiresAt':time.time()+86400}) private=root/'.runtime/retention/verify-fixture';private.mkdir(parents=True) (private/'data').write_bytes(b'x'*16384) br.atomic(private/'owner.json',{'kind':'private-quality-restore','directory':str(private)}) sample=ops.snapshot(root,support) self.assertGreater(sample['registeredTestBytes'],0); self.assertGreater(sample['privateVerifierBytes'],0) self.assertLess(sample['coreBytes'],sample['mysqlBytes']+sample['backupBytes']+sample['archiveBytes']) def test_monitor_still_collects_when_housekeeping_is_deferred_and_history_is_bounded(self): with tempfile.TemporaryDirectory() as d: root=Path(d).resolve();base=root/'.runtime/operations';base.mkdir(parents=True) br.atomic(base/'policy.json',ops.DEFAULTS) history=[self.sample(at=time.time()-i*100) for i in range(450)] br.atomic(base/'samples.json',history) with patch.object(ops,'snapshot',return_value=self.sample(at=time.time())), \ patch.object(ops,'janitor',return_value={'deferred':'Application jobs are active'}): result=ops.run(root,root/'support',lambda *_:{'submitted':True}) self.assertIn('sample',result);self.assertIn('deferred',result['cleanup']) self.assertLessEqual(len(json.loads((base/'samples.json').read_text())),330) class LogRetentionTests(unittest.TestCase): def setUp(self): self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve();self.support=self.root/'support' def tearDown(self):self.temp.cleanup() def file(self,name,age=0,size=100): p=self.root/'output/logs'/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(b'line\n'*(size//5)) os.utime(p,(time.time()-age,time.time()-age));return p def test_old_closed_logs_removed_but_active_symlinks_and_business_files_preserved(self): old=self.file('old.log',age=8*86400);active=self.file('backend.log',age=8*86400) other=self.file('worker.log',age=8*86400);outside=self.root/'important.xlsx';outside.write_bytes(b'keep') link=old.with_name('link.log');link.symlink_to(outside) result=logs.retain(self.root,self.support,open_paths={str(other)}) self.assertEqual(result['logsRemoved'],1);self.assertFalse(old.exists()) self.assertTrue(active.exists());self.assertTrue(other.exists());self.assertTrue(outside.exists());self.assertTrue(link.is_symlink()) def test_per_class_cap_prunes_old_archives_and_bounds_a_closed_console_tail(self): old=self.file('worker.log.1',age=100,size=100);new=self.file('worker.log',age=0,size=100) logs.retain(self.root,self.support,open_paths=set(),max_bytes=150) self.assertFalse(old.exists());self.assertTrue(new.exists()) big=self.file('huge.log',size=1000) logs.retain(self.root,self.support,open_paths=set(),max_bytes=100) self.assertLessEqual(big.stat().st_size,100) def test_mysql_rotation_uses_reopen_and_recovers_after_interruption(self): source=self.file('mysql-server.log',age=86400,size=100);original=source.read_bytes() with self.assertRaises(RuntimeError): logs.rotate_mysql(self.root,lambda *_:(_ for _ in ()).throw(RuntimeError('interrupted')),br) self.assertFalse(source.exists());self.assertTrue((self.root/'.runtime/operations/mysql-log-rotation.json').exists()) calls=[] def reopen(sql,_):calls.append(sql);source.write_bytes(b'new log') self.assertEqual(logs.rotate_mysql(self.root,reopen,br),1) self.assertEqual(calls,['FLUSH ERROR LOGS;']) archived=list(source.parent.glob('mysql-server.roll-*.log'));self.assertEqual(archived[0].read_bytes(),original) self.assertEqual(source.read_bytes(),b'new log') class ManagedResourceTests(unittest.TestCase): @classmethod def setUpClass(cls): if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured') def setUp(self): self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve() cfg=json.loads((ROOT/'.runtime/local-config.json').read_text()) br.atomic(self.root/'.runtime/local-config.json',{'mysqlDatabase':cfg['mysqlDatabase']}) self.names=[];self.users=[] def tearDown(self): for n in self.names:backup.client('DROP DATABASE IF EXISTS `'+n+'`;') for u in self.users:backup.client("DROP USER IF EXISTS '"+u+"'@'127.0.0.1';") self.temp.cleanup() def create(self,empty=False): name='quality_retention_test_'+uuid.uuid4().hex[:12];self.names.append(name) resources.create(self.root,name,backup.client,empty=empty);return name def test_successful_create_and_drop_unregisters_only_its_schema(self): name=self.create();self.assertTrue(resources.load(self.root,name)['ready']) resources.drop(self.root,name,backup.client);self.assertIsNone(resources.load(self.root,name)) def test_expired_dead_owner_is_cleaned_but_live_owner_is_protected(self): name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1 self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0) with patch.object(resources,'active_process',return_value=False): self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],1) self.assertIsNone(resources.load(self.root,name)) def test_unregistered_schema_wrong_server_and_wrong_marker_are_never_dropped(self): name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1 value['serverUuid']='different';br.atomic(resources.registry(self.root)/(name+'.json'),value) with patch.object(resources,'active_process',return_value=False): self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0) with self.assertRaises(ValueError):resources.drop(self.root,name,backup.client) value['serverUuid']=backup.client('SELECT @@server_uuid;').strip();br.atomic(resources.registry(self.root)/(name+'.json'),value) backup.client('UPDATE `'+name+'`.`'+resources.MARKER+"` SET token='different';") with patch.object(resources,'active_process',return_value=False): self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0) with self.assertRaises(ValueError):resources.create(self.root,'quality_platform',backup.client) def test_empty_schema_stays_empty_until_migrations_finish_then_seals(self): name=self.create(empty=True) self.assertEqual(backup.client('SHOW TABLES;',name).strip(),'') backup.client('CREATE TABLE business_data(id INT);',name) resources.seal(self.root,name,backup.client) self.assertTrue(resources.marker_matches(resources.load(self.root,name),backup.client)) resources.drop(self.root,name,backup.client) def test_empty_failed_schema_can_be_dropped_by_its_current_owner(self): name=self.create(empty=True);resources.drop(self.root,name,backup.client) self.assertIsNone(resources.load(self.root,name)) def test_registered_test_account_is_cleaned_after_expired_schema(self): name=self.create();user='qverify_'+uuid.uuid4().hex[:12];self.users.append(user) backup.client("CREATE USER '"+user+"'@'127.0.0.1' IDENTIFIED BY 'disposable-test-account';") backup.client('GRANT ALL ON `'+name+"`.* TO '"+user+"'@'127.0.0.1';") resources.attach_user(self.root,name,user) with patch.object(resources,'active_process',return_value=False): result=resources.cleanup(self.root,backup.client,resources.load(self.root,name)['expiresAt']+1) self.assertEqual(result['testDatabasesRemoved'],1);self.assertEqual(result['testAccountsRemoved'],1) self.assertIsNone(resources.load(self.root,name)) if __name__=='__main__':unittest.main()