Files

206 lines
13 KiB
Python

import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
import unittest
import uuid
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import backup
import backup_retention as br
import log_retention as logs
import managed_test_resources as resources
spec = importlib.util.spec_from_file_location('quality_operations', Path(__file__).with_name('operations.py'))
ops = importlib.util.module_from_spec(spec); spec.loader.exec_module(ops)
class OperationsTests(unittest.TestCase):
def test_native_logs_are_counted_once_and_do_not_hide_other_runtime_artifacts(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d).resolve();native=root/'.runtime/nao-users/actor'
for relative in ['logs/server.log','private/nao.sqlite']:
path=native/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'x'*8192)
(root/'.runtime/tool.bin').write_bytes(b'x'*16384)
sample=ops.snapshot(root,root/'support')
self.assertEqual(sample['coreBytes'],sum(sample[k] for k in ['mysqlBytes','archiveBytes','backupBytes','logBytes','nativeDataBytes']))
self.assertEqual(sample['artifactBytes'],ops.allocated(root/'.runtime')-ops.allocated(root/'.runtime/nao-users'))
def sample(self, **extra):
return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB,
'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra}
def test_capacity_levels_free_space_failures_and_growth_are_separate(self):
self.assertEqual(ops.conditions(self.sample(), ops.DEFAULTS, {}, []), {})
self.assertEqual(ops.conditions(self.sample(coreBytes=30*ops.GIB), ops.DEFAULTS, {}, [])['capacity']['severity'], 'warning')
self.assertIn('temporary',ops.conditions(self.sample(temporaryBytes=30*ops.GIB),ops.DEFAULTS,{},[]))
self.assertIn('artifacts',ops.conditions(self.sample(artifactBytes=50*ops.GIB),ops.DEFAULTS,{},[]))
found = ops.conditions(self.sample(coreBytes=50*ops.GIB, freeBytes=19*ops.GIB, overdueTestResources=1),
ops.DEFAULTS, {'consecutiveFailures': 3}, [])
self.assertEqual(set(found), {'capacity', 'disk', 'maintenance', 'tests'})
old = self.sample(at=200000-86400, permanentBytes=30*ops.GIB)
self.assertIn('growth', ops.conditions(self.sample(), ops.DEFAULTS, {}, [old]))
self.assertNotIn('maintenance', ops.conditions(self.sample(), ops.DEFAULTS, {'lastDeferred': {'reason': 'active'}}, []))
old = self.sample(at=200000-86400)
self.assertIn('growth',ops.conditions(self.sample(coreBytes=15*ops.GIB,permanentBytes=43*ops.GIB),ops.DEFAULTS,{},[old]))
def test_monitor_failure_notifies_after_three_attempts_without_repeated_noise(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d).resolve();sent=[]
def notify(*_):sent.append(True);return {'submitted':True}
for _ in range(4):ops.failed_run(root,RuntimeError('not logged'),notify)
self.assertEqual(len(sent),1)
self.assertEqual(json.loads((root/'.runtime/operations/failure.json').read_text())['count'],4)
def test_alert_deduplication_escalation_recovery_and_failed_delivery_retry(self):
sent = []
def notify(title, message): sent.append(message); return {'submitted': True}
warning = {'capacity': {'severity': 'warning', 'message': 'warning'}}
first, events = ops.dispatch(warning, {}, 100, ops.DEFAULTS, notify)
second, events = ops.dispatch(warning, first, 200, ops.DEFAULTS, notify)
self.assertEqual(len(sent), 1); self.assertFalse(events)
critical = {'capacity': {'severity': 'critical', 'message': 'critical'}}
third, _ = ops.dispatch(critical, second, 300, ops.DEFAULTS, notify)
self.assertEqual(len(sent), 2)
_, events = ops.dispatch({}, third, 400, ops.DEFAULTS, notify)
self.assertEqual(events[0]['kind'], 'resolved')
failed, _ = ops.dispatch(warning, {}, 500, ops.DEFAULTS, lambda *_: {'submitted': False})
ops.dispatch(warning, failed, 600, ops.DEFAULTS, notify)
self.assertEqual(len(sent), 3)
def test_registered_and_private_temporary_storage_is_not_permanent_growth(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d).resolve(); support = root / 'support'
name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
for path in [root/'.runtime/mysql-data/current/data.ibd', root/'.runtime/mysql-data'/name/'sample.ibd',
root/'output/backups/current/file.gz', root/'runtimedata/archive/source.xlsx']:
path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(b'x'*8192)
br.atomic(resources.registry(root)/(name+'.json'), {'kind':'quality-owned-test-db','root':str(root),
'database':name,'token':'1'*32,'created':True,'expiresAt':time.time()+86400})
private=root/'.runtime/retention/verify-fixture';private.mkdir(parents=True)
(private/'data').write_bytes(b'x'*16384)
br.atomic(private/'owner.json',{'kind':'private-quality-restore','directory':str(private)})
sample=ops.snapshot(root,support)
self.assertGreater(sample['registeredTestBytes'],0); self.assertGreater(sample['privateVerifierBytes'],0)
self.assertLess(sample['coreBytes'],sample['mysqlBytes']+sample['backupBytes']+sample['archiveBytes'])
def test_monitor_still_collects_when_housekeeping_is_deferred_and_history_is_bounded(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d).resolve();base=root/'.runtime/operations';base.mkdir(parents=True)
br.atomic(base/'policy.json',ops.DEFAULTS)
history=[self.sample(at=time.time()-i*100) for i in range(450)]
br.atomic(base/'samples.json',history)
with patch.object(ops,'snapshot',return_value=self.sample(at=time.time())), \
patch.object(ops,'janitor',return_value={'deferred':'Application jobs are active'}):
result=ops.run(root,root/'support',lambda *_:{'submitted':True})
self.assertIn('sample',result);self.assertIn('deferred',result['cleanup'])
self.assertLessEqual(len(json.loads((base/'samples.json').read_text())),330)
class LogRetentionTests(unittest.TestCase):
def setUp(self):
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve();self.support=self.root/'support'
def tearDown(self):self.temp.cleanup()
def file(self,name,age=0,size=100):
p=self.root/'output/logs'/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(b'line\n'*(size//5))
os.utime(p,(time.time()-age,time.time()-age));return p
def test_old_closed_logs_removed_but_active_symlinks_and_business_files_preserved(self):
old=self.file('old.log',age=8*86400);active=self.file('backend.log',age=8*86400)
other=self.file('worker.log',age=8*86400);outside=self.root/'important.xlsx';outside.write_bytes(b'keep')
link=old.with_name('link.log');link.symlink_to(outside)
result=logs.retain(self.root,self.support,open_paths={str(other)})
self.assertEqual(result['logsRemoved'],1);self.assertFalse(old.exists())
self.assertTrue(active.exists());self.assertTrue(other.exists());self.assertTrue(outside.exists());self.assertTrue(link.is_symlink())
def test_per_class_cap_prunes_old_archives_and_bounds_a_closed_console_tail(self):
old=self.file('worker.log.1',age=100,size=100);new=self.file('worker.log',age=0,size=100)
logs.retain(self.root,self.support,open_paths=set(),max_bytes=150)
self.assertFalse(old.exists());self.assertTrue(new.exists())
big=self.file('huge.log',size=1000)
logs.retain(self.root,self.support,open_paths=set(),max_bytes=100)
self.assertLessEqual(big.stat().st_size,100)
def test_mysql_rotation_uses_reopen_and_recovers_after_interruption(self):
source=self.file('mysql-server.log',age=86400,size=100);original=source.read_bytes()
with self.assertRaises(RuntimeError):
logs.rotate_mysql(self.root,lambda *_:(_ for _ in ()).throw(RuntimeError('interrupted')),br)
self.assertFalse(source.exists());self.assertTrue((self.root/'.runtime/operations/mysql-log-rotation.json').exists())
calls=[]
def reopen(sql,_):calls.append(sql);source.write_bytes(b'new log')
self.assertEqual(logs.rotate_mysql(self.root,reopen,br),1)
self.assertEqual(calls,['FLUSH ERROR LOGS;'])
archived=list(source.parent.glob('mysql-server.roll-*.log'));self.assertEqual(archived[0].read_bytes(),original)
self.assertEqual(source.read_bytes(),b'new log')
class ManagedResourceTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured')
def setUp(self):
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve()
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
br.atomic(self.root/'.runtime/local-config.json',{'mysqlDatabase':cfg['mysqlDatabase']})
self.names=[];self.users=[]
def tearDown(self):
for n in self.names:backup.client('DROP DATABASE IF EXISTS `'+n+'`;')
for u in self.users:backup.client("DROP USER IF EXISTS '"+u+"'@'127.0.0.1';")
self.temp.cleanup()
def create(self,empty=False):
name='quality_retention_test_'+uuid.uuid4().hex[:12];self.names.append(name)
resources.create(self.root,name,backup.client,empty=empty);return name
def test_successful_create_and_drop_unregisters_only_its_schema(self):
name=self.create();self.assertTrue(resources.load(self.root,name)['ready'])
resources.drop(self.root,name,backup.client);self.assertIsNone(resources.load(self.root,name))
def test_expired_dead_owner_is_cleaned_but_live_owner_is_protected(self):
name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0)
with patch.object(resources,'active_process',return_value=False):
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],1)
self.assertIsNone(resources.load(self.root,name))
def test_unregistered_schema_wrong_server_and_wrong_marker_are_never_dropped(self):
name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1
value['serverUuid']='different';br.atomic(resources.registry(self.root)/(name+'.json'),value)
with patch.object(resources,'active_process',return_value=False):
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0)
with self.assertRaises(ValueError):resources.drop(self.root,name,backup.client)
value['serverUuid']=backup.client('SELECT @@server_uuid;').strip();br.atomic(resources.registry(self.root)/(name+'.json'),value)
backup.client('UPDATE `'+name+'`.`'+resources.MARKER+"` SET token='different';")
with patch.object(resources,'active_process',return_value=False):
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0)
with self.assertRaises(ValueError):resources.create(self.root,'quality_platform',backup.client)
def test_empty_schema_stays_empty_until_migrations_finish_then_seals(self):
name=self.create(empty=True)
self.assertEqual(backup.client('SHOW TABLES;',name).strip(),'')
backup.client('CREATE TABLE business_data(id INT);',name)
resources.seal(self.root,name,backup.client)
self.assertTrue(resources.marker_matches(resources.load(self.root,name),backup.client))
resources.drop(self.root,name,backup.client)
def test_empty_failed_schema_can_be_dropped_by_its_current_owner(self):
name=self.create(empty=True);resources.drop(self.root,name,backup.client)
self.assertIsNone(resources.load(self.root,name))
def test_registered_test_account_is_cleaned_after_expired_schema(self):
name=self.create();user='qverify_'+uuid.uuid4().hex[:12];self.users.append(user)
backup.client("CREATE USER '"+user+"'@'127.0.0.1' IDENTIFIED BY 'disposable-test-account';")
backup.client('GRANT ALL ON `'+name+"`.* TO '"+user+"'@'127.0.0.1';")
resources.attach_user(self.root,name,user)
with patch.object(resources,'active_process',return_value=False):
result=resources.cleanup(self.root,backup.client,resources.load(self.root,name)['expiresAt']+1)
self.assertEqual(result['testDatabasesRemoved'],1);self.assertEqual(result['testAccountsRemoved'],1)
self.assertIsNone(resources.load(self.root,name))
if __name__=='__main__':unittest.main()