206 lines
13 KiB
Python
206 lines
13 KiB
Python
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
import uuid
|
|
from unittest.mock import patch
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
import backup
|
|
import backup_retention as br
|
|
import log_retention as logs
|
|
import managed_test_resources as resources
|
|
spec = importlib.util.spec_from_file_location('quality_operations', Path(__file__).with_name('operations.py'))
|
|
ops = importlib.util.module_from_spec(spec); spec.loader.exec_module(ops)
|
|
|
|
|
|
class OperationsTests(unittest.TestCase):
|
|
def test_native_logs_are_counted_once_and_do_not_hide_other_runtime_artifacts(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d).resolve();native=root/'.runtime/nao-users/actor'
|
|
for relative in ['logs/server.log','private/nao.sqlite']:
|
|
path=native/relative;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'x'*8192)
|
|
(root/'.runtime/tool.bin').write_bytes(b'x'*16384)
|
|
sample=ops.snapshot(root,root/'support')
|
|
self.assertEqual(sample['coreBytes'],sum(sample[k] for k in ['mysqlBytes','archiveBytes','backupBytes','logBytes','nativeDataBytes']))
|
|
self.assertEqual(sample['artifactBytes'],ops.allocated(root/'.runtime')-ops.allocated(root/'.runtime/nao-users'))
|
|
|
|
def sample(self, **extra):
|
|
return {'at': 200000., 'coreBytes': 12*ops.GIB, 'permanentBytes': 40*ops.GIB,
|
|
'freeBytes': 500*ops.GIB, 'overdueTestResources': 0, **extra}
|
|
|
|
def test_capacity_levels_free_space_failures_and_growth_are_separate(self):
|
|
self.assertEqual(ops.conditions(self.sample(), ops.DEFAULTS, {}, []), {})
|
|
self.assertEqual(ops.conditions(self.sample(coreBytes=30*ops.GIB), ops.DEFAULTS, {}, [])['capacity']['severity'], 'warning')
|
|
self.assertIn('temporary',ops.conditions(self.sample(temporaryBytes=30*ops.GIB),ops.DEFAULTS,{},[]))
|
|
self.assertIn('artifacts',ops.conditions(self.sample(artifactBytes=50*ops.GIB),ops.DEFAULTS,{},[]))
|
|
found = ops.conditions(self.sample(coreBytes=50*ops.GIB, freeBytes=19*ops.GIB, overdueTestResources=1),
|
|
ops.DEFAULTS, {'consecutiveFailures': 3}, [])
|
|
self.assertEqual(set(found), {'capacity', 'disk', 'maintenance', 'tests'})
|
|
old = self.sample(at=200000-86400, permanentBytes=30*ops.GIB)
|
|
self.assertIn('growth', ops.conditions(self.sample(), ops.DEFAULTS, {}, [old]))
|
|
self.assertNotIn('maintenance', ops.conditions(self.sample(), ops.DEFAULTS, {'lastDeferred': {'reason': 'active'}}, []))
|
|
old = self.sample(at=200000-86400)
|
|
self.assertIn('growth',ops.conditions(self.sample(coreBytes=15*ops.GIB,permanentBytes=43*ops.GIB),ops.DEFAULTS,{},[old]))
|
|
|
|
def test_monitor_failure_notifies_after_three_attempts_without_repeated_noise(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d).resolve();sent=[]
|
|
def notify(*_):sent.append(True);return {'submitted':True}
|
|
for _ in range(4):ops.failed_run(root,RuntimeError('not logged'),notify)
|
|
self.assertEqual(len(sent),1)
|
|
self.assertEqual(json.loads((root/'.runtime/operations/failure.json').read_text())['count'],4)
|
|
|
|
def test_alert_deduplication_escalation_recovery_and_failed_delivery_retry(self):
|
|
sent = []
|
|
def notify(title, message): sent.append(message); return {'submitted': True}
|
|
warning = {'capacity': {'severity': 'warning', 'message': 'warning'}}
|
|
first, events = ops.dispatch(warning, {}, 100, ops.DEFAULTS, notify)
|
|
second, events = ops.dispatch(warning, first, 200, ops.DEFAULTS, notify)
|
|
self.assertEqual(len(sent), 1); self.assertFalse(events)
|
|
critical = {'capacity': {'severity': 'critical', 'message': 'critical'}}
|
|
third, _ = ops.dispatch(critical, second, 300, ops.DEFAULTS, notify)
|
|
self.assertEqual(len(sent), 2)
|
|
_, events = ops.dispatch({}, third, 400, ops.DEFAULTS, notify)
|
|
self.assertEqual(events[0]['kind'], 'resolved')
|
|
failed, _ = ops.dispatch(warning, {}, 500, ops.DEFAULTS, lambda *_: {'submitted': False})
|
|
ops.dispatch(warning, failed, 600, ops.DEFAULTS, notify)
|
|
self.assertEqual(len(sent), 3)
|
|
|
|
def test_registered_and_private_temporary_storage_is_not_permanent_growth(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root = Path(d).resolve(); support = root / 'support'
|
|
name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
|
|
for path in [root/'.runtime/mysql-data/current/data.ibd', root/'.runtime/mysql-data'/name/'sample.ibd',
|
|
root/'output/backups/current/file.gz', root/'runtimedata/archive/source.xlsx']:
|
|
path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(b'x'*8192)
|
|
br.atomic(resources.registry(root)/(name+'.json'), {'kind':'quality-owned-test-db','root':str(root),
|
|
'database':name,'token':'1'*32,'created':True,'expiresAt':time.time()+86400})
|
|
private=root/'.runtime/retention/verify-fixture';private.mkdir(parents=True)
|
|
(private/'data').write_bytes(b'x'*16384)
|
|
br.atomic(private/'owner.json',{'kind':'private-quality-restore','directory':str(private)})
|
|
sample=ops.snapshot(root,support)
|
|
self.assertGreater(sample['registeredTestBytes'],0); self.assertGreater(sample['privateVerifierBytes'],0)
|
|
self.assertLess(sample['coreBytes'],sample['mysqlBytes']+sample['backupBytes']+sample['archiveBytes'])
|
|
|
|
def test_monitor_still_collects_when_housekeeping_is_deferred_and_history_is_bounded(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d).resolve();base=root/'.runtime/operations';base.mkdir(parents=True)
|
|
br.atomic(base/'policy.json',ops.DEFAULTS)
|
|
history=[self.sample(at=time.time()-i*100) for i in range(450)]
|
|
br.atomic(base/'samples.json',history)
|
|
with patch.object(ops,'snapshot',return_value=self.sample(at=time.time())), \
|
|
patch.object(ops,'janitor',return_value={'deferred':'Application jobs are active'}):
|
|
result=ops.run(root,root/'support',lambda *_:{'submitted':True})
|
|
self.assertIn('sample',result);self.assertIn('deferred',result['cleanup'])
|
|
self.assertLessEqual(len(json.loads((base/'samples.json').read_text())),330)
|
|
|
|
|
|
class LogRetentionTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve();self.support=self.root/'support'
|
|
def tearDown(self):self.temp.cleanup()
|
|
def file(self,name,age=0,size=100):
|
|
p=self.root/'output/logs'/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(b'line\n'*(size//5))
|
|
os.utime(p,(time.time()-age,time.time()-age));return p
|
|
|
|
def test_old_closed_logs_removed_but_active_symlinks_and_business_files_preserved(self):
|
|
old=self.file('old.log',age=8*86400);active=self.file('backend.log',age=8*86400)
|
|
other=self.file('worker.log',age=8*86400);outside=self.root/'important.xlsx';outside.write_bytes(b'keep')
|
|
link=old.with_name('link.log');link.symlink_to(outside)
|
|
result=logs.retain(self.root,self.support,open_paths={str(other)})
|
|
self.assertEqual(result['logsRemoved'],1);self.assertFalse(old.exists())
|
|
self.assertTrue(active.exists());self.assertTrue(other.exists());self.assertTrue(outside.exists());self.assertTrue(link.is_symlink())
|
|
|
|
def test_per_class_cap_prunes_old_archives_and_bounds_a_closed_console_tail(self):
|
|
old=self.file('worker.log.1',age=100,size=100);new=self.file('worker.log',age=0,size=100)
|
|
logs.retain(self.root,self.support,open_paths=set(),max_bytes=150)
|
|
self.assertFalse(old.exists());self.assertTrue(new.exists())
|
|
big=self.file('huge.log',size=1000)
|
|
logs.retain(self.root,self.support,open_paths=set(),max_bytes=100)
|
|
self.assertLessEqual(big.stat().st_size,100)
|
|
|
|
def test_mysql_rotation_uses_reopen_and_recovers_after_interruption(self):
|
|
source=self.file('mysql-server.log',age=86400,size=100);original=source.read_bytes()
|
|
with self.assertRaises(RuntimeError):
|
|
logs.rotate_mysql(self.root,lambda *_:(_ for _ in ()).throw(RuntimeError('interrupted')),br)
|
|
self.assertFalse(source.exists());self.assertTrue((self.root/'.runtime/operations/mysql-log-rotation.json').exists())
|
|
calls=[]
|
|
def reopen(sql,_):calls.append(sql);source.write_bytes(b'new log')
|
|
self.assertEqual(logs.rotate_mysql(self.root,reopen,br),1)
|
|
self.assertEqual(calls,['FLUSH ERROR LOGS;'])
|
|
archived=list(source.parent.glob('mysql-server.roll-*.log'));self.assertEqual(archived[0].read_bytes(),original)
|
|
self.assertEqual(source.read_bytes(),b'new log')
|
|
|
|
|
|
class ManagedResourceTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
if not (ROOT / '.runtime/local-config.json').is_file(): raise unittest.SkipTest('Local MySQL test runtime is not configured')
|
|
def setUp(self):
|
|
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name).resolve()
|
|
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
|
|
br.atomic(self.root/'.runtime/local-config.json',{'mysqlDatabase':cfg['mysqlDatabase']})
|
|
self.names=[];self.users=[]
|
|
def tearDown(self):
|
|
for n in self.names:backup.client('DROP DATABASE IF EXISTS `'+n+'`;')
|
|
for u in self.users:backup.client("DROP USER IF EXISTS '"+u+"'@'127.0.0.1';")
|
|
self.temp.cleanup()
|
|
def create(self,empty=False):
|
|
name='quality_retention_test_'+uuid.uuid4().hex[:12];self.names.append(name)
|
|
resources.create(self.root,name,backup.client,empty=empty);return name
|
|
|
|
def test_successful_create_and_drop_unregisters_only_its_schema(self):
|
|
name=self.create();self.assertTrue(resources.load(self.root,name)['ready'])
|
|
resources.drop(self.root,name,backup.client);self.assertIsNone(resources.load(self.root,name))
|
|
|
|
def test_expired_dead_owner_is_cleaned_but_live_owner_is_protected(self):
|
|
name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1
|
|
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0)
|
|
with patch.object(resources,'active_process',return_value=False):
|
|
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],1)
|
|
self.assertIsNone(resources.load(self.root,name))
|
|
|
|
def test_unregistered_schema_wrong_server_and_wrong_marker_are_never_dropped(self):
|
|
name=self.create();value=resources.load(self.root,name);now=value['expiresAt']+1
|
|
value['serverUuid']='different';br.atomic(resources.registry(self.root)/(name+'.json'),value)
|
|
with patch.object(resources,'active_process',return_value=False):
|
|
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0)
|
|
with self.assertRaises(ValueError):resources.drop(self.root,name,backup.client)
|
|
value['serverUuid']=backup.client('SELECT @@server_uuid;').strip();br.atomic(resources.registry(self.root)/(name+'.json'),value)
|
|
backup.client('UPDATE `'+name+'`.`'+resources.MARKER+"` SET token='different';")
|
|
with patch.object(resources,'active_process',return_value=False):
|
|
self.assertEqual(resources.cleanup(self.root,backup.client,now)['testDatabasesRemoved'],0)
|
|
with self.assertRaises(ValueError):resources.create(self.root,'quality_platform',backup.client)
|
|
|
|
def test_empty_schema_stays_empty_until_migrations_finish_then_seals(self):
|
|
name=self.create(empty=True)
|
|
self.assertEqual(backup.client('SHOW TABLES;',name).strip(),'')
|
|
backup.client('CREATE TABLE business_data(id INT);',name)
|
|
resources.seal(self.root,name,backup.client)
|
|
self.assertTrue(resources.marker_matches(resources.load(self.root,name),backup.client))
|
|
resources.drop(self.root,name,backup.client)
|
|
|
|
def test_empty_failed_schema_can_be_dropped_by_its_current_owner(self):
|
|
name=self.create(empty=True);resources.drop(self.root,name,backup.client)
|
|
self.assertIsNone(resources.load(self.root,name))
|
|
|
|
def test_registered_test_account_is_cleaned_after_expired_schema(self):
|
|
name=self.create();user='qverify_'+uuid.uuid4().hex[:12];self.users.append(user)
|
|
backup.client("CREATE USER '"+user+"'@'127.0.0.1' IDENTIFIED BY 'disposable-test-account';")
|
|
backup.client('GRANT ALL ON `'+name+"`.* TO '"+user+"'@'127.0.0.1';")
|
|
resources.attach_user(self.root,name,user)
|
|
with patch.object(resources,'active_process',return_value=False):
|
|
result=resources.cleanup(self.root,backup.client,resources.load(self.root,name)['expiresAt']+1)
|
|
self.assertEqual(result['testDatabasesRemoved'],1);self.assertEqual(result['testAccountsRemoved'],1)
|
|
self.assertIsNone(resources.load(self.root,name))
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|