Files
quality_analyse/scripts/test_backup_remediation.py
T

645 lines
35 KiB
Python

import contextlib
import gzip
import hashlib
import io
import json
import os
from pathlib import Path
import tarfile
import tempfile
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import backup
class BackupRemediationTest(unittest.TestCase):
@contextlib.contextmanager
def private_verifier_fixture(self, root):
"""Exercise verify's Darwin/container path without starting a MySQL server."""
import backup_verifier
config = root / '.runtime/retention/verify-fixture/client.cnf'
config.parent.mkdir(parents=True, exist_ok=True)
config.write_text('[client]\nuser=root\nsocket=/synthetic/private.sock\n')
registration = root / '.runtime/retention/scratch/fixture.json'
registered = []
child_pids = []
@contextlib.contextmanager
def fake_instance(instance_root):
self.assertEqual(Path(instance_root), root)
yield config
def fake_scratch_register(instance_root, database, server):
self.assertEqual(Path(instance_root), root)
self.assertRegex(database, r'^quality_restore_\d{14}_[a-f0-9]{8}$')
self.assertEqual(server, 'synthetic-server-uuid')
registration.parent.mkdir(parents=True, exist_ok=True)
registration.write_text('{}')
registered.append((database, server))
return registration
with patch.object(backup, 'sys', SimpleNamespace(platform='darwin')), \
patch.object(backup_verifier, 'instance', side_effect=fake_instance) as instance, \
patch.object(backup.backup_retention, 'scratch_register', side_effect=fake_scratch_register), \
patch.object(backup.backup_retention, 'scratch_created') as scratch_created, \
patch.object(backup.backup_retention, 'scratch_child',
side_effect=lambda path, pid: child_pids.append((Path(path), pid))):
yield {'config': config, 'instance': instance, 'registration': registration,
'registered': registered, 'childPids': child_pids,
'scratchCreated': scratch_created}
def test_create_preserves_binary_columns_as_utf8_safe_hex_literals(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self.make_root(root)
class BinaryDump:
def __init__(self, command, **kwargs):
blob = b'0x008BFF' if '--hex-blob' in command else b"'\x00\x8b\xff'"
self.stdout = io.BytesIO(b'CREATE TABLE `binary_fixture` (`payload` blob);\n'
b'INSERT INTO `binary_fixture` VALUES (' + blob + b');\n')
# candidate_child() records process creation time via psutil.
# Reuse this live test process as a synthetic child; no command runs.
self.pid = os.getpid()
def wait(self): return 0
def poll(self): return 0
def kill(self): pass
seen_client = []
def fake_client(statement, database=None):
seen_client.append((statement, database))
self.assertEqual(database, 'quality_fixture')
if statement == 'SHOW TABLES;':
return 'binary_fixture\n'
if statement == 'SELECT COUNT(*) FROM `binary_fixture`;':
return '1\n'
self.fail('Unexpected isolated export SQL: ' + statement)
with patch.object(backup, 'ROOT', root), \
patch.object(backup, 'BACKUP_ROOT', root / 'output/backups'), \
patch.object(backup, 'database_config', return_value={'mysqlDatabase': 'quality_fixture'}), \
patch.object(backup, 'verify_references', return_value={'verified': True}), \
patch.object(backup, 'query_evidence', return_value={}), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', BinaryDump):
folder = backup.create()
self.assertEqual(seen_client, [
('SHOW TABLES;', 'quality_fixture'),
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
('SHOW TABLES;', 'quality_fixture'),
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
])
owner = json.loads((folder / '.owner.json').read_text())
self.assertEqual(owner['ownerPid'], os.getpid())
self.assertIsInstance(owner['ownerCreated'], (int, float))
self.assertFalse(owner['childPending'])
self.assertEqual(len(owner['children']), 1)
self.assertEqual(owner['children'][0]['pid'], os.getpid())
self.assertIsInstance(owner['children'][0]['created'], (int, float))
manifest = json.loads((folder / 'manifest.json').read_text())
self.assertEqual(manifest['tableCounts']['binary_fixture'], 1)
with gzip.open(folder / 'database.sql.gz', 'rt', encoding='utf-8') as dump:
self.assertIn('VALUES (0x008BFF)', dump.read())
def make_root(self, root, missing=()):
for relative in backup.REQUIRED_APPLICATION_DIRECTORIES:
(root / relative).mkdir(parents=True, exist_ok=True)
for relative in backup.REQUIRED_APPLICATION_FILES:
if relative in missing:
continue
path = root / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text('fixture:' + relative)
def test_application_archive_requires_build_ai_and_nao_inputs(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self.make_root(root)
(root / '.runtime/nao-model-config.json').write_text('{"apiKey":"synthetic"}')
(root / '.runtime/nao-model-config.json').chmod(0o600)
with patch.object(backup, 'ROOT', root):
chosen = backup.application_files()
names = {p.relative_to(root).as_posix() for p in chosen}
self.assertTrue(set(backup.REQUIRED_APPLICATION_FILES).issubset(names))
self.assertIn('.runtime/nao-model-config.json', names)
output = root / 'backup'
output.mkdir()
manifest, _ = backup.archive_application(output)
self.assertIn('requirements-nao.lock', manifest)
self.assertIn('ai/nao-quality/agent/prompts/system.md', manifest)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self.make_root(root, missing={'web/index.html'})
with patch.object(backup, 'ROOT', root), self.assertRaises(backup.BackupFailure) as raised:
backup.application_files()
self.assertEqual((raised.exception.stage, raised.exception.code), ('INPUTS', 'APPLICATION_INPUT_MISSING'))
def test_verified_extract_rejects_traversal_and_links(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
archive = root / 'valid.tar.gz'
payload = b'synthetic source'
with tarfile.open(archive, 'w:gz') as tar:
info = tarfile.TarInfo('year/source.xlsx')
info.size = len(payload)
tar.addfile(info, io.BytesIO(payload))
extracted = root / 'safe'
actual = backup.extract_verified_tar(archive, extracted,
{'year/source.xlsx': hashlib.sha256(payload).hexdigest()}, private=True)
self.assertEqual(actual['year/source.xlsx'], hashlib.sha256(payload).hexdigest())
self.assertEqual((extracted / 'year/source.xlsx').read_bytes(), payload)
self.assertEqual(extracted.stat().st_mode & 0o777, 0o700)
bad = root / 'bad.tar.gz'
with tarfile.open(bad, 'w:gz') as tar:
info = tarfile.TarInfo('../escape.txt')
info.size = 1
tar.addfile(info, io.BytesIO(b'x'))
with self.assertRaises(backup.BackupFailure):
backup.extract_verified_tar(bad, root / 'bad-out', {'../escape.txt': hashlib.sha256(b'x').hexdigest()})
self.assertFalse((root / 'escape.txt').exists())
link = root / 'link.tar.gz'
with tarfile.open(link, 'w:gz') as tar:
info = tarfile.TarInfo('linked.xlsx')
info.type = tarfile.SYMTYPE
info.linkname = '../outside'
tar.addfile(info)
with self.assertRaises(backup.BackupFailure):
backup.extract_verified_tar(link, root / 'link-out', {'linked.xlsx': '0' * 64})
def test_absolute_archive_references_rebind_without_guessing_old_root(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
source_root = (root / 'old-project/runtimedata/archive').resolve()
relative = Path('2026/source.xlsx')
payload = b'synthetic workbook'
restored_root = root / 'new-project/runtimedata/archive'
restored_file = restored_root / relative
restored_file.parent.mkdir(parents=True)
restored_file.write_bytes(payload)
old_path = str(source_root / relative)
new_path = str(restored_file.resolve())
seen_updates = []
def fake_client(sql, database=None):
if sql.startswith('SELECT JSON_OBJECT') and "'path',archive_path,'sha256'" in sql:
row = {'path': old_path if not seen_updates else new_path,
'sha256': hashlib.sha256(payload).hexdigest(), 'size': len(payload)}
return json.dumps(row) + '\n'
if sql.startswith('UPDATE source_revision'):
seen_updates.append(sql)
return ''
if sql.startswith('SELECT JSON_OBJECT'):
return json.dumps({'path': new_path}) + '\n'
self.fail('Unexpected SQL in isolated rebinding fixture')
with patch.object(backup, 'client', side_effect=fake_client):
result = backup.verify_references('quality_restore_fixture', restored_root,
source_root, restored_root)
self.assertEqual(result, {'verified': True, 'rebound': True, 'count': 1})
self.assertEqual(len(seen_updates), 1)
self.assertIn(new_path.encode().hex(), seen_updates[0])
with patch.object(backup, 'client', return_value=json.dumps({'path': old_path, 'sha256': hashlib.sha256(payload).hexdigest(), 'size': len(payload)}) + '\n'):
legacy = backup.verify_references('quality_restore_fixture', restored_root, None, restored_root)
self.assertEqual(legacy, {'verified': False, 'rebound': False, 'count': 0})
def test_verify_records_data_file_and_relocation_evidence_separately(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
backup_root = root / 'output/backups'
folder = backup_root / 'fixture'
folder.mkdir(parents=True)
old_archive_root = (root / 'old-location/runtimedata/archive').resolve()
source_relative = Path('2026/synthetic.xlsx')
source_bytes = b'synthetic source bytes'
archive_file_map = {source_relative.as_posix(): hashlib.sha256(source_bytes).hexdigest()}
sql = folder / 'database.sql.gz'
with gzip.open(sql, 'wt') as stream:
stream.write('CREATE TABLE `fixture_table` (id INT);\nINSERT INTO `fixture_table` VALUES (1);\n')
archived = folder / 'archive.tar.gz'
with tarfile.open(archived, 'w:gz') as tar:
info = tarfile.TarInfo(source_relative.as_posix())
info.size = len(source_bytes)
tar.addfile(info, io.BytesIO(source_bytes))
application = folder / 'application.tar.gz'
app_files = {name: hashlib.sha256(('fixture:' + name).encode()).hexdigest()
for name in backup.REQUIRED_APPLICATION_FILES}
with tarfile.open(application, 'w:gz') as tar:
for name in backup.REQUIRED_APPLICATION_FILES:
data = ('fixture:' + name).encode()
info = tarfile.TarInfo(name)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
manifest = {
'formatVersion': 3,
'databaseDumpSha256': backup.file_sha256(sql),
'archiveSha256': backup.file_sha256(archived),
'archiveSourceRoot': str(old_archive_root),
'applicationSha256': backup.file_sha256(application),
'applicationFiles': app_files,
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES),
'files': archive_file_map,
'tableCounts': {'fixture_table': 1},
'queryEvidence': {'releaseId': None, 'months': []},
'verification': {},
}
(folder / 'manifest.json').write_text(json.dumps(manifest))
state = {'database': None, 'path': str(old_archive_root / source_relative), 'updated': False, 'dropped': False}
def fake_client(statement, database=None):
self.assertEqual(backup.client_config(), private['config'])
if statement == 'SELECT @@server_uuid;':
return 'synthetic-server-uuid\n'
if statement.startswith('CREATE DATABASE'):
state['database'] = statement.split('`')[1]
return ''
if statement == 'SHOW TABLES;':
return 'fixture_table\n'
if statement.startswith('SELECT COUNT(*)'):
return '1\n'
if "SELECT release_id FROM current_release" in statement:
return ''
if "'path',archive_path,'sha256'" in statement:
return json.dumps({'path': state['path'], 'sha256': archive_file_map[source_relative.as_posix()],
'size': len(source_bytes)}) + '\n'
if statement.startswith('UPDATE source_revision'):
state['updated'] = True
new_hex = statement.split('CONVERT(0x', 1)[1].split(' ', 1)[0]
state['path'] = bytes.fromhex(new_hex).decode('utf-8')
return ''
if "JSON_OBJECT('path',archive_path)" in statement:
return json.dumps({'path': state['path']}) + '\n'
if statement.startswith('DROP DATABASE'):
state['dropped'] = True
return ''
self.fail('Unexpected SQL in synthetic restore: ' + statement)
class RecordingInput(io.BytesIO):
def close(self):
state['restoredSql'] = self.getvalue()
super().close()
class FakeMysqlRestore:
def __init__(self, *args, **kwargs):
state['restoreCommand'] = list(args[0])
self.pid = 424242
self.stdin = RecordingInput()
self.returncode = 0
def wait(self):
return 0
def poll(self):
return self.returncode
def kill(self):
self.returncode = -9
with self.private_verifier_fixture(root) as private, \
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
result = backup.verify(folder)
saved = json.loads((folder / 'manifest.json').read_text())
self.assertIn('--init-command=SET SESSION autocommit=0', state['restoreCommand'])
self.assertTrue(state['restoredSql'].endswith(b'\nCOMMIT;\n'))
self.assertTrue(state['updated'])
self.assertTrue(state['dropped'])
self.assertTrue(private['instance'].called)
self.assertTrue(private['registered'])
self.assertEqual(private['childPids'], [(private['registration'], 424242)])
private['scratchCreated'].assert_called_once_with(private['registration'])
self.assertFalse(private['registration'].exists())
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
self.assertEqual(state['database'].split('_')[0], 'quality')
self.assertFalse(result['migrationReady'])
self.assertTrue(result['archiveReferencesRebound'])
self.assertEqual(saved['verification'], {
'databaseRestoreVerified': True,
'applicationFilesVerified': True,
'requiredApplicationInputsVerified': True,
'archiveFilesVerified': True,
'archiveReferencesVerified': True,
'applicationStartupVerified': False,
})
self.assertFalse(saved['verification']['applicationStartupVerified'])
def test_persistent_restore_rebinds_only_new_database_and_directory(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
backup_root = root / 'output/backups'
folder = backup_root / 'fixture-v3'
folder.mkdir(parents=True)
target = root / 'relocated-project'
source_root = (root / 'source-project/runtimedata/archive').resolve()
relative = Path('source-revisions/black/2026/sample.xlsx')
source = b'synthetic archived workbook'
source_hash = hashlib.sha256(source).hexdigest()
state = {'path': str(source_root / relative), 'created': None, 'dropped': False}
sql = folder / 'database.sql.gz'
with gzip.open(sql, 'wt') as stream:
stream.write('CREATE TABLE `fixture_table` (id INT);\nINSERT INTO `fixture_table` VALUES (1);\n')
archive = folder / 'archive.tar.gz'
with tarfile.open(archive, 'w:gz') as tar:
info = tarfile.TarInfo(relative.as_posix())
info.size = len(source)
tar.addfile(info, io.BytesIO(source))
app = folder / 'application.tar.gz'
app_files = {}
with tarfile.open(app, 'w:gz') as tar:
for name in backup.REQUIRED_APPLICATION_FILES:
if name == '.runtime/local-config.json':
data = b'{"mysqlDatabase":"quality_restore_source","databaseReady":true}\n'
else:
data = ('fixture:' + name).encode()
digest = hashlib.sha256(data).hexdigest()
app_files[name] = digest
info = tarfile.TarInfo(name)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
manifest = {'formatVersion': 3, 'archiveSourceRoot': str(source_root),
'databaseDumpSha256': backup.file_sha256(sql), 'archiveSha256': backup.file_sha256(archive),
'applicationSha256': backup.file_sha256(app), 'applicationFiles': app_files,
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES),
'files': {relative.as_posix(): source_hash}, 'tableCounts': {'fixture_table': 1},
'queryEvidence': {'releaseId': None, 'months': []}}
(folder / 'manifest.json').write_text(json.dumps(manifest))
def fake_client(statement, database=None):
if statement == 'SHOW DATABASES;':
return ''
if statement.startswith('CREATE DATABASE'):
state['created'] = statement.split('`')[1]
return ''
if statement == 'SHOW TABLES;':
return 'fixture_table\n'
if statement.startswith('SELECT COUNT(*)'):
return '1\n'
if "SELECT release_id FROM current_release" in statement:
return ''
if "'path',archive_path,'sha256'" in statement:
return json.dumps({'path': state['path'], 'sha256': source_hash, 'size': len(source)}) + '\n'
if statement.startswith('UPDATE source_revision'):
state['path'] = bytes.fromhex(statement.split('CONVERT(0x', 1)[1].split(' ', 1)[0]).decode()
return ''
if "JSON_OBJECT('path',archive_path)" in statement:
return json.dumps({'path': state['path']}) + '\n'
if statement.startswith('DROP DATABASE'):
state['dropped'] = True
return ''
self.fail('Unexpected persistent restore SQL: ' + statement)
class FakeMysqlRestore:
def __init__(self, *args, **kwargs):
self.stdin = io.BytesIO()
self.returncode = 0
def wait(self): return 0
def poll(self): return self.returncode
def kill(self): self.returncode = -9
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
result = backup.restore(folder, target, 'quality_restore_acceptance')
self.assertEqual(state['created'], 'quality_restore_acceptance')
self.assertFalse(state['dropped'])
self.assertEqual(state['path'], str((target / 'runtimedata/archive' / relative).resolve(strict=False)))
self.assertTrue((target / 'runtimedata/archive' / relative).is_file())
self.assertEqual(result['status'], 'ISOLATED_DATA_FILE_RESTORE_VERIFIED')
self.assertTrue(result['verification']['archiveReferencesVerified'])
self.assertFalse(result['verification']['applicationStartupVerified'])
restored_config = json.loads((target / '.runtime/local-config.json').read_text())
self.assertEqual(restored_config['mysqlDatabase'], 'quality_restore_acceptance')
self.assertTrue(restored_config['databaseReady'])
def test_persistent_restore_refuses_existing_target_or_database_without_removal(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
backup_root = root / 'output/backups'
folder = backup_root / 'fixture'
folder.mkdir(parents=True)
target = root / 'already-there'
target.mkdir()
marker = target / 'keep.txt'
marker.write_text('do not remove')
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
self.assertRaises(backup.BackupFailure) as raised:
backup.restore(folder, target, 'quality_restore_acceptance')
self.assertEqual((raised.exception.stage, raised.exception.code), ('RESTORE', 'RESTORE_TARGET_INVALID'))
self.assertEqual(marker.read_text(), 'do not remove')
target.unlink(missing_ok=True) if target.is_file() else None
marker.unlink()
target.rmdir()
target_database = 'quality_restore_existing'
(folder / 'database.sql.gz').write_bytes(gzip.compress(b''))
with tarfile.open(folder / 'archive.tar.gz', 'w:gz'):
pass
with tarfile.open(folder / 'application.tar.gz', 'w:gz'):
pass
(folder / 'manifest.json').write_text(json.dumps({
'formatVersion': 3, 'archiveSourceRoot': str(root / 'old/runtimedata/archive'),
'databaseDumpSha256': backup.file_sha256(folder / 'database.sql.gz'),
'archiveSha256': backup.file_sha256(folder / 'archive.tar.gz'),
'applicationSha256': backup.file_sha256(folder / 'application.tar.gz'),
'applicationFiles': {name: '0' * 64 for name in backup.REQUIRED_APPLICATION_FILES},
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES), 'files': {}, 'tableCounts': {},
}))
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', return_value=target_database + '\n'), \
self.assertRaises(backup.BackupFailure) as existing:
backup.restore(folder, target, target_database)
self.assertEqual((existing.exception.stage, existing.exception.code), ('RESTORE', 'RESTORE_DATABASE_EXISTS'))
self.assertFalse(target.exists())
def make_minimal_persistent_backup(self, root, backup_root):
folder = backup_root / 'cleanup-fixture'
folder.mkdir(parents=True)
sql = folder / 'database.sql.gz'
with gzip.open(sql, 'wb') as stream:
stream.write(b'')
archive = folder / 'archive.tar.gz'
with tarfile.open(archive, 'w:gz'):
pass
app = folder / 'application.tar.gz'
with tarfile.open(app, 'w:gz'):
pass
manifest = {'formatVersion': 3,
'archiveSourceRoot': str(root / 'old/runtimedata/archive'),
'databaseDumpSha256': backup.file_sha256(sql),
'archiveSha256': backup.file_sha256(archive),
'applicationSha256': backup.file_sha256(app),
'applicationFiles': {name: '0' * 64 for name in backup.REQUIRED_APPLICATION_FILES},
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES),
'files': {}, 'tableCounts': {}, 'queryEvidence': None}
(folder / 'manifest.json').write_text(json.dumps(manifest))
return folder
def test_persistent_restore_drop_failure_still_removes_private_stage(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
backup_root = root / 'output/backups'
backup_root.mkdir(parents=True)
folder = self.make_minimal_persistent_backup(root, backup_root)
target = root / 'target'
statements = []
def fake_client(statement, database=None):
statements.append(statement)
if statement == 'SHOW DATABASES;': return ''
if statement.startswith('CREATE DATABASE'): return ''
if statement.startswith('DROP DATABASE'):
raise OSError('synthetic drop failure with private details')
self.fail('Unexpected cleanup fixture SQL')
def partial_extract(archive, destination, expected, private=False, precreated=False):
self.assertTrue(precreated)
config = destination / '.runtime/local-config.json'
config.parent.mkdir(mode=0o700)
config.write_text('{"password":"synthetic"}')
raise backup.BackupFailure('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup, 'extract_verified_tar', side_effect=partial_extract), \
self.assertRaises(backup.BackupFailure) as raised:
backup.restore(folder, target, 'quality_restore_cleanup')
self.assertEqual((raised.exception.stage, raised.exception.code),
('CLEANUP', 'RESTORE_DATABASE_CLEANUP_FAILED'))
self.assertTrue(any(statement.startswith('DROP DATABASE') for statement in statements))
self.assertFalse(target.exists())
self.assertFalse(list(root.glob('.target.restore-*')))
def test_persistent_restore_reports_both_cleanup_failures_after_attempting_both(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
backup_root = root / 'output/backups'
backup_root.mkdir(parents=True)
folder = self.make_minimal_persistent_backup(root, backup_root)
target = root / 'target'
drop_attempted = []
def fake_client(statement, database=None):
if statement == 'SHOW DATABASES;': return ''
if statement.startswith('CREATE DATABASE'): return ''
if statement.startswith('DROP DATABASE'):
drop_attempted.append(True)
raise OSError('synthetic drop failure')
self.fail('Unexpected combined cleanup fixture SQL')
def partial_extract(archive, destination, expected, private=False, precreated=False):
(destination / '.runtime').mkdir()
(destination / '.runtime/local-config.json').write_text('{"password":"synthetic"}')
raise backup.BackupFailure('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup, 'extract_verified_tar', side_effect=partial_extract), \
patch.object(backup.shutil, 'rmtree', side_effect=OSError('synthetic stage cleanup failure')) as remove, \
self.assertRaises(backup.BackupFailure) as raised:
backup.restore(folder, target, 'quality_restore_cleanup_both')
self.assertEqual((raised.exception.stage, raised.exception.code),
('CLEANUP', 'RESTORE_DATABASE_AND_STAGE_CLEANUP_FAILED'))
self.assertTrue(drop_attempted)
remove.assert_called_once()
self.assertFalse(target.exists())
def test_failure_protocol_contains_only_whitelisted_safe_fields(self):
stdout = io.StringIO()
with patch.object(backup, 'create', side_effect=backup.BackupFailure('INPUTS', 'APPLICATION_INPUT_MISSING')):
with contextlib.redirect_stdout(stdout):
code = backup.main(['create'])
self.assertNotEqual(code, 0)
event = json.loads(stdout.getvalue())
self.assertEqual(event, {'event': 'failure', 'stage': 'INPUTS', 'code': 'APPLICATION_INPUT_MISSING',
'message': backup.SAFE_FAILURES[('INPUTS', 'APPLICATION_INPUT_MISSING')]})
self.assertNotIn('traceback', stdout.getvalue().lower())
self.assertNotIn('secret', stdout.getvalue().lower())
def test_checksum_sql_restore_and_missing_archive_failures_are_distinct(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
backup_root = root / 'output/backups'
folder = backup_root / 'fixture'
folder.mkdir(parents=True)
sql = folder / 'database.sql.gz'
with gzip.open(sql, 'wb') as stream:
stream.write(b'')
archive = folder / 'archive.tar.gz'
with tarfile.open(archive, 'w:gz'):
pass
base_manifest = {'databaseDumpSha256': backup.file_sha256(sql),
'archiveSha256': backup.file_sha256(archive), 'files': {},
'tableCounts': {}, 'queryEvidence': None,
'archiveSourceRoot': str(root / 'old/runtimedata/archive')}
(folder / 'manifest.json').write_text(json.dumps({**base_manifest, 'archiveSha256': '0' * 64}))
with self.private_verifier_fixture(root) as private, \
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
self.assertRaises(backup.BackupFailure) as checksum:
backup.verify(folder)
self.assertEqual((checksum.exception.stage, checksum.exception.code),
('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'))
private['instance'].assert_called_once_with(root)
self.assertEqual(private['registered'], [])
(folder / 'manifest.json').write_text(json.dumps(base_manifest))
state = {'dropped': False}
def fake_client(statement, database=None):
self.assertEqual(backup.client_config(), private['config'])
if statement.startswith('CREATE DATABASE'):
return ''
if statement == 'SELECT @@server_uuid;':
return 'synthetic-server-uuid\n'
if statement.startswith('DROP DATABASE'):
state['dropped'] = True
return ''
if statement == 'SHOW TABLES;':
return ''
self.fail('Unexpected SQL in failing import fixture')
class FailedMysqlRestore:
def __init__(self, *args, **kwargs):
self.pid = 434343
self.stdin = io.BytesIO()
self.returncode = 1
def wait(self):
return 1
def poll(self):
return self.returncode
def kill(self):
self.returncode = -9
with self.private_verifier_fixture(root) as private, \
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
patch.object(backup, 'client', side_effect=fake_client), \
patch.object(backup.subprocess, 'Popen', FailedMysqlRestore), \
self.assertRaises(backup.BackupFailure) as restore:
backup.verify(folder)
self.assertEqual((restore.exception.stage, restore.exception.code),
('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'))
self.assertTrue(state['dropped'], 'only the newly created scratch database is cleaned up')
private['instance'].assert_called_once_with(root)
self.assertEqual(private['childPids'], [(private['registration'], 434343)])
private['scratchCreated'].assert_called_once_with(private['registration'])
self.assertFalse(private['registration'].exists())
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
old_root = (root / 'old/runtimedata/archive').resolve()
row = {'path': str(old_root / 'missing.xlsx'), 'sha256': '1' * 64, 'size': 12}
with patch.object(backup, 'client', return_value=json.dumps(row) + '\n'), \
self.assertRaises(backup.BackupFailure) as missing_archive:
backup.verify_references('quality_restore_fixture', root / 'relocated/archive', old_root,
root / 'relocated/archive')
self.assertEqual((missing_archive.exception.stage, missing_archive.exception.code),
('ARCHIVE_RESTORE', 'SOURCE_ARCHIVE_INVALID'))
if __name__ == '__main__':
unittest.main()