645 lines
35 KiB
Python
645 lines
35 KiB
Python
import contextlib
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import backup
|
|
|
|
|
|
class BackupRemediationTest(unittest.TestCase):
|
|
@contextlib.contextmanager
|
|
def private_verifier_fixture(self, root):
|
|
"""Exercise verify's Darwin/container path without starting a MySQL server."""
|
|
import backup_verifier
|
|
|
|
config = root / '.runtime/retention/verify-fixture/client.cnf'
|
|
config.parent.mkdir(parents=True, exist_ok=True)
|
|
config.write_text('[client]\nuser=root\nsocket=/synthetic/private.sock\n')
|
|
registration = root / '.runtime/retention/scratch/fixture.json'
|
|
registered = []
|
|
child_pids = []
|
|
|
|
@contextlib.contextmanager
|
|
def fake_instance(instance_root):
|
|
self.assertEqual(Path(instance_root), root)
|
|
yield config
|
|
|
|
def fake_scratch_register(instance_root, database, server):
|
|
self.assertEqual(Path(instance_root), root)
|
|
self.assertRegex(database, r'^quality_restore_\d{14}_[a-f0-9]{8}$')
|
|
self.assertEqual(server, 'synthetic-server-uuid')
|
|
registration.parent.mkdir(parents=True, exist_ok=True)
|
|
registration.write_text('{}')
|
|
registered.append((database, server))
|
|
return registration
|
|
|
|
with patch.object(backup, 'sys', SimpleNamespace(platform='darwin')), \
|
|
patch.object(backup_verifier, 'instance', side_effect=fake_instance) as instance, \
|
|
patch.object(backup.backup_retention, 'scratch_register', side_effect=fake_scratch_register), \
|
|
patch.object(backup.backup_retention, 'scratch_created') as scratch_created, \
|
|
patch.object(backup.backup_retention, 'scratch_child',
|
|
side_effect=lambda path, pid: child_pids.append((Path(path), pid))):
|
|
yield {'config': config, 'instance': instance, 'registration': registration,
|
|
'registered': registered, 'childPids': child_pids,
|
|
'scratchCreated': scratch_created}
|
|
|
|
def test_create_preserves_binary_columns_as_utf8_safe_hex_literals(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
self.make_root(root)
|
|
|
|
class BinaryDump:
|
|
def __init__(self, command, **kwargs):
|
|
blob = b'0x008BFF' if '--hex-blob' in command else b"'\x00\x8b\xff'"
|
|
self.stdout = io.BytesIO(b'CREATE TABLE `binary_fixture` (`payload` blob);\n'
|
|
b'INSERT INTO `binary_fixture` VALUES (' + blob + b');\n')
|
|
# candidate_child() records process creation time via psutil.
|
|
# Reuse this live test process as a synthetic child; no command runs.
|
|
self.pid = os.getpid()
|
|
|
|
def wait(self): return 0
|
|
def poll(self): return 0
|
|
def kill(self): pass
|
|
|
|
seen_client = []
|
|
|
|
def fake_client(statement, database=None):
|
|
seen_client.append((statement, database))
|
|
self.assertEqual(database, 'quality_fixture')
|
|
if statement == 'SHOW TABLES;':
|
|
return 'binary_fixture\n'
|
|
if statement == 'SELECT COUNT(*) FROM `binary_fixture`;':
|
|
return '1\n'
|
|
self.fail('Unexpected isolated export SQL: ' + statement)
|
|
|
|
with patch.object(backup, 'ROOT', root), \
|
|
patch.object(backup, 'BACKUP_ROOT', root / 'output/backups'), \
|
|
patch.object(backup, 'database_config', return_value={'mysqlDatabase': 'quality_fixture'}), \
|
|
patch.object(backup, 'verify_references', return_value={'verified': True}), \
|
|
patch.object(backup, 'query_evidence', return_value={}), \
|
|
patch.object(backup, 'client', side_effect=fake_client), \
|
|
patch.object(backup.subprocess, 'Popen', BinaryDump):
|
|
folder = backup.create()
|
|
self.assertEqual(seen_client, [
|
|
('SHOW TABLES;', 'quality_fixture'),
|
|
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
|
|
('SHOW TABLES;', 'quality_fixture'),
|
|
('SELECT COUNT(*) FROM `binary_fixture`;', 'quality_fixture'),
|
|
])
|
|
owner = json.loads((folder / '.owner.json').read_text())
|
|
self.assertEqual(owner['ownerPid'], os.getpid())
|
|
self.assertIsInstance(owner['ownerCreated'], (int, float))
|
|
self.assertFalse(owner['childPending'])
|
|
self.assertEqual(len(owner['children']), 1)
|
|
self.assertEqual(owner['children'][0]['pid'], os.getpid())
|
|
self.assertIsInstance(owner['children'][0]['created'], (int, float))
|
|
manifest = json.loads((folder / 'manifest.json').read_text())
|
|
self.assertEqual(manifest['tableCounts']['binary_fixture'], 1)
|
|
with gzip.open(folder / 'database.sql.gz', 'rt', encoding='utf-8') as dump:
|
|
self.assertIn('VALUES (0x008BFF)', dump.read())
|
|
|
|
def make_root(self, root, missing=()):
|
|
for relative in backup.REQUIRED_APPLICATION_DIRECTORIES:
|
|
(root / relative).mkdir(parents=True, exist_ok=True)
|
|
for relative in backup.REQUIRED_APPLICATION_FILES:
|
|
if relative in missing:
|
|
continue
|
|
path = root / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text('fixture:' + relative)
|
|
|
|
def test_application_archive_requires_build_ai_and_nao_inputs(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
self.make_root(root)
|
|
(root / '.runtime/nao-model-config.json').write_text('{"apiKey":"synthetic"}')
|
|
(root / '.runtime/nao-model-config.json').chmod(0o600)
|
|
with patch.object(backup, 'ROOT', root):
|
|
chosen = backup.application_files()
|
|
names = {p.relative_to(root).as_posix() for p in chosen}
|
|
self.assertTrue(set(backup.REQUIRED_APPLICATION_FILES).issubset(names))
|
|
self.assertIn('.runtime/nao-model-config.json', names)
|
|
output = root / 'backup'
|
|
output.mkdir()
|
|
manifest, _ = backup.archive_application(output)
|
|
self.assertIn('requirements-nao.lock', manifest)
|
|
self.assertIn('ai/nao-quality/agent/prompts/system.md', manifest)
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
self.make_root(root, missing={'web/index.html'})
|
|
with patch.object(backup, 'ROOT', root), self.assertRaises(backup.BackupFailure) as raised:
|
|
backup.application_files()
|
|
self.assertEqual((raised.exception.stage, raised.exception.code), ('INPUTS', 'APPLICATION_INPUT_MISSING'))
|
|
|
|
def test_verified_extract_rejects_traversal_and_links(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
archive = root / 'valid.tar.gz'
|
|
payload = b'synthetic source'
|
|
with tarfile.open(archive, 'w:gz') as tar:
|
|
info = tarfile.TarInfo('year/source.xlsx')
|
|
info.size = len(payload)
|
|
tar.addfile(info, io.BytesIO(payload))
|
|
extracted = root / 'safe'
|
|
actual = backup.extract_verified_tar(archive, extracted,
|
|
{'year/source.xlsx': hashlib.sha256(payload).hexdigest()}, private=True)
|
|
self.assertEqual(actual['year/source.xlsx'], hashlib.sha256(payload).hexdigest())
|
|
self.assertEqual((extracted / 'year/source.xlsx').read_bytes(), payload)
|
|
self.assertEqual(extracted.stat().st_mode & 0o777, 0o700)
|
|
|
|
bad = root / 'bad.tar.gz'
|
|
with tarfile.open(bad, 'w:gz') as tar:
|
|
info = tarfile.TarInfo('../escape.txt')
|
|
info.size = 1
|
|
tar.addfile(info, io.BytesIO(b'x'))
|
|
with self.assertRaises(backup.BackupFailure):
|
|
backup.extract_verified_tar(bad, root / 'bad-out', {'../escape.txt': hashlib.sha256(b'x').hexdigest()})
|
|
self.assertFalse((root / 'escape.txt').exists())
|
|
|
|
link = root / 'link.tar.gz'
|
|
with tarfile.open(link, 'w:gz') as tar:
|
|
info = tarfile.TarInfo('linked.xlsx')
|
|
info.type = tarfile.SYMTYPE
|
|
info.linkname = '../outside'
|
|
tar.addfile(info)
|
|
with self.assertRaises(backup.BackupFailure):
|
|
backup.extract_verified_tar(link, root / 'link-out', {'linked.xlsx': '0' * 64})
|
|
|
|
def test_absolute_archive_references_rebind_without_guessing_old_root(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
source_root = (root / 'old-project/runtimedata/archive').resolve()
|
|
relative = Path('2026/source.xlsx')
|
|
payload = b'synthetic workbook'
|
|
restored_root = root / 'new-project/runtimedata/archive'
|
|
restored_file = restored_root / relative
|
|
restored_file.parent.mkdir(parents=True)
|
|
restored_file.write_bytes(payload)
|
|
old_path = str(source_root / relative)
|
|
new_path = str(restored_file.resolve())
|
|
seen_updates = []
|
|
|
|
def fake_client(sql, database=None):
|
|
if sql.startswith('SELECT JSON_OBJECT') and "'path',archive_path,'sha256'" in sql:
|
|
row = {'path': old_path if not seen_updates else new_path,
|
|
'sha256': hashlib.sha256(payload).hexdigest(), 'size': len(payload)}
|
|
return json.dumps(row) + '\n'
|
|
if sql.startswith('UPDATE source_revision'):
|
|
seen_updates.append(sql)
|
|
return ''
|
|
if sql.startswith('SELECT JSON_OBJECT'):
|
|
return json.dumps({'path': new_path}) + '\n'
|
|
self.fail('Unexpected SQL in isolated rebinding fixture')
|
|
|
|
with patch.object(backup, 'client', side_effect=fake_client):
|
|
result = backup.verify_references('quality_restore_fixture', restored_root,
|
|
source_root, restored_root)
|
|
self.assertEqual(result, {'verified': True, 'rebound': True, 'count': 1})
|
|
self.assertEqual(len(seen_updates), 1)
|
|
self.assertIn(new_path.encode().hex(), seen_updates[0])
|
|
|
|
with patch.object(backup, 'client', return_value=json.dumps({'path': old_path, 'sha256': hashlib.sha256(payload).hexdigest(), 'size': len(payload)}) + '\n'):
|
|
legacy = backup.verify_references('quality_restore_fixture', restored_root, None, restored_root)
|
|
self.assertEqual(legacy, {'verified': False, 'rebound': False, 'count': 0})
|
|
|
|
def test_verify_records_data_file_and_relocation_evidence_separately(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
backup_root = root / 'output/backups'
|
|
folder = backup_root / 'fixture'
|
|
folder.mkdir(parents=True)
|
|
old_archive_root = (root / 'old-location/runtimedata/archive').resolve()
|
|
source_relative = Path('2026/synthetic.xlsx')
|
|
source_bytes = b'synthetic source bytes'
|
|
archive_file_map = {source_relative.as_posix(): hashlib.sha256(source_bytes).hexdigest()}
|
|
|
|
sql = folder / 'database.sql.gz'
|
|
with gzip.open(sql, 'wt') as stream:
|
|
stream.write('CREATE TABLE `fixture_table` (id INT);\nINSERT INTO `fixture_table` VALUES (1);\n')
|
|
archived = folder / 'archive.tar.gz'
|
|
with tarfile.open(archived, 'w:gz') as tar:
|
|
info = tarfile.TarInfo(source_relative.as_posix())
|
|
info.size = len(source_bytes)
|
|
tar.addfile(info, io.BytesIO(source_bytes))
|
|
application = folder / 'application.tar.gz'
|
|
app_files = {name: hashlib.sha256(('fixture:' + name).encode()).hexdigest()
|
|
for name in backup.REQUIRED_APPLICATION_FILES}
|
|
with tarfile.open(application, 'w:gz') as tar:
|
|
for name in backup.REQUIRED_APPLICATION_FILES:
|
|
data = ('fixture:' + name).encode()
|
|
info = tarfile.TarInfo(name)
|
|
info.size = len(data)
|
|
tar.addfile(info, io.BytesIO(data))
|
|
manifest = {
|
|
'formatVersion': 3,
|
|
'databaseDumpSha256': backup.file_sha256(sql),
|
|
'archiveSha256': backup.file_sha256(archived),
|
|
'archiveSourceRoot': str(old_archive_root),
|
|
'applicationSha256': backup.file_sha256(application),
|
|
'applicationFiles': app_files,
|
|
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES),
|
|
'files': archive_file_map,
|
|
'tableCounts': {'fixture_table': 1},
|
|
'queryEvidence': {'releaseId': None, 'months': []},
|
|
'verification': {},
|
|
}
|
|
(folder / 'manifest.json').write_text(json.dumps(manifest))
|
|
state = {'database': None, 'path': str(old_archive_root / source_relative), 'updated': False, 'dropped': False}
|
|
|
|
def fake_client(statement, database=None):
|
|
self.assertEqual(backup.client_config(), private['config'])
|
|
if statement == 'SELECT @@server_uuid;':
|
|
return 'synthetic-server-uuid\n'
|
|
if statement.startswith('CREATE DATABASE'):
|
|
state['database'] = statement.split('`')[1]
|
|
return ''
|
|
if statement == 'SHOW TABLES;':
|
|
return 'fixture_table\n'
|
|
if statement.startswith('SELECT COUNT(*)'):
|
|
return '1\n'
|
|
if "SELECT release_id FROM current_release" in statement:
|
|
return ''
|
|
if "'path',archive_path,'sha256'" in statement:
|
|
return json.dumps({'path': state['path'], 'sha256': archive_file_map[source_relative.as_posix()],
|
|
'size': len(source_bytes)}) + '\n'
|
|
if statement.startswith('UPDATE source_revision'):
|
|
state['updated'] = True
|
|
new_hex = statement.split('CONVERT(0x', 1)[1].split(' ', 1)[0]
|
|
state['path'] = bytes.fromhex(new_hex).decode('utf-8')
|
|
return ''
|
|
if "JSON_OBJECT('path',archive_path)" in statement:
|
|
return json.dumps({'path': state['path']}) + '\n'
|
|
if statement.startswith('DROP DATABASE'):
|
|
state['dropped'] = True
|
|
return ''
|
|
self.fail('Unexpected SQL in synthetic restore: ' + statement)
|
|
|
|
class RecordingInput(io.BytesIO):
|
|
def close(self):
|
|
state['restoredSql'] = self.getvalue()
|
|
super().close()
|
|
|
|
class FakeMysqlRestore:
|
|
def __init__(self, *args, **kwargs):
|
|
state['restoreCommand'] = list(args[0])
|
|
self.pid = 424242
|
|
self.stdin = RecordingInput()
|
|
self.returncode = 0
|
|
def wait(self):
|
|
return 0
|
|
def poll(self):
|
|
return self.returncode
|
|
def kill(self):
|
|
self.returncode = -9
|
|
|
|
with self.private_verifier_fixture(root) as private, \
|
|
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
|
patch.object(backup, 'client', side_effect=fake_client), \
|
|
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
|
|
result = backup.verify(folder)
|
|
saved = json.loads((folder / 'manifest.json').read_text())
|
|
self.assertIn('--init-command=SET SESSION autocommit=0', state['restoreCommand'])
|
|
self.assertTrue(state['restoredSql'].endswith(b'\nCOMMIT;\n'))
|
|
self.assertTrue(state['updated'])
|
|
self.assertTrue(state['dropped'])
|
|
self.assertTrue(private['instance'].called)
|
|
self.assertTrue(private['registered'])
|
|
self.assertEqual(private['childPids'], [(private['registration'], 424242)])
|
|
private['scratchCreated'].assert_called_once_with(private['registration'])
|
|
self.assertFalse(private['registration'].exists())
|
|
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
|
|
self.assertEqual(state['database'].split('_')[0], 'quality')
|
|
self.assertFalse(result['migrationReady'])
|
|
self.assertTrue(result['archiveReferencesRebound'])
|
|
self.assertEqual(saved['verification'], {
|
|
'databaseRestoreVerified': True,
|
|
'applicationFilesVerified': True,
|
|
'requiredApplicationInputsVerified': True,
|
|
'archiveFilesVerified': True,
|
|
'archiveReferencesVerified': True,
|
|
'applicationStartupVerified': False,
|
|
})
|
|
self.assertFalse(saved['verification']['applicationStartupVerified'])
|
|
|
|
def test_persistent_restore_rebinds_only_new_database_and_directory(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
backup_root = root / 'output/backups'
|
|
folder = backup_root / 'fixture-v3'
|
|
folder.mkdir(parents=True)
|
|
target = root / 'relocated-project'
|
|
source_root = (root / 'source-project/runtimedata/archive').resolve()
|
|
relative = Path('source-revisions/black/2026/sample.xlsx')
|
|
source = b'synthetic archived workbook'
|
|
source_hash = hashlib.sha256(source).hexdigest()
|
|
state = {'path': str(source_root / relative), 'created': None, 'dropped': False}
|
|
|
|
sql = folder / 'database.sql.gz'
|
|
with gzip.open(sql, 'wt') as stream:
|
|
stream.write('CREATE TABLE `fixture_table` (id INT);\nINSERT INTO `fixture_table` VALUES (1);\n')
|
|
archive = folder / 'archive.tar.gz'
|
|
with tarfile.open(archive, 'w:gz') as tar:
|
|
info = tarfile.TarInfo(relative.as_posix())
|
|
info.size = len(source)
|
|
tar.addfile(info, io.BytesIO(source))
|
|
app = folder / 'application.tar.gz'
|
|
app_files = {}
|
|
with tarfile.open(app, 'w:gz') as tar:
|
|
for name in backup.REQUIRED_APPLICATION_FILES:
|
|
if name == '.runtime/local-config.json':
|
|
data = b'{"mysqlDatabase":"quality_restore_source","databaseReady":true}\n'
|
|
else:
|
|
data = ('fixture:' + name).encode()
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
app_files[name] = digest
|
|
info = tarfile.TarInfo(name)
|
|
info.size = len(data)
|
|
tar.addfile(info, io.BytesIO(data))
|
|
manifest = {'formatVersion': 3, 'archiveSourceRoot': str(source_root),
|
|
'databaseDumpSha256': backup.file_sha256(sql), 'archiveSha256': backup.file_sha256(archive),
|
|
'applicationSha256': backup.file_sha256(app), 'applicationFiles': app_files,
|
|
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES),
|
|
'files': {relative.as_posix(): source_hash}, 'tableCounts': {'fixture_table': 1},
|
|
'queryEvidence': {'releaseId': None, 'months': []}}
|
|
(folder / 'manifest.json').write_text(json.dumps(manifest))
|
|
|
|
def fake_client(statement, database=None):
|
|
if statement == 'SHOW DATABASES;':
|
|
return ''
|
|
if statement.startswith('CREATE DATABASE'):
|
|
state['created'] = statement.split('`')[1]
|
|
return ''
|
|
if statement == 'SHOW TABLES;':
|
|
return 'fixture_table\n'
|
|
if statement.startswith('SELECT COUNT(*)'):
|
|
return '1\n'
|
|
if "SELECT release_id FROM current_release" in statement:
|
|
return ''
|
|
if "'path',archive_path,'sha256'" in statement:
|
|
return json.dumps({'path': state['path'], 'sha256': source_hash, 'size': len(source)}) + '\n'
|
|
if statement.startswith('UPDATE source_revision'):
|
|
state['path'] = bytes.fromhex(statement.split('CONVERT(0x', 1)[1].split(' ', 1)[0]).decode()
|
|
return ''
|
|
if "JSON_OBJECT('path',archive_path)" in statement:
|
|
return json.dumps({'path': state['path']}) + '\n'
|
|
if statement.startswith('DROP DATABASE'):
|
|
state['dropped'] = True
|
|
return ''
|
|
self.fail('Unexpected persistent restore SQL: ' + statement)
|
|
|
|
class FakeMysqlRestore:
|
|
def __init__(self, *args, **kwargs):
|
|
self.stdin = io.BytesIO()
|
|
self.returncode = 0
|
|
def wait(self): return 0
|
|
def poll(self): return self.returncode
|
|
def kill(self): self.returncode = -9
|
|
|
|
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
|
patch.object(backup, 'client', side_effect=fake_client), \
|
|
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
|
|
result = backup.restore(folder, target, 'quality_restore_acceptance')
|
|
self.assertEqual(state['created'], 'quality_restore_acceptance')
|
|
self.assertFalse(state['dropped'])
|
|
self.assertEqual(state['path'], str((target / 'runtimedata/archive' / relative).resolve(strict=False)))
|
|
self.assertTrue((target / 'runtimedata/archive' / relative).is_file())
|
|
self.assertEqual(result['status'], 'ISOLATED_DATA_FILE_RESTORE_VERIFIED')
|
|
self.assertTrue(result['verification']['archiveReferencesVerified'])
|
|
self.assertFalse(result['verification']['applicationStartupVerified'])
|
|
restored_config = json.loads((target / '.runtime/local-config.json').read_text())
|
|
self.assertEqual(restored_config['mysqlDatabase'], 'quality_restore_acceptance')
|
|
self.assertTrue(restored_config['databaseReady'])
|
|
|
|
def test_persistent_restore_refuses_existing_target_or_database_without_removal(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
backup_root = root / 'output/backups'
|
|
folder = backup_root / 'fixture'
|
|
folder.mkdir(parents=True)
|
|
target = root / 'already-there'
|
|
target.mkdir()
|
|
marker = target / 'keep.txt'
|
|
marker.write_text('do not remove')
|
|
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
|
self.assertRaises(backup.BackupFailure) as raised:
|
|
backup.restore(folder, target, 'quality_restore_acceptance')
|
|
self.assertEqual((raised.exception.stage, raised.exception.code), ('RESTORE', 'RESTORE_TARGET_INVALID'))
|
|
self.assertEqual(marker.read_text(), 'do not remove')
|
|
|
|
target.unlink(missing_ok=True) if target.is_file() else None
|
|
marker.unlink()
|
|
target.rmdir()
|
|
target_database = 'quality_restore_existing'
|
|
(folder / 'database.sql.gz').write_bytes(gzip.compress(b''))
|
|
with tarfile.open(folder / 'archive.tar.gz', 'w:gz'):
|
|
pass
|
|
with tarfile.open(folder / 'application.tar.gz', 'w:gz'):
|
|
pass
|
|
(folder / 'manifest.json').write_text(json.dumps({
|
|
'formatVersion': 3, 'archiveSourceRoot': str(root / 'old/runtimedata/archive'),
|
|
'databaseDumpSha256': backup.file_sha256(folder / 'database.sql.gz'),
|
|
'archiveSha256': backup.file_sha256(folder / 'archive.tar.gz'),
|
|
'applicationSha256': backup.file_sha256(folder / 'application.tar.gz'),
|
|
'applicationFiles': {name: '0' * 64 for name in backup.REQUIRED_APPLICATION_FILES},
|
|
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES), 'files': {}, 'tableCounts': {},
|
|
}))
|
|
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
|
patch.object(backup, 'client', return_value=target_database + '\n'), \
|
|
self.assertRaises(backup.BackupFailure) as existing:
|
|
backup.restore(folder, target, target_database)
|
|
self.assertEqual((existing.exception.stage, existing.exception.code), ('RESTORE', 'RESTORE_DATABASE_EXISTS'))
|
|
self.assertFalse(target.exists())
|
|
|
|
def make_minimal_persistent_backup(self, root, backup_root):
|
|
folder = backup_root / 'cleanup-fixture'
|
|
folder.mkdir(parents=True)
|
|
sql = folder / 'database.sql.gz'
|
|
with gzip.open(sql, 'wb') as stream:
|
|
stream.write(b'')
|
|
archive = folder / 'archive.tar.gz'
|
|
with tarfile.open(archive, 'w:gz'):
|
|
pass
|
|
app = folder / 'application.tar.gz'
|
|
with tarfile.open(app, 'w:gz'):
|
|
pass
|
|
manifest = {'formatVersion': 3,
|
|
'archiveSourceRoot': str(root / 'old/runtimedata/archive'),
|
|
'databaseDumpSha256': backup.file_sha256(sql),
|
|
'archiveSha256': backup.file_sha256(archive),
|
|
'applicationSha256': backup.file_sha256(app),
|
|
'applicationFiles': {name: '0' * 64 for name in backup.REQUIRED_APPLICATION_FILES},
|
|
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES),
|
|
'files': {}, 'tableCounts': {}, 'queryEvidence': None}
|
|
(folder / 'manifest.json').write_text(json.dumps(manifest))
|
|
return folder
|
|
|
|
def test_persistent_restore_drop_failure_still_removes_private_stage(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
backup_root = root / 'output/backups'
|
|
backup_root.mkdir(parents=True)
|
|
folder = self.make_minimal_persistent_backup(root, backup_root)
|
|
target = root / 'target'
|
|
statements = []
|
|
|
|
def fake_client(statement, database=None):
|
|
statements.append(statement)
|
|
if statement == 'SHOW DATABASES;': return ''
|
|
if statement.startswith('CREATE DATABASE'): return ''
|
|
if statement.startswith('DROP DATABASE'):
|
|
raise OSError('synthetic drop failure with private details')
|
|
self.fail('Unexpected cleanup fixture SQL')
|
|
|
|
def partial_extract(archive, destination, expected, private=False, precreated=False):
|
|
self.assertTrue(precreated)
|
|
config = destination / '.runtime/local-config.json'
|
|
config.parent.mkdir(mode=0o700)
|
|
config.write_text('{"password":"synthetic"}')
|
|
raise backup.BackupFailure('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
|
|
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
|
patch.object(backup, 'client', side_effect=fake_client), \
|
|
patch.object(backup, 'extract_verified_tar', side_effect=partial_extract), \
|
|
self.assertRaises(backup.BackupFailure) as raised:
|
|
backup.restore(folder, target, 'quality_restore_cleanup')
|
|
self.assertEqual((raised.exception.stage, raised.exception.code),
|
|
('CLEANUP', 'RESTORE_DATABASE_CLEANUP_FAILED'))
|
|
self.assertTrue(any(statement.startswith('DROP DATABASE') for statement in statements))
|
|
self.assertFalse(target.exists())
|
|
self.assertFalse(list(root.glob('.target.restore-*')))
|
|
|
|
def test_persistent_restore_reports_both_cleanup_failures_after_attempting_both(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
backup_root = root / 'output/backups'
|
|
backup_root.mkdir(parents=True)
|
|
folder = self.make_minimal_persistent_backup(root, backup_root)
|
|
target = root / 'target'
|
|
drop_attempted = []
|
|
|
|
def fake_client(statement, database=None):
|
|
if statement == 'SHOW DATABASES;': return ''
|
|
if statement.startswith('CREATE DATABASE'): return ''
|
|
if statement.startswith('DROP DATABASE'):
|
|
drop_attempted.append(True)
|
|
raise OSError('synthetic drop failure')
|
|
self.fail('Unexpected combined cleanup fixture SQL')
|
|
|
|
def partial_extract(archive, destination, expected, private=False, precreated=False):
|
|
(destination / '.runtime').mkdir()
|
|
(destination / '.runtime/local-config.json').write_text('{"password":"synthetic"}')
|
|
raise backup.BackupFailure('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
|
|
|
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
|
patch.object(backup, 'client', side_effect=fake_client), \
|
|
patch.object(backup, 'extract_verified_tar', side_effect=partial_extract), \
|
|
patch.object(backup.shutil, 'rmtree', side_effect=OSError('synthetic stage cleanup failure')) as remove, \
|
|
self.assertRaises(backup.BackupFailure) as raised:
|
|
backup.restore(folder, target, 'quality_restore_cleanup_both')
|
|
self.assertEqual((raised.exception.stage, raised.exception.code),
|
|
('CLEANUP', 'RESTORE_DATABASE_AND_STAGE_CLEANUP_FAILED'))
|
|
self.assertTrue(drop_attempted)
|
|
remove.assert_called_once()
|
|
self.assertFalse(target.exists())
|
|
|
|
def test_failure_protocol_contains_only_whitelisted_safe_fields(self):
|
|
stdout = io.StringIO()
|
|
with patch.object(backup, 'create', side_effect=backup.BackupFailure('INPUTS', 'APPLICATION_INPUT_MISSING')):
|
|
with contextlib.redirect_stdout(stdout):
|
|
code = backup.main(['create'])
|
|
self.assertNotEqual(code, 0)
|
|
event = json.loads(stdout.getvalue())
|
|
self.assertEqual(event, {'event': 'failure', 'stage': 'INPUTS', 'code': 'APPLICATION_INPUT_MISSING',
|
|
'message': backup.SAFE_FAILURES[('INPUTS', 'APPLICATION_INPUT_MISSING')]})
|
|
self.assertNotIn('traceback', stdout.getvalue().lower())
|
|
self.assertNotIn('secret', stdout.getvalue().lower())
|
|
|
|
def test_checksum_sql_restore_and_missing_archive_failures_are_distinct(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
backup_root = root / 'output/backups'
|
|
folder = backup_root / 'fixture'
|
|
folder.mkdir(parents=True)
|
|
sql = folder / 'database.sql.gz'
|
|
with gzip.open(sql, 'wb') as stream:
|
|
stream.write(b'')
|
|
archive = folder / 'archive.tar.gz'
|
|
with tarfile.open(archive, 'w:gz'):
|
|
pass
|
|
base_manifest = {'databaseDumpSha256': backup.file_sha256(sql),
|
|
'archiveSha256': backup.file_sha256(archive), 'files': {},
|
|
'tableCounts': {}, 'queryEvidence': None,
|
|
'archiveSourceRoot': str(root / 'old/runtimedata/archive')}
|
|
(folder / 'manifest.json').write_text(json.dumps({**base_manifest, 'archiveSha256': '0' * 64}))
|
|
with self.private_verifier_fixture(root) as private, \
|
|
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
|
self.assertRaises(backup.BackupFailure) as checksum:
|
|
backup.verify(folder)
|
|
self.assertEqual((checksum.exception.stage, checksum.exception.code),
|
|
('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'))
|
|
private['instance'].assert_called_once_with(root)
|
|
self.assertEqual(private['registered'], [])
|
|
|
|
(folder / 'manifest.json').write_text(json.dumps(base_manifest))
|
|
state = {'dropped': False}
|
|
def fake_client(statement, database=None):
|
|
self.assertEqual(backup.client_config(), private['config'])
|
|
if statement.startswith('CREATE DATABASE'):
|
|
return ''
|
|
if statement == 'SELECT @@server_uuid;':
|
|
return 'synthetic-server-uuid\n'
|
|
if statement.startswith('DROP DATABASE'):
|
|
state['dropped'] = True
|
|
return ''
|
|
if statement == 'SHOW TABLES;':
|
|
return ''
|
|
self.fail('Unexpected SQL in failing import fixture')
|
|
|
|
class FailedMysqlRestore:
|
|
def __init__(self, *args, **kwargs):
|
|
self.pid = 434343
|
|
self.stdin = io.BytesIO()
|
|
self.returncode = 1
|
|
def wait(self):
|
|
return 1
|
|
def poll(self):
|
|
return self.returncode
|
|
def kill(self):
|
|
self.returncode = -9
|
|
|
|
with self.private_verifier_fixture(root) as private, \
|
|
patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
|
patch.object(backup, 'client', side_effect=fake_client), \
|
|
patch.object(backup.subprocess, 'Popen', FailedMysqlRestore), \
|
|
self.assertRaises(backup.BackupFailure) as restore:
|
|
backup.verify(folder)
|
|
self.assertEqual((restore.exception.stage, restore.exception.code),
|
|
('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'))
|
|
self.assertTrue(state['dropped'], 'only the newly created scratch database is cleaned up')
|
|
private['instance'].assert_called_once_with(root)
|
|
self.assertEqual(private['childPids'], [(private['registration'], 434343)])
|
|
private['scratchCreated'].assert_called_once_with(private['registration'])
|
|
self.assertFalse(private['registration'].exists())
|
|
self.assertIsNone(backup.CLIENT_CONFIG_OVERRIDE.get())
|
|
|
|
old_root = (root / 'old/runtimedata/archive').resolve()
|
|
row = {'path': str(old_root / 'missing.xlsx'), 'sha256': '1' * 64, 'size': 12}
|
|
with patch.object(backup, 'client', return_value=json.dumps(row) + '\n'), \
|
|
self.assertRaises(backup.BackupFailure) as missing_archive:
|
|
backup.verify_references('quality_restore_fixture', root / 'relocated/archive', old_root,
|
|
root / 'relocated/archive')
|
|
self.assertEqual((missing_archive.exception.stage, missing_archive.exception.code),
|
|
('ARCHIVE_RESTORE', 'SOURCE_ARCHIVE_INVALID'))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|