同步:完善质量分析工作台并更新主题字体
同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。 补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。 统一午夜黑及其他主题的控件细节、中文和业务数字字体。 排除本地业务数据、凭据、预览、发布与验收产物。 验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
This commit is contained in:
@@ -0,0 +1,554 @@
|
||||
import contextlib
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import backup
|
||||
|
||||
|
||||
class BackupRemediationTest(unittest.TestCase):
|
||||
def test_create_preserves_binary_columns_as_utf8_safe_hex_literals(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self.make_root(root)
|
||||
|
||||
class BinaryDump:
|
||||
def __init__(self, command, **kwargs):
|
||||
blob = b'0x008BFF' if '--hex-blob' in command else b"'\x00\x8b\xff'"
|
||||
self.stdout = io.BytesIO(b'CREATE TABLE `binary_fixture` (`payload` blob);\n'
|
||||
b'INSERT INTO `binary_fixture` VALUES (' + blob + b');\n')
|
||||
|
||||
def wait(self): return 0
|
||||
def poll(self): return 0
|
||||
def kill(self): pass
|
||||
|
||||
with patch.object(backup, 'ROOT', root), \
|
||||
patch.object(backup, 'BACKUP_ROOT', root / 'output/backups'), \
|
||||
patch.object(backup, 'database_config', return_value={'mysqlDatabase': 'quality_fixture'}), \
|
||||
patch.object(backup, 'verify_references', return_value={'verified': True}), \
|
||||
patch.object(backup, 'query_evidence', return_value={}), \
|
||||
patch.object(backup.subprocess, 'Popen', BinaryDump):
|
||||
folder = backup.create()
|
||||
manifest = json.loads((folder / 'manifest.json').read_text())
|
||||
self.assertEqual(manifest['tableCounts']['binary_fixture'], 1)
|
||||
with gzip.open(folder / 'database.sql.gz', 'rt', encoding='utf-8') as dump:
|
||||
self.assertIn('VALUES (0x008BFF)', dump.read())
|
||||
|
||||
def make_root(self, root, missing=()):
|
||||
for relative in backup.REQUIRED_APPLICATION_DIRECTORIES:
|
||||
(root / relative).mkdir(parents=True, exist_ok=True)
|
||||
for relative in backup.REQUIRED_APPLICATION_FILES:
|
||||
if relative in missing:
|
||||
continue
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text('fixture:' + relative)
|
||||
|
||||
def test_application_archive_requires_build_ai_and_nao_inputs(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self.make_root(root)
|
||||
(root / '.runtime/nao-model-config.json').write_text('{"apiKey":"synthetic"}')
|
||||
(root / '.runtime/nao-model-config.json').chmod(0o600)
|
||||
with patch.object(backup, 'ROOT', root):
|
||||
chosen = backup.application_files()
|
||||
names = {p.relative_to(root).as_posix() for p in chosen}
|
||||
self.assertTrue(set(backup.REQUIRED_APPLICATION_FILES).issubset(names))
|
||||
self.assertIn('.runtime/nao-model-config.json', names)
|
||||
output = root / 'backup'
|
||||
output.mkdir()
|
||||
manifest, _ = backup.archive_application(output)
|
||||
self.assertIn('requirements-nao.lock', manifest)
|
||||
self.assertIn('ai/nao-quality/agent/prompts/system.md', manifest)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self.make_root(root, missing={'web/index.html'})
|
||||
with patch.object(backup, 'ROOT', root), self.assertRaises(backup.BackupFailure) as raised:
|
||||
backup.application_files()
|
||||
self.assertEqual((raised.exception.stage, raised.exception.code), ('INPUTS', 'APPLICATION_INPUT_MISSING'))
|
||||
|
||||
def test_verified_extract_rejects_traversal_and_links(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
archive = root / 'valid.tar.gz'
|
||||
payload = b'synthetic source'
|
||||
with tarfile.open(archive, 'w:gz') as tar:
|
||||
info = tarfile.TarInfo('year/source.xlsx')
|
||||
info.size = len(payload)
|
||||
tar.addfile(info, io.BytesIO(payload))
|
||||
extracted = root / 'safe'
|
||||
actual = backup.extract_verified_tar(archive, extracted,
|
||||
{'year/source.xlsx': hashlib.sha256(payload).hexdigest()}, private=True)
|
||||
self.assertEqual(actual['year/source.xlsx'], hashlib.sha256(payload).hexdigest())
|
||||
self.assertEqual((extracted / 'year/source.xlsx').read_bytes(), payload)
|
||||
self.assertEqual(extracted.stat().st_mode & 0o777, 0o700)
|
||||
|
||||
bad = root / 'bad.tar.gz'
|
||||
with tarfile.open(bad, 'w:gz') as tar:
|
||||
info = tarfile.TarInfo('../escape.txt')
|
||||
info.size = 1
|
||||
tar.addfile(info, io.BytesIO(b'x'))
|
||||
with self.assertRaises(backup.BackupFailure):
|
||||
backup.extract_verified_tar(bad, root / 'bad-out', {'../escape.txt': hashlib.sha256(b'x').hexdigest()})
|
||||
self.assertFalse((root / 'escape.txt').exists())
|
||||
|
||||
link = root / 'link.tar.gz'
|
||||
with tarfile.open(link, 'w:gz') as tar:
|
||||
info = tarfile.TarInfo('linked.xlsx')
|
||||
info.type = tarfile.SYMTYPE
|
||||
info.linkname = '../outside'
|
||||
tar.addfile(info)
|
||||
with self.assertRaises(backup.BackupFailure):
|
||||
backup.extract_verified_tar(link, root / 'link-out', {'linked.xlsx': '0' * 64})
|
||||
|
||||
def test_absolute_archive_references_rebind_without_guessing_old_root(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
source_root = (root / 'old-project/runtimedata/archive').resolve()
|
||||
relative = Path('2026/source.xlsx')
|
||||
payload = b'synthetic workbook'
|
||||
restored_root = root / 'new-project/runtimedata/archive'
|
||||
restored_file = restored_root / relative
|
||||
restored_file.parent.mkdir(parents=True)
|
||||
restored_file.write_bytes(payload)
|
||||
old_path = str(source_root / relative)
|
||||
new_path = str(restored_file.resolve())
|
||||
seen_updates = []
|
||||
|
||||
def fake_client(sql, database=None):
|
||||
if sql.startswith('SELECT JSON_OBJECT') and "'path',archive_path,'sha256'" in sql:
|
||||
row = {'path': old_path if not seen_updates else new_path,
|
||||
'sha256': hashlib.sha256(payload).hexdigest(), 'size': len(payload)}
|
||||
return json.dumps(row) + '\n'
|
||||
if sql.startswith('UPDATE source_revision'):
|
||||
seen_updates.append(sql)
|
||||
return ''
|
||||
if sql.startswith('SELECT JSON_OBJECT'):
|
||||
return json.dumps({'path': new_path}) + '\n'
|
||||
self.fail('Unexpected SQL in isolated rebinding fixture')
|
||||
|
||||
with patch.object(backup, 'client', side_effect=fake_client):
|
||||
result = backup.verify_references('quality_restore_fixture', restored_root,
|
||||
source_root, restored_root)
|
||||
self.assertEqual(result, {'verified': True, 'rebound': True, 'count': 1})
|
||||
self.assertEqual(len(seen_updates), 1)
|
||||
self.assertIn(new_path.encode().hex(), seen_updates[0])
|
||||
|
||||
with patch.object(backup, 'client', return_value=json.dumps({'path': old_path, 'sha256': hashlib.sha256(payload).hexdigest(), 'size': len(payload)}) + '\n'):
|
||||
legacy = backup.verify_references('quality_restore_fixture', restored_root, None, restored_root)
|
||||
self.assertEqual(legacy, {'verified': False, 'rebound': False, 'count': 0})
|
||||
|
||||
def test_verify_records_data_file_and_relocation_evidence_separately(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
backup_root = root / 'output/backups'
|
||||
folder = backup_root / 'fixture'
|
||||
folder.mkdir(parents=True)
|
||||
old_archive_root = (root / 'old-location/runtimedata/archive').resolve()
|
||||
source_relative = Path('2026/synthetic.xlsx')
|
||||
source_bytes = b'synthetic source bytes'
|
||||
archive_file_map = {source_relative.as_posix(): hashlib.sha256(source_bytes).hexdigest()}
|
||||
|
||||
sql = folder / 'database.sql.gz'
|
||||
with gzip.open(sql, 'wt') as stream:
|
||||
stream.write('CREATE TABLE `fixture_table` (id INT);\nINSERT INTO `fixture_table` VALUES (1);\n')
|
||||
archived = folder / 'archive.tar.gz'
|
||||
with tarfile.open(archived, 'w:gz') as tar:
|
||||
info = tarfile.TarInfo(source_relative.as_posix())
|
||||
info.size = len(source_bytes)
|
||||
tar.addfile(info, io.BytesIO(source_bytes))
|
||||
application = folder / 'application.tar.gz'
|
||||
app_files = {name: hashlib.sha256(('fixture:' + name).encode()).hexdigest()
|
||||
for name in backup.REQUIRED_APPLICATION_FILES}
|
||||
with tarfile.open(application, 'w:gz') as tar:
|
||||
for name in backup.REQUIRED_APPLICATION_FILES:
|
||||
data = ('fixture:' + name).encode()
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(data)
|
||||
tar.addfile(info, io.BytesIO(data))
|
||||
manifest = {
|
||||
'formatVersion': 3,
|
||||
'databaseDumpSha256': backup.file_sha256(sql),
|
||||
'archiveSha256': backup.file_sha256(archived),
|
||||
'archiveSourceRoot': str(old_archive_root),
|
||||
'applicationSha256': backup.file_sha256(application),
|
||||
'applicationFiles': app_files,
|
||||
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES),
|
||||
'files': archive_file_map,
|
||||
'tableCounts': {'fixture_table': 1},
|
||||
'queryEvidence': {'releaseId': None, 'months': []},
|
||||
'verification': {},
|
||||
}
|
||||
(folder / 'manifest.json').write_text(json.dumps(manifest))
|
||||
state = {'database': None, 'path': str(old_archive_root / source_relative), 'updated': False, 'dropped': False}
|
||||
|
||||
def fake_client(statement, database=None):
|
||||
if statement.startswith('CREATE DATABASE'):
|
||||
state['database'] = statement.split('`')[1]
|
||||
return ''
|
||||
if statement == 'SHOW TABLES;':
|
||||
return 'fixture_table\n'
|
||||
if statement.startswith('SELECT COUNT(*)'):
|
||||
return '1\n'
|
||||
if "SELECT release_id FROM current_release" in statement:
|
||||
return ''
|
||||
if "'path',archive_path,'sha256'" in statement:
|
||||
return json.dumps({'path': state['path'], 'sha256': archive_file_map[source_relative.as_posix()],
|
||||
'size': len(source_bytes)}) + '\n'
|
||||
if statement.startswith('UPDATE source_revision'):
|
||||
state['updated'] = True
|
||||
new_hex = statement.split('CONVERT(0x', 1)[1].split(' ', 1)[0]
|
||||
state['path'] = bytes.fromhex(new_hex).decode('utf-8')
|
||||
return ''
|
||||
if "JSON_OBJECT('path',archive_path)" in statement:
|
||||
return json.dumps({'path': state['path']}) + '\n'
|
||||
if statement.startswith('DROP DATABASE'):
|
||||
state['dropped'] = True
|
||||
return ''
|
||||
self.fail('Unexpected SQL in synthetic restore: ' + statement)
|
||||
|
||||
class RecordingInput(io.BytesIO):
|
||||
def close(self):
|
||||
state['restoredSql'] = self.getvalue()
|
||||
super().close()
|
||||
|
||||
class FakeMysqlRestore:
|
||||
def __init__(self, *args, **kwargs):
|
||||
state['restoreCommand'] = list(args[0])
|
||||
self.stdin = RecordingInput()
|
||||
self.returncode = 0
|
||||
def wait(self):
|
||||
return 0
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
def kill(self):
|
||||
self.returncode = -9
|
||||
|
||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||
patch.object(backup, 'client', side_effect=fake_client), \
|
||||
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
|
||||
result = backup.verify(folder)
|
||||
saved = json.loads((folder / 'manifest.json').read_text())
|
||||
self.assertIn('--init-command=SET SESSION autocommit=0', state['restoreCommand'])
|
||||
self.assertTrue(state['restoredSql'].endswith(b'\nCOMMIT;\n'))
|
||||
self.assertTrue(state['updated'])
|
||||
self.assertTrue(state['dropped'])
|
||||
self.assertEqual(state['database'].split('_')[0], 'quality')
|
||||
self.assertFalse(result['migrationReady'])
|
||||
self.assertTrue(result['archiveReferencesRebound'])
|
||||
self.assertEqual(saved['verification'], {
|
||||
'databaseRestoreVerified': True,
|
||||
'applicationFilesVerified': True,
|
||||
'requiredApplicationInputsVerified': True,
|
||||
'archiveFilesVerified': True,
|
||||
'archiveReferencesVerified': True,
|
||||
'applicationStartupVerified': False,
|
||||
})
|
||||
self.assertFalse(saved['verification']['applicationStartupVerified'])
|
||||
|
||||
def test_persistent_restore_rebinds_only_new_database_and_directory(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
backup_root = root / 'output/backups'
|
||||
folder = backup_root / 'fixture-v3'
|
||||
folder.mkdir(parents=True)
|
||||
target = root / 'relocated-project'
|
||||
source_root = (root / 'source-project/runtimedata/archive').resolve()
|
||||
relative = Path('source-revisions/black/2026/sample.xlsx')
|
||||
source = b'synthetic archived workbook'
|
||||
source_hash = hashlib.sha256(source).hexdigest()
|
||||
state = {'path': str(source_root / relative), 'created': None, 'dropped': False}
|
||||
|
||||
sql = folder / 'database.sql.gz'
|
||||
with gzip.open(sql, 'wt') as stream:
|
||||
stream.write('CREATE TABLE `fixture_table` (id INT);\nINSERT INTO `fixture_table` VALUES (1);\n')
|
||||
archive = folder / 'archive.tar.gz'
|
||||
with tarfile.open(archive, 'w:gz') as tar:
|
||||
info = tarfile.TarInfo(relative.as_posix())
|
||||
info.size = len(source)
|
||||
tar.addfile(info, io.BytesIO(source))
|
||||
app = folder / 'application.tar.gz'
|
||||
app_files = {}
|
||||
with tarfile.open(app, 'w:gz') as tar:
|
||||
for name in backup.REQUIRED_APPLICATION_FILES:
|
||||
if name == '.runtime/local-config.json':
|
||||
data = b'{"mysqlDatabase":"quality_restore_source","databaseReady":true}\n'
|
||||
else:
|
||||
data = ('fixture:' + name).encode()
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
app_files[name] = digest
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(data)
|
||||
tar.addfile(info, io.BytesIO(data))
|
||||
manifest = {'formatVersion': 3, 'archiveSourceRoot': str(source_root),
|
||||
'databaseDumpSha256': backup.file_sha256(sql), 'archiveSha256': backup.file_sha256(archive),
|
||||
'applicationSha256': backup.file_sha256(app), 'applicationFiles': app_files,
|
||||
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES),
|
||||
'files': {relative.as_posix(): source_hash}, 'tableCounts': {'fixture_table': 1},
|
||||
'queryEvidence': {'releaseId': None, 'months': []}}
|
||||
(folder / 'manifest.json').write_text(json.dumps(manifest))
|
||||
|
||||
def fake_client(statement, database=None):
|
||||
if statement == 'SHOW DATABASES;':
|
||||
return ''
|
||||
if statement.startswith('CREATE DATABASE'):
|
||||
state['created'] = statement.split('`')[1]
|
||||
return ''
|
||||
if statement == 'SHOW TABLES;':
|
||||
return 'fixture_table\n'
|
||||
if statement.startswith('SELECT COUNT(*)'):
|
||||
return '1\n'
|
||||
if "SELECT release_id FROM current_release" in statement:
|
||||
return ''
|
||||
if "'path',archive_path,'sha256'" in statement:
|
||||
return json.dumps({'path': state['path'], 'sha256': source_hash, 'size': len(source)}) + '\n'
|
||||
if statement.startswith('UPDATE source_revision'):
|
||||
state['path'] = bytes.fromhex(statement.split('CONVERT(0x', 1)[1].split(' ', 1)[0]).decode()
|
||||
return ''
|
||||
if "JSON_OBJECT('path',archive_path)" in statement:
|
||||
return json.dumps({'path': state['path']}) + '\n'
|
||||
if statement.startswith('DROP DATABASE'):
|
||||
state['dropped'] = True
|
||||
return ''
|
||||
self.fail('Unexpected persistent restore SQL: ' + statement)
|
||||
|
||||
class FakeMysqlRestore:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.stdin = io.BytesIO()
|
||||
self.returncode = 0
|
||||
def wait(self): return 0
|
||||
def poll(self): return self.returncode
|
||||
def kill(self): self.returncode = -9
|
||||
|
||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||
patch.object(backup, 'client', side_effect=fake_client), \
|
||||
patch.object(backup.subprocess, 'Popen', FakeMysqlRestore):
|
||||
result = backup.restore(folder, target, 'quality_restore_acceptance')
|
||||
self.assertEqual(state['created'], 'quality_restore_acceptance')
|
||||
self.assertFalse(state['dropped'])
|
||||
self.assertEqual(state['path'], str((target / 'runtimedata/archive' / relative).resolve(strict=False)))
|
||||
self.assertTrue((target / 'runtimedata/archive' / relative).is_file())
|
||||
self.assertEqual(result['status'], 'ISOLATED_DATA_FILE_RESTORE_VERIFIED')
|
||||
self.assertTrue(result['verification']['archiveReferencesVerified'])
|
||||
self.assertFalse(result['verification']['applicationStartupVerified'])
|
||||
restored_config = json.loads((target / '.runtime/local-config.json').read_text())
|
||||
self.assertEqual(restored_config['mysqlDatabase'], 'quality_restore_acceptance')
|
||||
self.assertTrue(restored_config['databaseReady'])
|
||||
|
||||
def test_persistent_restore_refuses_existing_target_or_database_without_removal(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
backup_root = root / 'output/backups'
|
||||
folder = backup_root / 'fixture'
|
||||
folder.mkdir(parents=True)
|
||||
target = root / 'already-there'
|
||||
target.mkdir()
|
||||
marker = target / 'keep.txt'
|
||||
marker.write_text('do not remove')
|
||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||
self.assertRaises(backup.BackupFailure) as raised:
|
||||
backup.restore(folder, target, 'quality_restore_acceptance')
|
||||
self.assertEqual((raised.exception.stage, raised.exception.code), ('RESTORE', 'RESTORE_TARGET_INVALID'))
|
||||
self.assertEqual(marker.read_text(), 'do not remove')
|
||||
|
||||
target.unlink(missing_ok=True) if target.is_file() else None
|
||||
marker.unlink()
|
||||
target.rmdir()
|
||||
target_database = 'quality_restore_existing'
|
||||
(folder / 'database.sql.gz').write_bytes(gzip.compress(b''))
|
||||
with tarfile.open(folder / 'archive.tar.gz', 'w:gz'):
|
||||
pass
|
||||
with tarfile.open(folder / 'application.tar.gz', 'w:gz'):
|
||||
pass
|
||||
(folder / 'manifest.json').write_text(json.dumps({
|
||||
'formatVersion': 3, 'archiveSourceRoot': str(root / 'old/runtimedata/archive'),
|
||||
'databaseDumpSha256': backup.file_sha256(folder / 'database.sql.gz'),
|
||||
'archiveSha256': backup.file_sha256(folder / 'archive.tar.gz'),
|
||||
'applicationSha256': backup.file_sha256(folder / 'application.tar.gz'),
|
||||
'applicationFiles': {name: '0' * 64 for name in backup.REQUIRED_APPLICATION_FILES},
|
||||
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES), 'files': {}, 'tableCounts': {},
|
||||
}))
|
||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||
patch.object(backup, 'client', return_value=target_database + '\n'), \
|
||||
self.assertRaises(backup.BackupFailure) as existing:
|
||||
backup.restore(folder, target, target_database)
|
||||
self.assertEqual((existing.exception.stage, existing.exception.code), ('RESTORE', 'RESTORE_DATABASE_EXISTS'))
|
||||
self.assertFalse(target.exists())
|
||||
|
||||
def make_minimal_persistent_backup(self, root, backup_root):
|
||||
folder = backup_root / 'cleanup-fixture'
|
||||
folder.mkdir(parents=True)
|
||||
sql = folder / 'database.sql.gz'
|
||||
with gzip.open(sql, 'wb') as stream:
|
||||
stream.write(b'')
|
||||
archive = folder / 'archive.tar.gz'
|
||||
with tarfile.open(archive, 'w:gz'):
|
||||
pass
|
||||
app = folder / 'application.tar.gz'
|
||||
with tarfile.open(app, 'w:gz'):
|
||||
pass
|
||||
manifest = {'formatVersion': 3,
|
||||
'archiveSourceRoot': str(root / 'old/runtimedata/archive'),
|
||||
'databaseDumpSha256': backup.file_sha256(sql),
|
||||
'archiveSha256': backup.file_sha256(archive),
|
||||
'applicationSha256': backup.file_sha256(app),
|
||||
'applicationFiles': {name: '0' * 64 for name in backup.REQUIRED_APPLICATION_FILES},
|
||||
'requiredApplicationFiles': list(backup.REQUIRED_APPLICATION_FILES),
|
||||
'files': {}, 'tableCounts': {}, 'queryEvidence': None}
|
||||
(folder / 'manifest.json').write_text(json.dumps(manifest))
|
||||
return folder
|
||||
|
||||
def test_persistent_restore_drop_failure_still_removes_private_stage(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
backup_root = root / 'output/backups'
|
||||
backup_root.mkdir(parents=True)
|
||||
folder = self.make_minimal_persistent_backup(root, backup_root)
|
||||
target = root / 'target'
|
||||
statements = []
|
||||
|
||||
def fake_client(statement, database=None):
|
||||
statements.append(statement)
|
||||
if statement == 'SHOW DATABASES;': return ''
|
||||
if statement.startswith('CREATE DATABASE'): return ''
|
||||
if statement.startswith('DROP DATABASE'):
|
||||
raise OSError('synthetic drop failure with private details')
|
||||
self.fail('Unexpected cleanup fixture SQL')
|
||||
|
||||
def partial_extract(archive, destination, expected, private=False, precreated=False):
|
||||
self.assertTrue(precreated)
|
||||
config = destination / '.runtime/local-config.json'
|
||||
config.parent.mkdir(mode=0o700)
|
||||
config.write_text('{"password":"synthetic"}')
|
||||
raise backup.BackupFailure('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
||||
|
||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||
patch.object(backup, 'client', side_effect=fake_client), \
|
||||
patch.object(backup, 'extract_verified_tar', side_effect=partial_extract), \
|
||||
self.assertRaises(backup.BackupFailure) as raised:
|
||||
backup.restore(folder, target, 'quality_restore_cleanup')
|
||||
self.assertEqual((raised.exception.stage, raised.exception.code),
|
||||
('CLEANUP', 'RESTORE_DATABASE_CLEANUP_FAILED'))
|
||||
self.assertTrue(any(statement.startswith('DROP DATABASE') for statement in statements))
|
||||
self.assertFalse(target.exists())
|
||||
self.assertFalse(list(root.glob('.target.restore-*')))
|
||||
|
||||
def test_persistent_restore_reports_both_cleanup_failures_after_attempting_both(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
backup_root = root / 'output/backups'
|
||||
backup_root.mkdir(parents=True)
|
||||
folder = self.make_minimal_persistent_backup(root, backup_root)
|
||||
target = root / 'target'
|
||||
drop_attempted = []
|
||||
|
||||
def fake_client(statement, database=None):
|
||||
if statement == 'SHOW DATABASES;': return ''
|
||||
if statement.startswith('CREATE DATABASE'): return ''
|
||||
if statement.startswith('DROP DATABASE'):
|
||||
drop_attempted.append(True)
|
||||
raise OSError('synthetic drop failure')
|
||||
self.fail('Unexpected combined cleanup fixture SQL')
|
||||
|
||||
def partial_extract(archive, destination, expected, private=False, precreated=False):
|
||||
(destination / '.runtime').mkdir()
|
||||
(destination / '.runtime/local-config.json').write_text('{"password":"synthetic"}')
|
||||
raise backup.BackupFailure('APPLICATION_RESTORE', 'APPLICATION_ARCHIVE_INVALID')
|
||||
|
||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||
patch.object(backup, 'client', side_effect=fake_client), \
|
||||
patch.object(backup, 'extract_verified_tar', side_effect=partial_extract), \
|
||||
patch.object(backup.shutil, 'rmtree', side_effect=OSError('synthetic stage cleanup failure')) as remove, \
|
||||
self.assertRaises(backup.BackupFailure) as raised:
|
||||
backup.restore(folder, target, 'quality_restore_cleanup_both')
|
||||
self.assertEqual((raised.exception.stage, raised.exception.code),
|
||||
('CLEANUP', 'RESTORE_DATABASE_AND_STAGE_CLEANUP_FAILED'))
|
||||
self.assertTrue(drop_attempted)
|
||||
remove.assert_called_once()
|
||||
self.assertFalse(target.exists())
|
||||
|
||||
def test_failure_protocol_contains_only_whitelisted_safe_fields(self):
|
||||
stdout = io.StringIO()
|
||||
with patch.object(backup, 'create', side_effect=backup.BackupFailure('INPUTS', 'APPLICATION_INPUT_MISSING')):
|
||||
with contextlib.redirect_stdout(stdout):
|
||||
code = backup.main(['create'])
|
||||
self.assertNotEqual(code, 0)
|
||||
event = json.loads(stdout.getvalue())
|
||||
self.assertEqual(event, {'event': 'failure', 'stage': 'INPUTS', 'code': 'APPLICATION_INPUT_MISSING',
|
||||
'message': backup.SAFE_FAILURES[('INPUTS', 'APPLICATION_INPUT_MISSING')]})
|
||||
self.assertNotIn('traceback', stdout.getvalue().lower())
|
||||
self.assertNotIn('secret', stdout.getvalue().lower())
|
||||
|
||||
def test_checksum_sql_restore_and_missing_archive_failures_are_distinct(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
backup_root = root / 'output/backups'
|
||||
folder = backup_root / 'fixture'
|
||||
folder.mkdir(parents=True)
|
||||
sql = folder / 'database.sql.gz'
|
||||
with gzip.open(sql, 'wb') as stream:
|
||||
stream.write(b'')
|
||||
archive = folder / 'archive.tar.gz'
|
||||
with tarfile.open(archive, 'w:gz'):
|
||||
pass
|
||||
base_manifest = {'databaseDumpSha256': backup.file_sha256(sql),
|
||||
'archiveSha256': backup.file_sha256(archive), 'files': {},
|
||||
'tableCounts': {}, 'queryEvidence': None,
|
||||
'archiveSourceRoot': str(root / 'old/runtimedata/archive')}
|
||||
(folder / 'manifest.json').write_text(json.dumps({**base_manifest, 'archiveSha256': '0' * 64}))
|
||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||
self.assertRaises(backup.BackupFailure) as checksum:
|
||||
backup.verify(folder)
|
||||
self.assertEqual((checksum.exception.stage, checksum.exception.code),
|
||||
('CHECKSUM', 'BACKUP_CHECKSUM_MISMATCH'))
|
||||
|
||||
(folder / 'manifest.json').write_text(json.dumps(base_manifest))
|
||||
state = {'dropped': False}
|
||||
def fake_client(statement, database=None):
|
||||
if statement.startswith('CREATE DATABASE'):
|
||||
return ''
|
||||
if statement.startswith('DROP DATABASE'):
|
||||
state['dropped'] = True
|
||||
return ''
|
||||
if statement == 'SHOW TABLES;':
|
||||
return ''
|
||||
self.fail('Unexpected SQL in failing import fixture')
|
||||
|
||||
class FailedMysqlRestore:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.stdin = io.BytesIO()
|
||||
self.returncode = 1
|
||||
def wait(self):
|
||||
return 1
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
def kill(self):
|
||||
self.returncode = -9
|
||||
|
||||
with patch.object(backup, 'ROOT', root), patch.object(backup, 'BACKUP_ROOT', backup_root), \
|
||||
patch.object(backup, 'client', side_effect=fake_client), \
|
||||
patch.object(backup.subprocess, 'Popen', FailedMysqlRestore), \
|
||||
self.assertRaises(backup.BackupFailure) as restore:
|
||||
backup.verify(folder)
|
||||
self.assertEqual((restore.exception.stage, restore.exception.code),
|
||||
('DATABASE_RESTORE', 'DATABASE_RESTORE_FAILED'))
|
||||
self.assertTrue(state['dropped'], 'only the newly created scratch database is cleaned up')
|
||||
|
||||
old_root = (root / 'old/runtimedata/archive').resolve()
|
||||
row = {'path': str(old_root / 'missing.xlsx'), 'sha256': '1' * 64, 'size': 12}
|
||||
with patch.object(backup, 'client', return_value=json.dumps(row) + '\n'), \
|
||||
self.assertRaises(backup.BackupFailure) as missing_archive:
|
||||
backup.verify_references('quality_restore_fixture', root / 'relocated/archive', old_root,
|
||||
root / 'relocated/archive')
|
||||
self.assertEqual((missing_archive.exception.stage, missing_archive.exception.code),
|
||||
('ARCHIVE_RESTORE', 'SOURCE_ARCHIVE_INVALID'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user