230 lines
12 KiB
Python
230 lines
12 KiB
Python
import json
|
|
import zipfile
|
|
from pathlib import Path
|
|
import types
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import test_update_manager as fixtures
|
|
update = fixtures.update
|
|
|
|
|
|
class HardeningTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.f = fixtures.UpdateTests()
|
|
self.f.setUp()
|
|
self.addCleanup(self.f.doCleanups)
|
|
self.root = self.f.root
|
|
|
|
def backup(self):
|
|
b = self.root / '.runtime/update/backup-20261105T120000Z-abcdef12'
|
|
b.mkdir(parents=True)
|
|
update.backup_program(self.root, b, self.f.release)
|
|
(b / 'database.sql').write_bytes(b'-- database snapshot\n' + b'SELECT 1;\n' * 30)
|
|
update.atomic(self.root / '.runtime/update/journal.json', {
|
|
'backup': b.name, 'phase': 'installing', 'databaseBackedUp': True,
|
|
'databaseMayHaveChanged': True,
|
|
'backupManifestSha256': update.digest(b / 'manifest.json'),
|
|
'programInventorySha256': update.digest(b / 'program.json')})
|
|
return b
|
|
|
|
def test_completed_status_failure_never_rolls_back_committed_program(self):
|
|
original = update.write_status
|
|
def status(state, message, root=update.ROOT, **extra):
|
|
if state == 'completed':
|
|
raise PermissionError('status write failure')
|
|
return original(state, message, root, **extra)
|
|
with patch.object(update, 'write_status', side_effect=status):
|
|
calls, restores, _ = self.f.worker_harness()
|
|
self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence'])
|
|
self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase'])
|
|
self.assertFalse(restores)
|
|
self.assertEqual('completed', update.status(self.root)['state'])
|
|
|
|
def test_partially_completed_stop_attempt_restores_old_service(self):
|
|
failed = False
|
|
def hook(root, action, database_only, timeout, program_only):
|
|
nonlocal failed
|
|
if action == 'stop' and not failed:
|
|
failed = True
|
|
raise update.UpdateError('service still closing')
|
|
calls, restores, _ = self.f.worker_harness(control_hook=hook)
|
|
self.assertIn(('serve', False), calls)
|
|
self.assertFalse(restores)
|
|
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
|
|
self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase'])
|
|
|
|
def test_runtime_failure_occurs_before_stop_or_replacement(self):
|
|
def hook(root, action, database_only, timeout, program_only):
|
|
if action == 'verify-runtime' and not program_only:
|
|
raise update.UpdateError('runtime preflight failed')
|
|
calls, _, _ = self.f.worker_harness(control_hook=hook)
|
|
self.assertNotIn(('stop', False), calls)
|
|
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
|
|
self.assertEqual('failed', update.read(self.root / '.runtime/update/status.json')['state'])
|
|
|
|
def test_program_verification_failure_does_not_restore_unchanged_database(self):
|
|
calls, imports, queries = self.f.worker_harness(fail_verify=True)
|
|
self.assertFalse(imports)
|
|
self.assertFalse(any('CREATE DATABASE' in q for q in queries))
|
|
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
|
|
|
|
def test_durable_database_is_reused_after_startup_failure_and_bootstrap_replay(self):
|
|
b = self.backup()
|
|
imports, queries = [], []
|
|
fail = True
|
|
def ready(root, process, database_only=False):
|
|
nonlocal fail
|
|
if not database_only and fail:
|
|
fail = False
|
|
raise update.UpdateError('temporary startup failure')
|
|
with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \
|
|
patch.object(update, 'wait_ready', side_effect=ready), \
|
|
patch.object(update, 'mysql', side_effect=lambda *args, **kwargs: imports.append(kwargs['database'])), \
|
|
patch.object(update, 'sql', side_effect=lambda root, config, q: queries.append(q)):
|
|
with self.assertRaises(update.UpdateError):
|
|
update.rollback(self.root, b, True)
|
|
selected = update.read(self.root / '.runtime/deployment.json')['mysqlDatabase']
|
|
update.restore_program(self.root, b) # immutable bootstrap replays old program/private snapshot
|
|
update.recover_locked(self.root)
|
|
self.assertEqual(1, len(imports))
|
|
self.assertEqual(1, sum('CREATE DATABASE' in q for q in queries))
|
|
self.assertEqual(selected, update.read(self.root / '.runtime/deployment.json')['mysqlDatabase'])
|
|
|
|
def test_recovery_progress_rejects_changed_dump_before_any_service_mutation(self):
|
|
b = self.backup()
|
|
fail = True
|
|
def ready(root, process, database_only=False):
|
|
if not database_only:
|
|
raise update.UpdateError('temporary startup failure')
|
|
with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \
|
|
patch.object(update, 'wait_ready', side_effect=ready), \
|
|
patch.object(update, 'mysql'), patch.object(update, 'sql'):
|
|
with self.assertRaises(update.UpdateError):
|
|
update.rollback(self.root, b, True)
|
|
(b / 'database.sql').write_bytes(b'altered dump')
|
|
with patch.object(update, 'controller') as controller, self.assertRaises(update.UpdateError):
|
|
update.recover_locked(self.root)
|
|
controller.assert_not_called()
|
|
|
|
def test_old_terminal_status_cannot_override_new_operation(self):
|
|
update.atomic(self.root / '.runtime/update/journal.json', {
|
|
'phase': 'rolledBack', 'operationId': 'old-operation'})
|
|
update.write_status('downloading', 'downloading new attempt', self.root, operationId='new-operation')
|
|
with patch.object(update, 'live', return_value=True):
|
|
self.assertEqual('downloading', update.status(self.root)['state'])
|
|
|
|
def test_uncertain_complete_write_never_triggers_rollback(self):
|
|
original = update.atomic
|
|
def atomic(path, value):
|
|
original(path, value)
|
|
if path.name == 'journal.json' and value.get('phase') == 'complete':
|
|
raise OSError('write receipt lost after durable commit')
|
|
with patch.object(update, 'atomic', side_effect=atomic):
|
|
_, imports, _ = self.f.worker_harness()
|
|
self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence'])
|
|
self.assertFalse(imports)
|
|
journal = update.read(self.root / '.runtime/update/journal.json')
|
|
self.assertEqual('complete', journal['phase'])
|
|
self.assertIn('backupManifestSha256', journal)
|
|
|
|
def test_interrupted_import_attempts_are_bounded_without_dropping_databases(self):
|
|
b = self.backup()
|
|
imports, queries = [], []
|
|
def mysql(*args, **kwargs):
|
|
imports.append(kwargs['database'])
|
|
raise update.UpdateError('simulated partial import')
|
|
with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \
|
|
patch.object(update, 'wait_ready'), patch.object(update, 'mysql', side_effect=mysql), \
|
|
patch.object(update, 'sql', side_effect=lambda root, config, q: queries.append(q)):
|
|
for _ in range(update.MAX_RECOVERY_ATTEMPTS + 1):
|
|
with self.assertRaises(update.UpdateError):
|
|
update.recover_locked(self.root)
|
|
self.assertEqual(update.MAX_RECOVERY_ATTEMPTS, len(imports))
|
|
self.assertEqual(len(imports), len(set(imports)))
|
|
self.assertFalse(any('DROP' in q for q in queries))
|
|
self.assertNotIn(self.f.config['mysqlDatabase'], imports)
|
|
|
|
def test_database_is_not_durable_until_clean_stop_succeeds(self):
|
|
b = self.backup()
|
|
stops = 0
|
|
def controller(root, action, database_only=False, **kwargs):
|
|
nonlocal stops
|
|
if action == 'stop':
|
|
stops += 1
|
|
if stops == 2:
|
|
raise update.UpdateError('database still closing')
|
|
return types.SimpleNamespace(pid=1)
|
|
with patch.object(update, 'controller', side_effect=controller), \
|
|
patch.object(update, 'wait_ready'), patch.object(update, 'mysql'), patch.object(update, 'sql'):
|
|
with self.assertRaises(update.UpdateError):
|
|
update.rollback(self.root, b, True)
|
|
self.assertEqual('importing', update.read(self.root / '.runtime/update/recovery.json')['stage'])
|
|
|
|
def test_recovery_cannot_be_redirected_to_original_database(self):
|
|
b = self.backup()
|
|
binding = update.recovery_binding(b)
|
|
update.atomic(self.root / '.runtime/update/recovery.json', {
|
|
**binding, 'attempt': 1, 'stage': 'databaseDurable', 'database': self.f.config['mysqlDatabase']})
|
|
with patch.object(update, 'controller') as controller, self.assertRaises(update.UpdateError):
|
|
update.recover_locked(self.root)
|
|
controller.assert_not_called()
|
|
|
|
def test_backup_preflight_failure_keeps_live_service_and_program(self):
|
|
with patch.object(update, 'preflight_backup', side_effect=PermissionError('backup path not writable')):
|
|
calls, _, _ = self.f.worker_harness()
|
|
self.assertNotIn(('stop', False), calls)
|
|
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
|
|
|
|
def test_staged_protocol_blocks_old_or_mixed_package_before_stop(self):
|
|
stage = self.root / '.runtime/staged-protocol'
|
|
scripts = stage / 'deploy/offline'
|
|
scripts.mkdir(parents=True)
|
|
(stage / 'app').mkdir()
|
|
(scripts / 'update_manager.py').write_text('UPDATE_PROTOCOL = 2\n')
|
|
(scripts / 'control.py').write_text('UPDATE_CONTROL_PROTOCOL = 2\n')
|
|
with zipfile.ZipFile(stage / 'app/backend.jar', 'w') as jar:
|
|
jar.writestr('BOOT-INF/classes/update-protocol.json', '{"protocol":2}')
|
|
update.preflight_programs(stage)
|
|
(scripts / 'control.py').write_text('OLD_CONTROLLER = True\n')
|
|
with self.assertRaisesRegex(update.UpdateError, '服务尚未停止'):
|
|
update.preflight_programs(stage)
|
|
(scripts / 'control.py').write_text('UPDATE_CONTROL_PROTOCOL = 2\n')
|
|
with zipfile.ZipFile(stage / 'app/backend.jar', 'w') as jar:
|
|
jar.writestr('old-backend', 'old protocol')
|
|
with self.assertRaisesRegex(update.UpdateError, '服务尚未停止'):
|
|
update.preflight_programs(stage)
|
|
|
|
|
|
def test_actual_worker_rejects_incompatible_signed_payload_before_stopping(self):
|
|
calls, imports, _ = self.f.worker_harness(program_hook=update.preflight_programs)
|
|
self.assertNotIn(('stop', False), calls)
|
|
self.assertFalse(imports)
|
|
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
|
|
self.assertIn('更新包', update.read(self.root / '.runtime/update/status.json')['message'])
|
|
|
|
def test_legacy_backend_cannot_be_left_in_active_maintenance_after_final_status_failure(self):
|
|
original = update.write_status
|
|
def status(state, message, root=update.ROOT, **extra):
|
|
if state == 'failed':
|
|
raise PermissionError('final UI state blocked')
|
|
return original(state, message, root, **extra)
|
|
with patch.object(update, 'write_status', side_effect=status):
|
|
self.f.worker_harness(fail_start=True)
|
|
self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase'])
|
|
raw = update.read(self.root / '.runtime/update/status.json')['state']
|
|
self.assertNotIn(raw, {'backingUp', 'installing', 'restarting', 'rollingBack'})
|
|
|
|
def test_legacy_terminal_recovery_persists_status_reconciliation(self):
|
|
directory = self.root / '.runtime/update'
|
|
update.atomic(directory / 'journal.json', {'phase': 'rolledBack', 'operationId': 'same'})
|
|
update.atomic(directory / 'status.json', {'state': 'rollingBack', 'operationId': 'same'})
|
|
update.recover_locked(self.root)
|
|
self.assertEqual('failed', update.read(directory / 'status.json')['state'])
|
|
update.atomic(directory / 'status.json', {'state': 'rollingBack', 'operationId': 'same'})
|
|
with patch.object(update, 'write_status', side_effect=PermissionError('status blocked')), self.assertRaises(PermissionError):
|
|
update.recover_locked(self.root)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|