import hashlib import json import os from pathlib import Path import subprocess import sys import tempfile import textwrap import time import unittest from unittest.mock import patch import psutil sys.path.insert(0, str(Path(__file__).resolve().parent)) import update_bootstrap as bootstrap class ImmutableRecoveryTest(unittest.TestCase): def fixture(self, folder): root = Path(folder) directory = root / '.runtime/update' backup = directory / 'backup-20261104T120000Z-abcdef12' (backup / 'program/deploy/offline').mkdir(parents=True) (root / 'deploy/offline').mkdir(parents=True) old = { 'app/backend.jar': b'verified old backend jar', 'deploy/offline/control.py': b'VALID_OLD_VERSION = True\n', 'deploy/offline/update_manager.py': b'def recover_locked(root):\n (root / ".runtime/restored").write_text("done")\n return {"recovered": True}\n', } entries = [] for name, content in old.items(): path = backup / 'program' / name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) (root / name).parent.mkdir(parents=True, exist_ok=True) (root / name).write_bytes(b'def broken syntax\n') entries.append({'path': name, 'sha256': hashlib.sha256(content).hexdigest()}) (backup / 'manifest.json').write_text(json.dumps({'files': entries})) (backup / 'program.json').write_text(json.dumps({'all': list(old), 'present': list(old)})) journal = {'phase': 'installing', 'databaseBackedUp': True, 'backup': backup.name, 'backupManifestSha256': bootstrap.digest(backup / 'manifest.json'), 'programInventorySha256': bootstrap.digest(backup / 'program.json')} (directory / 'journal.json').write_text(json.dumps(journal)) return root, directory, backup, journal, old def runtime_aliases(self, root): for role in ('supervisor', 'backend'): executable = bootstrap.runtime_executable(root, role) executable.parent.mkdir(parents=True, exist_ok=True) executable.symlink_to(psutil.Process().exe()) def record(self, process, name=None): observed = psutil.Process(process.pid) value = {'pid': process.pid, 'created': observed.create_time(), 'executable': str(Path(observed.exe()).resolve()), 'args': observed.cmdline()[1:]} if name: value['name'] = name return value def stop_test_child(self, process): if process.poll() is not None: return descendants = [] try: descendants = psutil.Process(process.pid).children(recursive=True) except psutil.Error: pass process.terminate() try: process.wait(timeout=5) except subprocess.TimeoutExpired: process.kill() process.wait(timeout=5) for child in descendants: try: if child.is_running() and child.status() != psutil.STATUS_ZOMBIE: child.kill() except psutil.Error: pass def wait_file(self, path, process): deadline = time.monotonic() + 10 while time.monotonic() < deadline: if path.exists(): return self.assertIsNone(process.poll(), 'Owned test child exited before readiness') time.sleep(.02) self.fail('Owned test child did not become ready') def backend_script(self, root): path = root / 'backend-test.py' path.write_text(textwrap.dedent(''' import hashlib, os, signal, sys, time from pathlib import Path root = Path(__file__).parent jar = root / 'app/backend.jar' held = jar.open('rb') def stop(*ignored): (root / '.runtime/backend-stopped-before-copy').write_text(hashlib.sha256(jar.read_bytes()).hexdigest()) held.close() raise SystemExit(0) signal.signal(signal.SIGTERM, stop) (root / '.runtime/backend-ready').write_text(str(os.getpid())) while True: time.sleep(.02) ''')) return path def start_backend(self, root): self.runtime_aliases(root) script = self.backend_script(root) process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'backend')), str(script), '-jar', str(root / 'app/backend.jar')], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) self.addCleanup(self.stop_test_child, process) self.wait_file(root / '.runtime/backend-ready', process) return process def start_supervisor(self, root): self.runtime_aliases(root) self.backend_script(root) script = root / 'deploy/offline/control.py' script.write_text(textwrap.dedent(''' import json, os, signal, subprocess, sys, time from pathlib import Path import psutil root = Path(__file__).resolve().parents[2] token = 'a' * 48 java = root / 'runtimes/java/bin' / ('java.exe' if os.name == 'nt' else 'java') child = subprocess.Popen([str(java), str(root / 'backend-test.py'), '-jar', str(root / 'app/backend.jar')]) def record(pid): process = psutil.Process(pid) return {'pid': pid, 'created': process.create_time(), 'executable': str(Path(process.exe()).resolve()), 'args': process.cmdline()[1:]} state = {'supervisor': record(os.getpid()), 'token': token, 'children': [{**record(child.pid), 'name': 'backend'}], 'ready': True} def stop(*ignored): child.terminate() child.wait(timeout=5) (root / '.runtime/supervisor-stopped').write_text('done') (root / '.runtime/service.json').unlink(missing_ok=True) raise SystemExit(0) signal.signal(signal.SIGTERM, stop) (root / '.runtime/service.json').write_text(json.dumps(state)) (root / '.runtime/supervisor-ready').write_text('ready') while True: try: request = json.loads((root / '.runtime/stop-request.json').read_text()) if request.get('token') == token: (root / '.runtime/token-observed').write_text(token) stop() except (FileNotFoundError, ValueError): pass time.sleep(.02) ''')) process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'supervisor')), str(script), 'serve'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) self.addCleanup(self.stop_test_child, process) self.wait_file(root / '.runtime/supervisor-ready', process) self.wait_file(root / '.runtime/backend-ready', process) return process def test_recovers_before_importing_broken_control_and_updater(self): with tempfile.TemporaryDirectory() as folder: root, directory, backup, journal, old = self.fixture(folder) result = bootstrap.recover(root) self.assertTrue(result['recovered']) self.assertEqual((root / '.runtime/restored').read_text(), 'done') for name, content in old.items(): self.assertEqual((root / name).read_bytes(), content) self.assertFalse((directory / 'worker.json').exists()) def test_corrupt_backup_is_rejected_before_any_replacement(self): with tempfile.TemporaryDirectory() as folder: root, directory, backup, journal, old = self.fixture(folder) (backup / 'program/deploy/offline/control.py').write_text('corruption') with self.assertRaises(RuntimeError): bootstrap.recover(root) self.assertEqual((root / 'deploy/offline/control.py').read_text(), 'def broken syntax\n') def test_active_worker_cannot_be_overridden(self): with tempfile.TemporaryDirectory() as folder: root, directory, backup, journal, old = self.fixture(folder) (directory / 'worker.json').write_text(json.dumps({'pid': os.getpid(), 'created': psutil.Process().create_time()})) with self.assertRaises(RuntimeError): bootstrap.recover(root) self.assertFalse((root / '.runtime/restored').exists()) def test_program_protection_and_no_update_are_read_only(self): for name in ('deploy/bootstrap/windows-start.ps1', 'deploy/offline/update-public-key.pem', '.runtime/deployment.json', 'runtimedata/mysql/file'): self.assertFalse(bootstrap.program(name)) with tempfile.TemporaryDirectory() as folder: root = Path(folder) self.assertEqual(bootstrap.recover(root), {'recovered': False}) self.assertFalse((root / '.runtime').exists()) def test_orphaned_backend_stops_before_jar_copy_after_worker_crash(self): with tempfile.TemporaryDirectory() as folder: root, directory, _, _, old = self.fixture(folder) child = self.start_backend(root) state = {'supervisor': {'pid': 999999999, 'created': 1, 'executable': str(sys.executable), 'args': [str(root / 'deploy/offline/control.py'), 'serve']}, 'token': 'b' * 48, 'children': [self.record(child, 'backend')]} (root / '.runtime/service.json').write_text(json.dumps(state)) (directory / 'worker.json').write_text(json.dumps({'pid': 999999998, 'created': 1})) before = bootstrap.digest(root / 'app/backend.jar') real_copy = bootstrap.atomic_copy def checked_copy(source, target): self.assertIsNotNone(child.poll(), 'A mutable program was copied while the old backend still held its JAR') return real_copy(source, target) with patch.object(bootstrap, 'atomic_copy', side_effect=checked_copy): self.assertTrue(bootstrap.recover(root)['recovered']) self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text()) self.assertEqual(old['app/backend.jar'], (root / 'app/backend.jar').read_bytes()) def test_live_supervisor_receives_matching_stop_token_and_children_exit_before_copy(self): with tempfile.TemporaryDirectory() as folder: root, directory, _, _, _ = self.fixture(folder) supervisor = self.start_supervisor(root) (root / 'deploy/offline/control.py').write_text('def broken syntax\n') before = bootstrap.digest(root / 'app/backend.jar') result = bootstrap.recover(root) self.assertTrue(result['recovered']) self.assertIsNotNone(supervisor.poll()) self.assertEqual('a' * 48, (root / '.runtime/token-observed').read_text()) self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text()) self.assertFalse((root / '.runtime/stop-request.json').exists()) def test_stale_record_pointing_at_an_unrelated_live_pid_never_kills_or_replaces(self): with tempfile.TemporaryDirectory() as folder: root, _, _, _, _ = self.fixture(folder) self.runtime_aliases(root) child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) self.addCleanup(self.stop_test_child, child) info = self.record(child, 'backend') info['created'] -= 60 (root / '.runtime/service.json').write_text(json.dumps({'children': [info]})) with self.assertRaisesRegex(RuntimeError, '归属'): bootstrap.recover(root) self.assertIsNone(child.poll()) self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text()) self.assertFalse((root / '.runtime/restored').exists()) def test_unregistered_root_backend_is_refused_without_signalling(self): with tempfile.TemporaryDirectory() as folder: root, _, _, _, _ = self.fixture(folder) child = self.start_backend(root) with self.assertRaisesRegex(RuntimeError, '未登记'): bootstrap.recover(root) self.assertIsNone(child.poll()) self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text()) def test_other_deployment_root_cannot_be_claimed_by_a_valid_live_pid_record(self): with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second: root, _, _, _, _ = self.fixture(first) other, _, _, _, _ = self.fixture(second) self.runtime_aliases(root) child = self.start_backend(other) (root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]})) with self.assertRaisesRegex(RuntimeError, '归属'): bootstrap.recover(root) self.assertIsNone(child.poll()) self.assertFalse((root / '.runtime/restored').exists()) def test_invalid_supervisor_token_does_not_signal_any_owned_process(self): with tempfile.TemporaryDirectory() as folder: root, _, _, _, _ = self.fixture(folder) supervisor = self.start_supervisor(root) state = json.loads((root / '.runtime/service.json').read_text()) state['token'] = 'invalid-token' (root / '.runtime/service.json').write_text(json.dumps(state)) with self.assertRaisesRegex(RuntimeError, '令牌'): bootstrap.recover(root) self.assertIsNone(supervisor.poll()) self.assertFalse((root / '.runtime/stop-request.json').exists()) def test_dead_pid_records_do_not_block_safe_recovery(self): with tempfile.TemporaryDirectory() as folder: root, _, _, _, _ = self.fixture(folder) state = {'children': [{'pid': 999999999, 'created': 1, 'executable': str(sys.executable), 'args': ['missing'], 'name': 'backend'}]} (root / '.runtime/service.json').write_text(json.dumps(state)) self.assertTrue(bootstrap.recover(root)['recovered']) def test_a_process_owned_by_another_user_is_never_stopped(self): with tempfile.TemporaryDirectory() as folder: root, _, _, _, _ = self.fixture(folder) child = self.start_backend(root) (root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]})) with patch.object(bootstrap, 'current_user', return_value=False), self.assertRaisesRegex(RuntimeError, '归属'): bootstrap.recover(root) self.assertIsNone(child.poll()) self.assertFalse((root / '.runtime/restored').exists()) @unittest.skipIf(os.name == 'nt', 'Host-side immutable-client fixture uses a POSIX shebang; real Windows binaries need native acceptance') def test_orphaned_mysql_uses_immutable_shutdown_client_and_is_not_force_killed(self): with tempfile.TemporaryDirectory() as folder: root, _, _, _, _ = self.fixture(folder) executable = bootstrap.runtime_executable(root, 'mysql') executable.parent.mkdir(parents=True) executable.symlink_to(psutil.Process().exe()) server = root / 'mysql-test.py' server.write_text(textwrap.dedent(''' import os, signal, time from pathlib import Path root = Path(__file__).parent def stop(*ignored): (root / '.runtime/mysql-stopped-cleanly').write_text('done') raise SystemExit(0) signal.signal(signal.SIGTERM, stop) (root / '.runtime/mysql-test-pid').write_text(str(os.getpid())) while True: time.sleep(.02) ''')) child = subprocess.Popen([str(executable), str(server), '--datadir=' + str(root / 'runtimedata/mysql'), '--port=54489'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) self.addCleanup(self.stop_test_child, child) self.wait_file(root / '.runtime/mysql-test-pid', child) admin = executable.parent / 'mysqladmin' admin.write_text('#!' + psutil.Process().exe() + '\n' + textwrap.dedent(''' import os, signal, sys from pathlib import Path root = Path(__file__).resolve().parents[3] assert os.environ['MYSQL_PWD'] == 'isolated_password_value_1234' assert '--no-defaults' in sys.argv and '--port=54489' in sys.argv and sys.argv[-1] == 'shutdown' assert not any('password' in arg.lower() for arg in sys.argv) (root / '.runtime/immutable-mysqladmin-called').write_text('done') os.kill(int((root / '.runtime/mysql-test-pid').read_text()), signal.SIGTERM) ''')) admin.chmod(0o700) (root / '.runtime/deployment.json').write_text(json.dumps({'mysqlPort': 54489, 'rootPassword': 'isolated_password_value_1234'})) (root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'mysql')]})) with patch.object(psutil.Process, 'kill', side_effect=AssertionError('MySQL must not be force killed')): self.assertTrue(bootstrap.recover(root)['recovered']) self.assertIsNotNone(child.poll()) self.assertTrue((root / '.runtime/immutable-mysqladmin-called').exists()) self.assertTrue((root / '.runtime/mysql-stopped-cleanly').exists()) if __name__ == '__main__': unittest.main()