Files
quality_analyse/deploy/offline/test_update_bootstrap.py
T

340 lines
18 KiB
Python

import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import textwrap
import time
import unittest
from unittest.mock import patch
import psutil
sys.path.insert(0, str(Path(__file__).resolve().parent))
import update_bootstrap as bootstrap
class ImmutableRecoveryTest(unittest.TestCase):
def fixture(self, folder):
root = Path(folder)
directory = root / '.runtime/update'
backup = directory / 'backup-20261104T120000Z-abcdef12'
(backup / 'program/deploy/offline').mkdir(parents=True)
(root / 'deploy/offline').mkdir(parents=True)
old = {
'app/backend.jar': b'verified old backend jar',
'deploy/offline/control.py': b'VALID_OLD_VERSION = True\n',
'deploy/offline/update_manager.py': b'def recover_locked(root):\n (root / ".runtime/restored").write_text("done")\n return {"recovered": True}\n',
}
entries = []
for name, content in old.items():
path = backup / 'program' / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
(root / name).parent.mkdir(parents=True, exist_ok=True)
(root / name).write_bytes(b'def broken syntax\n')
entries.append({'path': name, 'sha256': hashlib.sha256(content).hexdigest()})
(backup / 'manifest.json').write_text(json.dumps({'files': entries}))
(backup / 'program.json').write_text(json.dumps({'all': list(old), 'present': list(old)}))
journal = {'phase': 'installing', 'databaseBackedUp': True, 'backup': backup.name,
'backupManifestSha256': bootstrap.digest(backup / 'manifest.json'),
'programInventorySha256': bootstrap.digest(backup / 'program.json')}
(directory / 'journal.json').write_text(json.dumps(journal))
return root, directory, backup, journal, old
def runtime_aliases(self, root):
for role in ('supervisor', 'backend'):
executable = bootstrap.runtime_executable(root, role)
executable.parent.mkdir(parents=True, exist_ok=True)
executable.symlink_to(psutil.Process().exe())
def record(self, process, name=None):
observed = psutil.Process(process.pid)
value = {'pid': process.pid, 'created': observed.create_time(),
'executable': str(Path(observed.exe()).resolve()), 'args': observed.cmdline()[1:]}
if name:
value['name'] = name
return value
def stop_test_child(self, process):
if process.poll() is not None:
return
descendants = []
try:
descendants = psutil.Process(process.pid).children(recursive=True)
except psutil.Error:
pass
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=5)
for child in descendants:
try:
if child.is_running() and child.status() != psutil.STATUS_ZOMBIE:
child.kill()
except psutil.Error:
pass
def wait_file(self, path, process):
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
if path.exists():
return
self.assertIsNone(process.poll(), 'Owned test child exited before readiness')
time.sleep(.02)
self.fail('Owned test child did not become ready')
def backend_script(self, root):
path = root / 'backend-test.py'
path.write_text(textwrap.dedent('''
import hashlib, os, signal, sys, time
from pathlib import Path
root = Path(__file__).parent
jar = root / 'app/backend.jar'
held = jar.open('rb')
def stop(*ignored):
(root / '.runtime/backend-stopped-before-copy').write_text(hashlib.sha256(jar.read_bytes()).hexdigest())
held.close()
raise SystemExit(0)
signal.signal(signal.SIGTERM, stop)
(root / '.runtime/backend-ready').write_text(str(os.getpid()))
while True:
time.sleep(.02)
'''))
return path
def start_backend(self, root):
self.runtime_aliases(root)
script = self.backend_script(root)
process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'backend')), str(script), '-jar', str(root / 'app/backend.jar')],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.addCleanup(self.stop_test_child, process)
self.wait_file(root / '.runtime/backend-ready', process)
return process
def start_supervisor(self, root):
self.runtime_aliases(root)
self.backend_script(root)
script = root / 'deploy/offline/control.py'
script.write_text(textwrap.dedent('''
import json, os, signal, subprocess, sys, time
from pathlib import Path
import psutil
root = Path(__file__).resolve().parents[2]
token = 'a' * 48
java = root / 'runtimes/java/bin' / ('java.exe' if os.name == 'nt' else 'java')
child = subprocess.Popen([str(java), str(root / 'backend-test.py'), '-jar', str(root / 'app/backend.jar')])
def record(pid):
process = psutil.Process(pid)
return {'pid': pid, 'created': process.create_time(), 'executable': str(Path(process.exe()).resolve()), 'args': process.cmdline()[1:]}
state = {'supervisor': record(os.getpid()), 'token': token, 'children': [{**record(child.pid), 'name': 'backend'}], 'ready': True}
def stop(*ignored):
child.terminate()
child.wait(timeout=5)
(root / '.runtime/supervisor-stopped').write_text('done')
(root / '.runtime/service.json').unlink(missing_ok=True)
raise SystemExit(0)
signal.signal(signal.SIGTERM, stop)
(root / '.runtime/service.json').write_text(json.dumps(state))
(root / '.runtime/supervisor-ready').write_text('ready')
while True:
try:
request = json.loads((root / '.runtime/stop-request.json').read_text())
if request.get('token') == token:
(root / '.runtime/token-observed').write_text(token)
stop()
except (FileNotFoundError, ValueError):
pass
time.sleep(.02)
'''))
process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'supervisor')), str(script), 'serve'],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.addCleanup(self.stop_test_child, process)
self.wait_file(root / '.runtime/supervisor-ready', process)
self.wait_file(root / '.runtime/backend-ready', process)
return process
def test_recovers_before_importing_broken_control_and_updater(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, backup, journal, old = self.fixture(folder)
result = bootstrap.recover(root)
self.assertTrue(result['recovered'])
self.assertEqual((root / '.runtime/restored').read_text(), 'done')
for name, content in old.items():
self.assertEqual((root / name).read_bytes(), content)
self.assertFalse((directory / 'worker.json').exists())
def test_corrupt_backup_is_rejected_before_any_replacement(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, backup, journal, old = self.fixture(folder)
(backup / 'program/deploy/offline/control.py').write_text('corruption')
with self.assertRaises(RuntimeError):
bootstrap.recover(root)
self.assertEqual((root / 'deploy/offline/control.py').read_text(), 'def broken syntax\n')
def test_active_worker_cannot_be_overridden(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, backup, journal, old = self.fixture(folder)
(directory / 'worker.json').write_text(json.dumps({'pid': os.getpid(), 'created': psutil.Process().create_time()}))
with self.assertRaises(RuntimeError):
bootstrap.recover(root)
self.assertFalse((root / '.runtime/restored').exists())
def test_program_protection_and_no_update_are_read_only(self):
for name in ('deploy/bootstrap/windows-start.ps1', 'deploy/offline/update-public-key.pem', '.runtime/deployment.json', 'runtimedata/mysql/file'):
self.assertFalse(bootstrap.program(name))
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
self.assertEqual(bootstrap.recover(root), {'recovered': False})
self.assertFalse((root / '.runtime').exists())
def test_orphaned_backend_stops_before_jar_copy_after_worker_crash(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, _, _, old = self.fixture(folder)
child = self.start_backend(root)
state = {'supervisor': {'pid': 999999999, 'created': 1, 'executable': str(sys.executable),
'args': [str(root / 'deploy/offline/control.py'), 'serve']}, 'token': 'b' * 48,
'children': [self.record(child, 'backend')]}
(root / '.runtime/service.json').write_text(json.dumps(state))
(directory / 'worker.json').write_text(json.dumps({'pid': 999999998, 'created': 1}))
before = bootstrap.digest(root / 'app/backend.jar')
real_copy = bootstrap.atomic_copy
def checked_copy(source, target):
self.assertIsNotNone(child.poll(), 'A mutable program was copied while the old backend still held its JAR')
return real_copy(source, target)
with patch.object(bootstrap, 'atomic_copy', side_effect=checked_copy):
self.assertTrue(bootstrap.recover(root)['recovered'])
self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text())
self.assertEqual(old['app/backend.jar'], (root / 'app/backend.jar').read_bytes())
def test_live_supervisor_receives_matching_stop_token_and_children_exit_before_copy(self):
with tempfile.TemporaryDirectory() as folder:
root, directory, _, _, _ = self.fixture(folder)
supervisor = self.start_supervisor(root)
(root / 'deploy/offline/control.py').write_text('def broken syntax\n')
before = bootstrap.digest(root / 'app/backend.jar')
result = bootstrap.recover(root)
self.assertTrue(result['recovered'])
self.assertIsNotNone(supervisor.poll())
self.assertEqual('a' * 48, (root / '.runtime/token-observed').read_text())
self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text())
self.assertFalse((root / '.runtime/stop-request.json').exists())
def test_stale_record_pointing_at_an_unrelated_live_pid_never_kills_or_replaces(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
self.runtime_aliases(root)
child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.addCleanup(self.stop_test_child, child)
info = self.record(child, 'backend')
info['created'] -= 60
(root / '.runtime/service.json').write_text(json.dumps({'children': [info]}))
with self.assertRaisesRegex(RuntimeError, '归属'):
bootstrap.recover(root)
self.assertIsNone(child.poll())
self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text())
self.assertFalse((root / '.runtime/restored').exists())
def test_unregistered_root_backend_is_refused_without_signalling(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
child = self.start_backend(root)
with self.assertRaisesRegex(RuntimeError, '未登记'):
bootstrap.recover(root)
self.assertIsNone(child.poll())
self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text())
def test_other_deployment_root_cannot_be_claimed_by_a_valid_live_pid_record(self):
with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second:
root, _, _, _, _ = self.fixture(first)
other, _, _, _, _ = self.fixture(second)
self.runtime_aliases(root)
child = self.start_backend(other)
(root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]}))
with self.assertRaisesRegex(RuntimeError, '归属'):
bootstrap.recover(root)
self.assertIsNone(child.poll())
self.assertFalse((root / '.runtime/restored').exists())
def test_invalid_supervisor_token_does_not_signal_any_owned_process(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
supervisor = self.start_supervisor(root)
state = json.loads((root / '.runtime/service.json').read_text())
state['token'] = 'invalid-token'
(root / '.runtime/service.json').write_text(json.dumps(state))
with self.assertRaisesRegex(RuntimeError, '令牌'):
bootstrap.recover(root)
self.assertIsNone(supervisor.poll())
self.assertFalse((root / '.runtime/stop-request.json').exists())
def test_dead_pid_records_do_not_block_safe_recovery(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
state = {'children': [{'pid': 999999999, 'created': 1, 'executable': str(sys.executable),
'args': ['missing'], 'name': 'backend'}]}
(root / '.runtime/service.json').write_text(json.dumps(state))
self.assertTrue(bootstrap.recover(root)['recovered'])
def test_a_process_owned_by_another_user_is_never_stopped(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
child = self.start_backend(root)
(root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]}))
with patch.object(bootstrap, 'current_user', return_value=False), self.assertRaisesRegex(RuntimeError, '归属'):
bootstrap.recover(root)
self.assertIsNone(child.poll())
self.assertFalse((root / '.runtime/restored').exists())
@unittest.skipIf(os.name == 'nt', 'Host-side immutable-client fixture uses a POSIX shebang; real Windows binaries need native acceptance')
def test_orphaned_mysql_uses_immutable_shutdown_client_and_is_not_force_killed(self):
with tempfile.TemporaryDirectory() as folder:
root, _, _, _, _ = self.fixture(folder)
executable = bootstrap.runtime_executable(root, 'mysql')
executable.parent.mkdir(parents=True)
executable.symlink_to(psutil.Process().exe())
server = root / 'mysql-test.py'
server.write_text(textwrap.dedent('''
import os, signal, time
from pathlib import Path
root = Path(__file__).parent
def stop(*ignored):
(root / '.runtime/mysql-stopped-cleanly').write_text('done')
raise SystemExit(0)
signal.signal(signal.SIGTERM, stop)
(root / '.runtime/mysql-test-pid').write_text(str(os.getpid()))
while True:
time.sleep(.02)
'''))
child = subprocess.Popen([str(executable), str(server), '--datadir=' + str(root / 'runtimedata/mysql'), '--port=54489'],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.addCleanup(self.stop_test_child, child)
self.wait_file(root / '.runtime/mysql-test-pid', child)
admin = executable.parent / 'mysqladmin'
admin.write_text('#!' + psutil.Process().exe() + '\n' + textwrap.dedent('''
import os, signal, sys
from pathlib import Path
root = Path(__file__).resolve().parents[3]
assert os.environ['MYSQL_PWD'] == 'isolated_password_value_1234'
assert '--no-defaults' in sys.argv and '--port=54489' in sys.argv and sys.argv[-1] == 'shutdown'
assert not any('password' in arg.lower() for arg in sys.argv)
(root / '.runtime/immutable-mysqladmin-called').write_text('done')
os.kill(int((root / '.runtime/mysql-test-pid').read_text()), signal.SIGTERM)
'''))
admin.chmod(0o700)
(root / '.runtime/deployment.json').write_text(json.dumps({'mysqlPort': 54489, 'rootPassword': 'isolated_password_value_1234'}))
(root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'mysql')]}))
with patch.object(psutil.Process, 'kill', side_effect=AssertionError('MySQL must not be force killed')):
self.assertTrue(bootstrap.recover(root)['recovered'])
self.assertIsNotNone(child.poll())
self.assertTrue((root / '.runtime/immutable-mysqladmin-called').exists())
self.assertTrue((root / '.runtime/mysql-stopped-cleanly').exists())
if __name__ == '__main__':
unittest.main()