完善:同月多文件导入与当前用户离线迁移更新
This commit is contained in:
@@ -0,0 +1,339 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import psutil
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import update_bootstrap as bootstrap
|
||||
|
||||
|
||||
class ImmutableRecoveryTest(unittest.TestCase):
|
||||
def fixture(self, folder):
|
||||
root = Path(folder)
|
||||
directory = root / '.runtime/update'
|
||||
backup = directory / 'backup-20261104T120000Z-abcdef12'
|
||||
(backup / 'program/deploy/offline').mkdir(parents=True)
|
||||
(root / 'deploy/offline').mkdir(parents=True)
|
||||
old = {
|
||||
'app/backend.jar': b'verified old backend jar',
|
||||
'deploy/offline/control.py': b'VALID_OLD_VERSION = True\n',
|
||||
'deploy/offline/update_manager.py': b'def recover_locked(root):\n (root / ".runtime/restored").write_text("done")\n return {"recovered": True}\n',
|
||||
}
|
||||
entries = []
|
||||
for name, content in old.items():
|
||||
path = backup / 'program' / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(content)
|
||||
(root / name).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / name).write_bytes(b'def broken syntax\n')
|
||||
entries.append({'path': name, 'sha256': hashlib.sha256(content).hexdigest()})
|
||||
(backup / 'manifest.json').write_text(json.dumps({'files': entries}))
|
||||
(backup / 'program.json').write_text(json.dumps({'all': list(old), 'present': list(old)}))
|
||||
journal = {'phase': 'installing', 'databaseBackedUp': True, 'backup': backup.name,
|
||||
'backupManifestSha256': bootstrap.digest(backup / 'manifest.json'),
|
||||
'programInventorySha256': bootstrap.digest(backup / 'program.json')}
|
||||
(directory / 'journal.json').write_text(json.dumps(journal))
|
||||
return root, directory, backup, journal, old
|
||||
|
||||
def runtime_aliases(self, root):
|
||||
for role in ('supervisor', 'backend'):
|
||||
executable = bootstrap.runtime_executable(root, role)
|
||||
executable.parent.mkdir(parents=True, exist_ok=True)
|
||||
executable.symlink_to(psutil.Process().exe())
|
||||
|
||||
def record(self, process, name=None):
|
||||
observed = psutil.Process(process.pid)
|
||||
value = {'pid': process.pid, 'created': observed.create_time(),
|
||||
'executable': str(Path(observed.exe()).resolve()), 'args': observed.cmdline()[1:]}
|
||||
if name:
|
||||
value['name'] = name
|
||||
return value
|
||||
|
||||
def stop_test_child(self, process):
|
||||
if process.poll() is not None:
|
||||
return
|
||||
descendants = []
|
||||
try:
|
||||
descendants = psutil.Process(process.pid).children(recursive=True)
|
||||
except psutil.Error:
|
||||
pass
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
for child in descendants:
|
||||
try:
|
||||
if child.is_running() and child.status() != psutil.STATUS_ZOMBIE:
|
||||
child.kill()
|
||||
except psutil.Error:
|
||||
pass
|
||||
|
||||
def wait_file(self, path, process):
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
if path.exists():
|
||||
return
|
||||
self.assertIsNone(process.poll(), 'Owned test child exited before readiness')
|
||||
time.sleep(.02)
|
||||
self.fail('Owned test child did not become ready')
|
||||
|
||||
def backend_script(self, root):
|
||||
path = root / 'backend-test.py'
|
||||
path.write_text(textwrap.dedent('''
|
||||
import hashlib, os, signal, sys, time
|
||||
from pathlib import Path
|
||||
root = Path(__file__).parent
|
||||
jar = root / 'app/backend.jar'
|
||||
held = jar.open('rb')
|
||||
def stop(*ignored):
|
||||
(root / '.runtime/backend-stopped-before-copy').write_text(hashlib.sha256(jar.read_bytes()).hexdigest())
|
||||
held.close()
|
||||
raise SystemExit(0)
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
(root / '.runtime/backend-ready').write_text(str(os.getpid()))
|
||||
while True:
|
||||
time.sleep(.02)
|
||||
'''))
|
||||
return path
|
||||
|
||||
def start_backend(self, root):
|
||||
self.runtime_aliases(root)
|
||||
script = self.backend_script(root)
|
||||
process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'backend')), str(script), '-jar', str(root / 'app/backend.jar')],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
self.addCleanup(self.stop_test_child, process)
|
||||
self.wait_file(root / '.runtime/backend-ready', process)
|
||||
return process
|
||||
|
||||
def start_supervisor(self, root):
|
||||
self.runtime_aliases(root)
|
||||
self.backend_script(root)
|
||||
script = root / 'deploy/offline/control.py'
|
||||
script.write_text(textwrap.dedent('''
|
||||
import json, os, signal, subprocess, sys, time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
token = 'a' * 48
|
||||
java = root / 'runtimes/java/bin' / ('java.exe' if os.name == 'nt' else 'java')
|
||||
child = subprocess.Popen([str(java), str(root / 'backend-test.py'), '-jar', str(root / 'app/backend.jar')])
|
||||
def record(pid):
|
||||
process = psutil.Process(pid)
|
||||
return {'pid': pid, 'created': process.create_time(), 'executable': str(Path(process.exe()).resolve()), 'args': process.cmdline()[1:]}
|
||||
state = {'supervisor': record(os.getpid()), 'token': token, 'children': [{**record(child.pid), 'name': 'backend'}], 'ready': True}
|
||||
def stop(*ignored):
|
||||
child.terminate()
|
||||
child.wait(timeout=5)
|
||||
(root / '.runtime/supervisor-stopped').write_text('done')
|
||||
(root / '.runtime/service.json').unlink(missing_ok=True)
|
||||
raise SystemExit(0)
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
(root / '.runtime/service.json').write_text(json.dumps(state))
|
||||
(root / '.runtime/supervisor-ready').write_text('ready')
|
||||
while True:
|
||||
try:
|
||||
request = json.loads((root / '.runtime/stop-request.json').read_text())
|
||||
if request.get('token') == token:
|
||||
(root / '.runtime/token-observed').write_text(token)
|
||||
stop()
|
||||
except (FileNotFoundError, ValueError):
|
||||
pass
|
||||
time.sleep(.02)
|
||||
'''))
|
||||
process = subprocess.Popen([str(bootstrap.runtime_executable(root, 'supervisor')), str(script), 'serve'],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
self.addCleanup(self.stop_test_child, process)
|
||||
self.wait_file(root / '.runtime/supervisor-ready', process)
|
||||
self.wait_file(root / '.runtime/backend-ready', process)
|
||||
return process
|
||||
|
||||
def test_recovers_before_importing_broken_control_and_updater(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, directory, backup, journal, old = self.fixture(folder)
|
||||
result = bootstrap.recover(root)
|
||||
self.assertTrue(result['recovered'])
|
||||
self.assertEqual((root / '.runtime/restored').read_text(), 'done')
|
||||
for name, content in old.items():
|
||||
self.assertEqual((root / name).read_bytes(), content)
|
||||
self.assertFalse((directory / 'worker.json').exists())
|
||||
|
||||
def test_corrupt_backup_is_rejected_before_any_replacement(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, directory, backup, journal, old = self.fixture(folder)
|
||||
(backup / 'program/deploy/offline/control.py').write_text('corruption')
|
||||
with self.assertRaises(RuntimeError):
|
||||
bootstrap.recover(root)
|
||||
self.assertEqual((root / 'deploy/offline/control.py').read_text(), 'def broken syntax\n')
|
||||
|
||||
def test_active_worker_cannot_be_overridden(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, directory, backup, journal, old = self.fixture(folder)
|
||||
(directory / 'worker.json').write_text(json.dumps({'pid': os.getpid(), 'created': psutil.Process().create_time()}))
|
||||
with self.assertRaises(RuntimeError):
|
||||
bootstrap.recover(root)
|
||||
self.assertFalse((root / '.runtime/restored').exists())
|
||||
|
||||
def test_program_protection_and_no_update_are_read_only(self):
|
||||
for name in ('deploy/bootstrap/windows-start.ps1', 'deploy/offline/update-public-key.pem', '.runtime/deployment.json', 'runtimedata/mysql/file'):
|
||||
self.assertFalse(bootstrap.program(name))
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
self.assertEqual(bootstrap.recover(root), {'recovered': False})
|
||||
self.assertFalse((root / '.runtime').exists())
|
||||
|
||||
def test_orphaned_backend_stops_before_jar_copy_after_worker_crash(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, directory, _, _, old = self.fixture(folder)
|
||||
child = self.start_backend(root)
|
||||
state = {'supervisor': {'pid': 999999999, 'created': 1, 'executable': str(sys.executable),
|
||||
'args': [str(root / 'deploy/offline/control.py'), 'serve']}, 'token': 'b' * 48,
|
||||
'children': [self.record(child, 'backend')]}
|
||||
(root / '.runtime/service.json').write_text(json.dumps(state))
|
||||
(directory / 'worker.json').write_text(json.dumps({'pid': 999999998, 'created': 1}))
|
||||
before = bootstrap.digest(root / 'app/backend.jar')
|
||||
real_copy = bootstrap.atomic_copy
|
||||
def checked_copy(source, target):
|
||||
self.assertIsNotNone(child.poll(), 'A mutable program was copied while the old backend still held its JAR')
|
||||
return real_copy(source, target)
|
||||
with patch.object(bootstrap, 'atomic_copy', side_effect=checked_copy):
|
||||
self.assertTrue(bootstrap.recover(root)['recovered'])
|
||||
self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text())
|
||||
self.assertEqual(old['app/backend.jar'], (root / 'app/backend.jar').read_bytes())
|
||||
|
||||
def test_live_supervisor_receives_matching_stop_token_and_children_exit_before_copy(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, directory, _, _, _ = self.fixture(folder)
|
||||
supervisor = self.start_supervisor(root)
|
||||
(root / 'deploy/offline/control.py').write_text('def broken syntax\n')
|
||||
before = bootstrap.digest(root / 'app/backend.jar')
|
||||
result = bootstrap.recover(root)
|
||||
self.assertTrue(result['recovered'])
|
||||
self.assertIsNotNone(supervisor.poll())
|
||||
self.assertEqual('a' * 48, (root / '.runtime/token-observed').read_text())
|
||||
self.assertEqual(before, (root / '.runtime/backend-stopped-before-copy').read_text())
|
||||
self.assertFalse((root / '.runtime/stop-request.json').exists())
|
||||
|
||||
def test_stale_record_pointing_at_an_unrelated_live_pid_never_kills_or_replaces(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, _, _, _, _ = self.fixture(folder)
|
||||
self.runtime_aliases(root)
|
||||
child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
self.addCleanup(self.stop_test_child, child)
|
||||
info = self.record(child, 'backend')
|
||||
info['created'] -= 60
|
||||
(root / '.runtime/service.json').write_text(json.dumps({'children': [info]}))
|
||||
with self.assertRaisesRegex(RuntimeError, '归属'):
|
||||
bootstrap.recover(root)
|
||||
self.assertIsNone(child.poll())
|
||||
self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text())
|
||||
self.assertFalse((root / '.runtime/restored').exists())
|
||||
|
||||
def test_unregistered_root_backend_is_refused_without_signalling(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, _, _, _, _ = self.fixture(folder)
|
||||
child = self.start_backend(root)
|
||||
with self.assertRaisesRegex(RuntimeError, '未登记'):
|
||||
bootstrap.recover(root)
|
||||
self.assertIsNone(child.poll())
|
||||
self.assertEqual('def broken syntax\n', (root / 'deploy/offline/control.py').read_text())
|
||||
|
||||
def test_other_deployment_root_cannot_be_claimed_by_a_valid_live_pid_record(self):
|
||||
with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second:
|
||||
root, _, _, _, _ = self.fixture(first)
|
||||
other, _, _, _, _ = self.fixture(second)
|
||||
self.runtime_aliases(root)
|
||||
child = self.start_backend(other)
|
||||
(root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]}))
|
||||
with self.assertRaisesRegex(RuntimeError, '归属'):
|
||||
bootstrap.recover(root)
|
||||
self.assertIsNone(child.poll())
|
||||
self.assertFalse((root / '.runtime/restored').exists())
|
||||
|
||||
def test_invalid_supervisor_token_does_not_signal_any_owned_process(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, _, _, _, _ = self.fixture(folder)
|
||||
supervisor = self.start_supervisor(root)
|
||||
state = json.loads((root / '.runtime/service.json').read_text())
|
||||
state['token'] = 'invalid-token'
|
||||
(root / '.runtime/service.json').write_text(json.dumps(state))
|
||||
with self.assertRaisesRegex(RuntimeError, '令牌'):
|
||||
bootstrap.recover(root)
|
||||
self.assertIsNone(supervisor.poll())
|
||||
self.assertFalse((root / '.runtime/stop-request.json').exists())
|
||||
|
||||
def test_dead_pid_records_do_not_block_safe_recovery(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, _, _, _, _ = self.fixture(folder)
|
||||
state = {'children': [{'pid': 999999999, 'created': 1, 'executable': str(sys.executable),
|
||||
'args': ['missing'], 'name': 'backend'}]}
|
||||
(root / '.runtime/service.json').write_text(json.dumps(state))
|
||||
self.assertTrue(bootstrap.recover(root)['recovered'])
|
||||
|
||||
def test_a_process_owned_by_another_user_is_never_stopped(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, _, _, _, _ = self.fixture(folder)
|
||||
child = self.start_backend(root)
|
||||
(root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'backend')]}))
|
||||
with patch.object(bootstrap, 'current_user', return_value=False), self.assertRaisesRegex(RuntimeError, '归属'):
|
||||
bootstrap.recover(root)
|
||||
self.assertIsNone(child.poll())
|
||||
self.assertFalse((root / '.runtime/restored').exists())
|
||||
|
||||
@unittest.skipIf(os.name == 'nt', 'Host-side immutable-client fixture uses a POSIX shebang; real Windows binaries need native acceptance')
|
||||
def test_orphaned_mysql_uses_immutable_shutdown_client_and_is_not_force_killed(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root, _, _, _, _ = self.fixture(folder)
|
||||
executable = bootstrap.runtime_executable(root, 'mysql')
|
||||
executable.parent.mkdir(parents=True)
|
||||
executable.symlink_to(psutil.Process().exe())
|
||||
server = root / 'mysql-test.py'
|
||||
server.write_text(textwrap.dedent('''
|
||||
import os, signal, time
|
||||
from pathlib import Path
|
||||
root = Path(__file__).parent
|
||||
def stop(*ignored):
|
||||
(root / '.runtime/mysql-stopped-cleanly').write_text('done')
|
||||
raise SystemExit(0)
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
(root / '.runtime/mysql-test-pid').write_text(str(os.getpid()))
|
||||
while True:
|
||||
time.sleep(.02)
|
||||
'''))
|
||||
child = subprocess.Popen([str(executable), str(server), '--datadir=' + str(root / 'runtimedata/mysql'), '--port=54489'],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
self.addCleanup(self.stop_test_child, child)
|
||||
self.wait_file(root / '.runtime/mysql-test-pid', child)
|
||||
admin = executable.parent / 'mysqladmin'
|
||||
admin.write_text('#!' + psutil.Process().exe() + '\n' + textwrap.dedent('''
|
||||
import os, signal, sys
|
||||
from pathlib import Path
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
assert os.environ['MYSQL_PWD'] == 'isolated_password_value_1234'
|
||||
assert '--no-defaults' in sys.argv and '--port=54489' in sys.argv and sys.argv[-1] == 'shutdown'
|
||||
assert not any('password' in arg.lower() for arg in sys.argv)
|
||||
(root / '.runtime/immutable-mysqladmin-called').write_text('done')
|
||||
os.kill(int((root / '.runtime/mysql-test-pid').read_text()), signal.SIGTERM)
|
||||
'''))
|
||||
admin.chmod(0o700)
|
||||
(root / '.runtime/deployment.json').write_text(json.dumps({'mysqlPort': 54489, 'rootPassword': 'isolated_password_value_1234'}))
|
||||
(root / '.runtime/service.json').write_text(json.dumps({'children': [self.record(child, 'mysql')]}))
|
||||
with patch.object(psutil.Process, 'kill', side_effect=AssertionError('MySQL must not be force killed')):
|
||||
self.assertTrue(bootstrap.recover(root)['recovered'])
|
||||
self.assertIsNotNone(child.poll())
|
||||
self.assertTrue((root / '.runtime/immutable-mysqladmin-called').exists())
|
||||
self.assertTrue((root / '.runtime/mysql-stopped-cleanly').exists())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user