358 lines
16 KiB
Python
358 lines
16 KiB
Python
"""Immutable stage-zero recovery. Installed under deploy/bootstrap, never updated."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import psutil # Bundled immutable Python environment, not an application module.
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def read(path, default=None):
|
|
return json.loads(path.read_text(encoding='utf-8')) if path.exists() else default
|
|
|
|
|
|
def digest(path):
|
|
with path.open('rb') as stream:
|
|
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
|
|
|
|
def atomic_copy(source, target):
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = target.with_name(target.name + '.recover-' + secrets.token_hex(6))
|
|
try:
|
|
with source.open('rb') as reader, temporary.open('xb') as writer:
|
|
shutil.copyfileobj(reader, writer)
|
|
writer.flush()
|
|
os.fsync(writer.fileno())
|
|
temporary.replace(target)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def safe(root, name):
|
|
if (not isinstance(name, str) or not name or '\\' in name or ':' in name or name.startswith('/')
|
|
or any(part in ('', '.', '..') or part.endswith((' ', '.')) for part in name.split('/'))):
|
|
raise RuntimeError('更新恢复路径无效。')
|
|
target = root.joinpath(*name.split('/'))
|
|
for path in (target, *target.parents):
|
|
if path == root:
|
|
break
|
|
if path.is_symlink() or (path.exists() and getattr(path.lstat(), 'st_file_attributes', 0) & 0x400):
|
|
raise RuntimeError('更新恢复目录包含无效链接。')
|
|
if not target.resolve().is_relative_to(root.resolve()):
|
|
raise RuntimeError('更新恢复路径越界。')
|
|
return target
|
|
|
|
|
|
def program(name):
|
|
if name == 'deploy/offline/update-public-key.pem' or name.startswith('deploy/bootstrap/'):
|
|
return False
|
|
path = Path(name)
|
|
return (name == 'app/backend.jar' or name.startswith('app/web/')
|
|
or name == 'backend/src/main/resources/analysis-runtime-policy.json'
|
|
or name.startswith('scripts/') and path.suffix == '.py'
|
|
or name.startswith('ai/nao-quality/')
|
|
or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md'))
|
|
|
|
|
|
def atomic_json(path, value):
|
|
temporary = path.with_name(path.name + '.stop-' + secrets.token_hex(6))
|
|
try:
|
|
with temporary.open('x', encoding='utf-8') as stream:
|
|
json.dump(value, stream)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
temporary.replace(path)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def same_path(left, right):
|
|
return os.path.normcase(str(Path(left).resolve())) == os.path.normcase(str(Path(right).resolve()))
|
|
|
|
|
|
def runtime_executable(root, role):
|
|
suffix = '.exe' if os.name == 'nt' else ''
|
|
relative = {'supervisor': 'runtimes/python/' + ('python.exe' if os.name == 'nt' else 'bin/python3'),
|
|
'backend': 'runtimes/java/bin/java' + suffix, 'mysql': 'runtimes/mysql/bin/mysqld' + suffix}[role]
|
|
return root / relative
|
|
|
|
|
|
def bound_to_root(root, role, args):
|
|
if not isinstance(args, list) or not all(isinstance(arg, str) for arg in args):
|
|
return False
|
|
if role == 'supervisor':
|
|
return (len(args) in (2, 3) and same_path(args[0], root / 'deploy/offline/control.py')
|
|
and args[1] == 'serve' and (len(args) == 2 or args[2] == '--database-only'))
|
|
if role == 'backend':
|
|
return len(args) >= 2 and args[-2] == '-jar' and same_path(args[-1], root / 'app/backend.jar')
|
|
datadirs = [arg.split('=', 1)[1] for arg in args if arg.startswith('--datadir=')]
|
|
return len(datadirs) == 1 and same_path(datadirs[0], root / 'runtimedata/mysql')
|
|
|
|
|
|
def current_user(process):
|
|
if os.name == 'nt':
|
|
return process.username().casefold() == psutil.Process().username().casefold()
|
|
return process.uids().real == os.getuid()
|
|
|
|
|
|
def owned_process(root, role, info):
|
|
"""Return a fully verified owned PID, None for a dead PID, never guess."""
|
|
if (not isinstance(info, dict) or type(info.get('pid')) is not int or info['pid'] <= 0
|
|
or not isinstance(info.get('created'), (int, float))
|
|
or not isinstance(info.get('executable'), str) or not isinstance(info.get('args'), list)):
|
|
raise RuntimeError('本包服务进程记录无效,未停止任何不明进程。')
|
|
try:
|
|
process = psutil.Process(info['pid'])
|
|
if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE:
|
|
return None
|
|
forbidden = {os.getpid(), *(parent.pid for parent in psutil.Process().parents())}
|
|
if (process.pid in forbidden or abs(process.create_time() - info['created']) >= .01
|
|
or not current_user(process) or not same_path(process.exe(), info['executable'])
|
|
or process.cmdline()[1:] != info['args']
|
|
or not same_path(info['executable'], runtime_executable(root, role))
|
|
or not bound_to_root(root, role, info['args'])):
|
|
raise RuntimeError('无法确认本包服务进程归属,未停止任何不明进程。')
|
|
return process
|
|
except psutil.NoSuchProcess:
|
|
return None
|
|
except psutil.Error:
|
|
raise RuntimeError('当前用户无法核验本包服务进程,恢复已安全停止。') from None
|
|
|
|
|
|
def root_markers(root, args):
|
|
"""Detect unregistered package processes without inspecting unrelated data."""
|
|
for arg in args:
|
|
if arg.startswith('--datadir='):
|
|
if same_path(arg.split('=', 1)[1], root / 'runtimedata/mysql'):
|
|
return True
|
|
elif Path(arg).is_absolute() and (same_path(arg, root / 'app/backend.jar')
|
|
or same_path(arg, root / 'deploy/offline/control.py')):
|
|
return True
|
|
return False
|
|
|
|
|
|
def refuse_unregistered(root, expected):
|
|
for process in psutil.process_iter():
|
|
try:
|
|
if process.pid == os.getpid() or process.status() == psutil.STATUS_ZOMBIE:
|
|
continue
|
|
args = process.cmdline()[1:]
|
|
if not root_markers(root, args):
|
|
continue
|
|
if process.pid not in expected:
|
|
raise RuntimeError('发现未登记的本包服务进程,已保留现场,请先检查该进程。')
|
|
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
|
continue
|
|
|
|
|
|
def wait_owned(root, role, info, seconds):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
if owned_process(root, role, info) is None:
|
|
return True
|
|
time.sleep(.05)
|
|
return owned_process(root, role, info) is None
|
|
|
|
|
|
def terminate_owned(root, role, info, seconds):
|
|
process = owned_process(root, role, info)
|
|
if process is None:
|
|
return
|
|
try:
|
|
process.terminate()
|
|
if wait_owned(root, role, info, seconds):
|
|
return
|
|
process = owned_process(root, role, info)
|
|
if process is not None:
|
|
process.kill()
|
|
if not wait_owned(root, role, info, 10):
|
|
raise RuntimeError('本包服务尚未退出,未替换任何程序文件。')
|
|
except psutil.NoSuchProcess:
|
|
return
|
|
except psutil.Error:
|
|
raise RuntimeError('当前用户无法停止本包服务,未替换任何程序文件。') from None
|
|
|
|
|
|
def shutdown_owned_mysql(root, info):
|
|
process = owned_process(root, 'mysql', info)
|
|
if process is None:
|
|
return
|
|
value = read(safe(root, '.runtime/deployment.json'), {})
|
|
port = value.get('mysqlPort')
|
|
if (type(port) is not int or not 1024 <= port <= 65535
|
|
or not re.fullmatch(r'[A-Za-z0-9_-]{20,100}', str(value.get('rootPassword', '')))
|
|
or info['args'].count('--port=' + str(port)) != 1):
|
|
raise RuntimeError('无法核验本包数据库关闭配置,未替换任何程序文件。')
|
|
executable = root / 'runtimes/mysql/bin' / ('mysqladmin.exe' if os.name == 'nt' else 'mysqladmin')
|
|
env = os.environ.copy()
|
|
env['MYSQL_PWD'] = value['rootPassword']
|
|
# Only the immutable bundled MySQL client is executed; no application code
|
|
# is loaded, and the current user's privileges are never elevated.
|
|
command = [str(executable), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
|
|
'--port=' + str(port), '--user=root', '--connect-timeout=3', 'shutdown']
|
|
try:
|
|
subprocess.run(command, env=env, cwd=root, stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=20)
|
|
except (OSError, subprocess.SubprocessError):
|
|
raise RuntimeError('本包数据库无法安全关闭,未替换任何程序文件。') from None
|
|
if not wait_owned(root, 'mysql', info, 120):
|
|
raise RuntimeError('本包数据库仍在关闭,未替换任何程序文件,请稍后重新启动。')
|
|
|
|
|
|
def stop_owned_services(root, grace_seconds=200):
|
|
"""Immutable implementation of service.json + stop-request token contract.
|
|
|
|
A crashed worker can leave a healthy new supervisor/Java behind. Validate
|
|
every recorded identity before requesting shutdown; a live stale/reused PID
|
|
is an error rather than permission to terminate it. Java must be gone before
|
|
a JAR is replaced. An orphaned MySQL is shut down through its immutable
|
|
client, never forcibly killed on Windows.
|
|
"""
|
|
state = read(safe(root, '.runtime/service.json'), {})
|
|
if not isinstance(state, dict) or not isinstance(state.get('children', []), list):
|
|
raise RuntimeError('本包服务状态无效,未替换任何程序文件。')
|
|
entries = []
|
|
supervisor = state.get('supervisor')
|
|
if supervisor:
|
|
entries.append(('supervisor', supervisor))
|
|
for info in state.get('children', []):
|
|
if not isinstance(info, dict) or info.get('name') not in ('backend', 'mysql'):
|
|
raise RuntimeError('本包服务子进程记录无效,未替换任何程序文件。')
|
|
entries.append((info['name'], info))
|
|
if len(entries) > 20 or len({info.get('pid') for _, info in entries}) != len(entries):
|
|
raise RuntimeError('本包服务进程记录重复,未替换任何程序文件。')
|
|
# Validate all owners before the first stop request or signal is sent.
|
|
active = {(role, info['pid']): owned_process(root, role, info) for role, info in entries}
|
|
refuse_unregistered(root, {pid for (_, pid), process in active.items() if process is not None})
|
|
if supervisor and active.get(('supervisor', supervisor['pid'])) is not None:
|
|
token = state.get('token')
|
|
if not isinstance(token, str) or not re.fullmatch(r'[a-f0-9]{48}', token):
|
|
raise RuntimeError('本包服务停止令牌无效,未停止任何进程。')
|
|
atomic_json(safe(root, '.runtime/stop-request.json'), {'token': token})
|
|
if not wait_owned(root, 'supervisor', supervisor, grace_seconds):
|
|
terminate_owned(root, 'supervisor', supervisor, 10)
|
|
# A dead supervisor cannot finish its own finally block. Only independently
|
|
# verified package children may be stopped; unrelated/reused PIDs are refused.
|
|
for role, info in entries:
|
|
if role == 'backend':
|
|
terminate_owned(root, role, info, 35)
|
|
for role, info in entries:
|
|
if role == 'mysql':
|
|
shutdown_owned_mysql(root, info)
|
|
for role, info in entries:
|
|
if owned_process(root, role, info) is not None:
|
|
raise RuntimeError('本包服务尚未完全退出,未替换任何程序文件。')
|
|
refuse_unregistered(root, set())
|
|
stop = safe(root, '.runtime/stop-request.json')
|
|
if isinstance(state.get('token'), str) and read(stop, {}).get('token') == state['token']:
|
|
stop.unlink(missing_ok=True)
|
|
|
|
|
|
def restore_program_before_import(root, directory, journal):
|
|
name = journal.get('backup', '')
|
|
if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name):
|
|
raise RuntimeError('更新恢复记录无效。')
|
|
backup = safe(directory, name)
|
|
manifest_file, inventory_file = backup / 'manifest.json', backup / 'program.json'
|
|
if (digest(manifest_file) != journal.get('backupManifestSha256')
|
|
or digest(inventory_file) != journal.get('programInventorySha256')):
|
|
raise RuntimeError('本机更新备份校验失败,请保留目录并联系管理员。')
|
|
manifest = read(manifest_file)
|
|
old = {entry['path']: entry for entry in manifest['files'] if program(entry['path'])}
|
|
inventory = read(inventory_file)
|
|
all_names, present = inventory['all'], set(inventory['present'])
|
|
if (not isinstance(all_names, list) or len(all_names) > 50000
|
|
or len(set(all_names)) != len(all_names) or not present.issubset(all_names)):
|
|
raise RuntimeError('更新恢复清单无效。')
|
|
for name in all_names:
|
|
if not program(name):
|
|
raise RuntimeError('更新恢复清单包含受保护路径。')
|
|
target = safe(root, name)
|
|
if name in present:
|
|
source = safe(backup / 'program', name)
|
|
if name not in old or not source.is_file() or digest(source) != old[name]['sha256']:
|
|
raise RuntimeError('原程序备份缺失或损坏。')
|
|
elif name in old:
|
|
raise RuntimeError('原程序备份不完整。')
|
|
# All backups are validated before any application file is replaced.
|
|
stop_owned_services(root)
|
|
for name in all_names:
|
|
target = safe(root, name)
|
|
if name in present:
|
|
atomic_copy(safe(backup / 'program', name), target)
|
|
else:
|
|
target.unlink(missing_ok=True)
|
|
atomic_copy(manifest_file, root / 'manifest.json')
|
|
|
|
|
|
def live(info):
|
|
try:
|
|
process = psutil.Process(info['pid'])
|
|
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
|
|
and abs(process.create_time() - info['created']) < .01)
|
|
except (psutil.Error, ValueError, TypeError, KeyError):
|
|
return False
|
|
|
|
|
|
def recover(root=ROOT):
|
|
directory = safe(root, '.runtime/update')
|
|
journal = read(directory / 'journal.json', {})
|
|
if journal.get('phase') in (None, 'complete', 'rolledBack'):
|
|
return {'recovered': False}
|
|
if live(read(directory / 'worker.json', {})):
|
|
raise RuntimeError('更新正在进行,请等待原窗口完成。')
|
|
with (directory / 'worker.lock').open('a+b') as lock:
|
|
try:
|
|
if os.name == 'nt':
|
|
import msvcrt
|
|
lock.seek(0)
|
|
msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
|
|
else:
|
|
import fcntl
|
|
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
raise RuntimeError('已有更新恢复正在进行,请等待完成。') from None
|
|
identity = {'pid': os.getpid(), 'created': psutil.Process().create_time()}
|
|
worker = directory / 'worker.json'
|
|
temporary = worker.with_suffix('.bootstrap.tmp')
|
|
temporary.write_text(json.dumps(identity), encoding='utf-8')
|
|
temporary.replace(worker)
|
|
try:
|
|
journal = read(directory / 'journal.json', {})
|
|
if journal.get('phase') == 'installing':
|
|
restore_program_before_import(root, directory, journal)
|
|
elif journal.get('phase') == 'stopping':
|
|
stop_owned_services(root)
|
|
else:
|
|
raise RuntimeError('更新恢复阶段无效,未替换任何程序文件。')
|
|
# This import occurs only after restoring a complete, verified old
|
|
# program set; no mutable application module was imported above.
|
|
path = safe(root, 'deploy/offline/update_manager.py')
|
|
spec = importlib.util.spec_from_file_location('quality_recovered_updater', path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module.recover_locked(root)
|
|
finally:
|
|
if read(worker, {}) == identity:
|
|
worker.unlink(missing_ok=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
print(json.dumps(recover(), ensure_ascii=False))
|
|
except Exception as error:
|
|
print(json.dumps({'error': str(error) if isinstance(error, RuntimeError) else '更新恢复未完成,请保留本机备份并联系管理员。'}, ensure_ascii=False))
|
|
raise SystemExit(1)
|