Files
quality_analyse/deploy/offline/update_manager.py
T

893 lines
44 KiB
Python

#!/usr/bin/env python3
"""Signed Windows standalone application updates. No caller-supplied URLs or paths.
The release envelope signs the *decoded payload bytes* with Ed25519. Runtime
identity is SHA256(canonical({runtimes, pythonWheels})). Program files are a full
snapshot; private data and runtimes are never update destinations.
"""
from __future__ import annotations
import argparse
import ast
import base64
import contextlib
import fnmatch
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import secrets
import shutil
import stat
import subprocess
import sys
import time
import traceback
import zipfile
from urllib.request import ProxyHandler, HTTPRedirectHandler, build_opener
import psutil
from cryptography.hazmat.primitives.serialization import load_pem_public_key
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
UPDATE_PROTOCOL = 2
ROOT = Path(__file__).resolve().parents[2]
BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/'
LATEST = BASE + 'latest.json'
KEY = 'deploy/offline/update-public-key.pem'
MAX_MANIFEST = 16 * 1024 * 1024
MAX_ARCHIVE = 2 * 1024**3
MAX_EXPANDED = 4 * 1024**3
ACTIVE = {'queued', 'downloading', 'backingUp', 'installing', 'restarting', 'rollingBack'}
SHA = re.compile(r'[a-f0-9]{64}')
class UpdateError(RuntimeError):
pass
class BusyUpdate(UpdateError):
pass
def canonical(value):
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8')
def unique_object(pairs):
value = {}
for key, item in pairs:
if key in value:
raise UpdateError('更新清单包含重复字段。')
value[key] = item
return value
def decode(raw):
return json.loads(raw, object_pairs_hook=unique_object)
def digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def atomic(path, value):
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
temporary = path.with_name(path.name + '.' + secrets.token_hex(6) + '.tmp')
with temporary.open('xb') as stream:
stream.write(canonical(value))
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
def copy_atomic(source, target):
source, target = Path(source), Path(target)
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_name(target.name + '.update-' + secrets.token_hex(6))
try:
with source.open('rb') as reader, temporary.open('xb') as writer:
shutil.copyfileobj(reader, writer)
writer.flush()
os.fsync(writer.fileno())
shutil.copystat(source, temporary)
temporary.replace(target)
finally:
temporary.unlink(missing_ok=True)
def read(path, default=None):
return decode(path.read_bytes()) if path.exists() else default
def runtime_fingerprint(manifest):
return hashlib.sha256(canonical({key: manifest.get(key, []) for key in ('runtimes', 'pythonWheels')})).hexdigest()
def safe_path(name):
if (not isinstance(name, str) or not name or len(name) > 230 or '\\' in name
or ':' in name or '\x00' in name or name.startswith('/')
or any(ord(c) < 32 for c in name)):
raise UpdateError('更新文件路径无效。')
parts = name.split('/')
if any(p in ('', '.', '..') or p.endswith((' ', '.'))
or re.fullmatch(r'(?i)(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?', p) for p in parts):
raise UpdateError('更新文件路径无效。')
return PurePosixPath(name)
def program_path(name):
path = safe_path(name)
if name == KEY or name.startswith('deploy/bootstrap/'):
return False # immutable trust anchor, even in a correctly signed release
return (name == 'app/backend.jar' or name.startswith('app/web/')
or name == 'backend/src/main/resources/analysis-runtime-policy.json'
or name.startswith('scripts/') and path.suffix == '.py'
or name.startswith('ai/nao-quality/')
or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md'))
def destination(root, name):
safe_path(name)
target = root.joinpath(*name.split('/'))
for path in (target, *target.parents):
if path == root:
break
if path.is_symlink():
raise UpdateError('更新目录包含符号链接,已停止更新。')
if os.name == 'nt' and path.exists() and getattr(path.lstat(), 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT:
raise UpdateError('更新目录包含重解析点,已停止更新。')
if not target.resolve().is_relative_to(root.resolve()):
raise UpdateError('更新文件超出程序目录。')
return target
def entry_map(entries, program_only=False):
if not isinstance(entries, list) or len(entries) > 50000:
raise UpdateError('更新文件清单无效。')
result, folded = {}, set()
for entry in entries:
if not isinstance(entry, dict):
raise UpdateError('更新文件清单无效。')
name = entry.get('path')
safe_path(name)
if name.casefold() in folded:
raise UpdateError('更新文件路径重复。')
folded.add(name.casefold())
if program_only and not program_path(name):
raise UpdateError('更新包含不允许替换的文件。')
if entry.get('symlink') is not None or not SHA.fullmatch(str(entry.get('sha256', ''))):
raise UpdateError('更新文件校验信息无效。')
result[name] = entry
return result
def validate_release(envelope, root=ROOT):
try:
raw = base64.b64decode(envelope['payload'], validate=True)
signature = base64.b64decode(envelope['signature'], validate=True)
key = load_pem_public_key(destination(root, KEY).read_bytes())
if not isinstance(key, Ed25519PublicKey):
raise ValueError('key type')
key.verify(signature, raw)
release = decode(raw)
except Exception as error:
raise UpdateError('更新签名验证失败。') from error
current = read(root / 'manifest.json')
if release.get('schema') != 1 or release.get('platform') != 'windows-x64':
raise UpdateError('更新包不适用于此系统。')
if not isinstance(release.get('version'), str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]{0,79}', release['version']):
raise UpdateError('更新版本无效。')
if type(release.get('sequence')) is not int or release['sequence'] <= 0:
raise UpdateError('更新版本序号无效。')
if release.get('runtimeFingerprint') != runtime_fingerprint(current):
raise UpdateError('运行环境版本不同,请使用完整安装包更新。')
archive = release.get('archive', {})
if (not re.fullmatch(r'quality-update-[A-Za-z0-9][A-Za-z0-9._-]{0,100}\.zip', str(archive.get('file', '')))
or type(archive.get('size')) is not int or not 0 < archive['size'] <= MAX_ARCHIVE
or not SHA.fullmatch(str(archive.get('sha256', '')))):
raise UpdateError('更新下载信息无效。')
files = entry_map(release.get('files'), True)
if not files or any(type(e.get('size')) is not int or e['size'] < 0 for e in files.values()) or sum(e['size'] for e in files.values()) > MAX_EXPANDED:
raise UpdateError('更新文件大小无效。')
if 'bundleManifest' in release:
raise UpdateError('更新清单不能包含部署数据。')
old_files = entry_map(current.get('files'))
required = {'app/backend.jar', 'deploy/offline/control.py', 'deploy/offline/update_manager.py',
'backend/src/main/resources/analysis-runtime-policy.json'}
if not required.issubset(files) or not any(name.startswith('app/web/') for name in files):
raise UpdateError('更新缺少必要程序文件。')
if KEY not in old_files or digest(destination(root, KEY)) != old_files[KEY]['sha256']:
raise UpdateError('更新信任公钥校验失败。')
# 只探测本次会备份、删除或替换的程序路径;运行环境保持原样。
# 完整清单仍逐项验证路径语法、重复项、摘要以及运行环境指纹。
old_program = {name for name in old_files if program_path(name)}
for name in old_program | set(files):
destination(root, name)
return release
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
raise UpdateError('更新服务器返回了不允许的跳转。')
def download(url, target=None, expected_size=None, expected_hash=None):
if url != LATEST and not (url.startswith(BASE) and re.fullmatch(r'quality-update-[A-Za-z0-9][A-Za-z0-9._-]{0,100}\.zip', url[len(BASE):])):
raise UpdateError('更新下载地址无效。')
limit = expected_size if expected_size is not None else MAX_MANIFEST
hasher, count, chunks = hashlib.sha256(), 0, []
with build_opener(ProxyHandler({}), NoRedirect()).open(url, timeout=30) as response:
if response.geturl() != url or response.status != 200:
raise UpdateError('更新服务器响应无效。')
with target.open('xb') if target is not None else contextlib.nullcontext() as stream:
while block := response.read(1024 * 1024):
count += len(block)
if count > limit:
raise UpdateError('更新下载大小不符合清单。')
hasher.update(block)
if stream is not None:
stream.write(block)
else:
chunks.append(block)
if expected_size is not None and (count != expected_size or hasher.hexdigest() != expected_hash):
raise UpdateError('更新下载校验失败。')
return b''.join(chunks)
def unpack(archive, stage, release):
entries = {e['path']: e for e in release['files']}
with zipfile.ZipFile(archive) as source:
seen = set()
for member in source.infolist():
name = member.filename
if name not in entries or name in seen or member.is_dir() or member.flag_bits & 1:
raise UpdateError('更新压缩包文件清单不符。')
seen.add(name)
if stat.S_ISLNK(member.external_attr >> 16) or member.file_size != entries[name]['size']:
raise UpdateError('更新压缩包文件类型或大小不符。')
target = destination(stage, name)
target.parent.mkdir(parents=True, exist_ok=True)
count = 0
with source.open(member) as reader, target.open('xb') as writer:
while block := reader.read(1024 * 1024):
count += len(block)
if count > entries[name]['size']:
raise UpdateError('更新解压大小不符。')
writer.write(block)
if digest(target) != entries[name]['sha256']:
raise UpdateError('更新解压校验失败。')
if seen != set(entries):
raise UpdateError('更新压缩包缺少文件。')
def enabled(root=ROOT):
return os.name == 'nt' and os.environ.get('QUALITY_UPDATE_ENABLED') == 'true' and (root / 'runtimes/python/python.exe').is_file() and (root / 'manifest.json').is_file()
def status(root=ROOT):
manifest = read(root / 'manifest.json', {})
try:
value = read(root / '.runtime/update/status.json', {})
except (OSError, ValueError):
value = {}
if not isinstance(value, dict):
value = {}
allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'}
result = {k: v for k, v in value.items() if k in allowed}
result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本')))
result.setdefault('state', 'idle')
journal = read(root / '.runtime/update/journal.json', {})
same_operation = bool(journal.get('operationId')) and journal.get('operationId') == value.get('operationId')
if (same_operation and journal.get('phase') == 'complete' and result['state'] in ACTIVE
and manifest.get('updateSequence') == journal.get('sequence')
and manifest.get('updateVersion') == journal.get('version')):
result.update(state='completed', message='更新完成。', available=False,
version=journal['version'], finishedAt=journal['finishedAt'])
elif same_operation and journal.get('phase') == 'rolledBack' and result['state'] in ACTIVE:
result.update(state='failed', message='更新未完成,已恢复原版本。', available=False)
if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})):
pending = read(root / '.runtime/update/request.json', {})
journal = read(root / '.runtime/update/journal.json', {})
if time.time() - pending.get('created', 0) > 120:
result.update(state='failed', available=False, message='更新进程已中断,请重新运行软件启动入口恢复。'
if journal.get('phase') not in (None, 'complete', 'rolledBack') else '更新进程已结束,请重新检查更新。')
if not result['enabled']:
result['message'] = '此部署不支持软件内更新。'
return result
def write_status(state, message, root=ROOT, **extra):
destination(root, '.runtime/update/status.json')
value = read(root / '.runtime/update/status.json', {})
value.update(extra, state=state, message=message)
atomic(root / '.runtime/update/status.json', value)
return status(root)
def check(root=ROOT):
release = validate_release(decode(download(LATEST)), root)
available = release['sequence'] > read(root / 'manifest.json').get('updateSequence', 0)
atomic(root / '.runtime/update/checked.json', release)
return write_status('available' if available else 'current', '发现新版本。' if available else '已是最新版本。', root,
version=release['version'], available=available, checkedAt=datetime.now(timezone.utc).isoformat())
def process_options():
return {'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_BREAKAWAY_FROM_JOB} if os.name == 'nt' else {'start_new_session': True}
@contextlib.contextmanager
def exclusive_update(root=ROOT):
directory = root / '.runtime/update'
destination(root, '.runtime/update')
directory.mkdir(parents=True, exist_ok=True)
if live(read(directory / 'worker.json', {})):
raise BusyUpdate('已有更新或恢复正在进行。')
sys.path.insert(0, str(root / 'scripts'))
from runtime_platform import fcntl
with (directory / 'worker.lock').open('a+b') as lock:
try:
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except (BlockingIOError, OSError):
raise BusyUpdate('已有更新或恢复正在进行。') from None
identity = {'pid': os.getpid(), 'created': psutil.Process().create_time()}
atomic(directory / 'worker.json', identity)
try:
yield
finally:
if read(directory / 'worker.json', {}) == identity:
try:
(directory / 'worker.json').unlink(missing_ok=True)
except OSError as error:
record_failure(root, 'cleanupWorker', error)
def launch(root=ROOT):
with exclusive_update(root):
return launch_locked(root)
def launch_locked(root=ROOT):
directory = root / '.runtime/update'
destination(root, '.runtime/update')
directory.mkdir(parents=True, exist_ok=True)
# This short launcher MUST exit before its worker may stop Java. The worker
# verifies launcher disappearance; DETACHED alone does not remove ancestry.
pending = read(directory / 'request.json', {})
if pending and time.time() - pending.get('created', 0) < 120:
raise BusyUpdate('已有更新正在启动。')
check(root) # refetch and verify; do not trust browser or cached JSON
if not status(root).get('available'):
return status(root)
token = secrets.token_hex(16)
atomic(directory / 'request.json', {'token': token, 'created': time.time()})
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False, operationId=token)
try:
with (directory / 'worker.log').open('ab') as log:
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token],
cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
except Exception:
(directory / 'request.json').unlink(missing_ok=True)
raise UpdateError('独立更新进程无法启动,服务仍在运行。') from None
return status(root)
def live(info):
try:
process = psutil.Process(info['pid'])
return process.is_running() and process.status() != psutil.STATUS_ZOMBIE and abs(process.create_time() - info['created']) < .01
except (psutil.Error, KeyError, TypeError):
return False
def controller(root, action, database_only=False, timeout=1100, program_only=False):
python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3')
command = [str(python), str(root / 'deploy/offline/control.py'), action]
if database_only:
command += ['--database-only']
if program_only:
command += ['--program-only']
if action == 'serve':
with (root / '.runtime/update/service.log').open('ab') as log:
return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
result = subprocess.run(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
if result.returncode:
raise UpdateError('服务启停失败,请查看本机更新日志。')
return result
def wait_ready(root, process, database_only=False):
deadline = time.monotonic() + (180 if database_only else 1000)
while time.monotonic() < deadline:
state = read(root / '.runtime/service.json', {})
if state.get('ready') and state.get('databaseOnly', False) == database_only and state.get('supervisor', {}).get('pid') == process.pid:
return
if process.poll() is not None:
raise UpdateError('更新后服务启动失败,正在恢复旧版本。')
time.sleep(.5)
raise UpdateError('更新后服务启动超时,正在恢复旧版本。')
def mysql(root, config, binary='mysql', database=None, stdin=None, stdout=subprocess.PIPE):
exe = '.exe' if os.name == 'nt' else ''
command = [str(root / 'runtimes/mysql/bin' / (binary + exe)), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
'--port=' + str(config['mysqlPort']), '--user=root', '--default-character-set=utf8mb4']
if binary == 'mysqldump':
command += ['--single-transaction', '--routines', '--events', '--triggers', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', config.get('mysqlDatabase', 'quality_platform')]
elif database:
command.append(database)
if binary == 'mysql':
command.append('--connect-timeout=10')
env = os.environ.copy()
env['MYSQL_PWD'] = config['rootPassword']
result = subprocess.run(command, cwd=root, env=env, stdin=stdin, stdout=stdout, stderr=subprocess.PIPE, timeout=3600)
if result.returncode:
error_log = destination(root, '.runtime/update/database-error.log')
with error_log.open('ab') as log:
log.write((binary + ' failed\n').encode() + result.stderr[-16384:] + b'\n')
raise UpdateError('数据库备份或恢复失败,已保留本机备份和原数据库。')
def sql(root, config, query):
# stdin must be a real file on Windows; do not place passwords in arguments.
path = root / '.runtime/update/query.sql'
try:
path.write_text(query, encoding='utf-8')
with path.open('rb') as source:
mysql(root, config, stdin=source)
finally:
path.unlink(missing_ok=True)
def backup_program(root, backup, release):
current = read(root / 'manifest.json')
names = sorted({e['path'] for e in current['files'] if program_path(e['path'])} | {e['path'] for e in release['files']})
present = []
for name in names:
source = destination(root, name)
if source.exists():
target = destination(backup / 'program', name)
target.parent.mkdir(parents=True, exist_ok=True)
copy_atomic(source, target)
present.append(name)
copy_atomic(root / 'manifest.json', backup / 'manifest.json')
# All private configuration (AI keys, deployment credentials and bootstrap
# flags) is backed up locally, but never becomes a release payload.
# 复制时在写句柄上刷盘;Windows 不支持在只读句柄上执行 _commit。
shutil.copytree(root / '.runtime', backup / 'private', ignore=shutil.ignore_patterns('update', '*.lock', '*.log', 'service.json', 'stop-request.json'), symlinks=False, copy_function=copy_atomic)
atomic(backup / 'program.json', {'all': names, 'present': present})
def replace_program(root, stage, backup, release):
current = read(root / 'manifest.json')
names = read(backup / 'program.json')['all']
wanted = {e['path'] for e in release['files']}
for name in names:
target = destination(root, name)
if name in wanted:
copy_atomic(stage / name, target)
else:
target.unlink(missing_ok=True)
current['files'] = ([e for e in current['files'] if not program_path(e['path'])]
+ [{'path': e['path'], 'sha256': e['sha256']} for e in release['files']])
current.update(updateSequence=release['sequence'], updateVersion=release['version'])
atomic(root / 'manifest.json', current)
def restore_program(root, backup):
names = read(backup / 'program.json')
for name in names['all']:
target = destination(root, name)
if name in names['present']:
copy_atomic(backup / 'program' / name, target)
else:
target.unlink(missing_ok=True)
copy_atomic(backup / 'manifest.json', destination(root, 'manifest.json'))
for source in (backup / 'private').rglob('*'):
if source.is_file():
relative = source.relative_to(backup / 'private').as_posix()
copy_atomic(source, destination(root, '.runtime/' + relative))
STOP_TIMEOUT = 360
MAX_RECOVERY_ATTEMPTS = 2
def best_effort_status(state, message, root=ROOT, **extra):
try:
return write_status(state, message, root, **extra)
except Exception as error:
record_failure(root, 'writeStatus', error)
def verify_backup(root, backup, journal=None):
journal = journal or read(root / '.runtime/update/journal.json', {})
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
raise UpdateError('更新恢复备份校验失败,已保留现场。')
manifest = entry_map(read(backup / 'manifest.json')['files'])
inventory = read(backup / 'program.json')
names, present = inventory.get('all'), inventory.get('present')
if (not isinstance(names, list) or not isinstance(present, list)
or len(names) != len(set(names)) or not set(present).issubset(names)):
raise UpdateError('更新恢复清单无效。')
for name in names:
if not program_path(name):
raise UpdateError('更新恢复清单包含受保护路径。')
destination(root, name)
if name in present:
source = destination(backup / 'program', name)
if name not in manifest or not source.is_file() or digest(source) != manifest[name]['sha256']:
raise UpdateError('原程序备份缺失或损坏。')
elif name in manifest:
raise UpdateError('原程序备份不完整。')
def recovery_binding(backup):
return {'backup': backup.name, 'backupManifestSha256': digest(backup / 'manifest.json'),
'programInventorySha256': digest(backup / 'program.json'),
'databaseSqlSha256': digest(backup / 'database.sql')}
def recovery_database(binding, attempt):
return 'quality_restore_' + hashlib.sha256(canonical({**binding, 'attempt': attempt})).hexdigest()[:24]
def recovery_progress(root, backup):
binding = recovery_binding(backup)
try:
value = read(root / '.runtime/update/recovery.json', {})
except (OSError, ValueError) as error:
raise UpdateError('恢复进度无法读取,已保留现场。') from error
if not value:
return binding, None
if (not isinstance(value, dict) or any(value.get(k) != v for k, v in binding.items())
or type(value.get('attempt')) is not int or not 1 <= value['attempt'] <= MAX_RECOVERY_ATTEMPTS
or value.get('database') != recovery_database(binding, value['attempt'])
or value.get('stage') not in ('importing', 'databaseDurable', 'published', 'ready')):
raise UpdateError('恢复进度与备份不一致,已保留现场。')
return binding, value
def prepare_old_backend_restart(root):
# 旧 JAR 不理解终态 journal 的优先级。先持久化不触发其维护锁的进度状态,
# 再启动旧服务;写失败则保留恢复记录,不声称已恢复。
write_status('queued', '正在启动原版本,完成后自动重新连接。', root)
def normalize_terminal_status(root, journal):
value = read(root / '.runtime/update/status.json', {})
if (not isinstance(value, dict) or value.get('state') not in ACTIVE
or journal.get('operationId') != value.get('operationId')):
return
pending = read(root / '.runtime/update/request.json', {})
if value.get('state') in ('queued', 'downloading') and time.time() - pending.get('created', 0) < 120:
return
if journal.get('phase') == 'rolledBack':
write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
elif journal.get('phase') == 'complete':
manifest = read(root / 'manifest.json')
if manifest.get('updateVersion') == journal.get('version', value.get('version')):
write_status('completed', '更新完成。', root, available=False)
def rollback(root, backup, database_backed_up):
verify_backup(root, backup)
binding, progress = recovery_progress(root, backup) if database_backed_up else ({}, None)
best_effort_status('rollingBack', '更新未完成,正在恢复原版本。', root)
controller(root, 'stop', timeout=STOP_TIMEOUT)
restore_program(root, backup)
if database_backed_up:
config = read(backup / 'private/deployment.json')
# 断电前未确认数据库安全关闭的导入,不能当作完整数据库复用。
if progress is None or progress['stage'] == 'importing':
attempt = 1 if progress is None else progress['attempt'] + 1
if attempt > MAX_RECOVERY_ATTEMPTS:
raise UpdateError('恢复导入多次中断,已停止自动重试并保留数据库,请核对本机恢复记录。')
reserve = (backup / 'database.sql').stat().st_size * 3 + 512 * 1024**2
if shutil.disk_usage(root).free < reserve:
raise UpdateError('空间不足以安全恢复数据库,已保留现场。')
database = recovery_database(binding, attempt)
if database == config.get('mysqlDatabase'):
raise UpdateError('恢复数据库与原库冲突,已停止。')
progress = {**binding, 'attempt': attempt, 'database': database, 'stage': 'importing'}
atomic(root / '.runtime/update/recovery.json', progress)
process = controller(root, 'serve', True)
wait_ready(root, process, True)
# 不删除任何已有数据库;名称冲突时 CREATE 会失败并保留现场。
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
with (backup / 'database.sql').open('rb') as source:
mysql(root, config, database=database, stdin=source)
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
controller(root, 'stop', timeout=STOP_TIMEOUT)
progress['stage'] = 'databaseDurable'
atomic(root / '.runtime/update/recovery.json', progress)
config['mysqlDatabase'] = progress['database']
atomic(root / '.runtime/deployment.json', config)
receipt_path = root / '.runtime/migration-receipt.json'
receipt = read(receipt_path, {})
if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'):
receipt['database'] = progress['database']
atomic(receipt_path, receipt)
progress['stage'] = 'published'
atomic(root / '.runtime/update/recovery.json', progress)
prepare_old_backend_restart(root)
process = controller(root, 'serve')
wait_ready(root, process)
if progress:
progress['stage'] = 'ready'
atomic(root / '.runtime/update/recovery.json', progress)
def preflight_programs(stage):
# 旧安装先加固恢复器,随后只能安装具备相同恢复协议的程序,防止覆盖回旧实现。
for name, marker in (('update_manager.py', 'UPDATE_PROTOCOL'), ('control.py', 'UPDATE_CONTROL_PROTOCOL')):
try:
tree = ast.parse((stage / 'deploy/offline' / name).read_bytes())
values = [node.value.value for node in tree.body if isinstance(node, ast.Assign)
and any(isinstance(target, ast.Name) and target.id == marker for target in node.targets)
and isinstance(node.value, ast.Constant) and type(node.value.value) is int]
if values != [UPDATE_PROTOCOL]:
raise ValueError('incompatible recovery protocol')
except (OSError, SyntaxError, ValueError) as error:
raise UpdateError('更新包未包含本轮加固的恢复流程,服务尚未停止;请等待配套更新包。') from error
try:
with zipfile.ZipFile(stage / 'app/backend.jar') as jar:
metadata = decode(jar.read('BOOT-INF/classes/update-protocol.json'))
if metadata != {'protocol': UPDATE_PROTOCOL}:
raise ValueError('incompatible backend protocol')
except (OSError, ValueError, KeyError, zipfile.BadZipFile) as error:
raise UpdateError('更新包后端不支持本轮恢复协议,服务尚未停止。') from error
def preflight_backup(root, backup, release):
paths = set(e['path'] for e in read(root / 'manifest.json')['files'] if program_path(e['path']))
paths.update(e['path'] for e in release['files'])
targets = [backup / 'program' / n for n in paths]
ignored = ('*.lock', '*.log', 'service.json', 'stop-request.json')
for folder, dirs, files in os.walk(root / '.runtime', followlinks=False):
dirs[:] = [n for n in dirs if n != 'update']
for n in dirs:
destination(root, (Path(folder) / n).relative_to(root).as_posix())
for n in files:
if any(fnmatch.fnmatch(n, pattern) for pattern in ignored):
continue
source = Path(folder) / n
destination(root, source.relative_to(root).as_posix())
with source.open('rb'):
pass
targets.append(backup / 'private' / source.relative_to(root / '.runtime'))
for target in targets:
destination(backup, target.relative_to(backup).as_posix())
if os.name == 'nt' and len(str(target.absolute())) + 24 > 248:
raise UpdateError('备份路径过长,请先缩短部署目录或历史路径;服务尚未停止。')
probe = backup / 'write-probe.tmp'
try:
with probe.open('xb') as stream:
stream.write(b'probe')
stream.flush()
os.fsync(stream.fileno())
finally:
probe.unlink(missing_ok=True)
def record_failure(root, stage, error, name='failure.json'):
# 仅记录阶段、异常类型、系统错误码和代码位置,不记录正文、凭据或异常参数。
receipt = {'stage': stage, 'errorType': type(error).__name__,
'occurredAt': datetime.now(timezone.utc).isoformat(),
'frames': [{'file': Path(frame.f_code.co_filename).name, 'function': frame.f_code.co_name, 'line': line}
for frame, line in traceback.walk_tb(error.__traceback__)]}
for key in ('errno', 'winerror'):
value = getattr(error, key, None)
if type(value) is int:
receipt[key] = value
try:
atomic(root / '.runtime/update' / name, receipt)
except Exception:
pass # 诊断写入失败不能阻止原有回退流程。
def worker(launcher, token, root=ROOT):
directory = root / '.runtime/update'
for _ in range(300):
if not psutil.pid_exists(launcher):
break
time.sleep(.1)
else:
raise UpdateError('更新启动器未退出,已安全停止。')
if any('java' in p.name().lower() for p in psutil.Process().parents()):
raise UpdateError('更新进程未独立启动,已安全停止。')
if read(directory / 'request.json', {}).get('token') != token:
raise UpdateError('更新请求已失效。')
with exclusive_update(root):
backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4))
stopped = backed_up = program_saved = changed = stop_requested = committed = False
failure_stage = 'validateRelease'
try:
envelope = decode(download(LATEST))
release = validate_release(envelope, root)
if release['sequence'] <= read(root / 'manifest.json').get('updateSequence', 0):
raise UpdateError('此更新版本已安装。')
backup.mkdir()
archive, stage = backup / 'update.zip', backup / 'stage'
expanded = sum(entry['size'] for entry in release['files'])
# Keep enough headroom for extraction, old program files and a
# logical dump; the dump itself must complete before replacement.
db_size = sum(p.stat().st_size for p in (root / 'runtimedata/mysql').rglob('*') if p.is_file())
old_program_size = sum((root / entry['path']).stat().st_size for entry in read(root / 'manifest.json')['files'] if program_path(entry['path']) and (root / entry['path']).is_file())
private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory))
if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2:
raise UpdateError('磁盘剩余空间不足,请释放空间后重试。')
write_status('downloading', '正在下载并校验更新。', root, operationId=token, version=release['version'])
info = release['archive']
failure_stage = 'downloadArchive'
download(BASE + info['file'], archive, info['size'], info['sha256'])
failure_stage = 'unpackArchive'
unpack(archive, stage, release)
failure_stage = 'programPreflight'
preflight_programs(stage)
failure_stage = 'runtimePreflight'
write_status('downloading', '正在检查运行环境和备份条件,服务仍可使用。', root)
controller(root, 'verify-runtime', timeout=1800)
failure_stage = 'backupPreflight'
preflight_backup(root, backup, release)
# Revalidate immediately before any service or disk mutation.
validate_release(envelope, root)
write_status('backingUp', '正在备份,服务即将重新启动。', root)
if (directory / 'recovery.json').exists():
copy_atomic(directory / 'recovery.json', backup / 'previous-recovery.json')
(directory / 'recovery.json').unlink()
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False,
'operationId': token, 'version': release['version'], 'sequence': release['sequence']})
failure_stage = 'stopService'
stop_requested = True
controller(root, 'stop', timeout=STOP_TIMEOUT)
stopped = True
failure_stage = 'backupProgram'
backup_program(root, backup, release)
program_saved = True
config = read(root / '.runtime/deployment.json')
failure_stage = 'startBackupDatabase'
database = controller(root, 'serve', True)
wait_ready(root, database, True)
failure_stage = 'backupDatabase'
with (backup / 'database.sql').open('xb') as output:
mysql(root, config, 'mysqldump', stdout=output)
output.flush()
os.fsync(output.fileno())
if (backup / 'database.sql').stat().st_size < 100:
raise UpdateError('数据库备份为空,已停止更新。')
backed_up = True
controller(root, 'stop', timeout=STOP_TIMEOUT)
# The actual dump can exceed the source's physical storage. Do not
# replace anything unless a separate rollback database still fits.
required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2
if shutil.disk_usage(root).free < required_recovery:
raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。')
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True,
'operationId': token, 'version': release['version'], 'sequence': release['sequence'],
'databaseMayHaveChanged': False,
'backupManifestSha256': digest(backup / 'manifest.json'),
'programInventorySha256': digest(backup / 'program.json')})
write_status('installing', '正在安装更新。', root)
changed = True
failure_stage = 'replaceProgram'
replace_program(root, stage, backup, release)
failure_stage = 'verifyProgram'
controller(root, 'verify-runtime', timeout=300, program_only=True)
write_status('restarting', '正在重新启动,请稍候。', root)
failure_stage = 'restartService'
journal = read(directory / 'journal.json')
journal['databaseMayHaveChanged'] = True
atomic(directory / 'journal.json', journal)
process = controller(root, 'serve')
wait_ready(root, process)
journal.update(phase='complete', version=release['version'], sequence=release['sequence'],
finishedAt=datetime.now(timezone.utc).isoformat())
atomic(directory / 'journal.json', journal)
committed = True
best_effort_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
except Exception as error:
record_failure(root, failure_stage, error)
message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。'
journal = read(directory / 'journal.json', {})
if committed or (journal.get('backup') == backup.name and journal.get('phase') == 'complete'):
return # 提交后的展示或清理失败绝不能发起回滚。
if stop_requested:
try:
if program_saved and changed:
rollback(root, backup, backed_up and journal.get('databaseMayHaveChanged', True))
else:
controller(root, 'stop', timeout=STOP_TIMEOUT)
prepare_old_backend_restart(root)
process = controller(root, 'serve')
wait_ready(root, process)
journal.update(backup=backup.name, phase='rolledBack', databaseBackedUp=backed_up)
atomic(directory / 'journal.json', journal)
message += ' 已恢复原版本。'
except Exception as rollback_error:
record_failure(root, 'rollback', rollback_error, 'rollback-failure.json')
message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。'
best_effort_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
finally:
try:
(directory / 'request.json').unlink(missing_ok=True)
except OSError as error:
record_failure(root, 'cleanupRequest', error)
def recover(root=ROOT):
with exclusive_update(root):
return recover_locked(root)
def recover_locked(root=ROOT):
directory = root / '.runtime/update'
journal = read(directory / 'journal.json', {})
if journal.get('phase') in (None, 'complete', 'rolledBack'):
if journal.get('phase') in ('complete', 'rolledBack'):
normalize_terminal_status(root, journal)
return status(root)
name = journal.get('backup', '')
if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name):
raise UpdateError('更新恢复记录无效。')
backup = directory / name
if journal.get('phase') not in ('installing', 'stopping'):
raise UpdateError('更新恢复阶段无效,已保留现场。')
if journal.get('phase') == 'installing':
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
raise UpdateError('更新恢复备份校验失败,已保留现场。')
verify_backup(root, backup, journal)
if journal.get('databaseBackedUp') is True:
recovery_progress(root, backup) # 先验证 sidecar 与 dump,尚未停止或替换任何文件。
rollback(root, backup, journal.get('databaseBackedUp') is True and journal.get('databaseMayHaveChanged', True))
else:
controller(root, 'stop', timeout=STOP_TIMEOUT)
prepare_old_backend_restart(root)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'})
best_effort_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
return status(root)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('status', 'check', 'launch', 'worker', 'recover'))
parser.add_argument('--launcher', type=int)
parser.add_argument('--token')
args = parser.parse_args()
if args.action == 'status':
return status()
if not enabled():
raise UpdateError('此部署不支持软件内更新。')
if args.action == 'worker':
if not args.launcher or not re.fullmatch(r'[a-f0-9]{32}', args.token or ''):
raise UpdateError('更新请求无效。')
worker(args.launcher, args.token)
return status()
if args.action == 'recover':
return recover()
journal = read(ROOT / '.runtime/update/journal.json', {})
if journal.get('phase') not in (None, 'complete', 'rolledBack'):
raise UpdateError('上次更新中断,请重新运行软件启动入口恢复。')
if live(read(ROOT / '.runtime/update/worker.json', {})):
raise BusyUpdate('已有更新正在进行。')
return check() if args.action == 'check' else launch()
if __name__ == '__main__':
try:
print(json.dumps(main(), ensure_ascii=False))
except Exception as error:
message = str(error) if isinstance(error, UpdateError) else '更新服务暂不可用,请稍后重试。'
if enabled() and not isinstance(error, BusyUpdate) and sys.argv[1:2] != ['status'] and not live(read(ROOT / '.runtime/update/worker.json', {})):
write_status('failed', message)
print(json.dumps({'error': message}, ensure_ascii=False))
raise SystemExit(1)