893 lines
44 KiB
Python
893 lines
44 KiB
Python
#!/usr/bin/env python3
|
|
"""Signed Windows standalone application updates. No caller-supplied URLs or paths.
|
|
|
|
The release envelope signs the *decoded payload bytes* with Ed25519. Runtime
|
|
identity is SHA256(canonical({runtimes, pythonWheels})). Program files are a full
|
|
snapshot; private data and runtimes are never update destinations.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import ast
|
|
import base64
|
|
import contextlib
|
|
import fnmatch
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import zipfile
|
|
from urllib.request import ProxyHandler, HTTPRedirectHandler, build_opener
|
|
|
|
import psutil
|
|
from cryptography.hazmat.primitives.serialization import load_pem_public_key
|
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
|
|
|
UPDATE_PROTOCOL = 2
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/'
|
|
LATEST = BASE + 'latest.json'
|
|
KEY = 'deploy/offline/update-public-key.pem'
|
|
MAX_MANIFEST = 16 * 1024 * 1024
|
|
MAX_ARCHIVE = 2 * 1024**3
|
|
MAX_EXPANDED = 4 * 1024**3
|
|
ACTIVE = {'queued', 'downloading', 'backingUp', 'installing', 'restarting', 'rollingBack'}
|
|
SHA = re.compile(r'[a-f0-9]{64}')
|
|
|
|
|
|
class UpdateError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class BusyUpdate(UpdateError):
|
|
pass
|
|
|
|
|
|
def canonical(value):
|
|
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8')
|
|
|
|
|
|
def unique_object(pairs):
|
|
value = {}
|
|
for key, item in pairs:
|
|
if key in value:
|
|
raise UpdateError('更新清单包含重复字段。')
|
|
value[key] = item
|
|
return value
|
|
|
|
|
|
def decode(raw):
|
|
return json.loads(raw, object_pairs_hook=unique_object)
|
|
|
|
|
|
def digest(path):
|
|
with path.open('rb') as stream:
|
|
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
|
|
|
|
def atomic(path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
temporary = path.with_name(path.name + '.' + secrets.token_hex(6) + '.tmp')
|
|
with temporary.open('xb') as stream:
|
|
stream.write(canonical(value))
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
temporary.replace(path)
|
|
|
|
|
|
def copy_atomic(source, target):
|
|
source, target = Path(source), Path(target)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = target.with_name(target.name + '.update-' + secrets.token_hex(6))
|
|
try:
|
|
with source.open('rb') as reader, temporary.open('xb') as writer:
|
|
shutil.copyfileobj(reader, writer)
|
|
writer.flush()
|
|
os.fsync(writer.fileno())
|
|
shutil.copystat(source, temporary)
|
|
temporary.replace(target)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def read(path, default=None):
|
|
return decode(path.read_bytes()) if path.exists() else default
|
|
|
|
|
|
def runtime_fingerprint(manifest):
|
|
return hashlib.sha256(canonical({key: manifest.get(key, []) for key in ('runtimes', 'pythonWheels')})).hexdigest()
|
|
|
|
|
|
def safe_path(name):
|
|
if (not isinstance(name, str) or not name or len(name) > 230 or '\\' in name
|
|
or ':' in name or '\x00' in name or name.startswith('/')
|
|
or any(ord(c) < 32 for c in name)):
|
|
raise UpdateError('更新文件路径无效。')
|
|
parts = name.split('/')
|
|
if any(p in ('', '.', '..') or p.endswith((' ', '.'))
|
|
or re.fullmatch(r'(?i)(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?', p) for p in parts):
|
|
raise UpdateError('更新文件路径无效。')
|
|
return PurePosixPath(name)
|
|
|
|
|
|
def program_path(name):
|
|
path = safe_path(name)
|
|
if name == KEY or name.startswith('deploy/bootstrap/'):
|
|
return False # immutable trust anchor, even in a correctly signed release
|
|
return (name == 'app/backend.jar' or name.startswith('app/web/')
|
|
or name == 'backend/src/main/resources/analysis-runtime-policy.json'
|
|
or name.startswith('scripts/') and path.suffix == '.py'
|
|
or name.startswith('ai/nao-quality/')
|
|
or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md'))
|
|
|
|
|
|
def destination(root, name):
|
|
safe_path(name)
|
|
target = root.joinpath(*name.split('/'))
|
|
for path in (target, *target.parents):
|
|
if path == root:
|
|
break
|
|
if path.is_symlink():
|
|
raise UpdateError('更新目录包含符号链接,已停止更新。')
|
|
if os.name == 'nt' and path.exists() and getattr(path.lstat(), 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT:
|
|
raise UpdateError('更新目录包含重解析点,已停止更新。')
|
|
if not target.resolve().is_relative_to(root.resolve()):
|
|
raise UpdateError('更新文件超出程序目录。')
|
|
return target
|
|
|
|
|
|
def entry_map(entries, program_only=False):
|
|
if not isinstance(entries, list) or len(entries) > 50000:
|
|
raise UpdateError('更新文件清单无效。')
|
|
result, folded = {}, set()
|
|
for entry in entries:
|
|
if not isinstance(entry, dict):
|
|
raise UpdateError('更新文件清单无效。')
|
|
name = entry.get('path')
|
|
safe_path(name)
|
|
if name.casefold() in folded:
|
|
raise UpdateError('更新文件路径重复。')
|
|
folded.add(name.casefold())
|
|
if program_only and not program_path(name):
|
|
raise UpdateError('更新包含不允许替换的文件。')
|
|
if entry.get('symlink') is not None or not SHA.fullmatch(str(entry.get('sha256', ''))):
|
|
raise UpdateError('更新文件校验信息无效。')
|
|
result[name] = entry
|
|
return result
|
|
|
|
|
|
def validate_release(envelope, root=ROOT):
|
|
try:
|
|
raw = base64.b64decode(envelope['payload'], validate=True)
|
|
signature = base64.b64decode(envelope['signature'], validate=True)
|
|
key = load_pem_public_key(destination(root, KEY).read_bytes())
|
|
if not isinstance(key, Ed25519PublicKey):
|
|
raise ValueError('key type')
|
|
key.verify(signature, raw)
|
|
release = decode(raw)
|
|
except Exception as error:
|
|
raise UpdateError('更新签名验证失败。') from error
|
|
current = read(root / 'manifest.json')
|
|
if release.get('schema') != 1 or release.get('platform') != 'windows-x64':
|
|
raise UpdateError('更新包不适用于此系统。')
|
|
if not isinstance(release.get('version'), str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]{0,79}', release['version']):
|
|
raise UpdateError('更新版本无效。')
|
|
if type(release.get('sequence')) is not int or release['sequence'] <= 0:
|
|
raise UpdateError('更新版本序号无效。')
|
|
if release.get('runtimeFingerprint') != runtime_fingerprint(current):
|
|
raise UpdateError('运行环境版本不同,请使用完整安装包更新。')
|
|
archive = release.get('archive', {})
|
|
if (not re.fullmatch(r'quality-update-[A-Za-z0-9][A-Za-z0-9._-]{0,100}\.zip', str(archive.get('file', '')))
|
|
or type(archive.get('size')) is not int or not 0 < archive['size'] <= MAX_ARCHIVE
|
|
or not SHA.fullmatch(str(archive.get('sha256', '')))):
|
|
raise UpdateError('更新下载信息无效。')
|
|
files = entry_map(release.get('files'), True)
|
|
if not files or any(type(e.get('size')) is not int or e['size'] < 0 for e in files.values()) or sum(e['size'] for e in files.values()) > MAX_EXPANDED:
|
|
raise UpdateError('更新文件大小无效。')
|
|
if 'bundleManifest' in release:
|
|
raise UpdateError('更新清单不能包含部署数据。')
|
|
old_files = entry_map(current.get('files'))
|
|
required = {'app/backend.jar', 'deploy/offline/control.py', 'deploy/offline/update_manager.py',
|
|
'backend/src/main/resources/analysis-runtime-policy.json'}
|
|
if not required.issubset(files) or not any(name.startswith('app/web/') for name in files):
|
|
raise UpdateError('更新缺少必要程序文件。')
|
|
if KEY not in old_files or digest(destination(root, KEY)) != old_files[KEY]['sha256']:
|
|
raise UpdateError('更新信任公钥校验失败。')
|
|
# 只探测本次会备份、删除或替换的程序路径;运行环境保持原样。
|
|
# 完整清单仍逐项验证路径语法、重复项、摘要以及运行环境指纹。
|
|
old_program = {name for name in old_files if program_path(name)}
|
|
for name in old_program | set(files):
|
|
destination(root, name)
|
|
return release
|
|
|
|
|
|
class NoRedirect(HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
raise UpdateError('更新服务器返回了不允许的跳转。')
|
|
|
|
|
|
def download(url, target=None, expected_size=None, expected_hash=None):
|
|
if url != LATEST and not (url.startswith(BASE) and re.fullmatch(r'quality-update-[A-Za-z0-9][A-Za-z0-9._-]{0,100}\.zip', url[len(BASE):])):
|
|
raise UpdateError('更新下载地址无效。')
|
|
limit = expected_size if expected_size is not None else MAX_MANIFEST
|
|
hasher, count, chunks = hashlib.sha256(), 0, []
|
|
with build_opener(ProxyHandler({}), NoRedirect()).open(url, timeout=30) as response:
|
|
if response.geturl() != url or response.status != 200:
|
|
raise UpdateError('更新服务器响应无效。')
|
|
with target.open('xb') if target is not None else contextlib.nullcontext() as stream:
|
|
while block := response.read(1024 * 1024):
|
|
count += len(block)
|
|
if count > limit:
|
|
raise UpdateError('更新下载大小不符合清单。')
|
|
hasher.update(block)
|
|
if stream is not None:
|
|
stream.write(block)
|
|
else:
|
|
chunks.append(block)
|
|
if expected_size is not None and (count != expected_size or hasher.hexdigest() != expected_hash):
|
|
raise UpdateError('更新下载校验失败。')
|
|
return b''.join(chunks)
|
|
|
|
|
|
def unpack(archive, stage, release):
|
|
entries = {e['path']: e for e in release['files']}
|
|
with zipfile.ZipFile(archive) as source:
|
|
seen = set()
|
|
for member in source.infolist():
|
|
name = member.filename
|
|
if name not in entries or name in seen or member.is_dir() or member.flag_bits & 1:
|
|
raise UpdateError('更新压缩包文件清单不符。')
|
|
seen.add(name)
|
|
if stat.S_ISLNK(member.external_attr >> 16) or member.file_size != entries[name]['size']:
|
|
raise UpdateError('更新压缩包文件类型或大小不符。')
|
|
target = destination(stage, name)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
count = 0
|
|
with source.open(member) as reader, target.open('xb') as writer:
|
|
while block := reader.read(1024 * 1024):
|
|
count += len(block)
|
|
if count > entries[name]['size']:
|
|
raise UpdateError('更新解压大小不符。')
|
|
writer.write(block)
|
|
if digest(target) != entries[name]['sha256']:
|
|
raise UpdateError('更新解压校验失败。')
|
|
if seen != set(entries):
|
|
raise UpdateError('更新压缩包缺少文件。')
|
|
|
|
|
|
def enabled(root=ROOT):
|
|
return os.name == 'nt' and os.environ.get('QUALITY_UPDATE_ENABLED') == 'true' and (root / 'runtimes/python/python.exe').is_file() and (root / 'manifest.json').is_file()
|
|
|
|
|
|
def status(root=ROOT):
|
|
manifest = read(root / 'manifest.json', {})
|
|
try:
|
|
value = read(root / '.runtime/update/status.json', {})
|
|
except (OSError, ValueError):
|
|
value = {}
|
|
if not isinstance(value, dict):
|
|
value = {}
|
|
allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'}
|
|
result = {k: v for k, v in value.items() if k in allowed}
|
|
result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本')))
|
|
result.setdefault('state', 'idle')
|
|
journal = read(root / '.runtime/update/journal.json', {})
|
|
same_operation = bool(journal.get('operationId')) and journal.get('operationId') == value.get('operationId')
|
|
if (same_operation and journal.get('phase') == 'complete' and result['state'] in ACTIVE
|
|
and manifest.get('updateSequence') == journal.get('sequence')
|
|
and manifest.get('updateVersion') == journal.get('version')):
|
|
result.update(state='completed', message='更新完成。', available=False,
|
|
version=journal['version'], finishedAt=journal['finishedAt'])
|
|
elif same_operation and journal.get('phase') == 'rolledBack' and result['state'] in ACTIVE:
|
|
result.update(state='failed', message='更新未完成,已恢复原版本。', available=False)
|
|
if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})):
|
|
pending = read(root / '.runtime/update/request.json', {})
|
|
journal = read(root / '.runtime/update/journal.json', {})
|
|
if time.time() - pending.get('created', 0) > 120:
|
|
result.update(state='failed', available=False, message='更新进程已中断,请重新运行软件启动入口恢复。'
|
|
if journal.get('phase') not in (None, 'complete', 'rolledBack') else '更新进程已结束,请重新检查更新。')
|
|
if not result['enabled']:
|
|
result['message'] = '此部署不支持软件内更新。'
|
|
return result
|
|
|
|
|
|
def write_status(state, message, root=ROOT, **extra):
|
|
destination(root, '.runtime/update/status.json')
|
|
value = read(root / '.runtime/update/status.json', {})
|
|
value.update(extra, state=state, message=message)
|
|
atomic(root / '.runtime/update/status.json', value)
|
|
return status(root)
|
|
|
|
|
|
def check(root=ROOT):
|
|
release = validate_release(decode(download(LATEST)), root)
|
|
available = release['sequence'] > read(root / 'manifest.json').get('updateSequence', 0)
|
|
atomic(root / '.runtime/update/checked.json', release)
|
|
return write_status('available' if available else 'current', '发现新版本。' if available else '已是最新版本。', root,
|
|
version=release['version'], available=available, checkedAt=datetime.now(timezone.utc).isoformat())
|
|
|
|
|
|
def process_options():
|
|
return {'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_BREAKAWAY_FROM_JOB} if os.name == 'nt' else {'start_new_session': True}
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def exclusive_update(root=ROOT):
|
|
directory = root / '.runtime/update'
|
|
destination(root, '.runtime/update')
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
if live(read(directory / 'worker.json', {})):
|
|
raise BusyUpdate('已有更新或恢复正在进行。')
|
|
sys.path.insert(0, str(root / 'scripts'))
|
|
from runtime_platform import fcntl
|
|
with (directory / 'worker.lock').open('a+b') as lock:
|
|
try:
|
|
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except (BlockingIOError, OSError):
|
|
raise BusyUpdate('已有更新或恢复正在进行。') from None
|
|
identity = {'pid': os.getpid(), 'created': psutil.Process().create_time()}
|
|
atomic(directory / 'worker.json', identity)
|
|
try:
|
|
yield
|
|
finally:
|
|
if read(directory / 'worker.json', {}) == identity:
|
|
try:
|
|
(directory / 'worker.json').unlink(missing_ok=True)
|
|
except OSError as error:
|
|
record_failure(root, 'cleanupWorker', error)
|
|
|
|
|
|
def launch(root=ROOT):
|
|
with exclusive_update(root):
|
|
return launch_locked(root)
|
|
|
|
|
|
def launch_locked(root=ROOT):
|
|
directory = root / '.runtime/update'
|
|
destination(root, '.runtime/update')
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
# This short launcher MUST exit before its worker may stop Java. The worker
|
|
# verifies launcher disappearance; DETACHED alone does not remove ancestry.
|
|
pending = read(directory / 'request.json', {})
|
|
if pending and time.time() - pending.get('created', 0) < 120:
|
|
raise BusyUpdate('已有更新正在启动。')
|
|
check(root) # refetch and verify; do not trust browser or cached JSON
|
|
if not status(root).get('available'):
|
|
return status(root)
|
|
token = secrets.token_hex(16)
|
|
atomic(directory / 'request.json', {'token': token, 'created': time.time()})
|
|
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False, operationId=token)
|
|
try:
|
|
with (directory / 'worker.log').open('ab') as log:
|
|
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token],
|
|
cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
|
|
except Exception:
|
|
(directory / 'request.json').unlink(missing_ok=True)
|
|
raise UpdateError('独立更新进程无法启动,服务仍在运行。') from None
|
|
return status(root)
|
|
|
|
|
|
def live(info):
|
|
try:
|
|
process = psutil.Process(info['pid'])
|
|
return process.is_running() and process.status() != psutil.STATUS_ZOMBIE and abs(process.create_time() - info['created']) < .01
|
|
except (psutil.Error, KeyError, TypeError):
|
|
return False
|
|
|
|
|
|
def controller(root, action, database_only=False, timeout=1100, program_only=False):
|
|
python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3')
|
|
command = [str(python), str(root / 'deploy/offline/control.py'), action]
|
|
if database_only:
|
|
command += ['--database-only']
|
|
if program_only:
|
|
command += ['--program-only']
|
|
if action == 'serve':
|
|
with (root / '.runtime/update/service.log').open('ab') as log:
|
|
return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
|
|
result = subprocess.run(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
|
if result.returncode:
|
|
raise UpdateError('服务启停失败,请查看本机更新日志。')
|
|
return result
|
|
|
|
|
|
def wait_ready(root, process, database_only=False):
|
|
deadline = time.monotonic() + (180 if database_only else 1000)
|
|
while time.monotonic() < deadline:
|
|
state = read(root / '.runtime/service.json', {})
|
|
if state.get('ready') and state.get('databaseOnly', False) == database_only and state.get('supervisor', {}).get('pid') == process.pid:
|
|
return
|
|
if process.poll() is not None:
|
|
raise UpdateError('更新后服务启动失败,正在恢复旧版本。')
|
|
time.sleep(.5)
|
|
raise UpdateError('更新后服务启动超时,正在恢复旧版本。')
|
|
|
|
|
|
def mysql(root, config, binary='mysql', database=None, stdin=None, stdout=subprocess.PIPE):
|
|
exe = '.exe' if os.name == 'nt' else ''
|
|
command = [str(root / 'runtimes/mysql/bin' / (binary + exe)), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
|
|
'--port=' + str(config['mysqlPort']), '--user=root', '--default-character-set=utf8mb4']
|
|
if binary == 'mysqldump':
|
|
command += ['--single-transaction', '--routines', '--events', '--triggers', '--hex-blob', '--no-tablespaces', '--set-gtid-purged=OFF', config.get('mysqlDatabase', 'quality_platform')]
|
|
elif database:
|
|
command.append(database)
|
|
if binary == 'mysql':
|
|
command.append('--connect-timeout=10')
|
|
env = os.environ.copy()
|
|
env['MYSQL_PWD'] = config['rootPassword']
|
|
result = subprocess.run(command, cwd=root, env=env, stdin=stdin, stdout=stdout, stderr=subprocess.PIPE, timeout=3600)
|
|
if result.returncode:
|
|
error_log = destination(root, '.runtime/update/database-error.log')
|
|
with error_log.open('ab') as log:
|
|
log.write((binary + ' failed\n').encode() + result.stderr[-16384:] + b'\n')
|
|
raise UpdateError('数据库备份或恢复失败,已保留本机备份和原数据库。')
|
|
|
|
|
|
def sql(root, config, query):
|
|
# stdin must be a real file on Windows; do not place passwords in arguments.
|
|
path = root / '.runtime/update/query.sql'
|
|
try:
|
|
path.write_text(query, encoding='utf-8')
|
|
with path.open('rb') as source:
|
|
mysql(root, config, stdin=source)
|
|
finally:
|
|
path.unlink(missing_ok=True)
|
|
|
|
|
|
def backup_program(root, backup, release):
|
|
current = read(root / 'manifest.json')
|
|
names = sorted({e['path'] for e in current['files'] if program_path(e['path'])} | {e['path'] for e in release['files']})
|
|
present = []
|
|
for name in names:
|
|
source = destination(root, name)
|
|
if source.exists():
|
|
target = destination(backup / 'program', name)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
copy_atomic(source, target)
|
|
present.append(name)
|
|
copy_atomic(root / 'manifest.json', backup / 'manifest.json')
|
|
# All private configuration (AI keys, deployment credentials and bootstrap
|
|
# flags) is backed up locally, but never becomes a release payload.
|
|
# 复制时在写句柄上刷盘;Windows 不支持在只读句柄上执行 _commit。
|
|
shutil.copytree(root / '.runtime', backup / 'private', ignore=shutil.ignore_patterns('update', '*.lock', '*.log', 'service.json', 'stop-request.json'), symlinks=False, copy_function=copy_atomic)
|
|
atomic(backup / 'program.json', {'all': names, 'present': present})
|
|
|
|
|
|
def replace_program(root, stage, backup, release):
|
|
current = read(root / 'manifest.json')
|
|
names = read(backup / 'program.json')['all']
|
|
wanted = {e['path'] for e in release['files']}
|
|
for name in names:
|
|
target = destination(root, name)
|
|
if name in wanted:
|
|
copy_atomic(stage / name, target)
|
|
else:
|
|
target.unlink(missing_ok=True)
|
|
current['files'] = ([e for e in current['files'] if not program_path(e['path'])]
|
|
+ [{'path': e['path'], 'sha256': e['sha256']} for e in release['files']])
|
|
current.update(updateSequence=release['sequence'], updateVersion=release['version'])
|
|
atomic(root / 'manifest.json', current)
|
|
|
|
|
|
def restore_program(root, backup):
|
|
names = read(backup / 'program.json')
|
|
for name in names['all']:
|
|
target = destination(root, name)
|
|
if name in names['present']:
|
|
copy_atomic(backup / 'program' / name, target)
|
|
else:
|
|
target.unlink(missing_ok=True)
|
|
copy_atomic(backup / 'manifest.json', destination(root, 'manifest.json'))
|
|
for source in (backup / 'private').rglob('*'):
|
|
if source.is_file():
|
|
relative = source.relative_to(backup / 'private').as_posix()
|
|
copy_atomic(source, destination(root, '.runtime/' + relative))
|
|
|
|
|
|
STOP_TIMEOUT = 360
|
|
MAX_RECOVERY_ATTEMPTS = 2
|
|
|
|
|
|
def best_effort_status(state, message, root=ROOT, **extra):
|
|
try:
|
|
return write_status(state, message, root, **extra)
|
|
except Exception as error:
|
|
record_failure(root, 'writeStatus', error)
|
|
|
|
|
|
def verify_backup(root, backup, journal=None):
|
|
journal = journal or read(root / '.runtime/update/journal.json', {})
|
|
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
|
|
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
|
|
raise UpdateError('更新恢复备份校验失败,已保留现场。')
|
|
manifest = entry_map(read(backup / 'manifest.json')['files'])
|
|
inventory = read(backup / 'program.json')
|
|
names, present = inventory.get('all'), inventory.get('present')
|
|
if (not isinstance(names, list) or not isinstance(present, list)
|
|
or len(names) != len(set(names)) or not set(present).issubset(names)):
|
|
raise UpdateError('更新恢复清单无效。')
|
|
for name in names:
|
|
if not program_path(name):
|
|
raise UpdateError('更新恢复清单包含受保护路径。')
|
|
destination(root, name)
|
|
if name in present:
|
|
source = destination(backup / 'program', name)
|
|
if name not in manifest or not source.is_file() or digest(source) != manifest[name]['sha256']:
|
|
raise UpdateError('原程序备份缺失或损坏。')
|
|
elif name in manifest:
|
|
raise UpdateError('原程序备份不完整。')
|
|
|
|
|
|
def recovery_binding(backup):
|
|
return {'backup': backup.name, 'backupManifestSha256': digest(backup / 'manifest.json'),
|
|
'programInventorySha256': digest(backup / 'program.json'),
|
|
'databaseSqlSha256': digest(backup / 'database.sql')}
|
|
|
|
|
|
def recovery_database(binding, attempt):
|
|
return 'quality_restore_' + hashlib.sha256(canonical({**binding, 'attempt': attempt})).hexdigest()[:24]
|
|
|
|
|
|
def recovery_progress(root, backup):
|
|
binding = recovery_binding(backup)
|
|
try:
|
|
value = read(root / '.runtime/update/recovery.json', {})
|
|
except (OSError, ValueError) as error:
|
|
raise UpdateError('恢复进度无法读取,已保留现场。') from error
|
|
if not value:
|
|
return binding, None
|
|
if (not isinstance(value, dict) or any(value.get(k) != v for k, v in binding.items())
|
|
or type(value.get('attempt')) is not int or not 1 <= value['attempt'] <= MAX_RECOVERY_ATTEMPTS
|
|
or value.get('database') != recovery_database(binding, value['attempt'])
|
|
or value.get('stage') not in ('importing', 'databaseDurable', 'published', 'ready')):
|
|
raise UpdateError('恢复进度与备份不一致,已保留现场。')
|
|
return binding, value
|
|
|
|
|
|
def prepare_old_backend_restart(root):
|
|
# 旧 JAR 不理解终态 journal 的优先级。先持久化不触发其维护锁的进度状态,
|
|
# 再启动旧服务;写失败则保留恢复记录,不声称已恢复。
|
|
write_status('queued', '正在启动原版本,完成后自动重新连接。', root)
|
|
|
|
|
|
def normalize_terminal_status(root, journal):
|
|
value = read(root / '.runtime/update/status.json', {})
|
|
if (not isinstance(value, dict) or value.get('state') not in ACTIVE
|
|
or journal.get('operationId') != value.get('operationId')):
|
|
return
|
|
pending = read(root / '.runtime/update/request.json', {})
|
|
if value.get('state') in ('queued', 'downloading') and time.time() - pending.get('created', 0) < 120:
|
|
return
|
|
if journal.get('phase') == 'rolledBack':
|
|
write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
|
|
elif journal.get('phase') == 'complete':
|
|
manifest = read(root / 'manifest.json')
|
|
if manifest.get('updateVersion') == journal.get('version', value.get('version')):
|
|
write_status('completed', '更新完成。', root, available=False)
|
|
|
|
|
|
def rollback(root, backup, database_backed_up):
|
|
verify_backup(root, backup)
|
|
binding, progress = recovery_progress(root, backup) if database_backed_up else ({}, None)
|
|
best_effort_status('rollingBack', '更新未完成,正在恢复原版本。', root)
|
|
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
|
restore_program(root, backup)
|
|
if database_backed_up:
|
|
config = read(backup / 'private/deployment.json')
|
|
# 断电前未确认数据库安全关闭的导入,不能当作完整数据库复用。
|
|
if progress is None or progress['stage'] == 'importing':
|
|
attempt = 1 if progress is None else progress['attempt'] + 1
|
|
if attempt > MAX_RECOVERY_ATTEMPTS:
|
|
raise UpdateError('恢复导入多次中断,已停止自动重试并保留数据库,请核对本机恢复记录。')
|
|
reserve = (backup / 'database.sql').stat().st_size * 3 + 512 * 1024**2
|
|
if shutil.disk_usage(root).free < reserve:
|
|
raise UpdateError('空间不足以安全恢复数据库,已保留现场。')
|
|
database = recovery_database(binding, attempt)
|
|
if database == config.get('mysqlDatabase'):
|
|
raise UpdateError('恢复数据库与原库冲突,已停止。')
|
|
progress = {**binding, 'attempt': attempt, 'database': database, 'stage': 'importing'}
|
|
atomic(root / '.runtime/update/recovery.json', progress)
|
|
process = controller(root, 'serve', True)
|
|
wait_ready(root, process, True)
|
|
# 不删除任何已有数据库;名称冲突时 CREATE 会失败并保留现场。
|
|
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
|
with (backup / 'database.sql').open('rb') as source:
|
|
mysql(root, config, database=database, stdin=source)
|
|
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
|
|
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
|
progress['stage'] = 'databaseDurable'
|
|
atomic(root / '.runtime/update/recovery.json', progress)
|
|
config['mysqlDatabase'] = progress['database']
|
|
atomic(root / '.runtime/deployment.json', config)
|
|
receipt_path = root / '.runtime/migration-receipt.json'
|
|
receipt = read(receipt_path, {})
|
|
if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'):
|
|
receipt['database'] = progress['database']
|
|
atomic(receipt_path, receipt)
|
|
progress['stage'] = 'published'
|
|
atomic(root / '.runtime/update/recovery.json', progress)
|
|
prepare_old_backend_restart(root)
|
|
process = controller(root, 'serve')
|
|
wait_ready(root, process)
|
|
if progress:
|
|
progress['stage'] = 'ready'
|
|
atomic(root / '.runtime/update/recovery.json', progress)
|
|
|
|
|
|
def preflight_programs(stage):
|
|
# 旧安装先加固恢复器,随后只能安装具备相同恢复协议的程序,防止覆盖回旧实现。
|
|
for name, marker in (('update_manager.py', 'UPDATE_PROTOCOL'), ('control.py', 'UPDATE_CONTROL_PROTOCOL')):
|
|
try:
|
|
tree = ast.parse((stage / 'deploy/offline' / name).read_bytes())
|
|
values = [node.value.value for node in tree.body if isinstance(node, ast.Assign)
|
|
and any(isinstance(target, ast.Name) and target.id == marker for target in node.targets)
|
|
and isinstance(node.value, ast.Constant) and type(node.value.value) is int]
|
|
if values != [UPDATE_PROTOCOL]:
|
|
raise ValueError('incompatible recovery protocol')
|
|
except (OSError, SyntaxError, ValueError) as error:
|
|
raise UpdateError('更新包未包含本轮加固的恢复流程,服务尚未停止;请等待配套更新包。') from error
|
|
try:
|
|
with zipfile.ZipFile(stage / 'app/backend.jar') as jar:
|
|
metadata = decode(jar.read('BOOT-INF/classes/update-protocol.json'))
|
|
if metadata != {'protocol': UPDATE_PROTOCOL}:
|
|
raise ValueError('incompatible backend protocol')
|
|
except (OSError, ValueError, KeyError, zipfile.BadZipFile) as error:
|
|
raise UpdateError('更新包后端不支持本轮恢复协议,服务尚未停止。') from error
|
|
|
|
|
|
def preflight_backup(root, backup, release):
|
|
paths = set(e['path'] for e in read(root / 'manifest.json')['files'] if program_path(e['path']))
|
|
paths.update(e['path'] for e in release['files'])
|
|
targets = [backup / 'program' / n for n in paths]
|
|
ignored = ('*.lock', '*.log', 'service.json', 'stop-request.json')
|
|
for folder, dirs, files in os.walk(root / '.runtime', followlinks=False):
|
|
dirs[:] = [n for n in dirs if n != 'update']
|
|
for n in dirs:
|
|
destination(root, (Path(folder) / n).relative_to(root).as_posix())
|
|
for n in files:
|
|
if any(fnmatch.fnmatch(n, pattern) for pattern in ignored):
|
|
continue
|
|
source = Path(folder) / n
|
|
destination(root, source.relative_to(root).as_posix())
|
|
with source.open('rb'):
|
|
pass
|
|
targets.append(backup / 'private' / source.relative_to(root / '.runtime'))
|
|
for target in targets:
|
|
destination(backup, target.relative_to(backup).as_posix())
|
|
if os.name == 'nt' and len(str(target.absolute())) + 24 > 248:
|
|
raise UpdateError('备份路径过长,请先缩短部署目录或历史路径;服务尚未停止。')
|
|
probe = backup / 'write-probe.tmp'
|
|
try:
|
|
with probe.open('xb') as stream:
|
|
stream.write(b'probe')
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
finally:
|
|
probe.unlink(missing_ok=True)
|
|
|
|
|
|
def record_failure(root, stage, error, name='failure.json'):
|
|
# 仅记录阶段、异常类型、系统错误码和代码位置,不记录正文、凭据或异常参数。
|
|
receipt = {'stage': stage, 'errorType': type(error).__name__,
|
|
'occurredAt': datetime.now(timezone.utc).isoformat(),
|
|
'frames': [{'file': Path(frame.f_code.co_filename).name, 'function': frame.f_code.co_name, 'line': line}
|
|
for frame, line in traceback.walk_tb(error.__traceback__)]}
|
|
for key in ('errno', 'winerror'):
|
|
value = getattr(error, key, None)
|
|
if type(value) is int:
|
|
receipt[key] = value
|
|
try:
|
|
atomic(root / '.runtime/update' / name, receipt)
|
|
except Exception:
|
|
pass # 诊断写入失败不能阻止原有回退流程。
|
|
|
|
|
|
def worker(launcher, token, root=ROOT):
|
|
directory = root / '.runtime/update'
|
|
for _ in range(300):
|
|
if not psutil.pid_exists(launcher):
|
|
break
|
|
time.sleep(.1)
|
|
else:
|
|
raise UpdateError('更新启动器未退出,已安全停止。')
|
|
if any('java' in p.name().lower() for p in psutil.Process().parents()):
|
|
raise UpdateError('更新进程未独立启动,已安全停止。')
|
|
if read(directory / 'request.json', {}).get('token') != token:
|
|
raise UpdateError('更新请求已失效。')
|
|
with exclusive_update(root):
|
|
backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4))
|
|
stopped = backed_up = program_saved = changed = stop_requested = committed = False
|
|
failure_stage = 'validateRelease'
|
|
try:
|
|
envelope = decode(download(LATEST))
|
|
release = validate_release(envelope, root)
|
|
if release['sequence'] <= read(root / 'manifest.json').get('updateSequence', 0):
|
|
raise UpdateError('此更新版本已安装。')
|
|
backup.mkdir()
|
|
archive, stage = backup / 'update.zip', backup / 'stage'
|
|
expanded = sum(entry['size'] for entry in release['files'])
|
|
# Keep enough headroom for extraction, old program files and a
|
|
# logical dump; the dump itself must complete before replacement.
|
|
db_size = sum(p.stat().st_size for p in (root / 'runtimedata/mysql').rglob('*') if p.is_file())
|
|
old_program_size = sum((root / entry['path']).stat().st_size for entry in read(root / 'manifest.json')['files'] if program_path(entry['path']) and (root / entry['path']).is_file())
|
|
private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory))
|
|
if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2:
|
|
raise UpdateError('磁盘剩余空间不足,请释放空间后重试。')
|
|
write_status('downloading', '正在下载并校验更新。', root, operationId=token, version=release['version'])
|
|
info = release['archive']
|
|
failure_stage = 'downloadArchive'
|
|
download(BASE + info['file'], archive, info['size'], info['sha256'])
|
|
failure_stage = 'unpackArchive'
|
|
unpack(archive, stage, release)
|
|
failure_stage = 'programPreflight'
|
|
preflight_programs(stage)
|
|
failure_stage = 'runtimePreflight'
|
|
write_status('downloading', '正在检查运行环境和备份条件,服务仍可使用。', root)
|
|
controller(root, 'verify-runtime', timeout=1800)
|
|
failure_stage = 'backupPreflight'
|
|
preflight_backup(root, backup, release)
|
|
# Revalidate immediately before any service or disk mutation.
|
|
validate_release(envelope, root)
|
|
write_status('backingUp', '正在备份,服务即将重新启动。', root)
|
|
if (directory / 'recovery.json').exists():
|
|
copy_atomic(directory / 'recovery.json', backup / 'previous-recovery.json')
|
|
(directory / 'recovery.json').unlink()
|
|
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False,
|
|
'operationId': token, 'version': release['version'], 'sequence': release['sequence']})
|
|
failure_stage = 'stopService'
|
|
stop_requested = True
|
|
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
|
stopped = True
|
|
failure_stage = 'backupProgram'
|
|
backup_program(root, backup, release)
|
|
program_saved = True
|
|
config = read(root / '.runtime/deployment.json')
|
|
failure_stage = 'startBackupDatabase'
|
|
database = controller(root, 'serve', True)
|
|
wait_ready(root, database, True)
|
|
failure_stage = 'backupDatabase'
|
|
with (backup / 'database.sql').open('xb') as output:
|
|
mysql(root, config, 'mysqldump', stdout=output)
|
|
output.flush()
|
|
os.fsync(output.fileno())
|
|
if (backup / 'database.sql').stat().st_size < 100:
|
|
raise UpdateError('数据库备份为空,已停止更新。')
|
|
backed_up = True
|
|
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
|
# The actual dump can exceed the source's physical storage. Do not
|
|
# replace anything unless a separate rollback database still fits.
|
|
required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2
|
|
if shutil.disk_usage(root).free < required_recovery:
|
|
raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。')
|
|
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True,
|
|
'operationId': token, 'version': release['version'], 'sequence': release['sequence'],
|
|
'databaseMayHaveChanged': False,
|
|
'backupManifestSha256': digest(backup / 'manifest.json'),
|
|
'programInventorySha256': digest(backup / 'program.json')})
|
|
write_status('installing', '正在安装更新。', root)
|
|
changed = True
|
|
failure_stage = 'replaceProgram'
|
|
replace_program(root, stage, backup, release)
|
|
failure_stage = 'verifyProgram'
|
|
controller(root, 'verify-runtime', timeout=300, program_only=True)
|
|
write_status('restarting', '正在重新启动,请稍候。', root)
|
|
failure_stage = 'restartService'
|
|
journal = read(directory / 'journal.json')
|
|
journal['databaseMayHaveChanged'] = True
|
|
atomic(directory / 'journal.json', journal)
|
|
process = controller(root, 'serve')
|
|
wait_ready(root, process)
|
|
journal.update(phase='complete', version=release['version'], sequence=release['sequence'],
|
|
finishedAt=datetime.now(timezone.utc).isoformat())
|
|
atomic(directory / 'journal.json', journal)
|
|
committed = True
|
|
best_effort_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
|
|
except Exception as error:
|
|
record_failure(root, failure_stage, error)
|
|
message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。'
|
|
journal = read(directory / 'journal.json', {})
|
|
if committed or (journal.get('backup') == backup.name and journal.get('phase') == 'complete'):
|
|
return # 提交后的展示或清理失败绝不能发起回滚。
|
|
if stop_requested:
|
|
try:
|
|
if program_saved and changed:
|
|
rollback(root, backup, backed_up and journal.get('databaseMayHaveChanged', True))
|
|
else:
|
|
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
|
prepare_old_backend_restart(root)
|
|
process = controller(root, 'serve')
|
|
wait_ready(root, process)
|
|
journal.update(backup=backup.name, phase='rolledBack', databaseBackedUp=backed_up)
|
|
atomic(directory / 'journal.json', journal)
|
|
message += ' 已恢复原版本。'
|
|
except Exception as rollback_error:
|
|
record_failure(root, 'rollback', rollback_error, 'rollback-failure.json')
|
|
message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。'
|
|
best_effort_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
|
|
finally:
|
|
try:
|
|
(directory / 'request.json').unlink(missing_ok=True)
|
|
except OSError as error:
|
|
record_failure(root, 'cleanupRequest', error)
|
|
|
|
|
|
def recover(root=ROOT):
|
|
with exclusive_update(root):
|
|
return recover_locked(root)
|
|
|
|
|
|
def recover_locked(root=ROOT):
|
|
directory = root / '.runtime/update'
|
|
journal = read(directory / 'journal.json', {})
|
|
if journal.get('phase') in (None, 'complete', 'rolledBack'):
|
|
if journal.get('phase') in ('complete', 'rolledBack'):
|
|
normalize_terminal_status(root, journal)
|
|
return status(root)
|
|
name = journal.get('backup', '')
|
|
if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name):
|
|
raise UpdateError('更新恢复记录无效。')
|
|
backup = directory / name
|
|
if journal.get('phase') not in ('installing', 'stopping'):
|
|
raise UpdateError('更新恢复阶段无效,已保留现场。')
|
|
if journal.get('phase') == 'installing':
|
|
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
|
|
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
|
|
raise UpdateError('更新恢复备份校验失败,已保留现场。')
|
|
verify_backup(root, backup, journal)
|
|
if journal.get('databaseBackedUp') is True:
|
|
recovery_progress(root, backup) # 先验证 sidecar 与 dump,尚未停止或替换任何文件。
|
|
rollback(root, backup, journal.get('databaseBackedUp') is True and journal.get('databaseMayHaveChanged', True))
|
|
else:
|
|
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
|
prepare_old_backend_restart(root)
|
|
process = controller(root, 'serve')
|
|
wait_ready(root, process)
|
|
atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'})
|
|
best_effort_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
|
|
return status(root)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('status', 'check', 'launch', 'worker', 'recover'))
|
|
parser.add_argument('--launcher', type=int)
|
|
parser.add_argument('--token')
|
|
args = parser.parse_args()
|
|
if args.action == 'status':
|
|
return status()
|
|
if not enabled():
|
|
raise UpdateError('此部署不支持软件内更新。')
|
|
if args.action == 'worker':
|
|
if not args.launcher or not re.fullmatch(r'[a-f0-9]{32}', args.token or ''):
|
|
raise UpdateError('更新请求无效。')
|
|
worker(args.launcher, args.token)
|
|
return status()
|
|
if args.action == 'recover':
|
|
return recover()
|
|
journal = read(ROOT / '.runtime/update/journal.json', {})
|
|
if journal.get('phase') not in (None, 'complete', 'rolledBack'):
|
|
raise UpdateError('上次更新中断,请重新运行软件启动入口恢复。')
|
|
if live(read(ROOT / '.runtime/update/worker.json', {})):
|
|
raise BusyUpdate('已有更新正在进行。')
|
|
return check() if args.action == 'check' else launch()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
print(json.dumps(main(), ensure_ascii=False))
|
|
except Exception as error:
|
|
message = str(error) if isinstance(error, UpdateError) else '更新服务暂不可用,请稍后重试。'
|
|
if enabled() and not isinstance(error, BusyUpdate) and sys.argv[1:2] != ['status'] and not live(read(ROOT / '.runtime/update/worker.json', {})):
|
|
write_status('failed', message)
|
|
print(json.dumps({'error': message}, ensure_ascii=False))
|
|
raise SystemExit(1)
|