修复:加固 Windows 更新预检、提交点和可恢复的数据库回退流程

This commit is contained in:
2026-11-05 16:54:30 +08:00
parent 2315c83067
commit 3962521a5c
13 changed files with 1821 additions and 236 deletions
+250 -34
View File
@@ -8,8 +8,10 @@ snapshot; private data and runtimes are never update destinations.
from __future__ import annotations
import argparse
import ast
import base64
import contextlib
import fnmatch
from datetime import datetime, timezone
import hashlib
import json
@@ -30,6 +32,7 @@ import psutil
from cryptography.hazmat.primitives.serialization import load_pem_public_key
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
UPDATE_PROTOCOL = 2
ROOT = Path(__file__).resolve().parents[2]
BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/'
LATEST = BASE + 'latest.json'
@@ -267,11 +270,25 @@ def enabled(root=ROOT):
def status(root=ROOT):
manifest = read(root / 'manifest.json', {})
value = read(root / '.runtime/update/status.json', {})
try:
value = read(root / '.runtime/update/status.json', {})
except (OSError, ValueError):
value = {}
if not isinstance(value, dict):
value = {}
allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'}
result = {k: v for k, v in value.items() if k in allowed}
result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本')))
result.setdefault('state', 'idle')
journal = read(root / '.runtime/update/journal.json', {})
same_operation = bool(journal.get('operationId')) and journal.get('operationId') == value.get('operationId')
if (same_operation and journal.get('phase') == 'complete' and result['state'] in ACTIVE
and manifest.get('updateSequence') == journal.get('sequence')
and manifest.get('updateVersion') == journal.get('version')):
result.update(state='completed', message='更新完成。', available=False,
version=journal['version'], finishedAt=journal['finishedAt'])
elif same_operation and journal.get('phase') == 'rolledBack' and result['state'] in ACTIVE:
result.update(state='failed', message='更新未完成,已恢复原版本。', available=False)
if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})):
pending = read(root / '.runtime/update/request.json', {})
journal = read(root / '.runtime/update/journal.json', {})
@@ -323,7 +340,10 @@ def exclusive_update(root=ROOT):
yield
finally:
if read(directory / 'worker.json', {}) == identity:
(directory / 'worker.json').unlink(missing_ok=True)
try:
(directory / 'worker.json').unlink(missing_ok=True)
except OSError as error:
record_failure(root, 'cleanupWorker', error)
def launch(root=ROOT):
@@ -345,7 +365,7 @@ def launch_locked(root=ROOT):
return status(root)
token = secrets.token_hex(16)
atomic(directory / 'request.json', {'token': token, 'created': time.time()})
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False)
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False, operationId=token)
try:
with (directory / 'worker.log').open('ab') as log:
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token],
@@ -364,11 +384,13 @@ def live(info):
return False
def controller(root, action, database_only=False, timeout=1100):
def controller(root, action, database_only=False, timeout=1100, program_only=False):
python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3')
command = [str(python), str(root / 'deploy/offline/control.py'), action]
if database_only:
command += ['--database-only']
if program_only:
command += ['--program-only']
if action == 'serve':
with (root / '.runtime/update/service.log').open('ab') as log:
return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
@@ -471,29 +493,186 @@ def restore_program(root, backup):
copy_atomic(source, destination(root, '.runtime/' + relative))
STOP_TIMEOUT = 360
MAX_RECOVERY_ATTEMPTS = 2
def best_effort_status(state, message, root=ROOT, **extra):
try:
return write_status(state, message, root, **extra)
except Exception as error:
record_failure(root, 'writeStatus', error)
def verify_backup(root, backup, journal=None):
journal = journal or read(root / '.runtime/update/journal.json', {})
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
raise UpdateError('更新恢复备份校验失败,已保留现场。')
manifest = entry_map(read(backup / 'manifest.json')['files'])
inventory = read(backup / 'program.json')
names, present = inventory.get('all'), inventory.get('present')
if (not isinstance(names, list) or not isinstance(present, list)
or len(names) != len(set(names)) or not set(present).issubset(names)):
raise UpdateError('更新恢复清单无效。')
for name in names:
if not program_path(name):
raise UpdateError('更新恢复清单包含受保护路径。')
destination(root, name)
if name in present:
source = destination(backup / 'program', name)
if name not in manifest or not source.is_file() or digest(source) != manifest[name]['sha256']:
raise UpdateError('原程序备份缺失或损坏。')
elif name in manifest:
raise UpdateError('原程序备份不完整。')
def recovery_binding(backup):
return {'backup': backup.name, 'backupManifestSha256': digest(backup / 'manifest.json'),
'programInventorySha256': digest(backup / 'program.json'),
'databaseSqlSha256': digest(backup / 'database.sql')}
def recovery_database(binding, attempt):
return 'quality_restore_' + hashlib.sha256(canonical({**binding, 'attempt': attempt})).hexdigest()[:24]
def recovery_progress(root, backup):
binding = recovery_binding(backup)
try:
value = read(root / '.runtime/update/recovery.json', {})
except (OSError, ValueError) as error:
raise UpdateError('恢复进度无法读取,已保留现场。') from error
if not value:
return binding, None
if (not isinstance(value, dict) or any(value.get(k) != v for k, v in binding.items())
or type(value.get('attempt')) is not int or not 1 <= value['attempt'] <= MAX_RECOVERY_ATTEMPTS
or value.get('database') != recovery_database(binding, value['attempt'])
or value.get('stage') not in ('importing', 'databaseDurable', 'published', 'ready')):
raise UpdateError('恢复进度与备份不一致,已保留现场。')
return binding, value
def prepare_old_backend_restart(root):
# 旧 JAR 不理解终态 journal 的优先级。先持久化不触发其维护锁的进度状态,
# 再启动旧服务;写失败则保留恢复记录,不声称已恢复。
write_status('queued', '正在启动原版本,完成后自动重新连接。', root)
def normalize_terminal_status(root, journal):
value = read(root / '.runtime/update/status.json', {})
if (not isinstance(value, dict) or value.get('state') not in ACTIVE
or journal.get('operationId') != value.get('operationId')):
return
pending = read(root / '.runtime/update/request.json', {})
if value.get('state') in ('queued', 'downloading') and time.time() - pending.get('created', 0) < 120:
return
if journal.get('phase') == 'rolledBack':
write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
elif journal.get('phase') == 'complete':
manifest = read(root / 'manifest.json')
if manifest.get('updateVersion') == journal.get('version', value.get('version')):
write_status('completed', '更新完成。', root, available=False)
def rollback(root, backup, database_backed_up):
write_status('rollingBack', '更新未完成,正在恢复原版本。', root)
controller(root, 'stop', timeout=180)
verify_backup(root, backup)
binding, progress = recovery_progress(root, backup) if database_backed_up else ({}, None)
best_effort_status('rollingBack', '更新未完成,正在恢复原版本。', root)
controller(root, 'stop', timeout=STOP_TIMEOUT)
restore_program(root, backup)
if database_backed_up:
config = read(root / '.runtime/deployment.json')
process = controller(root, 'serve', True)
wait_ready(root, process, True)
database = 'quality_restore_' + secrets.token_hex(12)
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
with (backup / 'database.sql').open('rb') as source:
mysql(root, config, database=database, stdin=source)
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
controller(root, 'stop', timeout=180)
config['mysqlDatabase'] = database
config = read(backup / 'private/deployment.json')
# 断电前未确认数据库安全关闭的导入,不能当作完整数据库复用。
if progress is None or progress['stage'] == 'importing':
attempt = 1 if progress is None else progress['attempt'] + 1
if attempt > MAX_RECOVERY_ATTEMPTS:
raise UpdateError('恢复导入多次中断,已停止自动重试并保留数据库,请核对本机恢复记录。')
reserve = (backup / 'database.sql').stat().st_size * 3 + 512 * 1024**2
if shutil.disk_usage(root).free < reserve:
raise UpdateError('空间不足以安全恢复数据库,已保留现场。')
database = recovery_database(binding, attempt)
if database == config.get('mysqlDatabase'):
raise UpdateError('恢复数据库与原库冲突,已停止。')
progress = {**binding, 'attempt': attempt, 'database': database, 'stage': 'importing'}
atomic(root / '.runtime/update/recovery.json', progress)
process = controller(root, 'serve', True)
wait_ready(root, process, True)
# 不删除任何已有数据库;名称冲突时 CREATE 会失败并保留现场。
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
with (backup / 'database.sql').open('rb') as source:
mysql(root, config, database=database, stdin=source)
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
controller(root, 'stop', timeout=STOP_TIMEOUT)
progress['stage'] = 'databaseDurable'
atomic(root / '.runtime/update/recovery.json', progress)
config['mysqlDatabase'] = progress['database']
atomic(root / '.runtime/deployment.json', config)
receipt_path = root / '.runtime/migration-receipt.json'
receipt = read(receipt_path, {})
if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'):
receipt['database'] = database
receipt['database'] = progress['database']
atomic(receipt_path, receipt)
progress['stage'] = 'published'
atomic(root / '.runtime/update/recovery.json', progress)
prepare_old_backend_restart(root)
process = controller(root, 'serve')
wait_ready(root, process)
if progress:
progress['stage'] = 'ready'
atomic(root / '.runtime/update/recovery.json', progress)
def preflight_programs(stage):
# 旧安装先加固恢复器,随后只能安装具备相同恢复协议的程序,防止覆盖回旧实现。
for name, marker in (('update_manager.py', 'UPDATE_PROTOCOL'), ('control.py', 'UPDATE_CONTROL_PROTOCOL')):
try:
tree = ast.parse((stage / 'deploy/offline' / name).read_bytes())
values = [node.value.value for node in tree.body if isinstance(node, ast.Assign)
and any(isinstance(target, ast.Name) and target.id == marker for target in node.targets)
and isinstance(node.value, ast.Constant) and type(node.value.value) is int]
if values != [UPDATE_PROTOCOL]:
raise ValueError('incompatible recovery protocol')
except (OSError, SyntaxError, ValueError) as error:
raise UpdateError('更新包未包含本轮加固的恢复流程,服务尚未停止;请等待配套更新包。') from error
try:
with zipfile.ZipFile(stage / 'app/backend.jar') as jar:
metadata = decode(jar.read('BOOT-INF/classes/update-protocol.json'))
if metadata != {'protocol': UPDATE_PROTOCOL}:
raise ValueError('incompatible backend protocol')
except (OSError, ValueError, KeyError, zipfile.BadZipFile) as error:
raise UpdateError('更新包后端不支持本轮恢复协议,服务尚未停止。') from error
def preflight_backup(root, backup, release):
paths = set(e['path'] for e in read(root / 'manifest.json')['files'] if program_path(e['path']))
paths.update(e['path'] for e in release['files'])
targets = [backup / 'program' / n for n in paths]
ignored = ('*.lock', '*.log', 'service.json', 'stop-request.json')
for folder, dirs, files in os.walk(root / '.runtime', followlinks=False):
dirs[:] = [n for n in dirs if n != 'update']
for n in dirs:
destination(root, (Path(folder) / n).relative_to(root).as_posix())
for n in files:
if any(fnmatch.fnmatch(n, pattern) for pattern in ignored):
continue
source = Path(folder) / n
destination(root, source.relative_to(root).as_posix())
with source.open('rb'):
pass
targets.append(backup / 'private' / source.relative_to(root / '.runtime'))
for target in targets:
destination(backup, target.relative_to(backup).as_posix())
if os.name == 'nt' and len(str(target.absolute())) + 24 > 248:
raise UpdateError('备份路径过长,请先缩短部署目录或历史路径;服务尚未停止。')
probe = backup / 'write-probe.tmp'
try:
with probe.open('xb') as stream:
stream.write(b'probe')
stream.flush()
os.fsync(stream.fileno())
finally:
probe.unlink(missing_ok=True)
def record_failure(root, stage, error, name='failure.json'):
@@ -526,7 +705,7 @@ def worker(launcher, token, root=ROOT):
raise UpdateError('更新请求已失效。')
with exclusive_update(root):
backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4))
stopped = backed_up = program_saved = changed = False
stopped = backed_up = program_saved = changed = stop_requested = committed = False
failure_stage = 'validateRelease'
try:
envelope = decode(download(LATEST))
@@ -543,18 +722,30 @@ def worker(launcher, token, root=ROOT):
private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory))
if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2:
raise UpdateError('磁盘剩余空间不足,请释放空间后重试。')
write_status('downloading', '正在下载并校验更新。', root)
write_status('downloading', '正在下载并校验更新。', root, operationId=token, version=release['version'])
info = release['archive']
failure_stage = 'downloadArchive'
download(BASE + info['file'], archive, info['size'], info['sha256'])
failure_stage = 'unpackArchive'
unpack(archive, stage, release)
failure_stage = 'programPreflight'
preflight_programs(stage)
failure_stage = 'runtimePreflight'
write_status('downloading', '正在检查运行环境和备份条件,服务仍可使用。', root)
controller(root, 'verify-runtime', timeout=1800)
failure_stage = 'backupPreflight'
preflight_backup(root, backup, release)
# Revalidate immediately before any service or disk mutation.
validate_release(envelope, root)
write_status('backingUp', '正在备份,服务即将重新启动。', root)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False})
if (directory / 'recovery.json').exists():
copy_atomic(directory / 'recovery.json', backup / 'previous-recovery.json')
(directory / 'recovery.json').unlink()
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False,
'operationId': token, 'version': release['version'], 'sequence': release['sequence']})
failure_stage = 'stopService'
controller(root, 'stop', timeout=180)
stop_requested = True
controller(root, 'stop', timeout=STOP_TIMEOUT)
stopped = True
failure_stage = 'backupProgram'
backup_program(root, backup, release)
@@ -571,13 +762,15 @@ def worker(launcher, token, root=ROOT):
if (backup / 'database.sql').stat().st_size < 100:
raise UpdateError('数据库备份为空,已停止更新。')
backed_up = True
controller(root, 'stop', timeout=180)
controller(root, 'stop', timeout=STOP_TIMEOUT)
# The actual dump can exceed the source's physical storage. Do not
# replace anything unless a separate rollback database still fits.
required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2
if shutil.disk_usage(root).free < required_recovery:
raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。')
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True,
'operationId': token, 'version': release['version'], 'sequence': release['sequence'],
'databaseMayHaveChanged': False,
'backupManifestSha256': digest(backup / 'manifest.json'),
'programInventorySha256': digest(backup / 'program.json')})
write_status('installing', '正在安装更新。', root)
@@ -585,32 +778,46 @@ def worker(launcher, token, root=ROOT):
failure_stage = 'replaceProgram'
replace_program(root, stage, backup, release)
failure_stage = 'verifyProgram'
controller(root, 'verify', timeout=180)
controller(root, 'verify-runtime', timeout=300, program_only=True)
write_status('restarting', '正在重新启动,请稍候。', root)
failure_stage = 'restartService'
journal = read(directory / 'journal.json')
journal['databaseMayHaveChanged'] = True
atomic(directory / 'journal.json', journal)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'complete', 'databaseBackedUp': True})
write_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
journal.update(phase='complete', version=release['version'], sequence=release['sequence'],
finishedAt=datetime.now(timezone.utc).isoformat())
atomic(directory / 'journal.json', journal)
committed = True
best_effort_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
except Exception as error:
record_failure(root, failure_stage, error)
message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。'
if stopped:
journal = read(directory / 'journal.json', {})
if committed or (journal.get('backup') == backup.name and journal.get('phase') == 'complete'):
return # 提交后的展示或清理失败绝不能发起回滚。
if stop_requested:
try:
if program_saved and changed:
rollback(root, backup, backed_up)
rollback(root, backup, backed_up and journal.get('databaseMayHaveChanged', True))
else:
controller(root, 'stop', timeout=180)
controller(root, 'stop', timeout=STOP_TIMEOUT)
prepare_old_backend_restart(root)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'rolledBack', 'databaseBackedUp': backed_up})
journal.update(backup=backup.name, phase='rolledBack', databaseBackedUp=backed_up)
atomic(directory / 'journal.json', journal)
message += ' 已恢复原版本。'
except Exception as rollback_error:
record_failure(root, 'rollback', rollback_error, 'rollback-failure.json')
message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。'
write_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
best_effort_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
finally:
(directory / 'request.json').unlink(missing_ok=True)
try:
(directory / 'request.json').unlink(missing_ok=True)
except OSError as error:
record_failure(root, 'cleanupRequest', error)
def recover(root=ROOT):
@@ -622,22 +829,31 @@ def recover_locked(root=ROOT):
directory = root / '.runtime/update'
journal = read(directory / 'journal.json', {})
if journal.get('phase') in (None, 'complete', 'rolledBack'):
if journal.get('phase') in ('complete', 'rolledBack'):
normalize_terminal_status(root, journal)
return status(root)
name = journal.get('backup', '')
if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name):
raise UpdateError('更新恢复记录无效。')
backup = directory / name
if journal.get('phase') not in ('installing', 'stopping'):
raise UpdateError('更新恢复阶段无效,已保留现场。')
if journal.get('phase') == 'installing':
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
raise UpdateError('更新恢复备份校验失败,已保留现场。')
rollback(root, backup, journal.get('databaseBackedUp') is True)
verify_backup(root, backup, journal)
if journal.get('databaseBackedUp') is True:
recovery_progress(root, backup) # 先验证 sidecar 与 dump,尚未停止或替换任何文件。
rollback(root, backup, journal.get('databaseBackedUp') is True and journal.get('databaseMayHaveChanged', True))
else:
controller(root, 'stop', timeout=180)
controller(root, 'stop', timeout=STOP_TIMEOUT)
prepare_old_backend_restart(root)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'})
return write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
best_effort_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
return status(root)
def main():