修复:加固 Windows 更新预检、提交点和可恢复的数据库回退流程
This commit is contained in:
@@ -8,8 +8,10 @@ snapshot; private data and runtimes are never update destinations.
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import base64
|
||||
import contextlib
|
||||
import fnmatch
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
@@ -30,6 +32,7 @@ import psutil
|
||||
from cryptography.hazmat.primitives.serialization import load_pem_public_key
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
||||
|
||||
UPDATE_PROTOCOL = 2
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/'
|
||||
LATEST = BASE + 'latest.json'
|
||||
@@ -267,11 +270,25 @@ def enabled(root=ROOT):
|
||||
|
||||
def status(root=ROOT):
|
||||
manifest = read(root / 'manifest.json', {})
|
||||
value = read(root / '.runtime/update/status.json', {})
|
||||
try:
|
||||
value = read(root / '.runtime/update/status.json', {})
|
||||
except (OSError, ValueError):
|
||||
value = {}
|
||||
if not isinstance(value, dict):
|
||||
value = {}
|
||||
allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'}
|
||||
result = {k: v for k, v in value.items() if k in allowed}
|
||||
result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本')))
|
||||
result.setdefault('state', 'idle')
|
||||
journal = read(root / '.runtime/update/journal.json', {})
|
||||
same_operation = bool(journal.get('operationId')) and journal.get('operationId') == value.get('operationId')
|
||||
if (same_operation and journal.get('phase') == 'complete' and result['state'] in ACTIVE
|
||||
and manifest.get('updateSequence') == journal.get('sequence')
|
||||
and manifest.get('updateVersion') == journal.get('version')):
|
||||
result.update(state='completed', message='更新完成。', available=False,
|
||||
version=journal['version'], finishedAt=journal['finishedAt'])
|
||||
elif same_operation and journal.get('phase') == 'rolledBack' and result['state'] in ACTIVE:
|
||||
result.update(state='failed', message='更新未完成,已恢复原版本。', available=False)
|
||||
if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})):
|
||||
pending = read(root / '.runtime/update/request.json', {})
|
||||
journal = read(root / '.runtime/update/journal.json', {})
|
||||
@@ -323,7 +340,10 @@ def exclusive_update(root=ROOT):
|
||||
yield
|
||||
finally:
|
||||
if read(directory / 'worker.json', {}) == identity:
|
||||
(directory / 'worker.json').unlink(missing_ok=True)
|
||||
try:
|
||||
(directory / 'worker.json').unlink(missing_ok=True)
|
||||
except OSError as error:
|
||||
record_failure(root, 'cleanupWorker', error)
|
||||
|
||||
|
||||
def launch(root=ROOT):
|
||||
@@ -345,7 +365,7 @@ def launch_locked(root=ROOT):
|
||||
return status(root)
|
||||
token = secrets.token_hex(16)
|
||||
atomic(directory / 'request.json', {'token': token, 'created': time.time()})
|
||||
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False)
|
||||
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False, operationId=token)
|
||||
try:
|
||||
with (directory / 'worker.log').open('ab') as log:
|
||||
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token],
|
||||
@@ -364,11 +384,13 @@ def live(info):
|
||||
return False
|
||||
|
||||
|
||||
def controller(root, action, database_only=False, timeout=1100):
|
||||
def controller(root, action, database_only=False, timeout=1100, program_only=False):
|
||||
python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3')
|
||||
command = [str(python), str(root / 'deploy/offline/control.py'), action]
|
||||
if database_only:
|
||||
command += ['--database-only']
|
||||
if program_only:
|
||||
command += ['--program-only']
|
||||
if action == 'serve':
|
||||
with (root / '.runtime/update/service.log').open('ab') as log:
|
||||
return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
|
||||
@@ -471,29 +493,186 @@ def restore_program(root, backup):
|
||||
copy_atomic(source, destination(root, '.runtime/' + relative))
|
||||
|
||||
|
||||
STOP_TIMEOUT = 360
|
||||
MAX_RECOVERY_ATTEMPTS = 2
|
||||
|
||||
|
||||
def best_effort_status(state, message, root=ROOT, **extra):
|
||||
try:
|
||||
return write_status(state, message, root, **extra)
|
||||
except Exception as error:
|
||||
record_failure(root, 'writeStatus', error)
|
||||
|
||||
|
||||
def verify_backup(root, backup, journal=None):
|
||||
journal = journal or read(root / '.runtime/update/journal.json', {})
|
||||
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
|
||||
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
|
||||
raise UpdateError('更新恢复备份校验失败,已保留现场。')
|
||||
manifest = entry_map(read(backup / 'manifest.json')['files'])
|
||||
inventory = read(backup / 'program.json')
|
||||
names, present = inventory.get('all'), inventory.get('present')
|
||||
if (not isinstance(names, list) or not isinstance(present, list)
|
||||
or len(names) != len(set(names)) or not set(present).issubset(names)):
|
||||
raise UpdateError('更新恢复清单无效。')
|
||||
for name in names:
|
||||
if not program_path(name):
|
||||
raise UpdateError('更新恢复清单包含受保护路径。')
|
||||
destination(root, name)
|
||||
if name in present:
|
||||
source = destination(backup / 'program', name)
|
||||
if name not in manifest or not source.is_file() or digest(source) != manifest[name]['sha256']:
|
||||
raise UpdateError('原程序备份缺失或损坏。')
|
||||
elif name in manifest:
|
||||
raise UpdateError('原程序备份不完整。')
|
||||
|
||||
|
||||
def recovery_binding(backup):
|
||||
return {'backup': backup.name, 'backupManifestSha256': digest(backup / 'manifest.json'),
|
||||
'programInventorySha256': digest(backup / 'program.json'),
|
||||
'databaseSqlSha256': digest(backup / 'database.sql')}
|
||||
|
||||
|
||||
def recovery_database(binding, attempt):
|
||||
return 'quality_restore_' + hashlib.sha256(canonical({**binding, 'attempt': attempt})).hexdigest()[:24]
|
||||
|
||||
|
||||
def recovery_progress(root, backup):
|
||||
binding = recovery_binding(backup)
|
||||
try:
|
||||
value = read(root / '.runtime/update/recovery.json', {})
|
||||
except (OSError, ValueError) as error:
|
||||
raise UpdateError('恢复进度无法读取,已保留现场。') from error
|
||||
if not value:
|
||||
return binding, None
|
||||
if (not isinstance(value, dict) or any(value.get(k) != v for k, v in binding.items())
|
||||
or type(value.get('attempt')) is not int or not 1 <= value['attempt'] <= MAX_RECOVERY_ATTEMPTS
|
||||
or value.get('database') != recovery_database(binding, value['attempt'])
|
||||
or value.get('stage') not in ('importing', 'databaseDurable', 'published', 'ready')):
|
||||
raise UpdateError('恢复进度与备份不一致,已保留现场。')
|
||||
return binding, value
|
||||
|
||||
|
||||
def prepare_old_backend_restart(root):
|
||||
# 旧 JAR 不理解终态 journal 的优先级。先持久化不触发其维护锁的进度状态,
|
||||
# 再启动旧服务;写失败则保留恢复记录,不声称已恢复。
|
||||
write_status('queued', '正在启动原版本,完成后自动重新连接。', root)
|
||||
|
||||
|
||||
def normalize_terminal_status(root, journal):
|
||||
value = read(root / '.runtime/update/status.json', {})
|
||||
if (not isinstance(value, dict) or value.get('state') not in ACTIVE
|
||||
or journal.get('operationId') != value.get('operationId')):
|
||||
return
|
||||
pending = read(root / '.runtime/update/request.json', {})
|
||||
if value.get('state') in ('queued', 'downloading') and time.time() - pending.get('created', 0) < 120:
|
||||
return
|
||||
if journal.get('phase') == 'rolledBack':
|
||||
write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
|
||||
elif journal.get('phase') == 'complete':
|
||||
manifest = read(root / 'manifest.json')
|
||||
if manifest.get('updateVersion') == journal.get('version', value.get('version')):
|
||||
write_status('completed', '更新完成。', root, available=False)
|
||||
|
||||
|
||||
def rollback(root, backup, database_backed_up):
|
||||
write_status('rollingBack', '更新未完成,正在恢复原版本。', root)
|
||||
controller(root, 'stop', timeout=180)
|
||||
verify_backup(root, backup)
|
||||
binding, progress = recovery_progress(root, backup) if database_backed_up else ({}, None)
|
||||
best_effort_status('rollingBack', '更新未完成,正在恢复原版本。', root)
|
||||
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
||||
restore_program(root, backup)
|
||||
if database_backed_up:
|
||||
config = read(root / '.runtime/deployment.json')
|
||||
process = controller(root, 'serve', True)
|
||||
wait_ready(root, process, True)
|
||||
database = 'quality_restore_' + secrets.token_hex(12)
|
||||
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
||||
with (backup / 'database.sql').open('rb') as source:
|
||||
mysql(root, config, database=database, stdin=source)
|
||||
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
|
||||
controller(root, 'stop', timeout=180)
|
||||
config['mysqlDatabase'] = database
|
||||
config = read(backup / 'private/deployment.json')
|
||||
# 断电前未确认数据库安全关闭的导入,不能当作完整数据库复用。
|
||||
if progress is None or progress['stage'] == 'importing':
|
||||
attempt = 1 if progress is None else progress['attempt'] + 1
|
||||
if attempt > MAX_RECOVERY_ATTEMPTS:
|
||||
raise UpdateError('恢复导入多次中断,已停止自动重试并保留数据库,请核对本机恢复记录。')
|
||||
reserve = (backup / 'database.sql').stat().st_size * 3 + 512 * 1024**2
|
||||
if shutil.disk_usage(root).free < reserve:
|
||||
raise UpdateError('空间不足以安全恢复数据库,已保留现场。')
|
||||
database = recovery_database(binding, attempt)
|
||||
if database == config.get('mysqlDatabase'):
|
||||
raise UpdateError('恢复数据库与原库冲突,已停止。')
|
||||
progress = {**binding, 'attempt': attempt, 'database': database, 'stage': 'importing'}
|
||||
atomic(root / '.runtime/update/recovery.json', progress)
|
||||
process = controller(root, 'serve', True)
|
||||
wait_ready(root, process, True)
|
||||
# 不删除任何已有数据库;名称冲突时 CREATE 会失败并保留现场。
|
||||
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
||||
with (backup / 'database.sql').open('rb') as source:
|
||||
mysql(root, config, database=database, stdin=source)
|
||||
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
|
||||
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
||||
progress['stage'] = 'databaseDurable'
|
||||
atomic(root / '.runtime/update/recovery.json', progress)
|
||||
config['mysqlDatabase'] = progress['database']
|
||||
atomic(root / '.runtime/deployment.json', config)
|
||||
receipt_path = root / '.runtime/migration-receipt.json'
|
||||
receipt = read(receipt_path, {})
|
||||
if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'):
|
||||
receipt['database'] = database
|
||||
receipt['database'] = progress['database']
|
||||
atomic(receipt_path, receipt)
|
||||
progress['stage'] = 'published'
|
||||
atomic(root / '.runtime/update/recovery.json', progress)
|
||||
prepare_old_backend_restart(root)
|
||||
process = controller(root, 'serve')
|
||||
wait_ready(root, process)
|
||||
if progress:
|
||||
progress['stage'] = 'ready'
|
||||
atomic(root / '.runtime/update/recovery.json', progress)
|
||||
|
||||
|
||||
def preflight_programs(stage):
|
||||
# 旧安装先加固恢复器,随后只能安装具备相同恢复协议的程序,防止覆盖回旧实现。
|
||||
for name, marker in (('update_manager.py', 'UPDATE_PROTOCOL'), ('control.py', 'UPDATE_CONTROL_PROTOCOL')):
|
||||
try:
|
||||
tree = ast.parse((stage / 'deploy/offline' / name).read_bytes())
|
||||
values = [node.value.value for node in tree.body if isinstance(node, ast.Assign)
|
||||
and any(isinstance(target, ast.Name) and target.id == marker for target in node.targets)
|
||||
and isinstance(node.value, ast.Constant) and type(node.value.value) is int]
|
||||
if values != [UPDATE_PROTOCOL]:
|
||||
raise ValueError('incompatible recovery protocol')
|
||||
except (OSError, SyntaxError, ValueError) as error:
|
||||
raise UpdateError('更新包未包含本轮加固的恢复流程,服务尚未停止;请等待配套更新包。') from error
|
||||
try:
|
||||
with zipfile.ZipFile(stage / 'app/backend.jar') as jar:
|
||||
metadata = decode(jar.read('BOOT-INF/classes/update-protocol.json'))
|
||||
if metadata != {'protocol': UPDATE_PROTOCOL}:
|
||||
raise ValueError('incompatible backend protocol')
|
||||
except (OSError, ValueError, KeyError, zipfile.BadZipFile) as error:
|
||||
raise UpdateError('更新包后端不支持本轮恢复协议,服务尚未停止。') from error
|
||||
|
||||
|
||||
def preflight_backup(root, backup, release):
|
||||
paths = set(e['path'] for e in read(root / 'manifest.json')['files'] if program_path(e['path']))
|
||||
paths.update(e['path'] for e in release['files'])
|
||||
targets = [backup / 'program' / n for n in paths]
|
||||
ignored = ('*.lock', '*.log', 'service.json', 'stop-request.json')
|
||||
for folder, dirs, files in os.walk(root / '.runtime', followlinks=False):
|
||||
dirs[:] = [n for n in dirs if n != 'update']
|
||||
for n in dirs:
|
||||
destination(root, (Path(folder) / n).relative_to(root).as_posix())
|
||||
for n in files:
|
||||
if any(fnmatch.fnmatch(n, pattern) for pattern in ignored):
|
||||
continue
|
||||
source = Path(folder) / n
|
||||
destination(root, source.relative_to(root).as_posix())
|
||||
with source.open('rb'):
|
||||
pass
|
||||
targets.append(backup / 'private' / source.relative_to(root / '.runtime'))
|
||||
for target in targets:
|
||||
destination(backup, target.relative_to(backup).as_posix())
|
||||
if os.name == 'nt' and len(str(target.absolute())) + 24 > 248:
|
||||
raise UpdateError('备份路径过长,请先缩短部署目录或历史路径;服务尚未停止。')
|
||||
probe = backup / 'write-probe.tmp'
|
||||
try:
|
||||
with probe.open('xb') as stream:
|
||||
stream.write(b'probe')
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
finally:
|
||||
probe.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def record_failure(root, stage, error, name='failure.json'):
|
||||
@@ -526,7 +705,7 @@ def worker(launcher, token, root=ROOT):
|
||||
raise UpdateError('更新请求已失效。')
|
||||
with exclusive_update(root):
|
||||
backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4))
|
||||
stopped = backed_up = program_saved = changed = False
|
||||
stopped = backed_up = program_saved = changed = stop_requested = committed = False
|
||||
failure_stage = 'validateRelease'
|
||||
try:
|
||||
envelope = decode(download(LATEST))
|
||||
@@ -543,18 +722,30 @@ def worker(launcher, token, root=ROOT):
|
||||
private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory))
|
||||
if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2:
|
||||
raise UpdateError('磁盘剩余空间不足,请释放空间后重试。')
|
||||
write_status('downloading', '正在下载并校验更新。', root)
|
||||
write_status('downloading', '正在下载并校验更新。', root, operationId=token, version=release['version'])
|
||||
info = release['archive']
|
||||
failure_stage = 'downloadArchive'
|
||||
download(BASE + info['file'], archive, info['size'], info['sha256'])
|
||||
failure_stage = 'unpackArchive'
|
||||
unpack(archive, stage, release)
|
||||
failure_stage = 'programPreflight'
|
||||
preflight_programs(stage)
|
||||
failure_stage = 'runtimePreflight'
|
||||
write_status('downloading', '正在检查运行环境和备份条件,服务仍可使用。', root)
|
||||
controller(root, 'verify-runtime', timeout=1800)
|
||||
failure_stage = 'backupPreflight'
|
||||
preflight_backup(root, backup, release)
|
||||
# Revalidate immediately before any service or disk mutation.
|
||||
validate_release(envelope, root)
|
||||
write_status('backingUp', '正在备份,服务即将重新启动。', root)
|
||||
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False})
|
||||
if (directory / 'recovery.json').exists():
|
||||
copy_atomic(directory / 'recovery.json', backup / 'previous-recovery.json')
|
||||
(directory / 'recovery.json').unlink()
|
||||
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False,
|
||||
'operationId': token, 'version': release['version'], 'sequence': release['sequence']})
|
||||
failure_stage = 'stopService'
|
||||
controller(root, 'stop', timeout=180)
|
||||
stop_requested = True
|
||||
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
||||
stopped = True
|
||||
failure_stage = 'backupProgram'
|
||||
backup_program(root, backup, release)
|
||||
@@ -571,13 +762,15 @@ def worker(launcher, token, root=ROOT):
|
||||
if (backup / 'database.sql').stat().st_size < 100:
|
||||
raise UpdateError('数据库备份为空,已停止更新。')
|
||||
backed_up = True
|
||||
controller(root, 'stop', timeout=180)
|
||||
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
||||
# The actual dump can exceed the source's physical storage. Do not
|
||||
# replace anything unless a separate rollback database still fits.
|
||||
required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2
|
||||
if shutil.disk_usage(root).free < required_recovery:
|
||||
raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。')
|
||||
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True,
|
||||
'operationId': token, 'version': release['version'], 'sequence': release['sequence'],
|
||||
'databaseMayHaveChanged': False,
|
||||
'backupManifestSha256': digest(backup / 'manifest.json'),
|
||||
'programInventorySha256': digest(backup / 'program.json')})
|
||||
write_status('installing', '正在安装更新。', root)
|
||||
@@ -585,32 +778,46 @@ def worker(launcher, token, root=ROOT):
|
||||
failure_stage = 'replaceProgram'
|
||||
replace_program(root, stage, backup, release)
|
||||
failure_stage = 'verifyProgram'
|
||||
controller(root, 'verify', timeout=180)
|
||||
controller(root, 'verify-runtime', timeout=300, program_only=True)
|
||||
write_status('restarting', '正在重新启动,请稍候。', root)
|
||||
failure_stage = 'restartService'
|
||||
journal = read(directory / 'journal.json')
|
||||
journal['databaseMayHaveChanged'] = True
|
||||
atomic(directory / 'journal.json', journal)
|
||||
process = controller(root, 'serve')
|
||||
wait_ready(root, process)
|
||||
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'complete', 'databaseBackedUp': True})
|
||||
write_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
|
||||
journal.update(phase='complete', version=release['version'], sequence=release['sequence'],
|
||||
finishedAt=datetime.now(timezone.utc).isoformat())
|
||||
atomic(directory / 'journal.json', journal)
|
||||
committed = True
|
||||
best_effort_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
|
||||
except Exception as error:
|
||||
record_failure(root, failure_stage, error)
|
||||
message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。'
|
||||
if stopped:
|
||||
journal = read(directory / 'journal.json', {})
|
||||
if committed or (journal.get('backup') == backup.name and journal.get('phase') == 'complete'):
|
||||
return # 提交后的展示或清理失败绝不能发起回滚。
|
||||
if stop_requested:
|
||||
try:
|
||||
if program_saved and changed:
|
||||
rollback(root, backup, backed_up)
|
||||
rollback(root, backup, backed_up and journal.get('databaseMayHaveChanged', True))
|
||||
else:
|
||||
controller(root, 'stop', timeout=180)
|
||||
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
||||
prepare_old_backend_restart(root)
|
||||
process = controller(root, 'serve')
|
||||
wait_ready(root, process)
|
||||
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'rolledBack', 'databaseBackedUp': backed_up})
|
||||
journal.update(backup=backup.name, phase='rolledBack', databaseBackedUp=backed_up)
|
||||
atomic(directory / 'journal.json', journal)
|
||||
message += ' 已恢复原版本。'
|
||||
except Exception as rollback_error:
|
||||
record_failure(root, 'rollback', rollback_error, 'rollback-failure.json')
|
||||
message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。'
|
||||
write_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
|
||||
best_effort_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
|
||||
finally:
|
||||
(directory / 'request.json').unlink(missing_ok=True)
|
||||
try:
|
||||
(directory / 'request.json').unlink(missing_ok=True)
|
||||
except OSError as error:
|
||||
record_failure(root, 'cleanupRequest', error)
|
||||
|
||||
|
||||
def recover(root=ROOT):
|
||||
@@ -622,22 +829,31 @@ def recover_locked(root=ROOT):
|
||||
directory = root / '.runtime/update'
|
||||
journal = read(directory / 'journal.json', {})
|
||||
if journal.get('phase') in (None, 'complete', 'rolledBack'):
|
||||
if journal.get('phase') in ('complete', 'rolledBack'):
|
||||
normalize_terminal_status(root, journal)
|
||||
return status(root)
|
||||
name = journal.get('backup', '')
|
||||
if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name):
|
||||
raise UpdateError('更新恢复记录无效。')
|
||||
backup = directory / name
|
||||
if journal.get('phase') not in ('installing', 'stopping'):
|
||||
raise UpdateError('更新恢复阶段无效,已保留现场。')
|
||||
if journal.get('phase') == 'installing':
|
||||
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
|
||||
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
|
||||
raise UpdateError('更新恢复备份校验失败,已保留现场。')
|
||||
rollback(root, backup, journal.get('databaseBackedUp') is True)
|
||||
verify_backup(root, backup, journal)
|
||||
if journal.get('databaseBackedUp') is True:
|
||||
recovery_progress(root, backup) # 先验证 sidecar 与 dump,尚未停止或替换任何文件。
|
||||
rollback(root, backup, journal.get('databaseBackedUp') is True and journal.get('databaseMayHaveChanged', True))
|
||||
else:
|
||||
controller(root, 'stop', timeout=180)
|
||||
controller(root, 'stop', timeout=STOP_TIMEOUT)
|
||||
prepare_old_backend_restart(root)
|
||||
process = controller(root, 'serve')
|
||||
wait_ready(root, process)
|
||||
atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'})
|
||||
return write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
|
||||
best_effort_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
|
||||
return status(root)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
Reference in New Issue
Block a user