修复:加固 Windows 更新预检、提交点和可恢复的数据库回退流程

This commit is contained in:
2026-11-05 16:54:30 +08:00
parent 2315c83067
commit 3962521a5c
13 changed files with 1821 additions and 236 deletions
@@ -55,9 +55,13 @@ public class SystemUpdateService {
if (!enabled) return false; if (!enabled) return false;
try { try {
Map<?, ?> state = readMetadata("status.json"); Map<?, ?> state = readMetadata("status.json");
Map<?, ?> journal = readMetadata("journal.json");
// 同一任务的持久提交记录优先于尚未成功刷新的进度提示。
if (journal.get("operationId") instanceof String operationId && !operationId.isBlank()
&& operationId.equals(state.get("operationId"))
&& Set.of("complete", "rolledBack").contains(journal.get("phase"))) return false;
if (state.get("state") instanceof String phase if (state.get("state") instanceof String phase
&& Set.of("backingUp", "installing", "restarting", "rollingBack").contains(phase)) return true; && Set.of("backingUp", "installing", "restarting", "rollingBack").contains(phase)) return true;
Map<?, ?> journal = readMetadata("journal.json");
return journal.get("phase") instanceof String phase && Set.of("stopping", "installing").contains(phase); return journal.get("phase") instanceof String phase && Set.of("stopping", "installing").contains(phase);
} catch (Exception failure) { } catch (Exception failure) {
return true; return true;
@@ -0,0 +1 @@
{"protocol":2}
@@ -60,4 +60,21 @@ class SystemUpdateServiceTest {
Files.writeString(directory.resolve("status.json"), "broken JSON"); Files.writeString(directory.resolve("status.json"), "broken JSON");
assertTrue(service.maintenanceActive()); assertTrue(service.maintenanceActive());
} }
@Test
void committedOperationOverridesItsStaleProgressButNotAnotherOperation(@TempDir Path root) throws Exception {
var service = new SystemUpdateService(JsonMapper.builder().build(), root.toString(), false);
var enabled = SystemUpdateService.class.getDeclaredField("enabled");
enabled.setAccessible(true);
enabled.setBoolean(service, true);
var directory = Files.createDirectories(root.resolve(".runtime/update"));
String current = "a".repeat(32), previous = "b".repeat(32);
for (String phase : new String[]{"complete", "rolledBack"}) {
Files.writeString(directory.resolve("status.json"), "{\"state\":\"restarting\",\"operationId\":\"" + current + "\"}");
Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"" + phase + "\",\"operationId\":\"" + current + "\"}");
assertFalse(service.maintenanceActive(), phase);
Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"" + phase + "\",\"operationId\":\"" + previous + "\"}");
assertTrue(service.maintenanceActive(), "another operation must not release maintenance");
}
}
} }
+13
View File
@@ -64,3 +64,16 @@ python3 deploy/offline/publish_update.py \
Windows 的刷盘操作要求写句柄。程序及私有配置备份统一使用写入、flush、fsync 后再原子替换,移除复制后以只读句柄刷盘的步骤。备份失败仍停止更新并保留旧程序与数据库;不降低备份持久化要求。 Windows 的刷盘操作要求写句柄。程序及私有配置备份统一使用写入、flush、fsync 后再原子替换,移除复制后以只读句柄刷盘的步骤。备份失败仍停止更新并保留旧程序与数据库;不降低备份持久化要求。
“修复Windows更新.cmd”适配原始更新器和已安装检查超时修复的更新器,保留原清单、缓存修复和版本信息。更新失败诊断写入 `.runtime/update/failure.json`,只包含阶段、异常类型、系统错误码和源码位置,不含业务正文、凭据或异常参数;回退异常单独记录在 `rollback-failure.json`。 “修复Windows更新.cmd”适配原始更新器和已安装检查超时修复的更新器,保留原清单、缓存修复和版本信息。更新失败诊断写入 `.runtime/update/failure.json`,只包含阶段、异常类型、系统错误码和源码位置,不含业务正文、凭据或异常参数;回退异常单独记录在 `rollback-failure.json`。
### Windows 更新流程加固(本地验证,尚未发布)
- 停服前完成签名、下载完整性、现有运行环境、备份路径长度及读写权限检查;预检失败保留当前服务运行。完整环境检查预算为30分钟,更新后仅核验程序和应用脚本(5分钟),不重复读取约4.6GB运行环境。
- 安全停服内部最多等待5分钟,更新器外层等待6分钟;已登记进程状态无法确认时停止后续操作,保留状态记录,不误报成功或结束未知进程。
- 仅当对应源码与原清单摘要一致时,忽略可再生的运行库字节码,不忽略真实源码损坏。
- `journal.phase=complete` 是不可逆提交点;状态展示或清理失败只记诊断,不再回退。后端按同一 operationId 的提交记录解除维护状态,旧任务记录不能解除新任务的维护。
- 新后端尚未启动时,回退不重新导入没有变化的数据库。可能发生数据库变更后,恢复进度写入独立 `recovery.json`,绑定备份程序清单及 SQL 摘要。导入及安全关闭确认后复用同一个恢复库,重复启动不重复导入,并幂等恢复配置和迁移回执。
- 不复用断电前未确认完整的导入;最多分配两次独立恢复尝试,且每次检查磁盘空间。超过次数停止自动重试并保留现场,不删除任何数据库或按前缀清理未知库。
- 旧安装修复入口同时更新 updater、control 和清单摘要;签名公钥、不可变 bootstrap、版本、业务数据和迁移回执不变,修复入口不启动服务。必须先安装该入口,保证不可变 bootstrap 未来恢复出的旧程序也包含加固后的恢复器。
- Windows 真机的 NTFS、进程权限和断电恢复仍需隔离副本验收。本轮本地故障注入及真实隔离 MySQL 验证不等同于生产 Windows 验收。
加固修复入口不能搭配已发布的旧0.3包使用:停服前会同时核对更新器、控制器及 JAR 中的恢复协议标记。不兼容的旧包会在服务仍运行时被拒绝;配套新包须包含本轮后端、控制器与更新器,禁止混装。
+434 -50
View File
@@ -6,6 +6,7 @@ import argparse
import contextlib import contextlib
import hashlib import hashlib
import json import json
import math
import os import os
from pathlib import Path from pathlib import Path
import platform import platform
@@ -15,8 +16,12 @@ import signal
import socket import socket
import subprocess import subprocess
import sys import sys
_old_dont_write_bytecode = sys.dont_write_bytecode
# Importing the controller must not create caches in the immutable package.
sys.dont_write_bytecode = True
import threading import threading
import time import time
from pathlib import PurePosixPath
from urllib.request import ProxyHandler, build_opener from urllib.request import ProxyHandler, build_opener
import psutil import psutil
@@ -25,12 +30,15 @@ ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts')) sys.path.insert(0, str(ROOT / 'scripts'))
import runtime_platform as host import runtime_platform as host
from gateway import make_server from gateway import make_server
sys.dont_write_bytecode = _old_dont_write_bytecode
del _old_dont_write_bytecode
PRIVATE = ROOT / '.runtime' PRIVATE = ROOT / '.runtime'
DATA = ROOT / 'runtimedata' DATA = ROOT / 'runtimedata'
CONFIG = PRIVATE / 'deployment.json' CONFIG = PRIVATE / 'deployment.json'
STATE = PRIVATE / 'service.json' STATE = PRIVATE / 'service.json'
STOP = PRIVATE / 'stop-request.json' STOP = PRIVATE / 'stop-request.json'
UPDATE_CONTROL_PROTOCOL = 2
IS_WINDOWS = os.name == 'nt' IS_WINDOWS = os.name == 'nt'
EXE = '.exe' if IS_WINDOWS else '' EXE = '.exe' if IS_WINDOWS else ''
PYTHON = ROOT / 'runtimes/python' / ('python.exe' if IS_WINDOWS else 'bin/python3') PYTHON = ROOT / 'runtimes/python' / ('python.exe' if IS_WINDOWS else 'bin/python3')
@@ -39,6 +47,7 @@ MYSQL = ROOT / 'runtimes/mysql/bin' / ('mysql' + EXE)
MYSQLD = ROOT / 'runtimes/mysql/bin' / ('mysqld' + EXE) MYSQLD = ROOT / 'runtimes/mysql/bin' / ('mysqld' + EXE)
ADMIN = ROOT / 'runtimes/mysql/bin' / ('mysqladmin' + EXE) ADMIN = ROOT / 'runtimes/mysql/bin' / ('mysqladmin' + EXE)
OPENER = build_opener(ProxyHandler({})) OPENER = build_opener(ProxyHandler({}))
STOP_TIMEOUT_SECONDS = 300
def atomic(path, value): def atomic(path, value):
@@ -65,7 +74,7 @@ def private_directory(path):
path.chmod(0o700) path.chmod(0o700)
def configuration(): def configuration(create=True):
if CONFIG.exists(): if CONFIG.exists():
value = json.loads(CONFIG.read_text(encoding='utf-8')) value = json.loads(CONFIG.read_text(encoding='utf-8'))
else: else:
@@ -73,6 +82,7 @@ def configuration():
'webBind': '0.0.0.0' if IS_WINDOWS else '127.0.0.1', 'webBind': '0.0.0.0' if IS_WINDOWS else '127.0.0.1',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32), 'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)} 'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)}
if create:
atomic(CONFIG, value) atomic(CONFIG, value)
value.setdefault('mysqlDatabase', 'quality_platform') value.setdefault('mysqlDatabase', 'quality_platform')
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', value['mysqlDatabase']): if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', value['mysqlDatabase']):
@@ -137,14 +147,159 @@ def record(process, command):
def matches(info): def matches(info):
"""Return True for this process, False for gone/replaced, None if unknown."""
if not isinstance(info, dict):
return None
try: try:
process = psutil.Process(int(info['pid'])) pid = info['pid']
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE created = info['created']
and abs(process.create_time() - info['created']) < .01 executable = info['executable']
and os.path.normcase(str(Path(process.exe()).resolve())) == os.path.normcase(info['executable']) arguments = info['args']
and process.cmdline()[1:] == info['args']) if (type(pid) is not int or pid <= 0 or type(created) not in (int, float)
except (psutil.Error, KeyError, ValueError, TypeError): or not math.isfinite(created) or not isinstance(executable, str)
or not isinstance(arguments, list) or not all(isinstance(item, str) for item in arguments)):
return None
except (KeyError, ValueError, TypeError):
return None
try:
process = psutil.Process(pid)
except psutil.NoSuchProcess:
return False return False
except Exception:
return None
try:
if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE:
return False
if abs(process.create_time() - created) >= .01:
return False
if os.path.normcase(str(Path(process.exe()).resolve())) != os.path.normcase(executable):
return False
if process.cmdline()[1:] != arguments:
return False
return True
except psutil.NoSuchProcess:
return False
except Exception:
return None
def registered_alive(records):
"""Keep both confirmed-live and uninspectable registered process records."""
if not isinstance(records, list):
return [], True
alive, unknown = [], False
for info in records:
status = matches(info)
if status is True:
alive.append(info)
elif status is None:
alive.append(info)
unknown = True
return alive, unknown
def retain_registered_children(state, records, unresolved=False):
survivors, unknown = registered_alive(records)
state['children'] = survivors
unresolved = unresolved or unknown
if unresolved:
state['unknownChildren'] = True
return survivors, unresolved
def registered_named(records, name):
confirmed, unknown = [], []
for info in records:
if not isinstance(info, dict) or info.get('name') != name:
continue
status = matches(info)
if status is True:
confirmed.append(info)
elif status is None:
unknown.append(info)
return confirmed, unknown
def snapshot_process(process, name):
try:
result = record(process, process.cmdline())
result['name'] = name
return result
except (psutil.Error, OSError):
return None
def process_object_matches(process, created):
try:
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
and abs(process.create_time() - created) < .01)
except psutil.NoSuchProcess:
return False
except Exception:
# Permission failures and unexpected inspection errors are unknown,
# not evidence that the observed child has exited.
return None
def terminate_observed(processes, timeout):
"""Stop only process objects captured below a verified service child."""
unverified = False
for process, created, info in processes:
observed = process_object_matches(process, created)
if observed is None:
if info is None:
unverified = True
elif observed is True:
with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
process.terminate()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
states = [process_object_matches(process, created) for process, created, _info in processes]
if any(observed is None for observed in states):
unverified = unverified or any(observed is None and info is None
for observed, (_process, _created, info) in zip(states, processes))
break
if not any(states):
break
time.sleep(.1)
for process, created, info in processes:
observed = process_object_matches(process, created)
if observed is None:
if info is None:
unverified = True
elif observed is True:
with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
process.kill()
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
states = [process_object_matches(process, created) for process, created, _info in processes]
if any(observed is None for observed in states):
unverified = unverified or any(observed is None and info is None
for observed, (_process, _created, info) in zip(states, processes))
break
if not any(states):
break
time.sleep(.1)
survivors = []
for process, created, info in processes:
observed = process_object_matches(process, created)
if observed is None:
if info is None:
unverified = True
else:
survivors.append(info)
elif observed is True:
identity = matches(info) if info is not None else None
if info is None:
unverified = True
elif identity is None:
survivors.append(info)
elif identity is False:
survivors.append(info)
unverified = True
else:
survivors.append(info)
return survivors, unverified
def read_state(): def read_state():
@@ -154,6 +309,45 @@ def read_state():
return {} return {}
def wait_for_stop(supervisor, request, timeout_seconds=STOP_TIMEOUT_SECONDS, interval=.5):
request_sent = False
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
supervisor_status = matches(supervisor)
if supervisor_status is None:
raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。')
if supervisor_status is True:
if not request_sent:
atomic(STOP, request)
request_sent = True
time.sleep(interval)
continue
state = read_state()
if 'supervisor' in state:
current_supervisor = state['supervisor']
current_status = matches(current_supervisor)
if current_status is None:
raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。')
if current_status is True:
if current_supervisor != supervisor:
raise RuntimeError('服务状态已变化,未停止任何不明进程。')
if not request_sent:
atomic(STOP, request)
request_sent = True
time.sleep(interval)
continue
survivors, unknown_children = registered_alive(state.get('children', []))
if unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
raise RuntimeError('服务监督程序已退出,但仍有未确认停止的已记录进程;未结束任何不明进程。')
if survivors:
time.sleep(interval)
continue
print('服务已停止。')
return
raise RuntimeError('服务仍在安全关闭,请稍后检查状态。')
def free_port(port, bind='127.0.0.1'): def free_port(port, bind='127.0.0.1'):
# A recently closed TCP connection may remain in TIME_WAIT. Reject a live # A recently closed TCP connection may remain in TIME_WAIT. Reject a live
# listener, but allow a clean restart without waiting for that state to expire. # listener, but allow a clean restart without waiting for that state to expire.
@@ -182,22 +376,136 @@ def sql(value, query):
env=env, timeout=10) env=env, timeout=10)
def verify(check_hashes=True): def manifest_name(name):
if (not isinstance(name, str) or not name or '\\' in name
or PurePosixPath(name).is_absolute()
or any(part in ('', '.', '..') for part in name.split('/'))):
raise RuntimeError('文件清单包含无效路径。')
return PurePosixPath(name)
def manifest_path(name):
parts = manifest_name(name).parts
path = ROOT.joinpath(*parts)
root = ROOT.resolve()
if not path.resolve(strict=False).is_relative_to(root):
raise RuntimeError('文件路径异常,已停止校验:' + name)
parent = path.parent
while parent != ROOT and parent != parent.parent:
if parent.is_symlink():
raise RuntimeError('文件路径异常,已停止校验:' + name)
parent = parent.parent
return path
def cache_source(name):
"""Return only timestamp-based bytecode sources from the bundled Python."""
if not isinstance(name, str) or '\\' in name:
return None
path = PurePosixPath(name)
if path.parts[:2] != ('runtimes', 'python') or any(part in ('.', '..') for part in name.split('/')):
return None
if path.parent.name != '__pycache__':
return None
match = re.fullmatch(r'(.+)\.cpython-\d+(?:\.opt-\d+)?\.pyc', path.name)
return (path.parent.parent / (match[1] + '.py')).as_posix() if match else None
def file_digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def regular_manifest_file(name):
path = manifest_path(name)
if path.is_symlink():
raise RuntimeError('文件路径异常,已停止校验:' + name)
return path
def program_file_path(name):
"""Mirror update_manager.program_path without importing code during a read-only check."""
path = manifest_name(name)
return (name == 'app/backend.jar' or name.startswith('app/web/')
or name == 'backend/src/main/resources/analysis-runtime-policy.json'
or name.startswith('scripts/') and path.suffix == '.py'
or name.startswith('ai/nao-quality/')
or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md'))
def verify_manifest_files(manifest, program_only=False):
entries = manifest.get('files')
if not isinstance(entries, list):
raise RuntimeError('文件清单无效。')
by_name = {}
for entry in entries:
if not isinstance(entry, dict) or not isinstance(entry.get('path'), str):
raise RuntimeError('文件清单无效。')
name = entry['path']
manifest_name(name)
if name in by_name:
raise RuntimeError('文件清单包含重复路径。')
by_name[name] = entry
ignored_caches = set()
if not program_only:
# Timestamp-based bytecode varies between Windows copies. Ignore only
# cache entries whose exact source is present and matches its manifest
# digest; ordinary Python source files are still hashed below.
for name in by_name:
source = cache_source(name)
if source is None:
continue
source_entry = by_name.get(source)
expected = source_entry.get('sha256') if isinstance(source_entry, dict) else None
source_path = regular_manifest_file(source)
if (not isinstance(expected, str) or not source_path.is_file()
or file_digest(source_path) != expected):
raise RuntimeError('对应源码缺失或校验失败,已停止校验:' + source)
cache_path = regular_manifest_file(name)
if cache_path.exists() and not cache_path.is_file():
raise RuntimeError('缓存文件路径异常,已停止校验:' + name)
ignored_caches.add(name)
for name, entry in by_name.items():
if program_only and not program_file_path(name):
continue
if name in ignored_caches:
continue
path = manifest_path(name)
expected_link = entry.get('symlink')
if expected_link is not None:
valid = path.is_symlink() and os.readlink(path) == expected_link
else:
if path.is_symlink():
valid = False
elif path.is_file():
with path.open('rb') as stream:
valid = hashlib.file_digest(stream, 'sha256').hexdigest() == entry.get('sha256')
else:
valid = False
if not valid:
raise RuntimeError('文件缺失或校验失败:' + name)
def verify_runtime(check_hashes=True, program_only=False):
manifest = json.loads((ROOT / 'manifest.json').read_text(encoding='utf-8')) manifest = json.loads((ROOT / 'manifest.json').read_text(encoding='utf-8'))
expected_os = 'windows' if IS_WINDOWS else platform.system().lower() expected_os = 'windows' if IS_WINDOWS else platform.system().lower()
if manifest['platform'] != expected_os or platform.machine().lower() not in ('amd64', 'x86_64'): if manifest['platform'] != expected_os or platform.machine().lower() not in ('amd64', 'x86_64'):
raise RuntimeError('此包仅用于 ' + manifest['platform'] + ' 的 Intel/AMD 64 位电脑。') raise RuntimeError('此包仅用于 ' + manifest['platform'] + ' 的 Intel/AMD 64 位电脑。')
if check_hashes: if check_hashes:
for entry in manifest['files']: verify_manifest_files(manifest, program_only=program_only)
path = ROOT / entry['path'] value = configuration(create=False)
if entry.get('symlink') is not None:
valid = path.is_symlink() and os.readlink(path) == entry['symlink']
else:
valid = path.is_file() and hashlib.file_digest(path.open('rb'), 'sha256').hexdigest() == entry['sha256']
if not valid:
raise RuntimeError('文件缺失或校验失败:' + entry['path'])
value = configuration()
env = environment(value) env = environment(value)
if program_only:
application_probe = subprocess.run([str(PYTHON), '-c',
'import sys;sys.path.insert(0,sys.argv[1]);import nao_quality_runtime as r;'
'import nao_native_run,nao_quality_runtime_mcp,nao_visual_recovery;'
'assert len(r._config_fingerprint("https://offline.invalid/v1","offline-check","offline-check"))==64',
str(ROOT / 'scripts')], capture_output=True, env=env, timeout=45)
if application_probe.returncode:
raise RuntimeError('AI 应用脚本或策略资源检查失败。')
return value
for command in ([JAVA, '-version'], [MYSQLD, '--no-defaults', '--version']): for command in ([JAVA, '-version'], [MYSQLD, '--no-defaults', '--version']):
result = subprocess.run([str(a) for a in command], capture_output=True, env=env, timeout=30) result = subprocess.run([str(a) for a in command], capture_output=True, env=env, timeout=30)
if result.returncode: if result.returncode:
@@ -221,6 +529,13 @@ def verify(check_hashes=True):
return value return value
def verify(check_hashes=True):
# Keep the historical full verification entry point and its configuration
# initialization behavior for the installer and migration recovery tools.
verify_runtime(check_hashes=check_hashes)
return configuration()
def addresses(value): def addresses(value):
ips = set() ips = set()
if value['webBind'] == '0.0.0.0': if value['webBind'] == '0.0.0.0':
@@ -360,38 +675,90 @@ def serve(database_only=False):
finally: finally:
state['ready'] = False state['ready'] = False
atomic(STATE, state) atomic(STATE, state)
# Stop backend first, then its isolated AI processes, then the database. cleanup_failures, extra_survivors = [], []
unknown_children = False
# Only signal the backend and descendants observed beneath its verified
# PID/start-time/executable/arguments identity.
for info in reversed(children): for info in reversed(children):
if info['name'] != 'backend' or not matches(info): if not isinstance(info, dict) or info.get('name') != 'backend':
continue
backend_status = matches(info)
if backend_status is None:
extra_survivors.append(info)
continue
if backend_status is not True:
continue continue
process = psutil.Process(info['pid'])
descendants = process.children(recursive=True)
process.terminate()
try: try:
process.wait(35) process = psutil.Process(info['pid'])
except psutil.TimeoutExpired: except psutil.NoSuchProcess:
if matches(info): continue
process.kill() except Exception:
extra_survivors.append(info)
continue
observed = []
try:
descendants = process.children(recursive=True)
except psutil.NoSuchProcess:
descendants = []
except psutil.Error:
descendants = []
unknown_children = True
for child in descendants: for child in descendants:
with contextlib.suppress(psutil.NoSuchProcess): try:
child.terminate() created = child.create_time()
with contextlib.suppress(OSError, subprocess.SubprocessError): observed.append((child, created, snapshot_process(child, 'backend-descendant')))
subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'], except (psutil.Error, OSError):
unknown_children = True
observed.append((process, info['created'], info))
survivors, unverified = terminate_observed(observed, 35)
extra_survivors.extend(item for item in survivors if item.get('name') == 'backend-descendant')
unknown_children = unknown_children or unverified
try:
stopped = subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'],
env=environment(value), cwd=ROOT, capture_output=True, timeout=25) env=environment(value), cwd=ROOT, capture_output=True, timeout=25)
if any(info['name'] == 'mysql' and matches(info) for info in children): if stopped.returncode:
cleanup_failures.append('analysisProcesses')
except (OSError, subprocess.SubprocessError):
cleanup_failures.append('analysisProcesses')
mysql_infos, unknown_mysql = registered_named(children, 'mysql')
extra_survivors.extend(unknown_mysql)
mysql_infos = [info for info in mysql_infos if matches(info) is True]
if mysql_infos:
env = environment(value) env = environment(value)
env['MYSQL_PWD'] = value['rootPassword'] env['MYSQL_PWD'] = value['rootPassword']
with contextlib.suppress(OSError, subprocess.SubprocessError): try:
subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env, shutdown = subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env,
capture_output=True, timeout=60) capture_output=True, timeout=60)
if shutdown.returncode:
cleanup_failures.append('databaseShutdown')
except (OSError, subprocess.SubprocessError):
cleanup_failures.append('databaseShutdown')
# mysqladmin can return before InnoDB has finished closing files. # mysqladmin can return before InnoDB has finished closing files.
for info in children: for info in mysql_infos:
if info['name'] == 'mysql' and matches(info): mysql_status = matches(info)
with contextlib.suppress(psutil.NoSuchProcess, psutil.TimeoutExpired): if mysql_status is None:
extra_survivors.append(info)
elif mysql_status is True:
try:
psutil.Process(info['pid']).wait(60) psutil.Process(info['pid']).wait(60)
survivors = [info for info in children if matches(info)] except (psutil.NoSuchProcess, psutil.TimeoutExpired):
if survivors: pass
state['children'] = survivors except Exception:
extra_survivors.append(info)
unique_records, seen_records = [], set()
for info in [*children, *extra_survivors]:
if not isinstance(info, dict):
unknown_children = True
continue
key = (info.get('pid'), info.get('created'))
if key not in seen_records:
seen_records.add(key)
unique_records.append(info)
survivors, unknown_children = retain_registered_children(state, unique_records, unknown_children)
if cleanup_failures:
state['cleanupFailures'] = cleanup_failures
if survivors or unknown_children or cleanup_failures:
atomic(STATE, state) atomic(STATE, state)
else: else:
STATE.unlink(missing_ok=True) STATE.unlink(missing_ok=True)
@@ -402,10 +769,25 @@ def serve(database_only=False):
def main(): def main():
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'serve')) parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'verify-runtime', 'serve'))
parser.add_argument('--lan', action='store_true') parser.add_argument('--lan', action='store_true')
parser.add_argument('--program-only', action='store_true')
parser.add_argument('--database-only', action='store_true', help=argparse.SUPPRESS) parser.add_argument('--database-only', action='store_true', help=argparse.SUPPRESS)
args = parser.parse_args() args = parser.parse_args()
if args.program_only and args.action != 'verify-runtime':
parser.error('--program-only 只能与 verify-runtime 一起使用。')
if args.action == 'verify-runtime':
verify_runtime(program_only=args.program_only)
print('程序文件与 AI 应用脚本检查通过。' if args.program_only else
'文件完整性、Java、MySQL、AI Python 与 AI 原生程序检查通过。')
return
if args.action == 'start':
initial_state = read_state()
if 'supervisor' in initial_state and matches(initial_state['supervisor']) is None:
raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。')
_children, unknown_children = registered_alive(initial_state.get('children', []))
if unknown_children or initial_state.get('unknownChildren') or initial_state.get('cleanupFailures'):
raise RuntimeError('发现未确认停止的已记录进程,未启动任何服务。')
if args.action in ('start', 'serve', 'verify'): if args.action in ('start', 'serve', 'verify'):
private_directory(PRIVATE) private_directory(PRIVATE)
private_directory(DATA) private_directory(DATA)
@@ -426,23 +808,24 @@ def main():
message = None message = None
raise RuntimeError(message or '程序更新正在进行或尚未安全恢复,请查看更新记录。') raise RuntimeError(message or '程序更新正在进行或尚未安全恢复,请查看更新记录。')
state = read_state() state = read_state()
running = matches(state.get('supervisor', {})) running = matches(state['supervisor']) if 'supervisor' in state else False
if args.action == 'status': if args.action == 'status':
if running is None:
print('无法确认服务状态')
return
print('已启动' if running and state.get('ready') else '正在启动或停止' if running else '已停止') print('已启动' if running and state.get('ready') else '正在启动或停止' if running else '已停止')
return return
if running is None:
raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。')
if args.action == 'stop': if args.action == 'stop':
if not running: if not running:
if any(matches(info) for info in state.get('children', [])): children, unknown_children = registered_alive(state.get('children', []))
raise RuntimeError('发现上次退出后遗留的进程,请根据日志检查;未结束任何不明进程。') if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
raise RuntimeError('发现上次退出后仍有未确认停止的进程,请根据日志检查;未结束任何不明进程。')
print('服务已停止。') print('服务已停止。')
return return
atomic(STOP, {'token': state['token']}) wait_for_stop(state['supervisor'], {'token': state['token']})
for _ in range(240):
if not matches(state['supervisor']):
print('服务已停止。')
return return
time.sleep(.5)
raise RuntimeError('服务仍在安全关闭,请稍后检查状态。')
if running: if running:
if args.lan and configuration()['webBind'] != '0.0.0.0': if args.lan and configuration()['webBind'] != '0.0.0.0':
raise RuntimeError('请先停止服务,再使用 --lan 启动。') raise RuntimeError('请先停止服务,再使用 --lan 启动。')
@@ -450,7 +833,8 @@ def main():
for url in addresses(configuration()): for url in addresses(configuration()):
print(url) print(url)
return return
if any(matches(info) for info in state.get('children', [])): children, unknown_children = registered_alive(state.get('children', []))
if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
raise RuntimeError('发现上次退出后的服务进程,原数据已保留;请检查日志。') raise RuntimeError('发现上次退出后的服务进程,原数据已保留;请检查日志。')
value = verify() value = verify()
if args.action == 'verify': if args.action == 'verify':
File diff suppressed because one or more lines are too long
@@ -0,0 +1,68 @@
import types
import unittest
from pathlib import Path
from unittest.mock import patch
import test_update_manager as fixtures
import update_bootstrap as bootstrap
class HardenedBootstrapTests(unittest.TestCase):
def test_immutable_bootstrap_restores_hardened_executor_and_reuses_durable_database(self):
f = fixtures.UpdateTests()
f.setUp()
self.addCleanup(f.doCleanups)
root, u = f.root, fixtures.update
# Firstaid preinstalls both executors before the old-version backup is made.
for name in ('control.py', 'update_manager.py'):
relative = 'deploy/offline/' + name
(root / relative).write_bytes((Path(__file__).parent / name).read_bytes())
manifest = u.read(root / 'manifest.json')
for entry in manifest['files']:
if entry['path'] in ('deploy/offline/control.py', 'deploy/offline/update_manager.py'):
entry['sha256'] = u.digest(root / entry['path'])
u.atomic(root / 'manifest.json', manifest)
backup = root / '.runtime/update/backup-20261105T170000Z-abcdef12'
backup.mkdir(parents=True)
u.backup_program(root, backup, f.release)
(backup / 'database.sql').write_bytes(b'-- restore snapshot\n' + b'SELECT 1;\n' * 30)
u.atomic(root / '.runtime/update/journal.json', {
'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True,
'databaseMayHaveChanged': True,
'backupManifestSha256': u.digest(backup / 'manifest.json'),
'programInventorySha256': u.digest(backup / 'program.json')})
imports, attempts = [], []
fail_once = True
real_spec = bootstrap.importlib.util.spec_from_file_location
def patched_spec(name, path):
spec = real_spec(name, path)
execute = spec.loader.exec_module
def load(module):
nonlocal fail_once
execute(module)
module.controller = lambda *a, **kw: types.SimpleNamespace(pid=1)
module.mysql = lambda *a, **kw: imports.append(kw['database'])
module.sql = lambda root, cfg, query: attempts.append(query)
def ready(root, process, database_only=False):
nonlocal fail_once
if not database_only and fail_once:
fail_once = False
raise module.UpdateError('simulated service restart failure')
module.wait_ready = ready
spec.loader.exec_module = load
return spec
with patch.object(bootstrap.importlib.util, 'spec_from_file_location', side_effect=patched_spec):
with self.assertRaisesRegex(RuntimeError, 'restart failure'):
bootstrap.recover(root)
chosen = u.read(root / '.runtime/deployment.json')['mysqlDatabase']
(root / 'deploy/offline/update_manager.py').write_bytes(b'broken new updater syntax')
bootstrap.recover(root)
self.assertEqual(1, len(imports))
self.assertEqual(1, sum('CREATE DATABASE' in query for query in attempts))
self.assertEqual(chosen, u.read(root / '.runtime/deployment.json')['mysqlDatabase'])
self.assertEqual((Path(__file__).parent / 'update_manager.py').read_bytes(), (root / 'deploy/offline/update_manager.py').read_bytes())
self.assertEqual('rolledBack', u.read(root / '.runtime/update/journal.json')['phase'])
self.assertEqual({'recovered': False}, bootstrap.recover(root))
if __name__ == '__main__':
unittest.main()
+299
View File
@@ -0,0 +1,299 @@
"""Read-only runtime verification and safe-stop regression tests."""
import hashlib
import importlib.util
import json
import platform
from pathlib import Path
import sys
import tempfile
import unittest
from unittest.mock import patch
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import control
class ControlEnvironmentTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.original = {
key: getattr(control, key)
for key in ('ROOT', 'PRIVATE', 'DATA', 'CONFIG', 'STATE', 'STOP', 'IS_WINDOWS',
'EXE', 'PYTHON', 'JAVA', 'MYSQL', 'MYSQLD', 'ADMIN')
}
control.ROOT = self.root
control.PRIVATE = self.root / '.runtime'
control.DATA = self.root / 'runtimedata'
control.CONFIG = control.PRIVATE / 'deployment.json'
control.STATE = control.PRIVATE / 'service.json'
control.STOP = control.PRIVATE / 'stop-request.json'
control.IS_WINDOWS = False
control.EXE = ''
control.PYTHON = self.root / 'runtimes/python/bin/python3'
control.JAVA = self.root / 'runtimes/java/bin/java'
control.MYSQL = self.root / 'runtimes/mysql/bin/mysql'
control.MYSQLD = self.root / 'runtimes/mysql/bin/mysqld'
control.ADMIN = self.root / 'runtimes/mysql/bin/mysqladmin'
self.addCleanup(self.restore_constants)
self.files = {
'scripts/app_probe.py': b'print("probe")',
'app/web/index.html': b'<html></html>',
'runtimes/python/bin/python3': b'python',
'runtimes/python/lib/python3.11/site-packages/nao_core/cache_source.py': b'good source',
'runtimes/python/lib/python3.11/site-packages/nao_core/__pycache__/cache_source.cpython-311.pyc': b'old timestamp cache',
}
for name, content in self.files.items():
self.write(name, content)
entries = [{'path': name, 'sha256': hashlib.sha256(content).hexdigest()}
for name, content in self.files.items()]
self.manifest = {'platform': platform.system().lower(), 'files': entries}
(self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8')
def restore_constants(self):
for key, value in self.original.items():
setattr(control, key, value)
def write(self, name, content):
path = self.root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
def successful_probe(self, command, **kwargs):
return type('Result', (), {'returncode': 0})()
def test_verify_runtime_is_read_only_and_skips_only_source_verified_pyc(self):
self.write('runtimes/python/lib/python3.11/site-packages/nao_core/__pycache__/cache_source.cpython-311.pyc',
b'bytecode changed by Windows')
with patch.object(control.platform, 'machine', return_value='x86_64'), \
patch.object(control.subprocess, 'run', side_effect=self.successful_probe):
control.verify_runtime()
self.assertFalse(control.CONFIG.exists())
self.assertFalse(control.DATA.exists())
def test_pyc_is_not_ignored_when_its_manifest_source_hash_fails(self):
self.write('runtimes/python/lib/python3.11/site-packages/nao_core/cache_source.py', b'corrupted source')
with patch.object(control.platform, 'machine', return_value='x86_64'), \
patch.object(control.subprocess, 'run', side_effect=self.successful_probe) as run:
with self.assertRaisesRegex(RuntimeError, '对应源码缺失或校验失败'):
control.verify_runtime()
run.assert_not_called()
def test_legacy_bundle_verification_needs_only_control_gateway_and_runtime_platform(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
offline = root / 'deploy/offline'
offline.mkdir(parents=True)
scripts = root / 'scripts'
scripts.mkdir()
(offline / 'control.py').write_bytes(Path(control.__file__).read_bytes())
(offline / 'gateway.py').write_text('def make_server(*args, **kwargs): return None\n', encoding='utf-8')
(scripts / 'runtime_platform.py').write_text('# minimal legacy runtime platform module\n', encoding='utf-8')
files = {
'runtimes/python/lib/python3.11/site-packages/nao_core/cache_source.py': b'legacy source',
'runtimes/python/lib/python3.11/site-packages/nao_core/__pycache__/cache_source.cpython-311.pyc': b'legacy cache',
}
entries = []
for name, content in files.items():
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
entries.append({'path': name, 'sha256': hashlib.sha256(content).hexdigest()})
(root / 'manifest.json').write_text(json.dumps({
'platform': platform.system().lower(), 'files': entries,
}), encoding='utf-8')
self.assertFalse((offline / 'repair_windows_cache.py').exists())
original_path = list(sys.path)
isolated_path = []
for item in original_path:
resolved = Path(item or '.').resolve()
if resolved != HERE:
isolated_path.append(item)
with patch.object(sys, 'path', [str(offline), *isolated_path]), patch.dict(sys.modules):
for module_name in ('gateway', 'runtime_platform', 'repair_windows_cache'):
sys.modules.pop(module_name, None)
spec = importlib.util.spec_from_file_location('control_legacy_fixture', offline / 'control.py')
isolated_control = importlib.util.module_from_spec(spec)
spec.loader.exec_module(isolated_control)
with patch.object(isolated_control.platform, 'machine', return_value='x86_64'), \
patch.object(isolated_control.subprocess, 'run', side_effect=self.successful_probe):
isolated_control.verify_runtime()
self.assertNotIn('repair_windows_cache', sys.modules)
self.assertFalse((root / '.runtime/deployment.json').exists())
self.assertFalse((root / 'runtimedata').exists())
def test_program_only_checks_program_files_and_application_probe_only(self):
self.write('runtimes/python/bin/python3', b'changed runtime should not be rehashed')
calls = []
def probe(command, **kwargs):
calls.append(command)
return self.successful_probe(command, **kwargs)
with patch.object(control.platform, 'machine', return_value='x86_64'), \
patch.object(control.subprocess, 'run', side_effect=probe):
control.verify_runtime(program_only=True)
self.assertEqual(1, len(calls))
self.assertIn('-c', calls[0])
self.assertFalse(control.CONFIG.exists())
self.assertFalse(control.DATA.exists())
def test_stop_waits_until_recorded_children_are_gone(self):
supervisor = {'pid': 10}
child = {'pid': 20}
state = {'supervisor': supervisor, 'children': [child]}
ticks = iter([0, 0, .2, .4])
matches = iter([True, False, False, True, False, False, False])
with patch.object(control, 'atomic') as atomic, \
patch.object(control, 'read_state', return_value=state), \
patch.object(control, 'matches', side_effect=lambda _info: next(matches)), \
patch.object(control.time, 'monotonic', side_effect=lambda: next(ticks)), \
patch.object(control.time, 'sleep'), \
patch('builtins.print') as output:
control.wait_for_stop(supervisor, {'token': 'safe-token'}, timeout_seconds=1, interval=0)
atomic.assert_called_once_with(control.STOP, {'token': 'safe-token'})
output.assert_called_once_with('服务已停止。')
def test_wait_for_stop_blocks_when_registered_supervisor_is_uninspectable(self):
supervisor = {'pid': 7301, 'created': 1.0, 'executable': '/service', 'args': []}
state = {'supervisor': supervisor, 'children': [], 'token': 'safe-token'}
control.STATE.parent.mkdir(parents=True, exist_ok=True)
control.STATE.write_text(json.dumps(state), encoding='utf-8')
with patch.object(control.psutil, 'Process', side_effect=lambda pid: (_ for _ in ()).throw(
control.psutil.AccessDenied(pid=pid))), \
patch.object(control, 'atomic') as request, patch('builtins.print') as output:
with self.assertRaisesRegex(RuntimeError, '无法确认.*监督进程身份'):
control.wait_for_stop(supervisor, {'token': state['token']}, timeout_seconds=1)
request.assert_not_called()
output.assert_not_called()
def test_main_stop_blocks_when_registered_supervisor_is_uninspectable(self):
supervisor = {'pid': 7302, 'created': 1.0, 'executable': '/service', 'args': []}
control.STATE.parent.mkdir(parents=True, exist_ok=True)
control.STATE.write_text(json.dumps({
'supervisor': supervisor, 'children': [], 'token': 'safe-token',
}), encoding='utf-8')
with patch.object(sys, 'argv', ['control.py', 'stop']), \
patch.object(control.psutil, 'Process', side_effect=lambda pid: (_ for _ in ()).throw(
control.psutil.AccessDenied(pid=pid))), \
patch.object(control, 'atomic') as request, patch('builtins.print') as output:
with self.assertRaisesRegex(RuntimeError, '无法确认.*监督进程身份'):
control.main()
request.assert_not_called()
output.assert_not_called()
def test_main_start_blocks_before_recovery_when_registered_supervisor_is_uninspectable(self):
supervisor = {'pid': 7303, 'created': 1.0, 'executable': '/service', 'args': []}
control.PRIVATE.mkdir(parents=True)
control.DATA.mkdir(parents=True)
control.STATE.write_text(json.dumps({
'supervisor': supervisor, 'children': [], 'token': 'safe-token',
}), encoding='utf-8')
updater = self.root / 'deploy/offline/update_manager.py'
updater.parent.mkdir(parents=True)
updater.write_text('# staged updater', encoding='utf-8')
with patch.object(sys, 'argv', ['control.py', 'start']), \
patch.object(control, 'IS_WINDOWS', True), \
patch.object(control, 'private_directory') as private_directory, \
patch.object(control.psutil, 'Process', side_effect=lambda pid: (_ for _ in ()).throw(
control.psutil.AccessDenied(pid=pid))), \
patch.object(control.subprocess, 'run') as recovery, \
patch('builtins.print') as output:
with self.assertRaisesRegex(RuntimeError, '无法确认.*监督进程身份'):
control.main()
recovery.assert_not_called()
private_directory.assert_not_called()
output.assert_not_called()
def test_stop_does_not_report_success_when_child_survives_budget(self):
supervisor = {'pid': 10}
child = {'pid': 20}
state = {'supervisor': supervisor, 'children': [child]}
ticks = iter([0, 0, .5, 1.1])
def is_running(info):
return info is child
with patch.object(control, 'atomic'), \
patch.object(control, 'read_state', return_value=state), \
patch.object(control, 'matches', side_effect=is_running), \
patch.object(control.time, 'monotonic', side_effect=lambda: next(ticks)), \
patch.object(control.time, 'sleep'), \
patch('builtins.print') as output:
with self.assertRaisesRegex(RuntimeError, '仍在安全关闭'):
control.wait_for_stop(supervisor, {'token': 'safe-token'}, timeout_seconds=1, interval=0)
output.assert_not_called()
def test_stop_has_three_hundred_second_internal_budget(self):
self.assertEqual(300, control.STOP_TIMEOUT_SECONDS)
def test_unknown_process_state_is_not_treated_as_exit_or_signalled(self):
class Uninspectable:
def is_running(self):
return True
def status(self):
raise control.psutil.AccessDenied(pid=4321)
def terminate(self):
raise AssertionError('unknown process must not be terminated')
def kill(self):
raise AssertionError('unknown process must not be killed')
process = Uninspectable()
with patch.object(control.time, 'monotonic', side_effect=[0, 0, 0, 0]):
survivors, unverified = control.terminate_observed([(process, 1.0, None)], timeout=0)
self.assertEqual([], survivors)
self.assertTrue(unverified)
def test_registered_backend_and_mysql_are_retained_when_identity_is_unknown(self):
backend = {'pid': 8101, 'created': 1.0, 'executable': '/java', 'args': ['-jar'] , 'name': 'backend'}
mysql = {'pid': 8102, 'created': 2.0, 'executable': '/mysqld', 'args': ['--port=53306'], 'name': 'mysql'}
def denied(pid):
raise control.psutil.AccessDenied(pid=pid)
with patch.object(control.psutil, 'Process', side_effect=denied):
state = {}
retained, unknown = control.retain_registered_children(state, [backend, mysql])
backend_confirmed, backend_unknown = control.registered_named([backend, mysql], 'backend')
mysql_confirmed, mysql_unknown = control.registered_named([backend, mysql], 'mysql')
self.assertEqual([backend, mysql], retained)
self.assertTrue(unknown)
self.assertEqual([backend, mysql], state['children'])
self.assertTrue(state['unknownChildren'])
self.assertEqual([], backend_confirmed)
self.assertEqual([backend], backend_unknown)
self.assertEqual([], mysql_confirmed)
self.assertEqual([mysql], mysql_unknown)
def test_no_such_process_is_confirmed_gone(self):
class Gone:
def is_running(self):
raise control.psutil.NoSuchProcess(pid=4321)
with patch.object(control.time, 'monotonic', side_effect=[0, 0, 0, 0]):
survivors, unverified = control.terminate_observed([(Gone(), 1.0, None)], timeout=0)
self.assertEqual([], survivors)
self.assertFalse(unverified)
def test_malformed_registered_identity_is_unknown(self):
self.assertIsNone(control.matches({'pid': 4321, 'created': float('nan'),
'executable': '/service', 'args': []}))
def test_verify_runtime_cli_bypasses_running_service_shortcut(self):
with patch.object(sys, 'argv', ['control.py', 'verify-runtime', '--program-only']), \
patch.object(control.platform, 'machine', return_value='x86_64'), \
patch.object(control, 'verify_runtime', return_value={}) as verify, \
patch.object(control, 'private_directory') as private_directory:
control.main()
verify.assert_called_once_with(program_only=True)
private_directory.assert_not_called()
if __name__ == '__main__':
unittest.main()
+229
View File
@@ -0,0 +1,229 @@
import json
import zipfile
from pathlib import Path
import types
import unittest
from unittest.mock import patch
import test_update_manager as fixtures
update = fixtures.update
class HardeningTests(unittest.TestCase):
def setUp(self):
self.f = fixtures.UpdateTests()
self.f.setUp()
self.addCleanup(self.f.doCleanups)
self.root = self.f.root
def backup(self):
b = self.root / '.runtime/update/backup-20261105T120000Z-abcdef12'
b.mkdir(parents=True)
update.backup_program(self.root, b, self.f.release)
(b / 'database.sql').write_bytes(b'-- database snapshot\n' + b'SELECT 1;\n' * 30)
update.atomic(self.root / '.runtime/update/journal.json', {
'backup': b.name, 'phase': 'installing', 'databaseBackedUp': True,
'databaseMayHaveChanged': True,
'backupManifestSha256': update.digest(b / 'manifest.json'),
'programInventorySha256': update.digest(b / 'program.json')})
return b
def test_completed_status_failure_never_rolls_back_committed_program(self):
original = update.write_status
def status(state, message, root=update.ROOT, **extra):
if state == 'completed':
raise PermissionError('status write failure')
return original(state, message, root, **extra)
with patch.object(update, 'write_status', side_effect=status):
calls, restores, _ = self.f.worker_harness()
self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence'])
self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase'])
self.assertFalse(restores)
self.assertEqual('completed', update.status(self.root)['state'])
def test_partially_completed_stop_attempt_restores_old_service(self):
failed = False
def hook(root, action, database_only, timeout, program_only):
nonlocal failed
if action == 'stop' and not failed:
failed = True
raise update.UpdateError('service still closing')
calls, restores, _ = self.f.worker_harness(control_hook=hook)
self.assertIn(('serve', False), calls)
self.assertFalse(restores)
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase'])
def test_runtime_failure_occurs_before_stop_or_replacement(self):
def hook(root, action, database_only, timeout, program_only):
if action == 'verify-runtime' and not program_only:
raise update.UpdateError('runtime preflight failed')
calls, _, _ = self.f.worker_harness(control_hook=hook)
self.assertNotIn(('stop', False), calls)
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
self.assertEqual('failed', update.read(self.root / '.runtime/update/status.json')['state'])
def test_program_verification_failure_does_not_restore_unchanged_database(self):
calls, imports, queries = self.f.worker_harness(fail_verify=True)
self.assertFalse(imports)
self.assertFalse(any('CREATE DATABASE' in q for q in queries))
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
def test_durable_database_is_reused_after_startup_failure_and_bootstrap_replay(self):
b = self.backup()
imports, queries = [], []
fail = True
def ready(root, process, database_only=False):
nonlocal fail
if not database_only and fail:
fail = False
raise update.UpdateError('temporary startup failure')
with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \
patch.object(update, 'wait_ready', side_effect=ready), \
patch.object(update, 'mysql', side_effect=lambda *args, **kwargs: imports.append(kwargs['database'])), \
patch.object(update, 'sql', side_effect=lambda root, config, q: queries.append(q)):
with self.assertRaises(update.UpdateError):
update.rollback(self.root, b, True)
selected = update.read(self.root / '.runtime/deployment.json')['mysqlDatabase']
update.restore_program(self.root, b) # immutable bootstrap replays old program/private snapshot
update.recover_locked(self.root)
self.assertEqual(1, len(imports))
self.assertEqual(1, sum('CREATE DATABASE' in q for q in queries))
self.assertEqual(selected, update.read(self.root / '.runtime/deployment.json')['mysqlDatabase'])
def test_recovery_progress_rejects_changed_dump_before_any_service_mutation(self):
b = self.backup()
fail = True
def ready(root, process, database_only=False):
if not database_only:
raise update.UpdateError('temporary startup failure')
with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \
patch.object(update, 'wait_ready', side_effect=ready), \
patch.object(update, 'mysql'), patch.object(update, 'sql'):
with self.assertRaises(update.UpdateError):
update.rollback(self.root, b, True)
(b / 'database.sql').write_bytes(b'altered dump')
with patch.object(update, 'controller') as controller, self.assertRaises(update.UpdateError):
update.recover_locked(self.root)
controller.assert_not_called()
def test_old_terminal_status_cannot_override_new_operation(self):
update.atomic(self.root / '.runtime/update/journal.json', {
'phase': 'rolledBack', 'operationId': 'old-operation'})
update.write_status('downloading', 'downloading new attempt', self.root, operationId='new-operation')
with patch.object(update, 'live', return_value=True):
self.assertEqual('downloading', update.status(self.root)['state'])
def test_uncertain_complete_write_never_triggers_rollback(self):
original = update.atomic
def atomic(path, value):
original(path, value)
if path.name == 'journal.json' and value.get('phase') == 'complete':
raise OSError('write receipt lost after durable commit')
with patch.object(update, 'atomic', side_effect=atomic):
_, imports, _ = self.f.worker_harness()
self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence'])
self.assertFalse(imports)
journal = update.read(self.root / '.runtime/update/journal.json')
self.assertEqual('complete', journal['phase'])
self.assertIn('backupManifestSha256', journal)
def test_interrupted_import_attempts_are_bounded_without_dropping_databases(self):
b = self.backup()
imports, queries = [], []
def mysql(*args, **kwargs):
imports.append(kwargs['database'])
raise update.UpdateError('simulated partial import')
with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \
patch.object(update, 'wait_ready'), patch.object(update, 'mysql', side_effect=mysql), \
patch.object(update, 'sql', side_effect=lambda root, config, q: queries.append(q)):
for _ in range(update.MAX_RECOVERY_ATTEMPTS + 1):
with self.assertRaises(update.UpdateError):
update.recover_locked(self.root)
self.assertEqual(update.MAX_RECOVERY_ATTEMPTS, len(imports))
self.assertEqual(len(imports), len(set(imports)))
self.assertFalse(any('DROP' in q for q in queries))
self.assertNotIn(self.f.config['mysqlDatabase'], imports)
def test_database_is_not_durable_until_clean_stop_succeeds(self):
b = self.backup()
stops = 0
def controller(root, action, database_only=False, **kwargs):
nonlocal stops
if action == 'stop':
stops += 1
if stops == 2:
raise update.UpdateError('database still closing')
return types.SimpleNamespace(pid=1)
with patch.object(update, 'controller', side_effect=controller), \
patch.object(update, 'wait_ready'), patch.object(update, 'mysql'), patch.object(update, 'sql'):
with self.assertRaises(update.UpdateError):
update.rollback(self.root, b, True)
self.assertEqual('importing', update.read(self.root / '.runtime/update/recovery.json')['stage'])
def test_recovery_cannot_be_redirected_to_original_database(self):
b = self.backup()
binding = update.recovery_binding(b)
update.atomic(self.root / '.runtime/update/recovery.json', {
**binding, 'attempt': 1, 'stage': 'databaseDurable', 'database': self.f.config['mysqlDatabase']})
with patch.object(update, 'controller') as controller, self.assertRaises(update.UpdateError):
update.recover_locked(self.root)
controller.assert_not_called()
def test_backup_preflight_failure_keeps_live_service_and_program(self):
with patch.object(update, 'preflight_backup', side_effect=PermissionError('backup path not writable')):
calls, _, _ = self.f.worker_harness()
self.assertNotIn(('stop', False), calls)
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
def test_staged_protocol_blocks_old_or_mixed_package_before_stop(self):
stage = self.root / '.runtime/staged-protocol'
scripts = stage / 'deploy/offline'
scripts.mkdir(parents=True)
(stage / 'app').mkdir()
(scripts / 'update_manager.py').write_text('UPDATE_PROTOCOL = 2\n')
(scripts / 'control.py').write_text('UPDATE_CONTROL_PROTOCOL = 2\n')
with zipfile.ZipFile(stage / 'app/backend.jar', 'w') as jar:
jar.writestr('BOOT-INF/classes/update-protocol.json', '{"protocol":2}')
update.preflight_programs(stage)
(scripts / 'control.py').write_text('OLD_CONTROLLER = True\n')
with self.assertRaisesRegex(update.UpdateError, '服务尚未停止'):
update.preflight_programs(stage)
(scripts / 'control.py').write_text('UPDATE_CONTROL_PROTOCOL = 2\n')
with zipfile.ZipFile(stage / 'app/backend.jar', 'w') as jar:
jar.writestr('old-backend', 'old protocol')
with self.assertRaisesRegex(update.UpdateError, '服务尚未停止'):
update.preflight_programs(stage)
def test_actual_worker_rejects_incompatible_signed_payload_before_stopping(self):
calls, imports, _ = self.f.worker_harness(program_hook=update.preflight_programs)
self.assertNotIn(('stop', False), calls)
self.assertFalse(imports)
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
self.assertIn('更新包', update.read(self.root / '.runtime/update/status.json')['message'])
def test_legacy_backend_cannot_be_left_in_active_maintenance_after_final_status_failure(self):
original = update.write_status
def status(state, message, root=update.ROOT, **extra):
if state == 'failed':
raise PermissionError('final UI state blocked')
return original(state, message, root, **extra)
with patch.object(update, 'write_status', side_effect=status):
self.f.worker_harness(fail_start=True)
self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase'])
raw = update.read(self.root / '.runtime/update/status.json')['state']
self.assertNotIn(raw, {'backingUp', 'installing', 'restarting', 'rollingBack'})
def test_legacy_terminal_recovery_persists_status_reconciliation(self):
directory = self.root / '.runtime/update'
update.atomic(directory / 'journal.json', {'phase': 'rolledBack', 'operationId': 'same'})
update.atomic(directory / 'status.json', {'state': 'rollingBack', 'operationId': 'same'})
update.recover_locked(self.root)
self.assertEqual('failed', update.read(directory / 'status.json')['state'])
update.atomic(directory / 'status.json', {'state': 'rollingBack', 'operationId': 'same'})
with patch.object(update, 'write_status', side_effect=PermissionError('status blocked')), self.assertRaises(PermissionError):
update.recover_locked(self.root)
if __name__ == '__main__':
unittest.main()
+13 -5
View File
@@ -194,13 +194,14 @@ class UpdateTests(unittest.TestCase):
self.assertEqual(self.manifest, update.read(self.root / 'manifest.json')) self.assertEqual(self.manifest, update.read(self.root / 'manifest.json'))
self.assertEqual(self.files['app/web/obsolete.js'], (self.root / 'app/web/obsolete.js').read_bytes()) self.assertEqual(self.files['app/web/obsolete.js'], (self.root / 'app/web/obsolete.js').read_bytes())
def worker_harness(self, fail_verify=False): def worker_harness(self, fail_verify=False, control_hook=None, fail_start=False, program_hook=None):
directory = self.root / '.runtime/update' directory = self.root / '.runtime/update'
update.atomic(directory / 'request.json', {'token': 'a' * 32}) update.atomic(directory / 'request.json', {'token': 'a' * 32})
archive = self.archive() archive = self.archive()
envelope = self.envelope() envelope = self.envelope()
original_download = update.download original_download = update.download
calls, restores, queries = [], [], [] calls, restores, queries = [], [], []
failed_start = False
def fake_download(url, target=None, *args): def fake_download(url, target=None, *args):
if target: if target:
@@ -208,9 +209,15 @@ class UpdateTests(unittest.TestCase):
return b'' return b''
return json.dumps(envelope).encode() return json.dumps(envelope).encode()
def fake_control(root, action, database_only=False, timeout=0): def fake_control(root, action, database_only=False, timeout=0, program_only=False):
nonlocal failed_start
calls.append((action, database_only)) calls.append((action, database_only))
if action == 'verify' and fail_verify: if fail_start and action == 'serve' and not database_only and not failed_start:
failed_start = True
raise update.UpdateError('模拟新版启动失败。')
if control_hook:
control_hook(root, action, database_only, timeout, program_only)
if action in ('verify', 'verify-runtime') and fail_verify and (program_only or action == 'verify'):
raise update.UpdateError('模拟新版启动校验失败。') raise update.UpdateError('模拟新版启动校验失败。')
return types.SimpleNamespace(pid=1234) return types.SimpleNamespace(pid=1234)
@@ -226,6 +233,7 @@ class UpdateTests(unittest.TestCase):
patch.object(update.psutil, 'Process', return_value=process), \ patch.object(update.psutil, 'Process', return_value=process), \
patch.object(update, 'download', side_effect=fake_download), \ patch.object(update, 'download', side_effect=fake_download), \
patch.object(update, 'controller', side_effect=fake_control), \ patch.object(update, 'controller', side_effect=fake_control), \
patch.object(update, 'preflight_programs', side_effect=program_hook), \
patch.object(update, 'wait_ready'), \ patch.object(update, 'wait_ready'), \
patch.object(update, 'mysql', side_effect=fake_mysql), \ patch.object(update, 'mysql', side_effect=fake_mysql), \
patch.object(update, 'sql', side_effect=lambda root, config, query: queries.append(query)): patch.object(update, 'sql', side_effect=lambda root, config, query: queries.append(query)):
@@ -265,7 +273,7 @@ class UpdateTests(unittest.TestCase):
def test_complete_update_orchestration_with_isolated_database_backup(self): def test_complete_update_orchestration_with_isolated_database_backup(self):
calls, restores, queries = self.worker_harness() calls, restores, queries = self.worker_harness()
self.assertEqual([('stop', False), ('serve', True), ('stop', False), ('verify', False), ('serve', False)], calls) self.assertEqual([('verify-runtime', False), ('stop', False), ('serve', True), ('stop', False), ('verify-runtime', False), ('serve', False)], calls)
self.assertFalse(restores) self.assertFalse(restores)
self.assertEqual('completed', update.read(self.root / '.runtime/update/status.json')['state']) self.assertEqual('completed', update.read(self.root / '.runtime/update/status.json')['state'])
self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase']) self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase'])
@@ -276,7 +284,7 @@ class UpdateTests(unittest.TestCase):
update.atomic(self.root / '.runtime/deployment.json', self.config) update.atomic(self.root / '.runtime/deployment.json', self.config)
update.atomic(self.root / '.runtime/migration-receipt.json', {'migrationId': 'isolated-test-migration', update.atomic(self.root / '.runtime/migration-receipt.json', {'migrationId': 'isolated-test-migration',
'database': self.config['mysqlDatabase'], 'migrationManifestSha256': 'b' * 64}) 'database': self.config['mysqlDatabase'], 'migrationManifestSha256': 'b' * 64})
calls, restores, queries = self.worker_harness(fail_verify=True) calls, restores, queries = self.worker_harness(fail_start=True)
self.assertEqual(self.manifest, update.read(self.root / 'manifest.json')) self.assertEqual(self.manifest, update.read(self.root / 'manifest.json'))
config = update.read(self.root / '.runtime/deployment.json') config = update.read(self.root / '.runtime/deployment.json')
self.assertRegex(config['mysqlDatabase'], r'^quality_restore_[a-f0-9]{24}$') self.assertRegex(config['mysqlDatabase'], r'^quality_restore_[a-f0-9]{24}$')
+2 -3
View File
@@ -54,10 +54,9 @@ class WindowsCacheRepairTest(unittest.TestCase):
patch.object(control.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0)): patch.object(control.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0)):
control.verify() control.verify()
def test_actual_startup_hash_check_fails_on_regenerated_cache_then_passes_after_repair(self): def test_startup_accepts_regenerated_cache_only_when_source_hash_is_valid(self):
(self.root / self.cache).write_bytes(b'new-bytecode-created-on-windows') (self.root / self.cache).write_bytes(b'new-bytecode-created-on-windows')
with self.assertRaisesRegex(RuntimeError, '__future__.*pyc'): self.verify() # 合法源码允许运行库重建缓存;原位修复仍可清除旧缓存清单。
self.verify()
result = fix.repair(self.root) result = fix.repair(self.root)
self.assertEqual(result['cacheEntriesRemoved'], 1) self.assertEqual(result['cacheEntriesRemoved'], 1)
self.verify() self.verify()
@@ -1,127 +1,253 @@
import hashlib import hashlib
import ctypes
import json import json
import os
from pathlib import Path from pathlib import Path
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import unittest import unittest
from unittest.mock import patch from unittest.mock import Mock, patch
HERE = Path(__file__).resolve().parent HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE)) sys.path.insert(0, str(HERE))
import repair_windows_update_check as fix import repair_windows_update_check as fix
ORIGINAL_SHA = fix.OLD_SHA
class UpdateCheckRepairTest(unittest.TestCase): class UpdateCheckRepairTest(unittest.TestCase):
def setUp(self): def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix='quality source path repair ') self.temp = tempfile.TemporaryDirectory(prefix='quality firstaid repair ')
self.addCleanup(self.temp.cleanup) self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name) / '中文 QualityCenter & files' self.root = Path(self.temp.name) / '中文 QualityCenter & files'
self.root.mkdir() self.root.mkdir()
self.original = b'original source fixture\n' self.old_manager = b'known old updater fixture\n'
self.old_sha = hashlib.sha256(self.original).hexdigest() self.old_control = b'known old controller fixture\n'
self.guard = patch.object(fix, 'OLD_SHA', self.old_sha) self.old_manager_sha = hashlib.sha256(self.old_manager).hexdigest()
self.guard.start() self.old_control_sha = hashlib.sha256(self.old_control).hexdigest()
self.addCleanup(self.guard.stop) self.reviewed_manager_hashes = fix.OLD_MANAGER_SHA256
for name, value in {fix.SCRIPT: self.original, self.reviewed_control_hashes = fix.OLD_CONTROL_SHA256
self.manager_guard = patch.object(fix, 'OLD_MANAGER_SHA256', frozenset({self.old_manager_sha}))
self.control_guard = patch.object(fix, 'OLD_CONTROL_SHA256', frozenset({self.old_control_sha}))
self.manager_guard.start()
self.control_guard.start()
self.addCleanup(self.manager_guard.stop)
self.addCleanup(self.control_guard.stop)
self.manager_payload, self.control_payload = fix.read_hardened_sources()
for name, value in {
fix.MANAGER: self.old_manager,
fix.CONTROL: self.old_control,
'deploy/bootstrap/windows-start.ps1': b'original bootstrap', 'deploy/bootstrap/windows-start.ps1': b'original bootstrap',
'migration/database.sql.gz': b'existing migration payload', 'migration/database.sql.gz': b'existing migration payload',
'.runtime/deployment.json': b'original private configuration', '.runtime/deployment.json': b'original private configuration',
'runtimedata/mysql/retained-database': b'imported database data'}.items(): 'runtimedata/mysql/retained-database': b'imported database data',
}.items():
target = self.root / name target = self.root / name
target.parent.mkdir(parents=True, exist_ok=True) target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(value) target.write_bytes(value)
self.manifest = {'platform': 'windows', 'updateVersion': '2026.11.04.1', 'updateSequence': 1, self.manifest = {
'platform': 'windows', 'updateVersion': '2026.11.04.1', 'updateSequence': 1,
'runtimeFingerprint': 'unchanged', 'otherMetadata': {'keep': True}, 'runtimeFingerprint': 'unchanged', 'otherMetadata': {'keep': True},
'files': [{'path': fix.SCRIPT, 'sha256': self.old_sha}, 'files': [
{'path': 'deploy/bootstrap/windows-start.ps1', 'sha256': 'unchanged'}]} {'path': fix.MANAGER, 'sha256': self.old_manager_sha},
{'path': fix.CONTROL, 'sha256': self.old_control_sha},
{'path': 'deploy/bootstrap/windows-start.ps1', 'sha256': 'unchanged'},
],
}
self.manifest_path = self.root / 'manifest.json' self.manifest_path = self.root / 'manifest.json'
self.manifest_path.write_text(json.dumps(self.manifest), encoding='utf-8') self.manifest_path.write_text(json.dumps(self.manifest), encoding='utf-8')
def test_backup_manifest_and_script_only_preserving_database_and_configuration(self): def test_backups_both_programs_and_manifest_and_preserves_private_data(self):
private = {name: (self.root / name).read_bytes() for name in private = {name: (self.root / name).read_bytes() for name in
['migration/database.sql.gz', '.runtime/deployment.json', 'runtimedata/mysql/retained-database']} ['migration/database.sql.gz', '.runtime/deployment.json',
result = fix.repair(self.root) 'runtimedata/mysql/retained-database']}
result = fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertTrue(result['updateCheckRepaired']) self.assertTrue(result['updateCheckRepaired'])
self.assertEqual((self.root / fix.SCRIPT).read_bytes(), fix.PATCHED_SCRIPT) self.assertTrue(result['controlRecoveryRepaired'])
self.assertEqual(result['changedFiles'], 3)
self.assertEqual((self.root / fix.MANAGER).read_bytes(), self.manager_payload)
self.assertEqual((self.root / fix.CONTROL).read_bytes(), self.control_payload)
expected = json.loads(json.dumps(self.manifest)) expected = json.loads(json.dumps(self.manifest))
expected['files'][0]['sha256'] = fix.NEW_SHA expected['files'][0]['sha256'] = hashlib.sha256(self.manager_payload).hexdigest()
expected['files'][1]['sha256'] = hashlib.sha256(self.control_payload).hexdigest()
self.assertEqual(json.loads(self.manifest_path.read_text()), expected) self.assertEqual(json.loads(self.manifest_path.read_text()), expected)
backup = self.root / result['backup'] backup = self.root / result['backup']
self.assertEqual((backup / 'update_manager.py').read_bytes(), self.original) self.assertEqual((backup / 'update_manager.py').read_bytes(), self.old_manager)
self.assertEqual((backup / 'control.py').read_bytes(), self.old_control)
self.assertEqual(json.loads((backup / 'manifest.json').read_text()), self.manifest) self.assertEqual(json.loads((backup / 'manifest.json').read_text()), self.manifest)
for name, value in private.items(): for name, value in private.items():
self.assertEqual((self.root / name).read_bytes(), value) self.assertEqual((self.root / name).read_bytes(), value)
self.assertTrue(fix.repair(self.root)['alreadyRepaired']) self.assertTrue(fix.repair(self.root, self.manager_payload, self.control_payload)['alreadyRepaired'])
def test_modified_script_or_manifest_is_not_approved_as_a_repair(self): def test_unknown_program_or_manifest_digest_is_never_overwritten(self):
before = self.manifest_path.read_bytes() original_manifest = self.manifest_path.read_bytes()
(self.root / fix.SCRIPT).write_bytes(b'unrecognized source') (self.root / fix.MANAGER).write_bytes(b'unrecognized updater')
with self.assertRaisesRegex(RuntimeError, '版本'): with self.assertRaisesRegex(RuntimeError, '版本'):
fix.repair(self.root) fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertEqual(self.manifest_path.read_bytes(), before) self.assertEqual(self.manifest_path.read_bytes(), original_manifest)
(self.root / fix.MANAGER).write_bytes(self.old_manager)
value = json.loads(original_manifest)
value['files'][0]['sha256'] = '0' * 64
self.manifest_path.write_text(json.dumps(value), encoding='utf-8')
with self.assertRaisesRegex(RuntimeError, '版本'):
fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertEqual((self.root / fix.MANAGER).read_bytes(), self.old_manager)
def test_failed_second_write_restores_original_program_and_manifest(self): def test_failed_manifest_write_rolls_back_both_programs_and_manifest(self):
before = self.manifest_path.read_bytes() before = {fix.MANAGER: (self.root / fix.MANAGER).read_bytes(),
fix.CONTROL: (self.root / fix.CONTROL).read_bytes(),
'manifest.json': self.manifest_path.read_bytes()}
original_atomic = fix.atomic original_atomic = fix.atomic
calls = [] calls = []
def failing(path, data): def failing(path, data):
calls.append(path) calls.append(path)
if len(calls) == 2: if len(calls) == 3:
raise OSError('simulated manifest write failure') raise OSError('simulated manifest write failure')
original_atomic(path, data) original_atomic(path, data)
with patch.object(fix, 'atomic', side_effect=failing), self.assertRaises(OSError): with patch.object(fix, 'atomic', side_effect=failing), self.assertRaises(OSError):
fix.repair(self.root) fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertEqual((self.root / fix.SCRIPT).read_bytes(), self.original) self.assertEqual((self.root / fix.MANAGER).read_bytes(), before[fix.MANAGER])
self.assertEqual(self.manifest_path.read_bytes(), before) self.assertEqual((self.root / fix.CONTROL).read_bytes(), before[fix.CONTROL])
self.assertEqual(self.manifest_path.read_bytes(), before['manifest.json'])
self.assertEqual(len(list((self.root / '.runtime/update-check-repair-backups').iterdir())), 1)
def test_retry_completes_an_interrupted_script_manifest_pair(self): def test_retry_completes_old_new_mix_and_accepts_rolled_back_journal(self):
(self.root / fix.SCRIPT).write_bytes(fix.PATCHED_SCRIPT) (self.root / fix.MANAGER).write_bytes(self.manager_payload)
fix.repair(self.root) journal = self.root / '.runtime/update/journal.json'
self.assertEqual(json.loads(self.manifest_path.read_text())['files'][0]['sha256'], fix.NEW_SHA) journal.parent.mkdir(parents=True)
journal.write_text(json.dumps({'phase': 'rolledBack'}), encoding='utf-8')
result = fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertTrue(result['updateCheckRepaired'])
self.assertEqual((self.root / fix.CONTROL).read_bytes(), self.control_payload)
manifest = json.loads(self.manifest_path.read_text())
self.assertEqual(manifest['files'][0]['sha256'], hashlib.sha256(self.manager_payload).hexdigest())
self.assertEqual(manifest['files'][1]['sha256'], hashlib.sha256(self.control_payload).hexdigest())
def test_cmd_payload_runs_with_bundled_standard_library_without_service_restart(self): def test_active_worker_and_unknown_or_unfinished_journal_block_repair(self):
# 真实原包脚本由交付包集成验证;此单元测试使用受控旧源码摘要。 worker = self.root / '.runtime/update/worker.json'
command = self.root / '修复更新检查.cmd' worker.parent.mkdir(parents=True)
body = fix.command_file().decode('utf-8').replace(ORIGINAL_SHA, self.old_sha) worker.write_text(json.dumps({'pid': __import__('os').getpid()}), encoding='utf-8')
command.write_bytes(body.encode('utf-8')) with self.assertRaisesRegex(RuntimeError, '进程'):
code = body.split(' -c "', 1)[1].split('" "%~f0"', 1)[0] fix.repair(self.root, self.manager_payload, self.control_payload)
result = subprocess.run([sys.executable, '-I', '-S', '-B', '-X', 'utf8', '-c', code, str(command)], capture_output=True, text=True) worker.unlink()
self.assertEqual(result.returncode, 0, result.stderr) journal = worker.parent / 'journal.json'
self.assertEqual((self.root / fix.SCRIPT).read_bytes(), fix.PATCHED_SCRIPT) for value in ({'phase': 'installing'}, {'phase': 'futurePhase'}, {'unknown': True}):
self.assertNotIn('powershell.exe', body) journal.write_text(json.dumps(value), encoding='utf-8')
self.assertIn('无需停止服务或重新导入数据库', body)
def test_active_update_lock_blocks_patching(self):
before = self.manifest_path.read_bytes()
with fix.update_lock(self.root), self.assertRaisesRegex(RuntimeError, '仍在运行'):
fix.repair(self.root)
self.assertEqual(self.manifest_path.read_bytes(), before)
def test_worker_or_unfinished_journal_blocks_repair(self):
for name, data in [('worker.json', {}), ('journal.json', {'phase': 'installing'})]:
path = self.root / '.runtime/update' / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data))
with self.assertRaises(RuntimeError): with self.assertRaises(RuntimeError):
fix.repair(self.root) fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertEqual((self.root / fix.SCRIPT).read_bytes(), self.original) self.assertEqual((self.root / fix.MANAGER).read_bytes(), self.old_manager)
path.unlink() journal.unlink()
def test_previous_check_repair_version_can_upgrade_without_losing_manifest_fields(self): def test_windows_worker_probe_is_read_only_for_missing_and_live_pids(self):
prior = b'previous optimized checker fixture' worker = self.root / '.runtime/update/worker.json'
digest = hashlib.sha256(prior).hexdigest() worker.parent.mkdir(parents=True)
(self.root / fix.SCRIPT).write_bytes(prior) worker.write_text(json.dumps({'pid': 424242}), encoding='utf-8')
missing_kernel = Mock()
missing_kernel.OpenProcess = Mock(return_value=None)
missing_kernel.GetExitCodeProcess = Mock()
missing_kernel.CloseHandle = Mock()
with (patch.object(fix.os, 'name', 'nt'),
patch.object(fix.os, 'kill', side_effect=AssertionError('os.kill must not run')) as kill,
patch('ctypes.WinDLL', return_value=missing_kernel, create=True),
patch('ctypes.get_last_error', return_value=87, create=True)):
self.assertFalse(fix.worker_active(self.root.resolve()))
kill.assert_not_called()
missing_kernel.GetExitCodeProcess.assert_not_called()
missing_kernel.CloseHandle.assert_not_called()
missing_kernel.TerminateProcess.assert_not_called()
missing_kernel.OpenProcess.assert_called_once_with(0x1000, False, 424242)
denied_kernel = Mock()
denied_kernel.OpenProcess = Mock(return_value=None)
denied_kernel.GetExitCodeProcess = Mock()
denied_kernel.CloseHandle = Mock()
with (patch.object(fix.os, 'name', 'nt'),
patch.object(fix.os, 'kill', side_effect=AssertionError('os.kill must not run')) as kill,
patch('ctypes.WinDLL', return_value=denied_kernel, create=True),
patch('ctypes.get_last_error', return_value=5, create=True)):
with self.assertRaisesRegex(RuntimeError, '无法只读确认'):
fix.worker_active(self.root.resolve())
kill.assert_not_called()
denied_kernel.TerminateProcess.assert_not_called()
live_kernel = Mock()
live_kernel.OpenProcess = Mock(return_value=123)
live_kernel.CloseHandle = Mock(return_value=1)
def set_still_active(_handle, output):
ctypes.cast(output, ctypes.POINTER(ctypes.c_ulong)).contents.value = 259
return 1
live_kernel.GetExitCodeProcess = Mock(side_effect=set_still_active)
with (patch.object(fix.os, 'name', 'nt'),
patch.object(fix.os, 'kill', side_effect=AssertionError('os.kill must not run')) as kill,
patch('ctypes.WinDLL', return_value=live_kernel, create=True)):
self.assertTrue(fix.worker_active(self.root.resolve()))
kill.assert_not_called()
live_kernel.CloseHandle.assert_called_once_with(123)
live_kernel.TerminateProcess.assert_not_called()
def test_stale_worker_pid_allows_repair_and_live_reused_pid_fails_closed(self):
worker = self.root / '.runtime/update/worker.json'
worker.parent.mkdir(parents=True)
worker.write_text(json.dumps({'pid': 424242}), encoding='utf-8')
with patch.object(fix.os, 'kill', side_effect=ProcessLookupError):
self.assertTrue(fix.repair(self.root, self.manager_payload, self.control_payload)['updateCheckRepaired'])
# A live process with a stale/reused PID is treated as active; no identity match is required.
(self.root / fix.MANAGER).write_bytes(self.old_manager)
(self.root / fix.CONTROL).write_bytes(self.old_control)
self.manifest_path.write_text(json.dumps(self.manifest), encoding='utf-8')
with patch.object(fix.os, 'kill', return_value=None):
with self.assertRaisesRegex(RuntimeError, '仍在运行'):
fix.repair(self.root, self.manager_payload, self.control_payload)
def test_update_lock_blocks_firstaid(self):
with fix.update_lock(self.root), self.assertRaisesRegex(RuntimeError, '仍在运行'):
fix.repair(self.root, self.manager_payload, self.control_payload)
def test_self_contained_cmd_uses_bundled_python_and_latest_source_bytes(self):
manager, control = fix.read_hardened_sources()
# This models a crash after source replacement but before manifest update.
(self.root / fix.MANAGER).write_bytes(manager)
(self.root / fix.CONTROL).write_bytes(control)
value = json.loads(self.manifest_path.read_text()) value = json.loads(self.manifest_path.read_text())
value['files'][0]['sha256'] = digest value['files'][0]['sha256'] = '29512f15a9ee329bc7c7777454bdd79aa7ffd11c3cffa9400b8a4585bdabf35b'
self.manifest_path.write_text(json.dumps(value)) value['files'][1]['sha256'] = 'd98fa24d17e9bdd90ff37f9886df280451eceabac0df35d2eeecf4fe615bc6f1'
with patch.object(fix, 'PREVIOUS_SHA', digest): self.manifest_path.write_text(json.dumps(value), encoding='utf-8')
self.assertTrue(fix.repair(self.root)['updateCheckRepaired']) command = self.root / '兼容旧安装修复.cmd'
self.assertEqual(fix.PATCHED_SCRIPT, (self.root / fix.SCRIPT).read_bytes()) body = fix.command_file()
command.write_bytes(body)
module, payload = body.rsplit(fix.MODULE_MARKER, 1)[1].rsplit(fix.PAYLOAD_MARKER, 1)
embedded = {}
exec(compile(payload.decode('ascii'), str(command), 'exec'), embedded)
self.assertEqual(embedded['PATCHED_MANAGER'], manager)
self.assertEqual(embedded['PATCHED_CONTROL'], control)
code = body.decode('utf-8').split(' -c "', 1)[1].split('" "%~f0"', 1)[0]
result = subprocess.run([sys.executable, '-I', '-S', '-B', '-X', 'utf8', '-c', code, str(command)],
capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual((self.root / fix.MANAGER).read_bytes(), manager)
self.assertEqual((self.root / fix.CONTROL).read_bytes(), control)
manifest = json.loads(self.manifest_path.read_text())
self.assertEqual(manifest['files'][0]['sha256'], hashlib.sha256(manager).hexdigest())
self.assertEqual(manifest['files'][1]['sha256'], hashlib.sha256(control).hexdigest())
command_header = body.split(fix.MODULE_MARKER, 1)[0].decode('utf-8')
self.assertNotIn('powershell.exe', command_header)
self.assertNotIn('windows-start.ps1', command_header)
self.assertIn(' -I -S -B ', command_header)
def test_embedded_program_matches_reviewed_source(self): def test_historical_allowlist_matches_known_release_inputs(self):
self.assertEqual(fix.PATCHED_SCRIPT, (HERE / 'update_manager.py').read_bytes()) self.assertEqual(self.reviewed_manager_hashes, frozenset({
'29512f15a9ee329bc7c7777454bdd79aa7ffd11c3cffa9400b8a4585bdabf35b',
'b91cb7fec10e183d2904d86f032ff0ffcf7f19f7cb27aecd2d803d3448685482',
'a4bd05789f559fede0bed083ce61123307d6b5cf52fa638b6bde1fa112a0348a',
}))
self.assertEqual(self.reviewed_control_hashes, frozenset({
'd98fa24d17e9bdd90ff37f9886df280451eceabac0df35d2eeecf4fe615bc6f1',
}))
if __name__ == '__main__': if __name__ == '__main__':
+241 -25
View File
@@ -8,8 +8,10 @@ snapshot; private data and runtimes are never update destinations.
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import ast
import base64 import base64
import contextlib import contextlib
import fnmatch
from datetime import datetime, timezone from datetime import datetime, timezone
import hashlib import hashlib
import json import json
@@ -30,6 +32,7 @@ import psutil
from cryptography.hazmat.primitives.serialization import load_pem_public_key from cryptography.hazmat.primitives.serialization import load_pem_public_key
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
UPDATE_PROTOCOL = 2
ROOT = Path(__file__).resolve().parents[2] ROOT = Path(__file__).resolve().parents[2]
BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/' BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/'
LATEST = BASE + 'latest.json' LATEST = BASE + 'latest.json'
@@ -267,11 +270,25 @@ def enabled(root=ROOT):
def status(root=ROOT): def status(root=ROOT):
manifest = read(root / 'manifest.json', {}) manifest = read(root / 'manifest.json', {})
try:
value = read(root / '.runtime/update/status.json', {}) value = read(root / '.runtime/update/status.json', {})
except (OSError, ValueError):
value = {}
if not isinstance(value, dict):
value = {}
allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'} allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'}
result = {k: v for k, v in value.items() if k in allowed} result = {k: v for k, v in value.items() if k in allowed}
result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本'))) result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本')))
result.setdefault('state', 'idle') result.setdefault('state', 'idle')
journal = read(root / '.runtime/update/journal.json', {})
same_operation = bool(journal.get('operationId')) and journal.get('operationId') == value.get('operationId')
if (same_operation and journal.get('phase') == 'complete' and result['state'] in ACTIVE
and manifest.get('updateSequence') == journal.get('sequence')
and manifest.get('updateVersion') == journal.get('version')):
result.update(state='completed', message='更新完成。', available=False,
version=journal['version'], finishedAt=journal['finishedAt'])
elif same_operation and journal.get('phase') == 'rolledBack' and result['state'] in ACTIVE:
result.update(state='failed', message='更新未完成,已恢复原版本。', available=False)
if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})): if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})):
pending = read(root / '.runtime/update/request.json', {}) pending = read(root / '.runtime/update/request.json', {})
journal = read(root / '.runtime/update/journal.json', {}) journal = read(root / '.runtime/update/journal.json', {})
@@ -323,7 +340,10 @@ def exclusive_update(root=ROOT):
yield yield
finally: finally:
if read(directory / 'worker.json', {}) == identity: if read(directory / 'worker.json', {}) == identity:
try:
(directory / 'worker.json').unlink(missing_ok=True) (directory / 'worker.json').unlink(missing_ok=True)
except OSError as error:
record_failure(root, 'cleanupWorker', error)
def launch(root=ROOT): def launch(root=ROOT):
@@ -345,7 +365,7 @@ def launch_locked(root=ROOT):
return status(root) return status(root)
token = secrets.token_hex(16) token = secrets.token_hex(16)
atomic(directory / 'request.json', {'token': token, 'created': time.time()}) atomic(directory / 'request.json', {'token': token, 'created': time.time()})
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False) write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False, operationId=token)
try: try:
with (directory / 'worker.log').open('ab') as log: with (directory / 'worker.log').open('ab') as log:
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token], subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token],
@@ -364,11 +384,13 @@ def live(info):
return False return False
def controller(root, action, database_only=False, timeout=1100): def controller(root, action, database_only=False, timeout=1100, program_only=False):
python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3') python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3')
command = [str(python), str(root / 'deploy/offline/control.py'), action] command = [str(python), str(root / 'deploy/offline/control.py'), action]
if database_only: if database_only:
command += ['--database-only'] command += ['--database-only']
if program_only:
command += ['--program-only']
if action == 'serve': if action == 'serve':
with (root / '.runtime/update/service.log').open('ab') as log: with (root / '.runtime/update/service.log').open('ab') as log:
return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options()) return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
@@ -471,29 +493,186 @@ def restore_program(root, backup):
copy_atomic(source, destination(root, '.runtime/' + relative)) copy_atomic(source, destination(root, '.runtime/' + relative))
STOP_TIMEOUT = 360
MAX_RECOVERY_ATTEMPTS = 2
def best_effort_status(state, message, root=ROOT, **extra):
try:
return write_status(state, message, root, **extra)
except Exception as error:
record_failure(root, 'writeStatus', error)
def verify_backup(root, backup, journal=None):
journal = journal or read(root / '.runtime/update/journal.json', {})
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
raise UpdateError('更新恢复备份校验失败,已保留现场。')
manifest = entry_map(read(backup / 'manifest.json')['files'])
inventory = read(backup / 'program.json')
names, present = inventory.get('all'), inventory.get('present')
if (not isinstance(names, list) or not isinstance(present, list)
or len(names) != len(set(names)) or not set(present).issubset(names)):
raise UpdateError('更新恢复清单无效。')
for name in names:
if not program_path(name):
raise UpdateError('更新恢复清单包含受保护路径。')
destination(root, name)
if name in present:
source = destination(backup / 'program', name)
if name not in manifest or not source.is_file() or digest(source) != manifest[name]['sha256']:
raise UpdateError('原程序备份缺失或损坏。')
elif name in manifest:
raise UpdateError('原程序备份不完整。')
def recovery_binding(backup):
return {'backup': backup.name, 'backupManifestSha256': digest(backup / 'manifest.json'),
'programInventorySha256': digest(backup / 'program.json'),
'databaseSqlSha256': digest(backup / 'database.sql')}
def recovery_database(binding, attempt):
return 'quality_restore_' + hashlib.sha256(canonical({**binding, 'attempt': attempt})).hexdigest()[:24]
def recovery_progress(root, backup):
binding = recovery_binding(backup)
try:
value = read(root / '.runtime/update/recovery.json', {})
except (OSError, ValueError) as error:
raise UpdateError('恢复进度无法读取,已保留现场。') from error
if not value:
return binding, None
if (not isinstance(value, dict) or any(value.get(k) != v for k, v in binding.items())
or type(value.get('attempt')) is not int or not 1 <= value['attempt'] <= MAX_RECOVERY_ATTEMPTS
or value.get('database') != recovery_database(binding, value['attempt'])
or value.get('stage') not in ('importing', 'databaseDurable', 'published', 'ready')):
raise UpdateError('恢复进度与备份不一致,已保留现场。')
return binding, value
def prepare_old_backend_restart(root):
# 旧 JAR 不理解终态 journal 的优先级。先持久化不触发其维护锁的进度状态,
# 再启动旧服务;写失败则保留恢复记录,不声称已恢复。
write_status('queued', '正在启动原版本,完成后自动重新连接。', root)
def normalize_terminal_status(root, journal):
value = read(root / '.runtime/update/status.json', {})
if (not isinstance(value, dict) or value.get('state') not in ACTIVE
or journal.get('operationId') != value.get('operationId')):
return
pending = read(root / '.runtime/update/request.json', {})
if value.get('state') in ('queued', 'downloading') and time.time() - pending.get('created', 0) < 120:
return
if journal.get('phase') == 'rolledBack':
write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
elif journal.get('phase') == 'complete':
manifest = read(root / 'manifest.json')
if manifest.get('updateVersion') == journal.get('version', value.get('version')):
write_status('completed', '更新完成。', root, available=False)
def rollback(root, backup, database_backed_up): def rollback(root, backup, database_backed_up):
write_status('rollingBack', '更新未完成,正在恢复原版本。', root) verify_backup(root, backup)
controller(root, 'stop', timeout=180) binding, progress = recovery_progress(root, backup) if database_backed_up else ({}, None)
best_effort_status('rollingBack', '更新未完成,正在恢复原版本。', root)
controller(root, 'stop', timeout=STOP_TIMEOUT)
restore_program(root, backup) restore_program(root, backup)
if database_backed_up: if database_backed_up:
config = read(root / '.runtime/deployment.json') config = read(backup / 'private/deployment.json')
# 断电前未确认数据库安全关闭的导入,不能当作完整数据库复用。
if progress is None or progress['stage'] == 'importing':
attempt = 1 if progress is None else progress['attempt'] + 1
if attempt > MAX_RECOVERY_ATTEMPTS:
raise UpdateError('恢复导入多次中断,已停止自动重试并保留数据库,请核对本机恢复记录。')
reserve = (backup / 'database.sql').stat().st_size * 3 + 512 * 1024**2
if shutil.disk_usage(root).free < reserve:
raise UpdateError('空间不足以安全恢复数据库,已保留现场。')
database = recovery_database(binding, attempt)
if database == config.get('mysqlDatabase'):
raise UpdateError('恢复数据库与原库冲突,已停止。')
progress = {**binding, 'attempt': attempt, 'database': database, 'stage': 'importing'}
atomic(root / '.runtime/update/recovery.json', progress)
process = controller(root, 'serve', True) process = controller(root, 'serve', True)
wait_ready(root, process, True) wait_ready(root, process, True)
database = 'quality_restore_' + secrets.token_hex(12) # 不删除任何已有数据库;名称冲突时 CREATE 会失败并保留现场。
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;') sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
with (backup / 'database.sql').open('rb') as source: with (backup / 'database.sql').open('rb') as source:
mysql(root, config, database=database, stdin=source) mysql(root, config, database=database, stdin=source)
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';") sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
controller(root, 'stop', timeout=180) controller(root, 'stop', timeout=STOP_TIMEOUT)
config['mysqlDatabase'] = database progress['stage'] = 'databaseDurable'
atomic(root / '.runtime/update/recovery.json', progress)
config['mysqlDatabase'] = progress['database']
atomic(root / '.runtime/deployment.json', config) atomic(root / '.runtime/deployment.json', config)
receipt_path = root / '.runtime/migration-receipt.json' receipt_path = root / '.runtime/migration-receipt.json'
receipt = read(receipt_path, {}) receipt = read(receipt_path, {})
if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'): if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'):
receipt['database'] = database receipt['database'] = progress['database']
atomic(receipt_path, receipt) atomic(receipt_path, receipt)
progress['stage'] = 'published'
atomic(root / '.runtime/update/recovery.json', progress)
prepare_old_backend_restart(root)
process = controller(root, 'serve') process = controller(root, 'serve')
wait_ready(root, process) wait_ready(root, process)
if progress:
progress['stage'] = 'ready'
atomic(root / '.runtime/update/recovery.json', progress)
def preflight_programs(stage):
# 旧安装先加固恢复器,随后只能安装具备相同恢复协议的程序,防止覆盖回旧实现。
for name, marker in (('update_manager.py', 'UPDATE_PROTOCOL'), ('control.py', 'UPDATE_CONTROL_PROTOCOL')):
try:
tree = ast.parse((stage / 'deploy/offline' / name).read_bytes())
values = [node.value.value for node in tree.body if isinstance(node, ast.Assign)
and any(isinstance(target, ast.Name) and target.id == marker for target in node.targets)
and isinstance(node.value, ast.Constant) and type(node.value.value) is int]
if values != [UPDATE_PROTOCOL]:
raise ValueError('incompatible recovery protocol')
except (OSError, SyntaxError, ValueError) as error:
raise UpdateError('更新包未包含本轮加固的恢复流程,服务尚未停止;请等待配套更新包。') from error
try:
with zipfile.ZipFile(stage / 'app/backend.jar') as jar:
metadata = decode(jar.read('BOOT-INF/classes/update-protocol.json'))
if metadata != {'protocol': UPDATE_PROTOCOL}:
raise ValueError('incompatible backend protocol')
except (OSError, ValueError, KeyError, zipfile.BadZipFile) as error:
raise UpdateError('更新包后端不支持本轮恢复协议,服务尚未停止。') from error
def preflight_backup(root, backup, release):
paths = set(e['path'] for e in read(root / 'manifest.json')['files'] if program_path(e['path']))
paths.update(e['path'] for e in release['files'])
targets = [backup / 'program' / n for n in paths]
ignored = ('*.lock', '*.log', 'service.json', 'stop-request.json')
for folder, dirs, files in os.walk(root / '.runtime', followlinks=False):
dirs[:] = [n for n in dirs if n != 'update']
for n in dirs:
destination(root, (Path(folder) / n).relative_to(root).as_posix())
for n in files:
if any(fnmatch.fnmatch(n, pattern) for pattern in ignored):
continue
source = Path(folder) / n
destination(root, source.relative_to(root).as_posix())
with source.open('rb'):
pass
targets.append(backup / 'private' / source.relative_to(root / '.runtime'))
for target in targets:
destination(backup, target.relative_to(backup).as_posix())
if os.name == 'nt' and len(str(target.absolute())) + 24 > 248:
raise UpdateError('备份路径过长,请先缩短部署目录或历史路径;服务尚未停止。')
probe = backup / 'write-probe.tmp'
try:
with probe.open('xb') as stream:
stream.write(b'probe')
stream.flush()
os.fsync(stream.fileno())
finally:
probe.unlink(missing_ok=True)
def record_failure(root, stage, error, name='failure.json'): def record_failure(root, stage, error, name='failure.json'):
@@ -526,7 +705,7 @@ def worker(launcher, token, root=ROOT):
raise UpdateError('更新请求已失效。') raise UpdateError('更新请求已失效。')
with exclusive_update(root): with exclusive_update(root):
backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4)) backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4))
stopped = backed_up = program_saved = changed = False stopped = backed_up = program_saved = changed = stop_requested = committed = False
failure_stage = 'validateRelease' failure_stage = 'validateRelease'
try: try:
envelope = decode(download(LATEST)) envelope = decode(download(LATEST))
@@ -543,18 +722,30 @@ def worker(launcher, token, root=ROOT):
private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory)) private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory))
if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2: if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2:
raise UpdateError('磁盘剩余空间不足,请释放空间后重试。') raise UpdateError('磁盘剩余空间不足,请释放空间后重试。')
write_status('downloading', '正在下载并校验更新。', root) write_status('downloading', '正在下载并校验更新。', root, operationId=token, version=release['version'])
info = release['archive'] info = release['archive']
failure_stage = 'downloadArchive' failure_stage = 'downloadArchive'
download(BASE + info['file'], archive, info['size'], info['sha256']) download(BASE + info['file'], archive, info['size'], info['sha256'])
failure_stage = 'unpackArchive' failure_stage = 'unpackArchive'
unpack(archive, stage, release) unpack(archive, stage, release)
failure_stage = 'programPreflight'
preflight_programs(stage)
failure_stage = 'runtimePreflight'
write_status('downloading', '正在检查运行环境和备份条件,服务仍可使用。', root)
controller(root, 'verify-runtime', timeout=1800)
failure_stage = 'backupPreflight'
preflight_backup(root, backup, release)
# Revalidate immediately before any service or disk mutation. # Revalidate immediately before any service or disk mutation.
validate_release(envelope, root) validate_release(envelope, root)
write_status('backingUp', '正在备份,服务即将重新启动。', root) write_status('backingUp', '正在备份,服务即将重新启动。', root)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False}) if (directory / 'recovery.json').exists():
copy_atomic(directory / 'recovery.json', backup / 'previous-recovery.json')
(directory / 'recovery.json').unlink()
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False,
'operationId': token, 'version': release['version'], 'sequence': release['sequence']})
failure_stage = 'stopService' failure_stage = 'stopService'
controller(root, 'stop', timeout=180) stop_requested = True
controller(root, 'stop', timeout=STOP_TIMEOUT)
stopped = True stopped = True
failure_stage = 'backupProgram' failure_stage = 'backupProgram'
backup_program(root, backup, release) backup_program(root, backup, release)
@@ -571,13 +762,15 @@ def worker(launcher, token, root=ROOT):
if (backup / 'database.sql').stat().st_size < 100: if (backup / 'database.sql').stat().st_size < 100:
raise UpdateError('数据库备份为空,已停止更新。') raise UpdateError('数据库备份为空,已停止更新。')
backed_up = True backed_up = True
controller(root, 'stop', timeout=180) controller(root, 'stop', timeout=STOP_TIMEOUT)
# The actual dump can exceed the source's physical storage. Do not # The actual dump can exceed the source's physical storage. Do not
# replace anything unless a separate rollback database still fits. # replace anything unless a separate rollback database still fits.
required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2 required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2
if shutil.disk_usage(root).free < required_recovery: if shutil.disk_usage(root).free < required_recovery:
raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。') raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。')
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True, atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True,
'operationId': token, 'version': release['version'], 'sequence': release['sequence'],
'databaseMayHaveChanged': False,
'backupManifestSha256': digest(backup / 'manifest.json'), 'backupManifestSha256': digest(backup / 'manifest.json'),
'programInventorySha256': digest(backup / 'program.json')}) 'programInventorySha256': digest(backup / 'program.json')})
write_status('installing', '正在安装更新。', root) write_status('installing', '正在安装更新。', root)
@@ -585,32 +778,46 @@ def worker(launcher, token, root=ROOT):
failure_stage = 'replaceProgram' failure_stage = 'replaceProgram'
replace_program(root, stage, backup, release) replace_program(root, stage, backup, release)
failure_stage = 'verifyProgram' failure_stage = 'verifyProgram'
controller(root, 'verify', timeout=180) controller(root, 'verify-runtime', timeout=300, program_only=True)
write_status('restarting', '正在重新启动,请稍候。', root) write_status('restarting', '正在重新启动,请稍候。', root)
failure_stage = 'restartService' failure_stage = 'restartService'
journal = read(directory / 'journal.json')
journal['databaseMayHaveChanged'] = True
atomic(directory / 'journal.json', journal)
process = controller(root, 'serve') process = controller(root, 'serve')
wait_ready(root, process) wait_ready(root, process)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'complete', 'databaseBackedUp': True}) journal.update(phase='complete', version=release['version'], sequence=release['sequence'],
write_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False) finishedAt=datetime.now(timezone.utc).isoformat())
atomic(directory / 'journal.json', journal)
committed = True
best_effort_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
except Exception as error: except Exception as error:
record_failure(root, failure_stage, error) record_failure(root, failure_stage, error)
message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。' message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。'
if stopped: journal = read(directory / 'journal.json', {})
if committed or (journal.get('backup') == backup.name and journal.get('phase') == 'complete'):
return # 提交后的展示或清理失败绝不能发起回滚。
if stop_requested:
try: try:
if program_saved and changed: if program_saved and changed:
rollback(root, backup, backed_up) rollback(root, backup, backed_up and journal.get('databaseMayHaveChanged', True))
else: else:
controller(root, 'stop', timeout=180) controller(root, 'stop', timeout=STOP_TIMEOUT)
prepare_old_backend_restart(root)
process = controller(root, 'serve') process = controller(root, 'serve')
wait_ready(root, process) wait_ready(root, process)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'rolledBack', 'databaseBackedUp': backed_up}) journal.update(backup=backup.name, phase='rolledBack', databaseBackedUp=backed_up)
atomic(directory / 'journal.json', journal)
message += ' 已恢复原版本。' message += ' 已恢复原版本。'
except Exception as rollback_error: except Exception as rollback_error:
record_failure(root, 'rollback', rollback_error, 'rollback-failure.json') record_failure(root, 'rollback', rollback_error, 'rollback-failure.json')
message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。' message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。'
write_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat()) best_effort_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
finally: finally:
try:
(directory / 'request.json').unlink(missing_ok=True) (directory / 'request.json').unlink(missing_ok=True)
except OSError as error:
record_failure(root, 'cleanupRequest', error)
def recover(root=ROOT): def recover(root=ROOT):
@@ -622,22 +829,31 @@ def recover_locked(root=ROOT):
directory = root / '.runtime/update' directory = root / '.runtime/update'
journal = read(directory / 'journal.json', {}) journal = read(directory / 'journal.json', {})
if journal.get('phase') in (None, 'complete', 'rolledBack'): if journal.get('phase') in (None, 'complete', 'rolledBack'):
if journal.get('phase') in ('complete', 'rolledBack'):
normalize_terminal_status(root, journal)
return status(root) return status(root)
name = journal.get('backup', '') name = journal.get('backup', '')
if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name): if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name):
raise UpdateError('更新恢复记录无效。') raise UpdateError('更新恢复记录无效。')
backup = directory / name backup = directory / name
if journal.get('phase') not in ('installing', 'stopping'):
raise UpdateError('更新恢复阶段无效,已保留现场。')
if journal.get('phase') == 'installing': if journal.get('phase') == 'installing':
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256') if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
or digest(backup / 'program.json') != journal.get('programInventorySha256')): or digest(backup / 'program.json') != journal.get('programInventorySha256')):
raise UpdateError('更新恢复备份校验失败,已保留现场。') raise UpdateError('更新恢复备份校验失败,已保留现场。')
rollback(root, backup, journal.get('databaseBackedUp') is True) verify_backup(root, backup, journal)
if journal.get('databaseBackedUp') is True:
recovery_progress(root, backup) # 先验证 sidecar 与 dump,尚未停止或替换任何文件。
rollback(root, backup, journal.get('databaseBackedUp') is True and journal.get('databaseMayHaveChanged', True))
else: else:
controller(root, 'stop', timeout=180) controller(root, 'stop', timeout=STOP_TIMEOUT)
prepare_old_backend_restart(root)
process = controller(root, 'serve') process = controller(root, 'serve')
wait_ready(root, process) wait_ready(root, process)
atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'}) atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'})
return write_status('failed', '上次更新中断,已恢复原版本。', root, available=False) best_effort_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
return status(root)
def main(): def main():