修复:加固 Windows 更新预检、提交点和可恢复的数据库回退流程

This commit is contained in:
2026-11-05 16:54:30 +08:00
parent 2315c83067
commit 3962521a5c
13 changed files with 1821 additions and 236 deletions
@@ -55,9 +55,13 @@ public class SystemUpdateService {
if (!enabled) return false;
try {
Map<?, ?> state = readMetadata("status.json");
Map<?, ?> journal = readMetadata("journal.json");
// 同一任务的持久提交记录优先于尚未成功刷新的进度提示。
if (journal.get("operationId") instanceof String operationId && !operationId.isBlank()
&& operationId.equals(state.get("operationId"))
&& Set.of("complete", "rolledBack").contains(journal.get("phase"))) return false;
if (state.get("state") instanceof String phase
&& Set.of("backingUp", "installing", "restarting", "rollingBack").contains(phase)) return true;
Map<?, ?> journal = readMetadata("journal.json");
return journal.get("phase") instanceof String phase && Set.of("stopping", "installing").contains(phase);
} catch (Exception failure) {
return true;
@@ -0,0 +1 @@
{"protocol":2}
@@ -60,4 +60,21 @@ class SystemUpdateServiceTest {
Files.writeString(directory.resolve("status.json"), "broken JSON");
assertTrue(service.maintenanceActive());
}
@Test
void committedOperationOverridesItsStaleProgressButNotAnotherOperation(@TempDir Path root) throws Exception {
var service = new SystemUpdateService(JsonMapper.builder().build(), root.toString(), false);
var enabled = SystemUpdateService.class.getDeclaredField("enabled");
enabled.setAccessible(true);
enabled.setBoolean(service, true);
var directory = Files.createDirectories(root.resolve(".runtime/update"));
String current = "a".repeat(32), previous = "b".repeat(32);
for (String phase : new String[]{"complete", "rolledBack"}) {
Files.writeString(directory.resolve("status.json"), "{\"state\":\"restarting\",\"operationId\":\"" + current + "\"}");
Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"" + phase + "\",\"operationId\":\"" + current + "\"}");
assertFalse(service.maintenanceActive(), phase);
Files.writeString(directory.resolve("journal.json"), "{\"phase\":\"" + phase + "\",\"operationId\":\"" + previous + "\"}");
assertTrue(service.maintenanceActive(), "another operation must not release maintenance");
}
}
}
+13
View File
@@ -64,3 +64,16 @@ python3 deploy/offline/publish_update.py \
Windows 的刷盘操作要求写句柄。程序及私有配置备份统一使用写入、flush、fsync 后再原子替换,移除复制后以只读句柄刷盘的步骤。备份失败仍停止更新并保留旧程序与数据库;不降低备份持久化要求。
“修复Windows更新.cmd”适配原始更新器和已安装检查超时修复的更新器,保留原清单、缓存修复和版本信息。更新失败诊断写入 `.runtime/update/failure.json`,只包含阶段、异常类型、系统错误码和源码位置,不含业务正文、凭据或异常参数;回退异常单独记录在 `rollback-failure.json`。
### Windows 更新流程加固(本地验证,尚未发布)
- 停服前完成签名、下载完整性、现有运行环境、备份路径长度及读写权限检查;预检失败保留当前服务运行。完整环境检查预算为30分钟,更新后仅核验程序和应用脚本(5分钟),不重复读取约4.6GB运行环境。
- 安全停服内部最多等待5分钟,更新器外层等待6分钟;已登记进程状态无法确认时停止后续操作,保留状态记录,不误报成功或结束未知进程。
- 仅当对应源码与原清单摘要一致时,忽略可再生的运行库字节码,不忽略真实源码损坏。
- `journal.phase=complete` 是不可逆提交点;状态展示或清理失败只记诊断,不再回退。后端按同一 operationId 的提交记录解除维护状态,旧任务记录不能解除新任务的维护。
- 新后端尚未启动时,回退不重新导入没有变化的数据库。可能发生数据库变更后,恢复进度写入独立 `recovery.json`,绑定备份程序清单及 SQL 摘要。导入及安全关闭确认后复用同一个恢复库,重复启动不重复导入,并幂等恢复配置和迁移回执。
- 不复用断电前未确认完整的导入;最多分配两次独立恢复尝试,且每次检查磁盘空间。超过次数停止自动重试并保留现场,不删除任何数据库或按前缀清理未知库。
- 旧安装修复入口同时更新 updater、control 和清单摘要;签名公钥、不可变 bootstrap、版本、业务数据和迁移回执不变,修复入口不启动服务。必须先安装该入口,保证不可变 bootstrap 未来恢复出的旧程序也包含加固后的恢复器。
- Windows 真机的 NTFS、进程权限和断电恢复仍需隔离副本验收。本轮本地故障注入及真实隔离 MySQL 验证不等同于生产 Windows 验收。
加固修复入口不能搭配已发布的旧0.3包使用:停服前会同时核对更新器、控制器及 JAR 中的恢复协议标记。不兼容的旧包会在服务仍运行时被拒绝;配套新包须包含本轮后端、控制器与更新器,禁止混装。
+438 -54
View File
@@ -6,6 +6,7 @@ import argparse
import contextlib
import hashlib
import json
import math
import os
from pathlib import Path
import platform
@@ -15,8 +16,12 @@ import signal
import socket
import subprocess
import sys
_old_dont_write_bytecode = sys.dont_write_bytecode
# Importing the controller must not create caches in the immutable package.
sys.dont_write_bytecode = True
import threading
import time
from pathlib import PurePosixPath
from urllib.request import ProxyHandler, build_opener
import psutil
@@ -25,12 +30,15 @@ ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import runtime_platform as host
from gateway import make_server
sys.dont_write_bytecode = _old_dont_write_bytecode
del _old_dont_write_bytecode
PRIVATE = ROOT / '.runtime'
DATA = ROOT / 'runtimedata'
CONFIG = PRIVATE / 'deployment.json'
STATE = PRIVATE / 'service.json'
STOP = PRIVATE / 'stop-request.json'
UPDATE_CONTROL_PROTOCOL = 2
IS_WINDOWS = os.name == 'nt'
EXE = '.exe' if IS_WINDOWS else ''
PYTHON = ROOT / 'runtimes/python' / ('python.exe' if IS_WINDOWS else 'bin/python3')
@@ -39,6 +47,7 @@ MYSQL = ROOT / 'runtimes/mysql/bin' / ('mysql' + EXE)
MYSQLD = ROOT / 'runtimes/mysql/bin' / ('mysqld' + EXE)
ADMIN = ROOT / 'runtimes/mysql/bin' / ('mysqladmin' + EXE)
OPENER = build_opener(ProxyHandler({}))
STOP_TIMEOUT_SECONDS = 300
def atomic(path, value):
@@ -65,7 +74,7 @@ def private_directory(path):
path.chmod(0o700)
def configuration():
def configuration(create=True):
if CONFIG.exists():
value = json.loads(CONFIG.read_text(encoding='utf-8'))
else:
@@ -73,7 +82,8 @@ def configuration():
'webBind': '0.0.0.0' if IS_WINDOWS else '127.0.0.1',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)}
atomic(CONFIG, value)
if create:
atomic(CONFIG, value)
value.setdefault('mysqlDatabase', 'quality_platform')
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', value['mysqlDatabase']):
raise RuntimeError('部署数据库名称无效。')
@@ -137,14 +147,159 @@ def record(process, command):
def matches(info):
"""Return True for this process, False for gone/replaced, None if unknown."""
if not isinstance(info, dict):
return None
try:
process = psutil.Process(int(info['pid']))
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
and abs(process.create_time() - info['created']) < .01
and os.path.normcase(str(Path(process.exe()).resolve())) == os.path.normcase(info['executable'])
and process.cmdline()[1:] == info['args'])
except (psutil.Error, KeyError, ValueError, TypeError):
pid = info['pid']
created = info['created']
executable = info['executable']
arguments = info['args']
if (type(pid) is not int or pid <= 0 or type(created) not in (int, float)
or not math.isfinite(created) or not isinstance(executable, str)
or not isinstance(arguments, list) or not all(isinstance(item, str) for item in arguments)):
return None
except (KeyError, ValueError, TypeError):
return None
try:
process = psutil.Process(pid)
except psutil.NoSuchProcess:
return False
except Exception:
return None
try:
if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE:
return False
if abs(process.create_time() - created) >= .01:
return False
if os.path.normcase(str(Path(process.exe()).resolve())) != os.path.normcase(executable):
return False
if process.cmdline()[1:] != arguments:
return False
return True
except psutil.NoSuchProcess:
return False
except Exception:
return None
def registered_alive(records):
"""Keep both confirmed-live and uninspectable registered process records."""
if not isinstance(records, list):
return [], True
alive, unknown = [], False
for info in records:
status = matches(info)
if status is True:
alive.append(info)
elif status is None:
alive.append(info)
unknown = True
return alive, unknown
def retain_registered_children(state, records, unresolved=False):
survivors, unknown = registered_alive(records)
state['children'] = survivors
unresolved = unresolved or unknown
if unresolved:
state['unknownChildren'] = True
return survivors, unresolved
def registered_named(records, name):
confirmed, unknown = [], []
for info in records:
if not isinstance(info, dict) or info.get('name') != name:
continue
status = matches(info)
if status is True:
confirmed.append(info)
elif status is None:
unknown.append(info)
return confirmed, unknown
def snapshot_process(process, name):
try:
result = record(process, process.cmdline())
result['name'] = name
return result
except (psutil.Error, OSError):
return None
def process_object_matches(process, created):
try:
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
and abs(process.create_time() - created) < .01)
except psutil.NoSuchProcess:
return False
except Exception:
# Permission failures and unexpected inspection errors are unknown,
# not evidence that the observed child has exited.
return None
def terminate_observed(processes, timeout):
"""Stop only process objects captured below a verified service child."""
unverified = False
for process, created, info in processes:
observed = process_object_matches(process, created)
if observed is None:
if info is None:
unverified = True
elif observed is True:
with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
process.terminate()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
states = [process_object_matches(process, created) for process, created, _info in processes]
if any(observed is None for observed in states):
unverified = unverified or any(observed is None and info is None
for observed, (_process, _created, info) in zip(states, processes))
break
if not any(states):
break
time.sleep(.1)
for process, created, info in processes:
observed = process_object_matches(process, created)
if observed is None:
if info is None:
unverified = True
elif observed is True:
with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
process.kill()
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
states = [process_object_matches(process, created) for process, created, _info in processes]
if any(observed is None for observed in states):
unverified = unverified or any(observed is None and info is None
for observed, (_process, _created, info) in zip(states, processes))
break
if not any(states):
break
time.sleep(.1)
survivors = []
for process, created, info in processes:
observed = process_object_matches(process, created)
if observed is None:
if info is None:
unverified = True
else:
survivors.append(info)
elif observed is True:
identity = matches(info) if info is not None else None
if info is None:
unverified = True
elif identity is None:
survivors.append(info)
elif identity is False:
survivors.append(info)
unverified = True
else:
survivors.append(info)
return survivors, unverified
def read_state():
@@ -154,6 +309,45 @@ def read_state():
return {}
def wait_for_stop(supervisor, request, timeout_seconds=STOP_TIMEOUT_SECONDS, interval=.5):
request_sent = False
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
supervisor_status = matches(supervisor)
if supervisor_status is None:
raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。')
if supervisor_status is True:
if not request_sent:
atomic(STOP, request)
request_sent = True
time.sleep(interval)
continue
state = read_state()
if 'supervisor' in state:
current_supervisor = state['supervisor']
current_status = matches(current_supervisor)
if current_status is None:
raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。')
if current_status is True:
if current_supervisor != supervisor:
raise RuntimeError('服务状态已变化,未停止任何不明进程。')
if not request_sent:
atomic(STOP, request)
request_sent = True
time.sleep(interval)
continue
survivors, unknown_children = registered_alive(state.get('children', []))
if unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
raise RuntimeError('服务监督程序已退出,但仍有未确认停止的已记录进程;未结束任何不明进程。')
if survivors:
time.sleep(interval)
continue
print('服务已停止。')
return
raise RuntimeError('服务仍在安全关闭,请稍后检查状态。')
def free_port(port, bind='127.0.0.1'):
# A recently closed TCP connection may remain in TIME_WAIT. Reject a live
# listener, but allow a clean restart without waiting for that state to expire.
@@ -182,22 +376,136 @@ def sql(value, query):
env=env, timeout=10)
def verify(check_hashes=True):
def manifest_name(name):
if (not isinstance(name, str) or not name or '\\' in name
or PurePosixPath(name).is_absolute()
or any(part in ('', '.', '..') for part in name.split('/'))):
raise RuntimeError('文件清单包含无效路径。')
return PurePosixPath(name)
def manifest_path(name):
parts = manifest_name(name).parts
path = ROOT.joinpath(*parts)
root = ROOT.resolve()
if not path.resolve(strict=False).is_relative_to(root):
raise RuntimeError('文件路径异常,已停止校验:' + name)
parent = path.parent
while parent != ROOT and parent != parent.parent:
if parent.is_symlink():
raise RuntimeError('文件路径异常,已停止校验:' + name)
parent = parent.parent
return path
def cache_source(name):
"""Return only timestamp-based bytecode sources from the bundled Python."""
if not isinstance(name, str) or '\\' in name:
return None
path = PurePosixPath(name)
if path.parts[:2] != ('runtimes', 'python') or any(part in ('.', '..') for part in name.split('/')):
return None
if path.parent.name != '__pycache__':
return None
match = re.fullmatch(r'(.+)\.cpython-\d+(?:\.opt-\d+)?\.pyc', path.name)
return (path.parent.parent / (match[1] + '.py')).as_posix() if match else None
def file_digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def regular_manifest_file(name):
path = manifest_path(name)
if path.is_symlink():
raise RuntimeError('文件路径异常,已停止校验:' + name)
return path
def program_file_path(name):
"""Mirror update_manager.program_path without importing code during a read-only check."""
path = manifest_name(name)
return (name == 'app/backend.jar' or name.startswith('app/web/')
or name == 'backend/src/main/resources/analysis-runtime-policy.json'
or name.startswith('scripts/') and path.suffix == '.py'
or name.startswith('ai/nao-quality/')
or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md'))
def verify_manifest_files(manifest, program_only=False):
entries = manifest.get('files')
if not isinstance(entries, list):
raise RuntimeError('文件清单无效。')
by_name = {}
for entry in entries:
if not isinstance(entry, dict) or not isinstance(entry.get('path'), str):
raise RuntimeError('文件清单无效。')
name = entry['path']
manifest_name(name)
if name in by_name:
raise RuntimeError('文件清单包含重复路径。')
by_name[name] = entry
ignored_caches = set()
if not program_only:
# Timestamp-based bytecode varies between Windows copies. Ignore only
# cache entries whose exact source is present and matches its manifest
# digest; ordinary Python source files are still hashed below.
for name in by_name:
source = cache_source(name)
if source is None:
continue
source_entry = by_name.get(source)
expected = source_entry.get('sha256') if isinstance(source_entry, dict) else None
source_path = regular_manifest_file(source)
if (not isinstance(expected, str) or not source_path.is_file()
or file_digest(source_path) != expected):
raise RuntimeError('对应源码缺失或校验失败,已停止校验:' + source)
cache_path = regular_manifest_file(name)
if cache_path.exists() and not cache_path.is_file():
raise RuntimeError('缓存文件路径异常,已停止校验:' + name)
ignored_caches.add(name)
for name, entry in by_name.items():
if program_only and not program_file_path(name):
continue
if name in ignored_caches:
continue
path = manifest_path(name)
expected_link = entry.get('symlink')
if expected_link is not None:
valid = path.is_symlink() and os.readlink(path) == expected_link
else:
if path.is_symlink():
valid = False
elif path.is_file():
with path.open('rb') as stream:
valid = hashlib.file_digest(stream, 'sha256').hexdigest() == entry.get('sha256')
else:
valid = False
if not valid:
raise RuntimeError('文件缺失或校验失败:' + name)
def verify_runtime(check_hashes=True, program_only=False):
manifest = json.loads((ROOT / 'manifest.json').read_text(encoding='utf-8'))
expected_os = 'windows' if IS_WINDOWS else platform.system().lower()
if manifest['platform'] != expected_os or platform.machine().lower() not in ('amd64', 'x86_64'):
raise RuntimeError('此包仅用于 ' + manifest['platform'] + ' 的 Intel/AMD 64 位电脑。')
if check_hashes:
for entry in manifest['files']:
path = ROOT / entry['path']
if entry.get('symlink') is not None:
valid = path.is_symlink() and os.readlink(path) == entry['symlink']
else:
valid = path.is_file() and hashlib.file_digest(path.open('rb'), 'sha256').hexdigest() == entry['sha256']
if not valid:
raise RuntimeError('文件缺失或校验失败:' + entry['path'])
value = configuration()
verify_manifest_files(manifest, program_only=program_only)
value = configuration(create=False)
env = environment(value)
if program_only:
application_probe = subprocess.run([str(PYTHON), '-c',
'import sys;sys.path.insert(0,sys.argv[1]);import nao_quality_runtime as r;'
'import nao_native_run,nao_quality_runtime_mcp,nao_visual_recovery;'
'assert len(r._config_fingerprint("https://offline.invalid/v1","offline-check","offline-check"))==64',
str(ROOT / 'scripts')], capture_output=True, env=env, timeout=45)
if application_probe.returncode:
raise RuntimeError('AI 应用脚本或策略资源检查失败。')
return value
for command in ([JAVA, '-version'], [MYSQLD, '--no-defaults', '--version']):
result = subprocess.run([str(a) for a in command], capture_output=True, env=env, timeout=30)
if result.returncode:
@@ -221,6 +529,13 @@ def verify(check_hashes=True):
return value
def verify(check_hashes=True):
# Keep the historical full verification entry point and its configuration
# initialization behavior for the installer and migration recovery tools.
verify_runtime(check_hashes=check_hashes)
return configuration()
def addresses(value):
ips = set()
if value['webBind'] == '0.0.0.0':
@@ -360,38 +675,90 @@ def serve(database_only=False):
finally:
state['ready'] = False
atomic(STATE, state)
# Stop backend first, then its isolated AI processes, then the database.
cleanup_failures, extra_survivors = [], []
unknown_children = False
# Only signal the backend and descendants observed beneath its verified
# PID/start-time/executable/arguments identity.
for info in reversed(children):
if info['name'] != 'backend' or not matches(info):
if not isinstance(info, dict) or info.get('name') != 'backend':
continue
backend_status = matches(info)
if backend_status is None:
extra_survivors.append(info)
continue
if backend_status is not True:
continue
process = psutil.Process(info['pid'])
descendants = process.children(recursive=True)
process.terminate()
try:
process.wait(35)
except psutil.TimeoutExpired:
if matches(info):
process.kill()
process = psutil.Process(info['pid'])
except psutil.NoSuchProcess:
continue
except Exception:
extra_survivors.append(info)
continue
observed = []
try:
descendants = process.children(recursive=True)
except psutil.NoSuchProcess:
descendants = []
except psutil.Error:
descendants = []
unknown_children = True
for child in descendants:
with contextlib.suppress(psutil.NoSuchProcess):
child.terminate()
with contextlib.suppress(OSError, subprocess.SubprocessError):
subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'],
env=environment(value), cwd=ROOT, capture_output=True, timeout=25)
if any(info['name'] == 'mysql' and matches(info) for info in children):
try:
created = child.create_time()
observed.append((child, created, snapshot_process(child, 'backend-descendant')))
except (psutil.Error, OSError):
unknown_children = True
observed.append((process, info['created'], info))
survivors, unverified = terminate_observed(observed, 35)
extra_survivors.extend(item for item in survivors if item.get('name') == 'backend-descendant')
unknown_children = unknown_children or unverified
try:
stopped = subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'],
env=environment(value), cwd=ROOT, capture_output=True, timeout=25)
if stopped.returncode:
cleanup_failures.append('analysisProcesses')
except (OSError, subprocess.SubprocessError):
cleanup_failures.append('analysisProcesses')
mysql_infos, unknown_mysql = registered_named(children, 'mysql')
extra_survivors.extend(unknown_mysql)
mysql_infos = [info for info in mysql_infos if matches(info) is True]
if mysql_infos:
env = environment(value)
env['MYSQL_PWD'] = value['rootPassword']
with contextlib.suppress(OSError, subprocess.SubprocessError):
subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env,
capture_output=True, timeout=60)
try:
shutdown = subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env,
capture_output=True, timeout=60)
if shutdown.returncode:
cleanup_failures.append('databaseShutdown')
except (OSError, subprocess.SubprocessError):
cleanup_failures.append('databaseShutdown')
# mysqladmin can return before InnoDB has finished closing files.
for info in children:
if info['name'] == 'mysql' and matches(info):
with contextlib.suppress(psutil.NoSuchProcess, psutil.TimeoutExpired):
for info in mysql_infos:
mysql_status = matches(info)
if mysql_status is None:
extra_survivors.append(info)
elif mysql_status is True:
try:
psutil.Process(info['pid']).wait(60)
survivors = [info for info in children if matches(info)]
if survivors:
state['children'] = survivors
except (psutil.NoSuchProcess, psutil.TimeoutExpired):
pass
except Exception:
extra_survivors.append(info)
unique_records, seen_records = [], set()
for info in [*children, *extra_survivors]:
if not isinstance(info, dict):
unknown_children = True
continue
key = (info.get('pid'), info.get('created'))
if key not in seen_records:
seen_records.add(key)
unique_records.append(info)
survivors, unknown_children = retain_registered_children(state, unique_records, unknown_children)
if cleanup_failures:
state['cleanupFailures'] = cleanup_failures
if survivors or unknown_children or cleanup_failures:
atomic(STATE, state)
else:
STATE.unlink(missing_ok=True)
@@ -402,10 +769,25 @@ def serve(database_only=False):
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'serve'))
parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'verify-runtime', 'serve'))
parser.add_argument('--lan', action='store_true')
parser.add_argument('--program-only', action='store_true')
parser.add_argument('--database-only', action='store_true', help=argparse.SUPPRESS)
args = parser.parse_args()
if args.program_only and args.action != 'verify-runtime':
parser.error('--program-only 只能与 verify-runtime 一起使用。')
if args.action == 'verify-runtime':
verify_runtime(program_only=args.program_only)
print('程序文件与 AI 应用脚本检查通过。' if args.program_only else
'文件完整性、Java、MySQL、AI Python 与 AI 原生程序检查通过。')
return
if args.action == 'start':
initial_state = read_state()
if 'supervisor' in initial_state and matches(initial_state['supervisor']) is None:
raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。')
_children, unknown_children = registered_alive(initial_state.get('children', []))
if unknown_children or initial_state.get('unknownChildren') or initial_state.get('cleanupFailures'):
raise RuntimeError('发现未确认停止的已记录进程,未启动任何服务。')
if args.action in ('start', 'serve', 'verify'):
private_directory(PRIVATE)
private_directory(DATA)
@@ -426,23 +808,24 @@ def main():
message = None
raise RuntimeError(message or '程序更新正在进行或尚未安全恢复,请查看更新记录。')
state = read_state()
running = matches(state.get('supervisor', {}))
running = matches(state['supervisor']) if 'supervisor' in state else False
if args.action == 'status':
if running is None:
print('无法确认服务状态')
return
print('已启动' if running and state.get('ready') else '正在启动或停止' if running else '已停止')
return
if running is None:
raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。')
if args.action == 'stop':
if not running:
if any(matches(info) for info in state.get('children', [])):
raise RuntimeError('发现上次退出后遗留的进程,请根据日志检查;未结束任何不明进程。')
children, unknown_children = registered_alive(state.get('children', []))
if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
raise RuntimeError('发现上次退出后仍有未确认停止的进程,请根据日志检查;未结束任何不明进程。')
print('服务已停止。')
return
atomic(STOP, {'token': state['token']})
for _ in range(240):
if not matches(state['supervisor']):
print('服务已停止。')
return
time.sleep(.5)
raise RuntimeError('服务仍在安全关闭,请稍后检查状态。')
wait_for_stop(state['supervisor'], {'token': state['token']})
return
if running:
if args.lan and configuration()['webBind'] != '0.0.0.0':
raise RuntimeError('请先停止服务,再使用 --lan 启动。')
@@ -450,7 +833,8 @@ def main():
for url in addresses(configuration()):
print(url)
return
if any(matches(info) for info in state.get('children', [])):
children, unknown_children = registered_alive(state.get('children', []))
if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
raise RuntimeError('发现上次退出后的服务进程,原数据已保留;请检查日志。')
value = verify()
if args.action == 'verify':
File diff suppressed because one or more lines are too long
@@ -0,0 +1,68 @@
import types
import unittest
from pathlib import Path
from unittest.mock import patch
import test_update_manager as fixtures
import update_bootstrap as bootstrap
class HardenedBootstrapTests(unittest.TestCase):
def test_immutable_bootstrap_restores_hardened_executor_and_reuses_durable_database(self):
f = fixtures.UpdateTests()
f.setUp()
self.addCleanup(f.doCleanups)
root, u = f.root, fixtures.update
# Firstaid preinstalls both executors before the old-version backup is made.
for name in ('control.py', 'update_manager.py'):
relative = 'deploy/offline/' + name
(root / relative).write_bytes((Path(__file__).parent / name).read_bytes())
manifest = u.read(root / 'manifest.json')
for entry in manifest['files']:
if entry['path'] in ('deploy/offline/control.py', 'deploy/offline/update_manager.py'):
entry['sha256'] = u.digest(root / entry['path'])
u.atomic(root / 'manifest.json', manifest)
backup = root / '.runtime/update/backup-20261105T170000Z-abcdef12'
backup.mkdir(parents=True)
u.backup_program(root, backup, f.release)
(backup / 'database.sql').write_bytes(b'-- restore snapshot\n' + b'SELECT 1;\n' * 30)
u.atomic(root / '.runtime/update/journal.json', {
'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True,
'databaseMayHaveChanged': True,
'backupManifestSha256': u.digest(backup / 'manifest.json'),
'programInventorySha256': u.digest(backup / 'program.json')})
imports, attempts = [], []
fail_once = True
real_spec = bootstrap.importlib.util.spec_from_file_location
def patched_spec(name, path):
spec = real_spec(name, path)
execute = spec.loader.exec_module
def load(module):
nonlocal fail_once
execute(module)
module.controller = lambda *a, **kw: types.SimpleNamespace(pid=1)
module.mysql = lambda *a, **kw: imports.append(kw['database'])
module.sql = lambda root, cfg, query: attempts.append(query)
def ready(root, process, database_only=False):
nonlocal fail_once
if not database_only and fail_once:
fail_once = False
raise module.UpdateError('simulated service restart failure')
module.wait_ready = ready
spec.loader.exec_module = load
return spec
with patch.object(bootstrap.importlib.util, 'spec_from_file_location', side_effect=patched_spec):
with self.assertRaisesRegex(RuntimeError, 'restart failure'):
bootstrap.recover(root)
chosen = u.read(root / '.runtime/deployment.json')['mysqlDatabase']
(root / 'deploy/offline/update_manager.py').write_bytes(b'broken new updater syntax')
bootstrap.recover(root)
self.assertEqual(1, len(imports))
self.assertEqual(1, sum('CREATE DATABASE' in query for query in attempts))
self.assertEqual(chosen, u.read(root / '.runtime/deployment.json')['mysqlDatabase'])
self.assertEqual((Path(__file__).parent / 'update_manager.py').read_bytes(), (root / 'deploy/offline/update_manager.py').read_bytes())
self.assertEqual('rolledBack', u.read(root / '.runtime/update/journal.json')['phase'])
self.assertEqual({'recovered': False}, bootstrap.recover(root))
if __name__ == '__main__':
unittest.main()
+299
View File
@@ -0,0 +1,299 @@
"""Read-only runtime verification and safe-stop regression tests."""
import hashlib
import importlib.util
import json
import platform
from pathlib import Path
import sys
import tempfile
import unittest
from unittest.mock import patch
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import control
class ControlEnvironmentTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.original = {
key: getattr(control, key)
for key in ('ROOT', 'PRIVATE', 'DATA', 'CONFIG', 'STATE', 'STOP', 'IS_WINDOWS',
'EXE', 'PYTHON', 'JAVA', 'MYSQL', 'MYSQLD', 'ADMIN')
}
control.ROOT = self.root
control.PRIVATE = self.root / '.runtime'
control.DATA = self.root / 'runtimedata'
control.CONFIG = control.PRIVATE / 'deployment.json'
control.STATE = control.PRIVATE / 'service.json'
control.STOP = control.PRIVATE / 'stop-request.json'
control.IS_WINDOWS = False
control.EXE = ''
control.PYTHON = self.root / 'runtimes/python/bin/python3'
control.JAVA = self.root / 'runtimes/java/bin/java'
control.MYSQL = self.root / 'runtimes/mysql/bin/mysql'
control.MYSQLD = self.root / 'runtimes/mysql/bin/mysqld'
control.ADMIN = self.root / 'runtimes/mysql/bin/mysqladmin'
self.addCleanup(self.restore_constants)
self.files = {
'scripts/app_probe.py': b'print("probe")',
'app/web/index.html': b'<html></html>',
'runtimes/python/bin/python3': b'python',
'runtimes/python/lib/python3.11/site-packages/nao_core/cache_source.py': b'good source',
'runtimes/python/lib/python3.11/site-packages/nao_core/__pycache__/cache_source.cpython-311.pyc': b'old timestamp cache',
}
for name, content in self.files.items():
self.write(name, content)
entries = [{'path': name, 'sha256': hashlib.sha256(content).hexdigest()}
for name, content in self.files.items()]
self.manifest = {'platform': platform.system().lower(), 'files': entries}
(self.root / 'manifest.json').write_text(json.dumps(self.manifest), encoding='utf-8')
def restore_constants(self):
for key, value in self.original.items():
setattr(control, key, value)
def write(self, name, content):
path = self.root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
def successful_probe(self, command, **kwargs):
return type('Result', (), {'returncode': 0})()
def test_verify_runtime_is_read_only_and_skips_only_source_verified_pyc(self):
self.write('runtimes/python/lib/python3.11/site-packages/nao_core/__pycache__/cache_source.cpython-311.pyc',
b'bytecode changed by Windows')
with patch.object(control.platform, 'machine', return_value='x86_64'), \
patch.object(control.subprocess, 'run', side_effect=self.successful_probe):
control.verify_runtime()
self.assertFalse(control.CONFIG.exists())
self.assertFalse(control.DATA.exists())
def test_pyc_is_not_ignored_when_its_manifest_source_hash_fails(self):
self.write('runtimes/python/lib/python3.11/site-packages/nao_core/cache_source.py', b'corrupted source')
with patch.object(control.platform, 'machine', return_value='x86_64'), \
patch.object(control.subprocess, 'run', side_effect=self.successful_probe) as run:
with self.assertRaisesRegex(RuntimeError, '对应源码缺失或校验失败'):
control.verify_runtime()
run.assert_not_called()
def test_legacy_bundle_verification_needs_only_control_gateway_and_runtime_platform(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
offline = root / 'deploy/offline'
offline.mkdir(parents=True)
scripts = root / 'scripts'
scripts.mkdir()
(offline / 'control.py').write_bytes(Path(control.__file__).read_bytes())
(offline / 'gateway.py').write_text('def make_server(*args, **kwargs): return None\n', encoding='utf-8')
(scripts / 'runtime_platform.py').write_text('# minimal legacy runtime platform module\n', encoding='utf-8')
files = {
'runtimes/python/lib/python3.11/site-packages/nao_core/cache_source.py': b'legacy source',
'runtimes/python/lib/python3.11/site-packages/nao_core/__pycache__/cache_source.cpython-311.pyc': b'legacy cache',
}
entries = []
for name, content in files.items():
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
entries.append({'path': name, 'sha256': hashlib.sha256(content).hexdigest()})
(root / 'manifest.json').write_text(json.dumps({
'platform': platform.system().lower(), 'files': entries,
}), encoding='utf-8')
self.assertFalse((offline / 'repair_windows_cache.py').exists())
original_path = list(sys.path)
isolated_path = []
for item in original_path:
resolved = Path(item or '.').resolve()
if resolved != HERE:
isolated_path.append(item)
with patch.object(sys, 'path', [str(offline), *isolated_path]), patch.dict(sys.modules):
for module_name in ('gateway', 'runtime_platform', 'repair_windows_cache'):
sys.modules.pop(module_name, None)
spec = importlib.util.spec_from_file_location('control_legacy_fixture', offline / 'control.py')
isolated_control = importlib.util.module_from_spec(spec)
spec.loader.exec_module(isolated_control)
with patch.object(isolated_control.platform, 'machine', return_value='x86_64'), \
patch.object(isolated_control.subprocess, 'run', side_effect=self.successful_probe):
isolated_control.verify_runtime()
self.assertNotIn('repair_windows_cache', sys.modules)
self.assertFalse((root / '.runtime/deployment.json').exists())
self.assertFalse((root / 'runtimedata').exists())
def test_program_only_checks_program_files_and_application_probe_only(self):
self.write('runtimes/python/bin/python3', b'changed runtime should not be rehashed')
calls = []
def probe(command, **kwargs):
calls.append(command)
return self.successful_probe(command, **kwargs)
with patch.object(control.platform, 'machine', return_value='x86_64'), \
patch.object(control.subprocess, 'run', side_effect=probe):
control.verify_runtime(program_only=True)
self.assertEqual(1, len(calls))
self.assertIn('-c', calls[0])
self.assertFalse(control.CONFIG.exists())
self.assertFalse(control.DATA.exists())
def test_stop_waits_until_recorded_children_are_gone(self):
supervisor = {'pid': 10}
child = {'pid': 20}
state = {'supervisor': supervisor, 'children': [child]}
ticks = iter([0, 0, .2, .4])
matches = iter([True, False, False, True, False, False, False])
with patch.object(control, 'atomic') as atomic, \
patch.object(control, 'read_state', return_value=state), \
patch.object(control, 'matches', side_effect=lambda _info: next(matches)), \
patch.object(control.time, 'monotonic', side_effect=lambda: next(ticks)), \
patch.object(control.time, 'sleep'), \
patch('builtins.print') as output:
control.wait_for_stop(supervisor, {'token': 'safe-token'}, timeout_seconds=1, interval=0)
atomic.assert_called_once_with(control.STOP, {'token': 'safe-token'})
output.assert_called_once_with('服务已停止。')
def test_wait_for_stop_blocks_when_registered_supervisor_is_uninspectable(self):
supervisor = {'pid': 7301, 'created': 1.0, 'executable': '/service', 'args': []}
state = {'supervisor': supervisor, 'children': [], 'token': 'safe-token'}
control.STATE.parent.mkdir(parents=True, exist_ok=True)
control.STATE.write_text(json.dumps(state), encoding='utf-8')
with patch.object(control.psutil, 'Process', side_effect=lambda pid: (_ for _ in ()).throw(
control.psutil.AccessDenied(pid=pid))), \
patch.object(control, 'atomic') as request, patch('builtins.print') as output:
with self.assertRaisesRegex(RuntimeError, '无法确认.*监督进程身份'):
control.wait_for_stop(supervisor, {'token': state['token']}, timeout_seconds=1)
request.assert_not_called()
output.assert_not_called()
def test_main_stop_blocks_when_registered_supervisor_is_uninspectable(self):
supervisor = {'pid': 7302, 'created': 1.0, 'executable': '/service', 'args': []}
control.STATE.parent.mkdir(parents=True, exist_ok=True)
control.STATE.write_text(json.dumps({
'supervisor': supervisor, 'children': [], 'token': 'safe-token',
}), encoding='utf-8')
with patch.object(sys, 'argv', ['control.py', 'stop']), \
patch.object(control.psutil, 'Process', side_effect=lambda pid: (_ for _ in ()).throw(
control.psutil.AccessDenied(pid=pid))), \
patch.object(control, 'atomic') as request, patch('builtins.print') as output:
with self.assertRaisesRegex(RuntimeError, '无法确认.*监督进程身份'):
control.main()
request.assert_not_called()
output.assert_not_called()
def test_main_start_blocks_before_recovery_when_registered_supervisor_is_uninspectable(self):
supervisor = {'pid': 7303, 'created': 1.0, 'executable': '/service', 'args': []}
control.PRIVATE.mkdir(parents=True)
control.DATA.mkdir(parents=True)
control.STATE.write_text(json.dumps({
'supervisor': supervisor, 'children': [], 'token': 'safe-token',
}), encoding='utf-8')
updater = self.root / 'deploy/offline/update_manager.py'
updater.parent.mkdir(parents=True)
updater.write_text('# staged updater', encoding='utf-8')
with patch.object(sys, 'argv', ['control.py', 'start']), \
patch.object(control, 'IS_WINDOWS', True), \
patch.object(control, 'private_directory') as private_directory, \
patch.object(control.psutil, 'Process', side_effect=lambda pid: (_ for _ in ()).throw(
control.psutil.AccessDenied(pid=pid))), \
patch.object(control.subprocess, 'run') as recovery, \
patch('builtins.print') as output:
with self.assertRaisesRegex(RuntimeError, '无法确认.*监督进程身份'):
control.main()
recovery.assert_not_called()
private_directory.assert_not_called()
output.assert_not_called()
def test_stop_does_not_report_success_when_child_survives_budget(self):
supervisor = {'pid': 10}
child = {'pid': 20}
state = {'supervisor': supervisor, 'children': [child]}
ticks = iter([0, 0, .5, 1.1])
def is_running(info):
return info is child
with patch.object(control, 'atomic'), \
patch.object(control, 'read_state', return_value=state), \
patch.object(control, 'matches', side_effect=is_running), \
patch.object(control.time, 'monotonic', side_effect=lambda: next(ticks)), \
patch.object(control.time, 'sleep'), \
patch('builtins.print') as output:
with self.assertRaisesRegex(RuntimeError, '仍在安全关闭'):
control.wait_for_stop(supervisor, {'token': 'safe-token'}, timeout_seconds=1, interval=0)
output.assert_not_called()
def test_stop_has_three_hundred_second_internal_budget(self):
self.assertEqual(300, control.STOP_TIMEOUT_SECONDS)
def test_unknown_process_state_is_not_treated_as_exit_or_signalled(self):
class Uninspectable:
def is_running(self):
return True
def status(self):
raise control.psutil.AccessDenied(pid=4321)
def terminate(self):
raise AssertionError('unknown process must not be terminated')
def kill(self):
raise AssertionError('unknown process must not be killed')
process = Uninspectable()
with patch.object(control.time, 'monotonic', side_effect=[0, 0, 0, 0]):
survivors, unverified = control.terminate_observed([(process, 1.0, None)], timeout=0)
self.assertEqual([], survivors)
self.assertTrue(unverified)
def test_registered_backend_and_mysql_are_retained_when_identity_is_unknown(self):
backend = {'pid': 8101, 'created': 1.0, 'executable': '/java', 'args': ['-jar'] , 'name': 'backend'}
mysql = {'pid': 8102, 'created': 2.0, 'executable': '/mysqld', 'args': ['--port=53306'], 'name': 'mysql'}
def denied(pid):
raise control.psutil.AccessDenied(pid=pid)
with patch.object(control.psutil, 'Process', side_effect=denied):
state = {}
retained, unknown = control.retain_registered_children(state, [backend, mysql])
backend_confirmed, backend_unknown = control.registered_named([backend, mysql], 'backend')
mysql_confirmed, mysql_unknown = control.registered_named([backend, mysql], 'mysql')
self.assertEqual([backend, mysql], retained)
self.assertTrue(unknown)
self.assertEqual([backend, mysql], state['children'])
self.assertTrue(state['unknownChildren'])
self.assertEqual([], backend_confirmed)
self.assertEqual([backend], backend_unknown)
self.assertEqual([], mysql_confirmed)
self.assertEqual([mysql], mysql_unknown)
def test_no_such_process_is_confirmed_gone(self):
class Gone:
def is_running(self):
raise control.psutil.NoSuchProcess(pid=4321)
with patch.object(control.time, 'monotonic', side_effect=[0, 0, 0, 0]):
survivors, unverified = control.terminate_observed([(Gone(), 1.0, None)], timeout=0)
self.assertEqual([], survivors)
self.assertFalse(unverified)
def test_malformed_registered_identity_is_unknown(self):
self.assertIsNone(control.matches({'pid': 4321, 'created': float('nan'),
'executable': '/service', 'args': []}))
def test_verify_runtime_cli_bypasses_running_service_shortcut(self):
with patch.object(sys, 'argv', ['control.py', 'verify-runtime', '--program-only']), \
patch.object(control.platform, 'machine', return_value='x86_64'), \
patch.object(control, 'verify_runtime', return_value={}) as verify, \
patch.object(control, 'private_directory') as private_directory:
control.main()
verify.assert_called_once_with(program_only=True)
private_directory.assert_not_called()
if __name__ == '__main__':
unittest.main()
+229
View File
@@ -0,0 +1,229 @@
import json
import zipfile
from pathlib import Path
import types
import unittest
from unittest.mock import patch
import test_update_manager as fixtures
update = fixtures.update
class HardeningTests(unittest.TestCase):
def setUp(self):
self.f = fixtures.UpdateTests()
self.f.setUp()
self.addCleanup(self.f.doCleanups)
self.root = self.f.root
def backup(self):
b = self.root / '.runtime/update/backup-20261105T120000Z-abcdef12'
b.mkdir(parents=True)
update.backup_program(self.root, b, self.f.release)
(b / 'database.sql').write_bytes(b'-- database snapshot\n' + b'SELECT 1;\n' * 30)
update.atomic(self.root / '.runtime/update/journal.json', {
'backup': b.name, 'phase': 'installing', 'databaseBackedUp': True,
'databaseMayHaveChanged': True,
'backupManifestSha256': update.digest(b / 'manifest.json'),
'programInventorySha256': update.digest(b / 'program.json')})
return b
def test_completed_status_failure_never_rolls_back_committed_program(self):
original = update.write_status
def status(state, message, root=update.ROOT, **extra):
if state == 'completed':
raise PermissionError('status write failure')
return original(state, message, root, **extra)
with patch.object(update, 'write_status', side_effect=status):
calls, restores, _ = self.f.worker_harness()
self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence'])
self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase'])
self.assertFalse(restores)
self.assertEqual('completed', update.status(self.root)['state'])
def test_partially_completed_stop_attempt_restores_old_service(self):
failed = False
def hook(root, action, database_only, timeout, program_only):
nonlocal failed
if action == 'stop' and not failed:
failed = True
raise update.UpdateError('service still closing')
calls, restores, _ = self.f.worker_harness(control_hook=hook)
self.assertIn(('serve', False), calls)
self.assertFalse(restores)
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase'])
def test_runtime_failure_occurs_before_stop_or_replacement(self):
def hook(root, action, database_only, timeout, program_only):
if action == 'verify-runtime' and not program_only:
raise update.UpdateError('runtime preflight failed')
calls, _, _ = self.f.worker_harness(control_hook=hook)
self.assertNotIn(('stop', False), calls)
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
self.assertEqual('failed', update.read(self.root / '.runtime/update/status.json')['state'])
def test_program_verification_failure_does_not_restore_unchanged_database(self):
calls, imports, queries = self.f.worker_harness(fail_verify=True)
self.assertFalse(imports)
self.assertFalse(any('CREATE DATABASE' in q for q in queries))
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
def test_durable_database_is_reused_after_startup_failure_and_bootstrap_replay(self):
b = self.backup()
imports, queries = [], []
fail = True
def ready(root, process, database_only=False):
nonlocal fail
if not database_only and fail:
fail = False
raise update.UpdateError('temporary startup failure')
with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \
patch.object(update, 'wait_ready', side_effect=ready), \
patch.object(update, 'mysql', side_effect=lambda *args, **kwargs: imports.append(kwargs['database'])), \
patch.object(update, 'sql', side_effect=lambda root, config, q: queries.append(q)):
with self.assertRaises(update.UpdateError):
update.rollback(self.root, b, True)
selected = update.read(self.root / '.runtime/deployment.json')['mysqlDatabase']
update.restore_program(self.root, b) # immutable bootstrap replays old program/private snapshot
update.recover_locked(self.root)
self.assertEqual(1, len(imports))
self.assertEqual(1, sum('CREATE DATABASE' in q for q in queries))
self.assertEqual(selected, update.read(self.root / '.runtime/deployment.json')['mysqlDatabase'])
def test_recovery_progress_rejects_changed_dump_before_any_service_mutation(self):
b = self.backup()
fail = True
def ready(root, process, database_only=False):
if not database_only:
raise update.UpdateError('temporary startup failure')
with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \
patch.object(update, 'wait_ready', side_effect=ready), \
patch.object(update, 'mysql'), patch.object(update, 'sql'):
with self.assertRaises(update.UpdateError):
update.rollback(self.root, b, True)
(b / 'database.sql').write_bytes(b'altered dump')
with patch.object(update, 'controller') as controller, self.assertRaises(update.UpdateError):
update.recover_locked(self.root)
controller.assert_not_called()
def test_old_terminal_status_cannot_override_new_operation(self):
update.atomic(self.root / '.runtime/update/journal.json', {
'phase': 'rolledBack', 'operationId': 'old-operation'})
update.write_status('downloading', 'downloading new attempt', self.root, operationId='new-operation')
with patch.object(update, 'live', return_value=True):
self.assertEqual('downloading', update.status(self.root)['state'])
def test_uncertain_complete_write_never_triggers_rollback(self):
original = update.atomic
def atomic(path, value):
original(path, value)
if path.name == 'journal.json' and value.get('phase') == 'complete':
raise OSError('write receipt lost after durable commit')
with patch.object(update, 'atomic', side_effect=atomic):
_, imports, _ = self.f.worker_harness()
self.assertEqual(6, update.read(self.root / 'manifest.json')['updateSequence'])
self.assertFalse(imports)
journal = update.read(self.root / '.runtime/update/journal.json')
self.assertEqual('complete', journal['phase'])
self.assertIn('backupManifestSha256', journal)
def test_interrupted_import_attempts_are_bounded_without_dropping_databases(self):
b = self.backup()
imports, queries = [], []
def mysql(*args, **kwargs):
imports.append(kwargs['database'])
raise update.UpdateError('simulated partial import')
with patch.object(update, 'controller', return_value=types.SimpleNamespace(pid=1)), \
patch.object(update, 'wait_ready'), patch.object(update, 'mysql', side_effect=mysql), \
patch.object(update, 'sql', side_effect=lambda root, config, q: queries.append(q)):
for _ in range(update.MAX_RECOVERY_ATTEMPTS + 1):
with self.assertRaises(update.UpdateError):
update.recover_locked(self.root)
self.assertEqual(update.MAX_RECOVERY_ATTEMPTS, len(imports))
self.assertEqual(len(imports), len(set(imports)))
self.assertFalse(any('DROP' in q for q in queries))
self.assertNotIn(self.f.config['mysqlDatabase'], imports)
def test_database_is_not_durable_until_clean_stop_succeeds(self):
b = self.backup()
stops = 0
def controller(root, action, database_only=False, **kwargs):
nonlocal stops
if action == 'stop':
stops += 1
if stops == 2:
raise update.UpdateError('database still closing')
return types.SimpleNamespace(pid=1)
with patch.object(update, 'controller', side_effect=controller), \
patch.object(update, 'wait_ready'), patch.object(update, 'mysql'), patch.object(update, 'sql'):
with self.assertRaises(update.UpdateError):
update.rollback(self.root, b, True)
self.assertEqual('importing', update.read(self.root / '.runtime/update/recovery.json')['stage'])
def test_recovery_cannot_be_redirected_to_original_database(self):
b = self.backup()
binding = update.recovery_binding(b)
update.atomic(self.root / '.runtime/update/recovery.json', {
**binding, 'attempt': 1, 'stage': 'databaseDurable', 'database': self.f.config['mysqlDatabase']})
with patch.object(update, 'controller') as controller, self.assertRaises(update.UpdateError):
update.recover_locked(self.root)
controller.assert_not_called()
def test_backup_preflight_failure_keeps_live_service_and_program(self):
with patch.object(update, 'preflight_backup', side_effect=PermissionError('backup path not writable')):
calls, _, _ = self.f.worker_harness()
self.assertNotIn(('stop', False), calls)
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
def test_staged_protocol_blocks_old_or_mixed_package_before_stop(self):
stage = self.root / '.runtime/staged-protocol'
scripts = stage / 'deploy/offline'
scripts.mkdir(parents=True)
(stage / 'app').mkdir()
(scripts / 'update_manager.py').write_text('UPDATE_PROTOCOL = 2\n')
(scripts / 'control.py').write_text('UPDATE_CONTROL_PROTOCOL = 2\n')
with zipfile.ZipFile(stage / 'app/backend.jar', 'w') as jar:
jar.writestr('BOOT-INF/classes/update-protocol.json', '{"protocol":2}')
update.preflight_programs(stage)
(scripts / 'control.py').write_text('OLD_CONTROLLER = True\n')
with self.assertRaisesRegex(update.UpdateError, '服务尚未停止'):
update.preflight_programs(stage)
(scripts / 'control.py').write_text('UPDATE_CONTROL_PROTOCOL = 2\n')
with zipfile.ZipFile(stage / 'app/backend.jar', 'w') as jar:
jar.writestr('old-backend', 'old protocol')
with self.assertRaisesRegex(update.UpdateError, '服务尚未停止'):
update.preflight_programs(stage)
def test_actual_worker_rejects_incompatible_signed_payload_before_stopping(self):
calls, imports, _ = self.f.worker_harness(program_hook=update.preflight_programs)
self.assertNotIn(('stop', False), calls)
self.assertFalse(imports)
self.assertEqual(self.f.manifest, update.read(self.root / 'manifest.json'))
self.assertIn('更新包', update.read(self.root / '.runtime/update/status.json')['message'])
def test_legacy_backend_cannot_be_left_in_active_maintenance_after_final_status_failure(self):
original = update.write_status
def status(state, message, root=update.ROOT, **extra):
if state == 'failed':
raise PermissionError('final UI state blocked')
return original(state, message, root, **extra)
with patch.object(update, 'write_status', side_effect=status):
self.f.worker_harness(fail_start=True)
self.assertEqual('rolledBack', update.read(self.root / '.runtime/update/journal.json')['phase'])
raw = update.read(self.root / '.runtime/update/status.json')['state']
self.assertNotIn(raw, {'backingUp', 'installing', 'restarting', 'rollingBack'})
def test_legacy_terminal_recovery_persists_status_reconciliation(self):
directory = self.root / '.runtime/update'
update.atomic(directory / 'journal.json', {'phase': 'rolledBack', 'operationId': 'same'})
update.atomic(directory / 'status.json', {'state': 'rollingBack', 'operationId': 'same'})
update.recover_locked(self.root)
self.assertEqual('failed', update.read(directory / 'status.json')['state'])
update.atomic(directory / 'status.json', {'state': 'rollingBack', 'operationId': 'same'})
with patch.object(update, 'write_status', side_effect=PermissionError('status blocked')), self.assertRaises(PermissionError):
update.recover_locked(self.root)
if __name__ == '__main__':
unittest.main()
+13 -5
View File
@@ -194,13 +194,14 @@ class UpdateTests(unittest.TestCase):
self.assertEqual(self.manifest, update.read(self.root / 'manifest.json'))
self.assertEqual(self.files['app/web/obsolete.js'], (self.root / 'app/web/obsolete.js').read_bytes())
def worker_harness(self, fail_verify=False):
def worker_harness(self, fail_verify=False, control_hook=None, fail_start=False, program_hook=None):
directory = self.root / '.runtime/update'
update.atomic(directory / 'request.json', {'token': 'a' * 32})
archive = self.archive()
envelope = self.envelope()
original_download = update.download
calls, restores, queries = [], [], []
failed_start = False
def fake_download(url, target=None, *args):
if target:
@@ -208,9 +209,15 @@ class UpdateTests(unittest.TestCase):
return b''
return json.dumps(envelope).encode()
def fake_control(root, action, database_only=False, timeout=0):
def fake_control(root, action, database_only=False, timeout=0, program_only=False):
nonlocal failed_start
calls.append((action, database_only))
if action == 'verify' and fail_verify:
if fail_start and action == 'serve' and not database_only and not failed_start:
failed_start = True
raise update.UpdateError('模拟新版启动失败。')
if control_hook:
control_hook(root, action, database_only, timeout, program_only)
if action in ('verify', 'verify-runtime') and fail_verify and (program_only or action == 'verify'):
raise update.UpdateError('模拟新版启动校验失败。')
return types.SimpleNamespace(pid=1234)
@@ -226,6 +233,7 @@ class UpdateTests(unittest.TestCase):
patch.object(update.psutil, 'Process', return_value=process), \
patch.object(update, 'download', side_effect=fake_download), \
patch.object(update, 'controller', side_effect=fake_control), \
patch.object(update, 'preflight_programs', side_effect=program_hook), \
patch.object(update, 'wait_ready'), \
patch.object(update, 'mysql', side_effect=fake_mysql), \
patch.object(update, 'sql', side_effect=lambda root, config, query: queries.append(query)):
@@ -265,7 +273,7 @@ class UpdateTests(unittest.TestCase):
def test_complete_update_orchestration_with_isolated_database_backup(self):
calls, restores, queries = self.worker_harness()
self.assertEqual([('stop', False), ('serve', True), ('stop', False), ('verify', False), ('serve', False)], calls)
self.assertEqual([('verify-runtime', False), ('stop', False), ('serve', True), ('stop', False), ('verify-runtime', False), ('serve', False)], calls)
self.assertFalse(restores)
self.assertEqual('completed', update.read(self.root / '.runtime/update/status.json')['state'])
self.assertEqual('complete', update.read(self.root / '.runtime/update/journal.json')['phase'])
@@ -276,7 +284,7 @@ class UpdateTests(unittest.TestCase):
update.atomic(self.root / '.runtime/deployment.json', self.config)
update.atomic(self.root / '.runtime/migration-receipt.json', {'migrationId': 'isolated-test-migration',
'database': self.config['mysqlDatabase'], 'migrationManifestSha256': 'b' * 64})
calls, restores, queries = self.worker_harness(fail_verify=True)
calls, restores, queries = self.worker_harness(fail_start=True)
self.assertEqual(self.manifest, update.read(self.root / 'manifest.json'))
config = update.read(self.root / '.runtime/deployment.json')
self.assertRegex(config['mysqlDatabase'], r'^quality_restore_[a-f0-9]{24}$')
+2 -3
View File
@@ -54,10 +54,9 @@ class WindowsCacheRepairTest(unittest.TestCase):
patch.object(control.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0)):
control.verify()
def test_actual_startup_hash_check_fails_on_regenerated_cache_then_passes_after_repair(self):
def test_startup_accepts_regenerated_cache_only_when_source_hash_is_valid(self):
(self.root / self.cache).write_bytes(b'new-bytecode-created-on-windows')
with self.assertRaisesRegex(RuntimeError, '__future__.*pyc'):
self.verify()
self.verify() # 合法源码允许运行库重建缓存;原位修复仍可清除旧缓存清单。
result = fix.repair(self.root)
self.assertEqual(result['cacheEntriesRemoved'], 1)
self.verify()
@@ -1,127 +1,253 @@
import hashlib
import ctypes
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
from unittest.mock import patch
from unittest.mock import Mock, patch
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import repair_windows_update_check as fix
ORIGINAL_SHA = fix.OLD_SHA
class UpdateCheckRepairTest(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix='quality source path repair ')
self.temp = tempfile.TemporaryDirectory(prefix='quality firstaid repair ')
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name) / '中文 QualityCenter & files'
self.root.mkdir()
self.original = b'original source fixture\n'
self.old_sha = hashlib.sha256(self.original).hexdigest()
self.guard = patch.object(fix, 'OLD_SHA', self.old_sha)
self.guard.start()
self.addCleanup(self.guard.stop)
for name, value in {fix.SCRIPT: self.original,
'deploy/bootstrap/windows-start.ps1': b'original bootstrap',
'migration/database.sql.gz': b'existing migration payload',
'.runtime/deployment.json': b'original private configuration',
'runtimedata/mysql/retained-database': b'imported database data'}.items():
self.old_manager = b'known old updater fixture\n'
self.old_control = b'known old controller fixture\n'
self.old_manager_sha = hashlib.sha256(self.old_manager).hexdigest()
self.old_control_sha = hashlib.sha256(self.old_control).hexdigest()
self.reviewed_manager_hashes = fix.OLD_MANAGER_SHA256
self.reviewed_control_hashes = fix.OLD_CONTROL_SHA256
self.manager_guard = patch.object(fix, 'OLD_MANAGER_SHA256', frozenset({self.old_manager_sha}))
self.control_guard = patch.object(fix, 'OLD_CONTROL_SHA256', frozenset({self.old_control_sha}))
self.manager_guard.start()
self.control_guard.start()
self.addCleanup(self.manager_guard.stop)
self.addCleanup(self.control_guard.stop)
self.manager_payload, self.control_payload = fix.read_hardened_sources()
for name, value in {
fix.MANAGER: self.old_manager,
fix.CONTROL: self.old_control,
'deploy/bootstrap/windows-start.ps1': b'original bootstrap',
'migration/database.sql.gz': b'existing migration payload',
'.runtime/deployment.json': b'original private configuration',
'runtimedata/mysql/retained-database': b'imported database data',
}.items():
target = self.root / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(value)
self.manifest = {'platform': 'windows', 'updateVersion': '2026.11.04.1', 'updateSequence': 1,
'runtimeFingerprint': 'unchanged', 'otherMetadata': {'keep': True},
'files': [{'path': fix.SCRIPT, 'sha256': self.old_sha},
{'path': 'deploy/bootstrap/windows-start.ps1', 'sha256': 'unchanged'}]}
self.manifest = {
'platform': 'windows', 'updateVersion': '2026.11.04.1', 'updateSequence': 1,
'runtimeFingerprint': 'unchanged', 'otherMetadata': {'keep': True},
'files': [
{'path': fix.MANAGER, 'sha256': self.old_manager_sha},
{'path': fix.CONTROL, 'sha256': self.old_control_sha},
{'path': 'deploy/bootstrap/windows-start.ps1', 'sha256': 'unchanged'},
],
}
self.manifest_path = self.root / 'manifest.json'
self.manifest_path.write_text(json.dumps(self.manifest), encoding='utf-8')
def test_backup_manifest_and_script_only_preserving_database_and_configuration(self):
def test_backups_both_programs_and_manifest_and_preserves_private_data(self):
private = {name: (self.root / name).read_bytes() for name in
['migration/database.sql.gz', '.runtime/deployment.json', 'runtimedata/mysql/retained-database']}
result = fix.repair(self.root)
['migration/database.sql.gz', '.runtime/deployment.json',
'runtimedata/mysql/retained-database']}
result = fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertTrue(result['updateCheckRepaired'])
self.assertEqual((self.root / fix.SCRIPT).read_bytes(), fix.PATCHED_SCRIPT)
self.assertTrue(result['controlRecoveryRepaired'])
self.assertEqual(result['changedFiles'], 3)
self.assertEqual((self.root / fix.MANAGER).read_bytes(), self.manager_payload)
self.assertEqual((self.root / fix.CONTROL).read_bytes(), self.control_payload)
expected = json.loads(json.dumps(self.manifest))
expected['files'][0]['sha256'] = fix.NEW_SHA
expected['files'][0]['sha256'] = hashlib.sha256(self.manager_payload).hexdigest()
expected['files'][1]['sha256'] = hashlib.sha256(self.control_payload).hexdigest()
self.assertEqual(json.loads(self.manifest_path.read_text()), expected)
backup = self.root / result['backup']
self.assertEqual((backup / 'update_manager.py').read_bytes(), self.original)
self.assertEqual((backup / 'update_manager.py').read_bytes(), self.old_manager)
self.assertEqual((backup / 'control.py').read_bytes(), self.old_control)
self.assertEqual(json.loads((backup / 'manifest.json').read_text()), self.manifest)
for name, value in private.items():
self.assertEqual((self.root / name).read_bytes(), value)
self.assertTrue(fix.repair(self.root)['alreadyRepaired'])
self.assertTrue(fix.repair(self.root, self.manager_payload, self.control_payload)['alreadyRepaired'])
def test_modified_script_or_manifest_is_not_approved_as_a_repair(self):
before = self.manifest_path.read_bytes()
(self.root / fix.SCRIPT).write_bytes(b'unrecognized source')
def test_unknown_program_or_manifest_digest_is_never_overwritten(self):
original_manifest = self.manifest_path.read_bytes()
(self.root / fix.MANAGER).write_bytes(b'unrecognized updater')
with self.assertRaisesRegex(RuntimeError, '版本'):
fix.repair(self.root)
self.assertEqual(self.manifest_path.read_bytes(), before)
fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertEqual(self.manifest_path.read_bytes(), original_manifest)
(self.root / fix.MANAGER).write_bytes(self.old_manager)
value = json.loads(original_manifest)
value['files'][0]['sha256'] = '0' * 64
self.manifest_path.write_text(json.dumps(value), encoding='utf-8')
with self.assertRaisesRegex(RuntimeError, '版本'):
fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertEqual((self.root / fix.MANAGER).read_bytes(), self.old_manager)
def test_failed_second_write_restores_original_program_and_manifest(self):
before = self.manifest_path.read_bytes()
def test_failed_manifest_write_rolls_back_both_programs_and_manifest(self):
before = {fix.MANAGER: (self.root / fix.MANAGER).read_bytes(),
fix.CONTROL: (self.root / fix.CONTROL).read_bytes(),
'manifest.json': self.manifest_path.read_bytes()}
original_atomic = fix.atomic
calls = []
def failing(path, data):
calls.append(path)
if len(calls) == 2:
if len(calls) == 3:
raise OSError('simulated manifest write failure')
original_atomic(path, data)
with patch.object(fix, 'atomic', side_effect=failing), self.assertRaises(OSError):
fix.repair(self.root)
self.assertEqual((self.root / fix.SCRIPT).read_bytes(), self.original)
self.assertEqual(self.manifest_path.read_bytes(), before)
fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertEqual((self.root / fix.MANAGER).read_bytes(), before[fix.MANAGER])
self.assertEqual((self.root / fix.CONTROL).read_bytes(), before[fix.CONTROL])
self.assertEqual(self.manifest_path.read_bytes(), before['manifest.json'])
self.assertEqual(len(list((self.root / '.runtime/update-check-repair-backups').iterdir())), 1)
def test_retry_completes_an_interrupted_script_manifest_pair(self):
(self.root / fix.SCRIPT).write_bytes(fix.PATCHED_SCRIPT)
fix.repair(self.root)
self.assertEqual(json.loads(self.manifest_path.read_text())['files'][0]['sha256'], fix.NEW_SHA)
def test_retry_completes_old_new_mix_and_accepts_rolled_back_journal(self):
(self.root / fix.MANAGER).write_bytes(self.manager_payload)
journal = self.root / '.runtime/update/journal.json'
journal.parent.mkdir(parents=True)
journal.write_text(json.dumps({'phase': 'rolledBack'}), encoding='utf-8')
result = fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertTrue(result['updateCheckRepaired'])
self.assertEqual((self.root / fix.CONTROL).read_bytes(), self.control_payload)
manifest = json.loads(self.manifest_path.read_text())
self.assertEqual(manifest['files'][0]['sha256'], hashlib.sha256(self.manager_payload).hexdigest())
self.assertEqual(manifest['files'][1]['sha256'], hashlib.sha256(self.control_payload).hexdigest())
def test_cmd_payload_runs_with_bundled_standard_library_without_service_restart(self):
# 真实原包脚本由交付包集成验证;此单元测试使用受控旧源码摘要。
command = self.root / '修复更新检查.cmd'
body = fix.command_file().decode('utf-8').replace(ORIGINAL_SHA, self.old_sha)
command.write_bytes(body.encode('utf-8'))
code = body.split(' -c "', 1)[1].split('" "%~f0"', 1)[0]
result = subprocess.run([sys.executable, '-I', '-S', '-B', '-X', 'utf8', '-c', code, str(command)], capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual((self.root / fix.SCRIPT).read_bytes(), fix.PATCHED_SCRIPT)
self.assertNotIn('powershell.exe', body)
self.assertIn('无需停止服务或重新导入数据库', body)
def test_active_update_lock_blocks_patching(self):
before = self.manifest_path.read_bytes()
with fix.update_lock(self.root), self.assertRaisesRegex(RuntimeError, '仍在运行'):
fix.repair(self.root)
self.assertEqual(self.manifest_path.read_bytes(), before)
def test_worker_or_unfinished_journal_blocks_repair(self):
for name, data in [('worker.json', {}), ('journal.json', {'phase': 'installing'})]:
path = self.root / '.runtime/update' / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data))
def test_active_worker_and_unknown_or_unfinished_journal_block_repair(self):
worker = self.root / '.runtime/update/worker.json'
worker.parent.mkdir(parents=True)
worker.write_text(json.dumps({'pid': __import__('os').getpid()}), encoding='utf-8')
with self.assertRaisesRegex(RuntimeError, '进程'):
fix.repair(self.root, self.manager_payload, self.control_payload)
worker.unlink()
journal = worker.parent / 'journal.json'
for value in ({'phase': 'installing'}, {'phase': 'futurePhase'}, {'unknown': True}):
journal.write_text(json.dumps(value), encoding='utf-8')
with self.assertRaises(RuntimeError):
fix.repair(self.root)
self.assertEqual((self.root / fix.SCRIPT).read_bytes(), self.original)
path.unlink()
fix.repair(self.root, self.manager_payload, self.control_payload)
self.assertEqual((self.root / fix.MANAGER).read_bytes(), self.old_manager)
journal.unlink()
def test_previous_check_repair_version_can_upgrade_without_losing_manifest_fields(self):
prior = b'previous optimized checker fixture'
digest = hashlib.sha256(prior).hexdigest()
(self.root / fix.SCRIPT).write_bytes(prior)
def test_windows_worker_probe_is_read_only_for_missing_and_live_pids(self):
worker = self.root / '.runtime/update/worker.json'
worker.parent.mkdir(parents=True)
worker.write_text(json.dumps({'pid': 424242}), encoding='utf-8')
missing_kernel = Mock()
missing_kernel.OpenProcess = Mock(return_value=None)
missing_kernel.GetExitCodeProcess = Mock()
missing_kernel.CloseHandle = Mock()
with (patch.object(fix.os, 'name', 'nt'),
patch.object(fix.os, 'kill', side_effect=AssertionError('os.kill must not run')) as kill,
patch('ctypes.WinDLL', return_value=missing_kernel, create=True),
patch('ctypes.get_last_error', return_value=87, create=True)):
self.assertFalse(fix.worker_active(self.root.resolve()))
kill.assert_not_called()
missing_kernel.GetExitCodeProcess.assert_not_called()
missing_kernel.CloseHandle.assert_not_called()
missing_kernel.TerminateProcess.assert_not_called()
missing_kernel.OpenProcess.assert_called_once_with(0x1000, False, 424242)
denied_kernel = Mock()
denied_kernel.OpenProcess = Mock(return_value=None)
denied_kernel.GetExitCodeProcess = Mock()
denied_kernel.CloseHandle = Mock()
with (patch.object(fix.os, 'name', 'nt'),
patch.object(fix.os, 'kill', side_effect=AssertionError('os.kill must not run')) as kill,
patch('ctypes.WinDLL', return_value=denied_kernel, create=True),
patch('ctypes.get_last_error', return_value=5, create=True)):
with self.assertRaisesRegex(RuntimeError, '无法只读确认'):
fix.worker_active(self.root.resolve())
kill.assert_not_called()
denied_kernel.TerminateProcess.assert_not_called()
live_kernel = Mock()
live_kernel.OpenProcess = Mock(return_value=123)
live_kernel.CloseHandle = Mock(return_value=1)
def set_still_active(_handle, output):
ctypes.cast(output, ctypes.POINTER(ctypes.c_ulong)).contents.value = 259
return 1
live_kernel.GetExitCodeProcess = Mock(side_effect=set_still_active)
with (patch.object(fix.os, 'name', 'nt'),
patch.object(fix.os, 'kill', side_effect=AssertionError('os.kill must not run')) as kill,
patch('ctypes.WinDLL', return_value=live_kernel, create=True)):
self.assertTrue(fix.worker_active(self.root.resolve()))
kill.assert_not_called()
live_kernel.CloseHandle.assert_called_once_with(123)
live_kernel.TerminateProcess.assert_not_called()
def test_stale_worker_pid_allows_repair_and_live_reused_pid_fails_closed(self):
worker = self.root / '.runtime/update/worker.json'
worker.parent.mkdir(parents=True)
worker.write_text(json.dumps({'pid': 424242}), encoding='utf-8')
with patch.object(fix.os, 'kill', side_effect=ProcessLookupError):
self.assertTrue(fix.repair(self.root, self.manager_payload, self.control_payload)['updateCheckRepaired'])
# A live process with a stale/reused PID is treated as active; no identity match is required.
(self.root / fix.MANAGER).write_bytes(self.old_manager)
(self.root / fix.CONTROL).write_bytes(self.old_control)
self.manifest_path.write_text(json.dumps(self.manifest), encoding='utf-8')
with patch.object(fix.os, 'kill', return_value=None):
with self.assertRaisesRegex(RuntimeError, '仍在运行'):
fix.repair(self.root, self.manager_payload, self.control_payload)
def test_update_lock_blocks_firstaid(self):
with fix.update_lock(self.root), self.assertRaisesRegex(RuntimeError, '仍在运行'):
fix.repair(self.root, self.manager_payload, self.control_payload)
def test_self_contained_cmd_uses_bundled_python_and_latest_source_bytes(self):
manager, control = fix.read_hardened_sources()
# This models a crash after source replacement but before manifest update.
(self.root / fix.MANAGER).write_bytes(manager)
(self.root / fix.CONTROL).write_bytes(control)
value = json.loads(self.manifest_path.read_text())
value['files'][0]['sha256'] = digest
self.manifest_path.write_text(json.dumps(value))
with patch.object(fix, 'PREVIOUS_SHA', digest):
self.assertTrue(fix.repair(self.root)['updateCheckRepaired'])
self.assertEqual(fix.PATCHED_SCRIPT, (self.root / fix.SCRIPT).read_bytes())
value['files'][0]['sha256'] = '29512f15a9ee329bc7c7777454bdd79aa7ffd11c3cffa9400b8a4585bdabf35b'
value['files'][1]['sha256'] = 'd98fa24d17e9bdd90ff37f9886df280451eceabac0df35d2eeecf4fe615bc6f1'
self.manifest_path.write_text(json.dumps(value), encoding='utf-8')
command = self.root / '兼容旧安装修复.cmd'
body = fix.command_file()
command.write_bytes(body)
module, payload = body.rsplit(fix.MODULE_MARKER, 1)[1].rsplit(fix.PAYLOAD_MARKER, 1)
embedded = {}
exec(compile(payload.decode('ascii'), str(command), 'exec'), embedded)
self.assertEqual(embedded['PATCHED_MANAGER'], manager)
self.assertEqual(embedded['PATCHED_CONTROL'], control)
code = body.decode('utf-8').split(' -c "', 1)[1].split('" "%~f0"', 1)[0]
result = subprocess.run([sys.executable, '-I', '-S', '-B', '-X', 'utf8', '-c', code, str(command)],
capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual((self.root / fix.MANAGER).read_bytes(), manager)
self.assertEqual((self.root / fix.CONTROL).read_bytes(), control)
manifest = json.loads(self.manifest_path.read_text())
self.assertEqual(manifest['files'][0]['sha256'], hashlib.sha256(manager).hexdigest())
self.assertEqual(manifest['files'][1]['sha256'], hashlib.sha256(control).hexdigest())
command_header = body.split(fix.MODULE_MARKER, 1)[0].decode('utf-8')
self.assertNotIn('powershell.exe', command_header)
self.assertNotIn('windows-start.ps1', command_header)
self.assertIn(' -I -S -B ', command_header)
def test_embedded_program_matches_reviewed_source(self):
self.assertEqual(fix.PATCHED_SCRIPT, (HERE / 'update_manager.py').read_bytes())
def test_historical_allowlist_matches_known_release_inputs(self):
self.assertEqual(self.reviewed_manager_hashes, frozenset({
'29512f15a9ee329bc7c7777454bdd79aa7ffd11c3cffa9400b8a4585bdabf35b',
'b91cb7fec10e183d2904d86f032ff0ffcf7f19f7cb27aecd2d803d3448685482',
'a4bd05789f559fede0bed083ce61123307d6b5cf52fa638b6bde1fa112a0348a',
}))
self.assertEqual(self.reviewed_control_hashes, frozenset({
'd98fa24d17e9bdd90ff37f9886df280451eceabac0df35d2eeecf4fe615bc6f1',
}))
if __name__ == '__main__':
+250 -34
View File
@@ -8,8 +8,10 @@ snapshot; private data and runtimes are never update destinations.
from __future__ import annotations
import argparse
import ast
import base64
import contextlib
import fnmatch
from datetime import datetime, timezone
import hashlib
import json
@@ -30,6 +32,7 @@ import psutil
from cryptography.hazmat.primitives.serialization import load_pem_public_key
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
UPDATE_PROTOCOL = 2
ROOT = Path(__file__).resolve().parents[2]
BASE = 'https://quality.lyunlab.me/quality-updates/windows-x64/'
LATEST = BASE + 'latest.json'
@@ -267,11 +270,25 @@ def enabled(root=ROOT):
def status(root=ROOT):
manifest = read(root / 'manifest.json', {})
value = read(root / '.runtime/update/status.json', {})
try:
value = read(root / '.runtime/update/status.json', {})
except (OSError, ValueError):
value = {}
if not isinstance(value, dict):
value = {}
allowed = {'state', 'message', 'checkedAt', 'version', 'available', 'startedAt', 'finishedAt'}
result = {k: v for k, v in value.items() if k in allowed}
result.update(enabled=enabled(root), currentVersion=manifest.get('updateVersion', manifest.get('builtAtUtc', '当前版本')))
result.setdefault('state', 'idle')
journal = read(root / '.runtime/update/journal.json', {})
same_operation = bool(journal.get('operationId')) and journal.get('operationId') == value.get('operationId')
if (same_operation and journal.get('phase') == 'complete' and result['state'] in ACTIVE
and manifest.get('updateSequence') == journal.get('sequence')
and manifest.get('updateVersion') == journal.get('version')):
result.update(state='completed', message='更新完成。', available=False,
version=journal['version'], finishedAt=journal['finishedAt'])
elif same_operation and journal.get('phase') == 'rolledBack' and result['state'] in ACTIVE:
result.update(state='failed', message='更新未完成,已恢复原版本。', available=False)
if result['state'] in ACTIVE and not live(read(root / '.runtime/update/worker.json', {})):
pending = read(root / '.runtime/update/request.json', {})
journal = read(root / '.runtime/update/journal.json', {})
@@ -323,7 +340,10 @@ def exclusive_update(root=ROOT):
yield
finally:
if read(directory / 'worker.json', {}) == identity:
(directory / 'worker.json').unlink(missing_ok=True)
try:
(directory / 'worker.json').unlink(missing_ok=True)
except OSError as error:
record_failure(root, 'cleanupWorker', error)
def launch(root=ROOT):
@@ -345,7 +365,7 @@ def launch_locked(root=ROOT):
return status(root)
token = secrets.token_hex(16)
atomic(directory / 'request.json', {'token': token, 'created': time.time()})
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False)
write_status('queued', '更新已开始,完成后将自动重新连接。', root, startedAt=datetime.now(timezone.utc).isoformat(), available=False, operationId=token)
try:
with (directory / 'worker.log').open('ab') as log:
subprocess.Popen([sys.executable, str(Path(__file__).resolve()), 'worker', '--launcher', str(os.getpid()), '--token', token],
@@ -364,11 +384,13 @@ def live(info):
return False
def controller(root, action, database_only=False, timeout=1100):
def controller(root, action, database_only=False, timeout=1100, program_only=False):
python = root / 'runtimes/python' / ('python.exe' if os.name == 'nt' else 'bin/python3')
command = [str(python), str(root / 'deploy/offline/control.py'), action]
if database_only:
command += ['--database-only']
if program_only:
command += ['--program-only']
if action == 'serve':
with (root / '.runtime/update/service.log').open('ab') as log:
return subprocess.Popen(command, cwd=root, env=os.environ.copy(), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **process_options())
@@ -471,29 +493,186 @@ def restore_program(root, backup):
copy_atomic(source, destination(root, '.runtime/' + relative))
STOP_TIMEOUT = 360
MAX_RECOVERY_ATTEMPTS = 2
def best_effort_status(state, message, root=ROOT, **extra):
try:
return write_status(state, message, root, **extra)
except Exception as error:
record_failure(root, 'writeStatus', error)
def verify_backup(root, backup, journal=None):
journal = journal or read(root / '.runtime/update/journal.json', {})
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
raise UpdateError('更新恢复备份校验失败,已保留现场。')
manifest = entry_map(read(backup / 'manifest.json')['files'])
inventory = read(backup / 'program.json')
names, present = inventory.get('all'), inventory.get('present')
if (not isinstance(names, list) or not isinstance(present, list)
or len(names) != len(set(names)) or not set(present).issubset(names)):
raise UpdateError('更新恢复清单无效。')
for name in names:
if not program_path(name):
raise UpdateError('更新恢复清单包含受保护路径。')
destination(root, name)
if name in present:
source = destination(backup / 'program', name)
if name not in manifest or not source.is_file() or digest(source) != manifest[name]['sha256']:
raise UpdateError('原程序备份缺失或损坏。')
elif name in manifest:
raise UpdateError('原程序备份不完整。')
def recovery_binding(backup):
return {'backup': backup.name, 'backupManifestSha256': digest(backup / 'manifest.json'),
'programInventorySha256': digest(backup / 'program.json'),
'databaseSqlSha256': digest(backup / 'database.sql')}
def recovery_database(binding, attempt):
return 'quality_restore_' + hashlib.sha256(canonical({**binding, 'attempt': attempt})).hexdigest()[:24]
def recovery_progress(root, backup):
binding = recovery_binding(backup)
try:
value = read(root / '.runtime/update/recovery.json', {})
except (OSError, ValueError) as error:
raise UpdateError('恢复进度无法读取,已保留现场。') from error
if not value:
return binding, None
if (not isinstance(value, dict) or any(value.get(k) != v for k, v in binding.items())
or type(value.get('attempt')) is not int or not 1 <= value['attempt'] <= MAX_RECOVERY_ATTEMPTS
or value.get('database') != recovery_database(binding, value['attempt'])
or value.get('stage') not in ('importing', 'databaseDurable', 'published', 'ready')):
raise UpdateError('恢复进度与备份不一致,已保留现场。')
return binding, value
def prepare_old_backend_restart(root):
# 旧 JAR 不理解终态 journal 的优先级。先持久化不触发其维护锁的进度状态,
# 再启动旧服务;写失败则保留恢复记录,不声称已恢复。
write_status('queued', '正在启动原版本,完成后自动重新连接。', root)
def normalize_terminal_status(root, journal):
value = read(root / '.runtime/update/status.json', {})
if (not isinstance(value, dict) or value.get('state') not in ACTIVE
or journal.get('operationId') != value.get('operationId')):
return
pending = read(root / '.runtime/update/request.json', {})
if value.get('state') in ('queued', 'downloading') and time.time() - pending.get('created', 0) < 120:
return
if journal.get('phase') == 'rolledBack':
write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
elif journal.get('phase') == 'complete':
manifest = read(root / 'manifest.json')
if manifest.get('updateVersion') == journal.get('version', value.get('version')):
write_status('completed', '更新完成。', root, available=False)
def rollback(root, backup, database_backed_up):
write_status('rollingBack', '更新未完成,正在恢复原版本。', root)
controller(root, 'stop', timeout=180)
verify_backup(root, backup)
binding, progress = recovery_progress(root, backup) if database_backed_up else ({}, None)
best_effort_status('rollingBack', '更新未完成,正在恢复原版本。', root)
controller(root, 'stop', timeout=STOP_TIMEOUT)
restore_program(root, backup)
if database_backed_up:
config = read(root / '.runtime/deployment.json')
process = controller(root, 'serve', True)
wait_ready(root, process, True)
database = 'quality_restore_' + secrets.token_hex(12)
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
with (backup / 'database.sql').open('rb') as source:
mysql(root, config, database=database, stdin=source)
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
controller(root, 'stop', timeout=180)
config['mysqlDatabase'] = database
config = read(backup / 'private/deployment.json')
# 断电前未确认数据库安全关闭的导入,不能当作完整数据库复用。
if progress is None or progress['stage'] == 'importing':
attempt = 1 if progress is None else progress['attempt'] + 1
if attempt > MAX_RECOVERY_ATTEMPTS:
raise UpdateError('恢复导入多次中断,已停止自动重试并保留数据库,请核对本机恢复记录。')
reserve = (backup / 'database.sql').stat().st_size * 3 + 512 * 1024**2
if shutil.disk_usage(root).free < reserve:
raise UpdateError('空间不足以安全恢复数据库,已保留现场。')
database = recovery_database(binding, attempt)
if database == config.get('mysqlDatabase'):
raise UpdateError('恢复数据库与原库冲突,已停止。')
progress = {**binding, 'attempt': attempt, 'database': database, 'stage': 'importing'}
atomic(root / '.runtime/update/recovery.json', progress)
process = controller(root, 'serve', True)
wait_ready(root, process, True)
# 不删除任何已有数据库;名称冲突时 CREATE 会失败并保留现场。
sql(root, config, f'CREATE DATABASE `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
with (backup / 'database.sql').open('rb') as source:
mysql(root, config, database=database, stdin=source)
sql(root, config, f"GRANT ALL PRIVILEGES ON `{database}`.* TO 'quality_app'@'127.0.0.1';")
controller(root, 'stop', timeout=STOP_TIMEOUT)
progress['stage'] = 'databaseDurable'
atomic(root / '.runtime/update/recovery.json', progress)
config['mysqlDatabase'] = progress['database']
atomic(root / '.runtime/deployment.json', config)
receipt_path = root / '.runtime/migration-receipt.json'
receipt = read(receipt_path, {})
if receipt and receipt.get('migrationId') == config.get('restoredFromMigration'):
receipt['database'] = database
receipt['database'] = progress['database']
atomic(receipt_path, receipt)
progress['stage'] = 'published'
atomic(root / '.runtime/update/recovery.json', progress)
prepare_old_backend_restart(root)
process = controller(root, 'serve')
wait_ready(root, process)
if progress:
progress['stage'] = 'ready'
atomic(root / '.runtime/update/recovery.json', progress)
def preflight_programs(stage):
# 旧安装先加固恢复器,随后只能安装具备相同恢复协议的程序,防止覆盖回旧实现。
for name, marker in (('update_manager.py', 'UPDATE_PROTOCOL'), ('control.py', 'UPDATE_CONTROL_PROTOCOL')):
try:
tree = ast.parse((stage / 'deploy/offline' / name).read_bytes())
values = [node.value.value for node in tree.body if isinstance(node, ast.Assign)
and any(isinstance(target, ast.Name) and target.id == marker for target in node.targets)
and isinstance(node.value, ast.Constant) and type(node.value.value) is int]
if values != [UPDATE_PROTOCOL]:
raise ValueError('incompatible recovery protocol')
except (OSError, SyntaxError, ValueError) as error:
raise UpdateError('更新包未包含本轮加固的恢复流程,服务尚未停止;请等待配套更新包。') from error
try:
with zipfile.ZipFile(stage / 'app/backend.jar') as jar:
metadata = decode(jar.read('BOOT-INF/classes/update-protocol.json'))
if metadata != {'protocol': UPDATE_PROTOCOL}:
raise ValueError('incompatible backend protocol')
except (OSError, ValueError, KeyError, zipfile.BadZipFile) as error:
raise UpdateError('更新包后端不支持本轮恢复协议,服务尚未停止。') from error
def preflight_backup(root, backup, release):
paths = set(e['path'] for e in read(root / 'manifest.json')['files'] if program_path(e['path']))
paths.update(e['path'] for e in release['files'])
targets = [backup / 'program' / n for n in paths]
ignored = ('*.lock', '*.log', 'service.json', 'stop-request.json')
for folder, dirs, files in os.walk(root / '.runtime', followlinks=False):
dirs[:] = [n for n in dirs if n != 'update']
for n in dirs:
destination(root, (Path(folder) / n).relative_to(root).as_posix())
for n in files:
if any(fnmatch.fnmatch(n, pattern) for pattern in ignored):
continue
source = Path(folder) / n
destination(root, source.relative_to(root).as_posix())
with source.open('rb'):
pass
targets.append(backup / 'private' / source.relative_to(root / '.runtime'))
for target in targets:
destination(backup, target.relative_to(backup).as_posix())
if os.name == 'nt' and len(str(target.absolute())) + 24 > 248:
raise UpdateError('备份路径过长,请先缩短部署目录或历史路径;服务尚未停止。')
probe = backup / 'write-probe.tmp'
try:
with probe.open('xb') as stream:
stream.write(b'probe')
stream.flush()
os.fsync(stream.fileno())
finally:
probe.unlink(missing_ok=True)
def record_failure(root, stage, error, name='failure.json'):
@@ -526,7 +705,7 @@ def worker(launcher, token, root=ROOT):
raise UpdateError('更新请求已失效。')
with exclusive_update(root):
backup = directory / ('backup-' + datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + secrets.token_hex(4))
stopped = backed_up = program_saved = changed = False
stopped = backed_up = program_saved = changed = stop_requested = committed = False
failure_stage = 'validateRelease'
try:
envelope = decode(download(LATEST))
@@ -543,18 +722,30 @@ def worker(launcher, token, root=ROOT):
private_size = sum(p.stat().st_size for p in (root / '.runtime').rglob('*') if p.is_file() and not p.is_relative_to(directory))
if shutil.disk_usage(root).free < release['archive']['size'] + expanded + old_program_size + private_size + db_size * 3 + 512 * 1024**2:
raise UpdateError('磁盘剩余空间不足,请释放空间后重试。')
write_status('downloading', '正在下载并校验更新。', root)
write_status('downloading', '正在下载并校验更新。', root, operationId=token, version=release['version'])
info = release['archive']
failure_stage = 'downloadArchive'
download(BASE + info['file'], archive, info['size'], info['sha256'])
failure_stage = 'unpackArchive'
unpack(archive, stage, release)
failure_stage = 'programPreflight'
preflight_programs(stage)
failure_stage = 'runtimePreflight'
write_status('downloading', '正在检查运行环境和备份条件,服务仍可使用。', root)
controller(root, 'verify-runtime', timeout=1800)
failure_stage = 'backupPreflight'
preflight_backup(root, backup, release)
# Revalidate immediately before any service or disk mutation.
validate_release(envelope, root)
write_status('backingUp', '正在备份,服务即将重新启动。', root)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False})
if (directory / 'recovery.json').exists():
copy_atomic(directory / 'recovery.json', backup / 'previous-recovery.json')
(directory / 'recovery.json').unlink()
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'stopping', 'databaseBackedUp': False,
'operationId': token, 'version': release['version'], 'sequence': release['sequence']})
failure_stage = 'stopService'
controller(root, 'stop', timeout=180)
stop_requested = True
controller(root, 'stop', timeout=STOP_TIMEOUT)
stopped = True
failure_stage = 'backupProgram'
backup_program(root, backup, release)
@@ -571,13 +762,15 @@ def worker(launcher, token, root=ROOT):
if (backup / 'database.sql').stat().st_size < 100:
raise UpdateError('数据库备份为空,已停止更新。')
backed_up = True
controller(root, 'stop', timeout=180)
controller(root, 'stop', timeout=STOP_TIMEOUT)
# The actual dump can exceed the source's physical storage. Do not
# replace anything unless a separate rollback database still fits.
required_recovery = max(db_size, (backup / 'database.sql').stat().st_size * 3) + 512 * 1024**2
if shutil.disk_usage(root).free < required_recovery:
raise UpdateError('备份已完成,但剩余磁盘空间不足以安全回退,已停止更新。')
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'installing', 'databaseBackedUp': True,
'operationId': token, 'version': release['version'], 'sequence': release['sequence'],
'databaseMayHaveChanged': False,
'backupManifestSha256': digest(backup / 'manifest.json'),
'programInventorySha256': digest(backup / 'program.json')})
write_status('installing', '正在安装更新。', root)
@@ -585,32 +778,46 @@ def worker(launcher, token, root=ROOT):
failure_stage = 'replaceProgram'
replace_program(root, stage, backup, release)
failure_stage = 'verifyProgram'
controller(root, 'verify', timeout=180)
controller(root, 'verify-runtime', timeout=300, program_only=True)
write_status('restarting', '正在重新启动,请稍候。', root)
failure_stage = 'restartService'
journal = read(directory / 'journal.json')
journal['databaseMayHaveChanged'] = True
atomic(directory / 'journal.json', journal)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'complete', 'databaseBackedUp': True})
write_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
journal.update(phase='complete', version=release['version'], sequence=release['sequence'],
finishedAt=datetime.now(timezone.utc).isoformat())
atomic(directory / 'journal.json', journal)
committed = True
best_effort_status('completed', '更新完成。', root, version=release['version'], finishedAt=datetime.now(timezone.utc).isoformat(), available=False)
except Exception as error:
record_failure(root, failure_stage, error)
message = str(error) if isinstance(error, UpdateError) else '更新未完成,请查看本机更新日志。'
if stopped:
journal = read(directory / 'journal.json', {})
if committed or (journal.get('backup') == backup.name and journal.get('phase') == 'complete'):
return # 提交后的展示或清理失败绝不能发起回滚。
if stop_requested:
try:
if program_saved and changed:
rollback(root, backup, backed_up)
rollback(root, backup, backed_up and journal.get('databaseMayHaveChanged', True))
else:
controller(root, 'stop', timeout=180)
controller(root, 'stop', timeout=STOP_TIMEOUT)
prepare_old_backend_restart(root)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {'backup': backup.name, 'phase': 'rolledBack', 'databaseBackedUp': backed_up})
journal.update(backup=backup.name, phase='rolledBack', databaseBackedUp=backed_up)
atomic(directory / 'journal.json', journal)
message += ' 已恢复原版本。'
except Exception as rollback_error:
record_failure(root, 'rollback', rollback_error, 'rollback-failure.json')
message += ' 自动恢复未完成,请联系管理员使用本机更新备份恢复。'
write_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
best_effort_status('failed', message, root, available=False, finishedAt=datetime.now(timezone.utc).isoformat())
finally:
(directory / 'request.json').unlink(missing_ok=True)
try:
(directory / 'request.json').unlink(missing_ok=True)
except OSError as error:
record_failure(root, 'cleanupRequest', error)
def recover(root=ROOT):
@@ -622,22 +829,31 @@ def recover_locked(root=ROOT):
directory = root / '.runtime/update'
journal = read(directory / 'journal.json', {})
if journal.get('phase') in (None, 'complete', 'rolledBack'):
if journal.get('phase') in ('complete', 'rolledBack'):
normalize_terminal_status(root, journal)
return status(root)
name = journal.get('backup', '')
if not re.fullmatch(r'backup-[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}', name):
raise UpdateError('更新恢复记录无效。')
backup = directory / name
if journal.get('phase') not in ('installing', 'stopping'):
raise UpdateError('更新恢复阶段无效,已保留现场。')
if journal.get('phase') == 'installing':
if (digest(backup / 'manifest.json') != journal.get('backupManifestSha256')
or digest(backup / 'program.json') != journal.get('programInventorySha256')):
raise UpdateError('更新恢复备份校验失败,已保留现场。')
rollback(root, backup, journal.get('databaseBackedUp') is True)
verify_backup(root, backup, journal)
if journal.get('databaseBackedUp') is True:
recovery_progress(root, backup) # 先验证 sidecar 与 dump,尚未停止或替换任何文件。
rollback(root, backup, journal.get('databaseBackedUp') is True and journal.get('databaseMayHaveChanged', True))
else:
controller(root, 'stop', timeout=180)
controller(root, 'stop', timeout=STOP_TIMEOUT)
prepare_old_backend_restart(root)
process = controller(root, 'serve')
wait_ready(root, process)
atomic(directory / 'journal.json', {**journal, 'phase': 'rolledBack'})
return write_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
best_effort_status('failed', '上次更新中断,已恢复原版本。', root, available=False)
return status(root)
def main():