修复:加固 Windows 更新预检、提交点和可恢复的数据库回退流程

This commit is contained in:
2026-11-05 16:54:30 +08:00
parent 2315c83067
commit 3962521a5c
13 changed files with 1821 additions and 236 deletions
+438 -54
View File
@@ -6,6 +6,7 @@ import argparse
import contextlib
import hashlib
import json
import math
import os
from pathlib import Path
import platform
@@ -15,8 +16,12 @@ import signal
import socket
import subprocess
import sys
_old_dont_write_bytecode = sys.dont_write_bytecode
# Importing the controller must not create caches in the immutable package.
sys.dont_write_bytecode = True
import threading
import time
from pathlib import PurePosixPath
from urllib.request import ProxyHandler, build_opener
import psutil
@@ -25,12 +30,15 @@ ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import runtime_platform as host
from gateway import make_server
sys.dont_write_bytecode = _old_dont_write_bytecode
del _old_dont_write_bytecode
PRIVATE = ROOT / '.runtime'
DATA = ROOT / 'runtimedata'
CONFIG = PRIVATE / 'deployment.json'
STATE = PRIVATE / 'service.json'
STOP = PRIVATE / 'stop-request.json'
UPDATE_CONTROL_PROTOCOL = 2
IS_WINDOWS = os.name == 'nt'
EXE = '.exe' if IS_WINDOWS else ''
PYTHON = ROOT / 'runtimes/python' / ('python.exe' if IS_WINDOWS else 'bin/python3')
@@ -39,6 +47,7 @@ MYSQL = ROOT / 'runtimes/mysql/bin' / ('mysql' + EXE)
MYSQLD = ROOT / 'runtimes/mysql/bin' / ('mysqld' + EXE)
ADMIN = ROOT / 'runtimes/mysql/bin' / ('mysqladmin' + EXE)
OPENER = build_opener(ProxyHandler({}))
STOP_TIMEOUT_SECONDS = 300
def atomic(path, value):
@@ -65,7 +74,7 @@ def private_directory(path):
path.chmod(0o700)
def configuration():
def configuration(create=True):
if CONFIG.exists():
value = json.loads(CONFIG.read_text(encoding='utf-8'))
else:
@@ -73,7 +82,8 @@ def configuration():
'webBind': '0.0.0.0' if IS_WINDOWS else '127.0.0.1',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)}
atomic(CONFIG, value)
if create:
atomic(CONFIG, value)
value.setdefault('mysqlDatabase', 'quality_platform')
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', value['mysqlDatabase']):
raise RuntimeError('部署数据库名称无效。')
@@ -137,14 +147,159 @@ def record(process, command):
def matches(info):
"""Return True for this process, False for gone/replaced, None if unknown."""
if not isinstance(info, dict):
return None
try:
process = psutil.Process(int(info['pid']))
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
and abs(process.create_time() - info['created']) < .01
and os.path.normcase(str(Path(process.exe()).resolve())) == os.path.normcase(info['executable'])
and process.cmdline()[1:] == info['args'])
except (psutil.Error, KeyError, ValueError, TypeError):
pid = info['pid']
created = info['created']
executable = info['executable']
arguments = info['args']
if (type(pid) is not int or pid <= 0 or type(created) not in (int, float)
or not math.isfinite(created) or not isinstance(executable, str)
or not isinstance(arguments, list) or not all(isinstance(item, str) for item in arguments)):
return None
except (KeyError, ValueError, TypeError):
return None
try:
process = psutil.Process(pid)
except psutil.NoSuchProcess:
return False
except Exception:
return None
try:
if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE:
return False
if abs(process.create_time() - created) >= .01:
return False
if os.path.normcase(str(Path(process.exe()).resolve())) != os.path.normcase(executable):
return False
if process.cmdline()[1:] != arguments:
return False
return True
except psutil.NoSuchProcess:
return False
except Exception:
return None
def registered_alive(records):
"""Keep both confirmed-live and uninspectable registered process records."""
if not isinstance(records, list):
return [], True
alive, unknown = [], False
for info in records:
status = matches(info)
if status is True:
alive.append(info)
elif status is None:
alive.append(info)
unknown = True
return alive, unknown
def retain_registered_children(state, records, unresolved=False):
survivors, unknown = registered_alive(records)
state['children'] = survivors
unresolved = unresolved or unknown
if unresolved:
state['unknownChildren'] = True
return survivors, unresolved
def registered_named(records, name):
confirmed, unknown = [], []
for info in records:
if not isinstance(info, dict) or info.get('name') != name:
continue
status = matches(info)
if status is True:
confirmed.append(info)
elif status is None:
unknown.append(info)
return confirmed, unknown
def snapshot_process(process, name):
try:
result = record(process, process.cmdline())
result['name'] = name
return result
except (psutil.Error, OSError):
return None
def process_object_matches(process, created):
try:
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
and abs(process.create_time() - created) < .01)
except psutil.NoSuchProcess:
return False
except Exception:
# Permission failures and unexpected inspection errors are unknown,
# not evidence that the observed child has exited.
return None
def terminate_observed(processes, timeout):
"""Stop only process objects captured below a verified service child."""
unverified = False
for process, created, info in processes:
observed = process_object_matches(process, created)
if observed is None:
if info is None:
unverified = True
elif observed is True:
with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
process.terminate()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
states = [process_object_matches(process, created) for process, created, _info in processes]
if any(observed is None for observed in states):
unverified = unverified or any(observed is None and info is None
for observed, (_process, _created, info) in zip(states, processes))
break
if not any(states):
break
time.sleep(.1)
for process, created, info in processes:
observed = process_object_matches(process, created)
if observed is None:
if info is None:
unverified = True
elif observed is True:
with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
process.kill()
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
states = [process_object_matches(process, created) for process, created, _info in processes]
if any(observed is None for observed in states):
unverified = unverified or any(observed is None and info is None
for observed, (_process, _created, info) in zip(states, processes))
break
if not any(states):
break
time.sleep(.1)
survivors = []
for process, created, info in processes:
observed = process_object_matches(process, created)
if observed is None:
if info is None:
unverified = True
else:
survivors.append(info)
elif observed is True:
identity = matches(info) if info is not None else None
if info is None:
unverified = True
elif identity is None:
survivors.append(info)
elif identity is False:
survivors.append(info)
unverified = True
else:
survivors.append(info)
return survivors, unverified
def read_state():
@@ -154,6 +309,45 @@ def read_state():
return {}
def wait_for_stop(supervisor, request, timeout_seconds=STOP_TIMEOUT_SECONDS, interval=.5):
request_sent = False
deadline = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
supervisor_status = matches(supervisor)
if supervisor_status is None:
raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。')
if supervisor_status is True:
if not request_sent:
atomic(STOP, request)
request_sent = True
time.sleep(interval)
continue
state = read_state()
if 'supervisor' in state:
current_supervisor = state['supervisor']
current_status = matches(current_supervisor)
if current_status is None:
raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。')
if current_status is True:
if current_supervisor != supervisor:
raise RuntimeError('服务状态已变化,未停止任何不明进程。')
if not request_sent:
atomic(STOP, request)
request_sent = True
time.sleep(interval)
continue
survivors, unknown_children = registered_alive(state.get('children', []))
if unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
raise RuntimeError('服务监督程序已退出,但仍有未确认停止的已记录进程;未结束任何不明进程。')
if survivors:
time.sleep(interval)
continue
print('服务已停止。')
return
raise RuntimeError('服务仍在安全关闭,请稍后检查状态。')
def free_port(port, bind='127.0.0.1'):
# A recently closed TCP connection may remain in TIME_WAIT. Reject a live
# listener, but allow a clean restart without waiting for that state to expire.
@@ -182,22 +376,136 @@ def sql(value, query):
env=env, timeout=10)
def verify(check_hashes=True):
def manifest_name(name):
if (not isinstance(name, str) or not name or '\\' in name
or PurePosixPath(name).is_absolute()
or any(part in ('', '.', '..') for part in name.split('/'))):
raise RuntimeError('文件清单包含无效路径。')
return PurePosixPath(name)
def manifest_path(name):
parts = manifest_name(name).parts
path = ROOT.joinpath(*parts)
root = ROOT.resolve()
if not path.resolve(strict=False).is_relative_to(root):
raise RuntimeError('文件路径异常,已停止校验:' + name)
parent = path.parent
while parent != ROOT and parent != parent.parent:
if parent.is_symlink():
raise RuntimeError('文件路径异常,已停止校验:' + name)
parent = parent.parent
return path
def cache_source(name):
"""Return only timestamp-based bytecode sources from the bundled Python."""
if not isinstance(name, str) or '\\' in name:
return None
path = PurePosixPath(name)
if path.parts[:2] != ('runtimes', 'python') or any(part in ('.', '..') for part in name.split('/')):
return None
if path.parent.name != '__pycache__':
return None
match = re.fullmatch(r'(.+)\.cpython-\d+(?:\.opt-\d+)?\.pyc', path.name)
return (path.parent.parent / (match[1] + '.py')).as_posix() if match else None
def file_digest(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def regular_manifest_file(name):
path = manifest_path(name)
if path.is_symlink():
raise RuntimeError('文件路径异常,已停止校验:' + name)
return path
def program_file_path(name):
"""Mirror update_manager.program_path without importing code during a read-only check."""
path = manifest_name(name)
return (name == 'app/backend.jar' or name.startswith('app/web/')
or name == 'backend/src/main/resources/analysis-runtime-policy.json'
or name.startswith('scripts/') and path.suffix == '.py'
or name.startswith('ai/nao-quality/')
or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md'))
def verify_manifest_files(manifest, program_only=False):
entries = manifest.get('files')
if not isinstance(entries, list):
raise RuntimeError('文件清单无效。')
by_name = {}
for entry in entries:
if not isinstance(entry, dict) or not isinstance(entry.get('path'), str):
raise RuntimeError('文件清单无效。')
name = entry['path']
manifest_name(name)
if name in by_name:
raise RuntimeError('文件清单包含重复路径。')
by_name[name] = entry
ignored_caches = set()
if not program_only:
# Timestamp-based bytecode varies between Windows copies. Ignore only
# cache entries whose exact source is present and matches its manifest
# digest; ordinary Python source files are still hashed below.
for name in by_name:
source = cache_source(name)
if source is None:
continue
source_entry = by_name.get(source)
expected = source_entry.get('sha256') if isinstance(source_entry, dict) else None
source_path = regular_manifest_file(source)
if (not isinstance(expected, str) or not source_path.is_file()
or file_digest(source_path) != expected):
raise RuntimeError('对应源码缺失或校验失败,已停止校验:' + source)
cache_path = regular_manifest_file(name)
if cache_path.exists() and not cache_path.is_file():
raise RuntimeError('缓存文件路径异常,已停止校验:' + name)
ignored_caches.add(name)
for name, entry in by_name.items():
if program_only and not program_file_path(name):
continue
if name in ignored_caches:
continue
path = manifest_path(name)
expected_link = entry.get('symlink')
if expected_link is not None:
valid = path.is_symlink() and os.readlink(path) == expected_link
else:
if path.is_symlink():
valid = False
elif path.is_file():
with path.open('rb') as stream:
valid = hashlib.file_digest(stream, 'sha256').hexdigest() == entry.get('sha256')
else:
valid = False
if not valid:
raise RuntimeError('文件缺失或校验失败:' + name)
def verify_runtime(check_hashes=True, program_only=False):
manifest = json.loads((ROOT / 'manifest.json').read_text(encoding='utf-8'))
expected_os = 'windows' if IS_WINDOWS else platform.system().lower()
if manifest['platform'] != expected_os or platform.machine().lower() not in ('amd64', 'x86_64'):
raise RuntimeError('此包仅用于 ' + manifest['platform'] + ' 的 Intel/AMD 64 位电脑。')
if check_hashes:
for entry in manifest['files']:
path = ROOT / entry['path']
if entry.get('symlink') is not None:
valid = path.is_symlink() and os.readlink(path) == entry['symlink']
else:
valid = path.is_file() and hashlib.file_digest(path.open('rb'), 'sha256').hexdigest() == entry['sha256']
if not valid:
raise RuntimeError('文件缺失或校验失败:' + entry['path'])
value = configuration()
verify_manifest_files(manifest, program_only=program_only)
value = configuration(create=False)
env = environment(value)
if program_only:
application_probe = subprocess.run([str(PYTHON), '-c',
'import sys;sys.path.insert(0,sys.argv[1]);import nao_quality_runtime as r;'
'import nao_native_run,nao_quality_runtime_mcp,nao_visual_recovery;'
'assert len(r._config_fingerprint("https://offline.invalid/v1","offline-check","offline-check"))==64',
str(ROOT / 'scripts')], capture_output=True, env=env, timeout=45)
if application_probe.returncode:
raise RuntimeError('AI 应用脚本或策略资源检查失败。')
return value
for command in ([JAVA, '-version'], [MYSQLD, '--no-defaults', '--version']):
result = subprocess.run([str(a) for a in command], capture_output=True, env=env, timeout=30)
if result.returncode:
@@ -221,6 +529,13 @@ def verify(check_hashes=True):
return value
def verify(check_hashes=True):
# Keep the historical full verification entry point and its configuration
# initialization behavior for the installer and migration recovery tools.
verify_runtime(check_hashes=check_hashes)
return configuration()
def addresses(value):
ips = set()
if value['webBind'] == '0.0.0.0':
@@ -360,38 +675,90 @@ def serve(database_only=False):
finally:
state['ready'] = False
atomic(STATE, state)
# Stop backend first, then its isolated AI processes, then the database.
cleanup_failures, extra_survivors = [], []
unknown_children = False
# Only signal the backend and descendants observed beneath its verified
# PID/start-time/executable/arguments identity.
for info in reversed(children):
if info['name'] != 'backend' or not matches(info):
if not isinstance(info, dict) or info.get('name') != 'backend':
continue
backend_status = matches(info)
if backend_status is None:
extra_survivors.append(info)
continue
if backend_status is not True:
continue
process = psutil.Process(info['pid'])
descendants = process.children(recursive=True)
process.terminate()
try:
process.wait(35)
except psutil.TimeoutExpired:
if matches(info):
process.kill()
process = psutil.Process(info['pid'])
except psutil.NoSuchProcess:
continue
except Exception:
extra_survivors.append(info)
continue
observed = []
try:
descendants = process.children(recursive=True)
except psutil.NoSuchProcess:
descendants = []
except psutil.Error:
descendants = []
unknown_children = True
for child in descendants:
with contextlib.suppress(psutil.NoSuchProcess):
child.terminate()
with contextlib.suppress(OSError, subprocess.SubprocessError):
subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'],
env=environment(value), cwd=ROOT, capture_output=True, timeout=25)
if any(info['name'] == 'mysql' and matches(info) for info in children):
try:
created = child.create_time()
observed.append((child, created, snapshot_process(child, 'backend-descendant')))
except (psutil.Error, OSError):
unknown_children = True
observed.append((process, info['created'], info))
survivors, unverified = terminate_observed(observed, 35)
extra_survivors.extend(item for item in survivors if item.get('name') == 'backend-descendant')
unknown_children = unknown_children or unverified
try:
stopped = subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'],
env=environment(value), cwd=ROOT, capture_output=True, timeout=25)
if stopped.returncode:
cleanup_failures.append('analysisProcesses')
except (OSError, subprocess.SubprocessError):
cleanup_failures.append('analysisProcesses')
mysql_infos, unknown_mysql = registered_named(children, 'mysql')
extra_survivors.extend(unknown_mysql)
mysql_infos = [info for info in mysql_infos if matches(info) is True]
if mysql_infos:
env = environment(value)
env['MYSQL_PWD'] = value['rootPassword']
with contextlib.suppress(OSError, subprocess.SubprocessError):
subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env,
capture_output=True, timeout=60)
try:
shutdown = subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env,
capture_output=True, timeout=60)
if shutdown.returncode:
cleanup_failures.append('databaseShutdown')
except (OSError, subprocess.SubprocessError):
cleanup_failures.append('databaseShutdown')
# mysqladmin can return before InnoDB has finished closing files.
for info in children:
if info['name'] == 'mysql' and matches(info):
with contextlib.suppress(psutil.NoSuchProcess, psutil.TimeoutExpired):
for info in mysql_infos:
mysql_status = matches(info)
if mysql_status is None:
extra_survivors.append(info)
elif mysql_status is True:
try:
psutil.Process(info['pid']).wait(60)
survivors = [info for info in children if matches(info)]
if survivors:
state['children'] = survivors
except (psutil.NoSuchProcess, psutil.TimeoutExpired):
pass
except Exception:
extra_survivors.append(info)
unique_records, seen_records = [], set()
for info in [*children, *extra_survivors]:
if not isinstance(info, dict):
unknown_children = True
continue
key = (info.get('pid'), info.get('created'))
if key not in seen_records:
seen_records.add(key)
unique_records.append(info)
survivors, unknown_children = retain_registered_children(state, unique_records, unknown_children)
if cleanup_failures:
state['cleanupFailures'] = cleanup_failures
if survivors or unknown_children or cleanup_failures:
atomic(STATE, state)
else:
STATE.unlink(missing_ok=True)
@@ -402,10 +769,25 @@ def serve(database_only=False):
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'serve'))
parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'verify-runtime', 'serve'))
parser.add_argument('--lan', action='store_true')
parser.add_argument('--program-only', action='store_true')
parser.add_argument('--database-only', action='store_true', help=argparse.SUPPRESS)
args = parser.parse_args()
if args.program_only and args.action != 'verify-runtime':
parser.error('--program-only 只能与 verify-runtime 一起使用。')
if args.action == 'verify-runtime':
verify_runtime(program_only=args.program_only)
print('程序文件与 AI 应用脚本检查通过。' if args.program_only else
'文件完整性、Java、MySQL、AI Python 与 AI 原生程序检查通过。')
return
if args.action == 'start':
initial_state = read_state()
if 'supervisor' in initial_state and matches(initial_state['supervisor']) is None:
raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。')
_children, unknown_children = registered_alive(initial_state.get('children', []))
if unknown_children or initial_state.get('unknownChildren') or initial_state.get('cleanupFailures'):
raise RuntimeError('发现未确认停止的已记录进程,未启动任何服务。')
if args.action in ('start', 'serve', 'verify'):
private_directory(PRIVATE)
private_directory(DATA)
@@ -426,23 +808,24 @@ def main():
message = None
raise RuntimeError(message or '程序更新正在进行或尚未安全恢复,请查看更新记录。')
state = read_state()
running = matches(state.get('supervisor', {}))
running = matches(state['supervisor']) if 'supervisor' in state else False
if args.action == 'status':
if running is None:
print('无法确认服务状态')
return
print('已启动' if running and state.get('ready') else '正在启动或停止' if running else '已停止')
return
if running is None:
raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。')
if args.action == 'stop':
if not running:
if any(matches(info) for info in state.get('children', [])):
raise RuntimeError('发现上次退出后遗留的进程,请根据日志检查;未结束任何不明进程。')
children, unknown_children = registered_alive(state.get('children', []))
if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
raise RuntimeError('发现上次退出后仍有未确认停止的进程,请根据日志检查;未结束任何不明进程。')
print('服务已停止。')
return
atomic(STOP, {'token': state['token']})
for _ in range(240):
if not matches(state['supervisor']):
print('服务已停止。')
return
time.sleep(.5)
raise RuntimeError('服务仍在安全关闭,请稍后检查状态。')
wait_for_stop(state['supervisor'], {'token': state['token']})
return
if running:
if args.lan and configuration()['webBind'] != '0.0.0.0':
raise RuntimeError('请先停止服务,再使用 --lan 启动。')
@@ -450,7 +833,8 @@ def main():
for url in addresses(configuration()):
print(url)
return
if any(matches(info) for info in state.get('children', [])):
children, unknown_children = registered_alive(state.get('children', []))
if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
raise RuntimeError('发现上次退出后的服务进程,原数据已保留;请检查日志。')
value = verify()
if args.action == 'verify':