871 lines
40 KiB
Python
871 lines
40 KiB
Python
#!/usr/bin/env python3
|
|
"""Offline, unprivileged service supervisor for the standalone distribution."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import secrets
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
_old_dont_write_bytecode = sys.dont_write_bytecode
|
|
# Importing the controller must not create caches in the immutable package.
|
|
sys.dont_write_bytecode = True
|
|
import threading
|
|
import time
|
|
from pathlib import PurePosixPath
|
|
from urllib.request import ProxyHandler, build_opener
|
|
|
|
import psutil
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
import runtime_platform as host
|
|
from gateway import make_server
|
|
sys.dont_write_bytecode = _old_dont_write_bytecode
|
|
del _old_dont_write_bytecode
|
|
|
|
PRIVATE = ROOT / '.runtime'
|
|
DATA = ROOT / 'runtimedata'
|
|
CONFIG = PRIVATE / 'deployment.json'
|
|
STATE = PRIVATE / 'service.json'
|
|
STOP = PRIVATE / 'stop-request.json'
|
|
UPDATE_CONTROL_PROTOCOL = 2
|
|
IS_WINDOWS = os.name == 'nt'
|
|
EXE = '.exe' if IS_WINDOWS else ''
|
|
PYTHON = ROOT / 'runtimes/python' / ('python.exe' if IS_WINDOWS else 'bin/python3')
|
|
JAVA = ROOT / 'runtimes/java/bin' / ('java' + EXE)
|
|
MYSQL = ROOT / 'runtimes/mysql/bin' / ('mysql' + EXE)
|
|
MYSQLD = ROOT / 'runtimes/mysql/bin' / ('mysqld' + EXE)
|
|
ADMIN = ROOT / 'runtimes/mysql/bin' / ('mysqladmin' + EXE)
|
|
OPENER = build_opener(ProxyHandler({}))
|
|
STOP_TIMEOUT_SECONDS = 300
|
|
|
|
|
|
def atomic(path, value):
|
|
temporary = path.with_name(path.name + '.' + secrets.token_hex(5) + '.tmp')
|
|
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(fd, 'w', encoding='utf-8') as stream:
|
|
json.dump(value, stream, ensure_ascii=False, indent=2)
|
|
temporary.replace(path)
|
|
|
|
|
|
def private_directory(path):
|
|
if path.is_symlink():
|
|
raise RuntimeError('运行目录不能是符号链接。')
|
|
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
if IS_WINDOWS:
|
|
result = subprocess.run(['whoami', '/user', '/fo', 'csv', '/nh'], capture_output=True, check=True)
|
|
sid = re.search(rb'S-1-\d+(?:-\d+)+', result.stdout)
|
|
if not sid:
|
|
raise RuntimeError('无法确定当前 Windows 用户。')
|
|
subprocess.run(['icacls', str(path), '/inheritance:r', '/grant:r',
|
|
'*' + sid.group().decode() + ':(OI)(CI)F', '*S-1-5-18:(OI)(CI)F', '/Q'],
|
|
stdout=subprocess.DEVNULL, check=True)
|
|
else:
|
|
path.chmod(0o700)
|
|
|
|
|
|
def configuration(create=True):
|
|
if CONFIG.exists():
|
|
value = json.loads(CONFIG.read_text(encoding='utf-8'))
|
|
else:
|
|
value = {'mysqlPort': 53306, 'backendPort': 5281, 'webPort': 5280,
|
|
'webBind': '0.0.0.0' if IS_WINDOWS else '127.0.0.1',
|
|
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
|
|
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)}
|
|
if create:
|
|
atomic(CONFIG, value)
|
|
value.setdefault('mysqlDatabase', 'quality_platform')
|
|
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', value['mysqlDatabase']):
|
|
raise RuntimeError('部署数据库名称无效。')
|
|
relative_archive = Path(value.get('archiveDirectory', 'runtimedata/archive'))
|
|
if relative_archive.is_absolute() or '..' in relative_archive.parts or not (ROOT / relative_archive).resolve().is_relative_to(DATA.resolve()):
|
|
raise RuntimeError('来源归档目录无效。')
|
|
for key in ('mysqlPort', 'backendPort', 'webPort'):
|
|
if type(value.get(key)) is not int or not 1024 <= value[key] <= 65535:
|
|
raise RuntimeError('部署端口配置无效。')
|
|
if value['backendPort'] not in (5281, 5283, 5284, 5285):
|
|
raise RuntimeError('接口端口必须为 5281、5283、5284 或 5285。')
|
|
if len({value[k] for k in ('mysqlPort', 'backendPort', 'webPort')}) != 3:
|
|
raise RuntimeError('数据库、接口和网页端口不能重复。')
|
|
if value.get('webBind') not in ('127.0.0.1', '0.0.0.0'):
|
|
raise RuntimeError('网页监听配置无效。')
|
|
for key in ('mysqlPassword', 'rootPassword', 'adminPassword'):
|
|
if not re.fullmatch(r'[A-Za-z0-9_-]{20,100}', value.get(key, '')):
|
|
raise RuntimeError('本机随机凭据配置无效,原配置已保留。')
|
|
return value
|
|
|
|
|
|
def environment(value):
|
|
env = os.environ.copy()
|
|
package = ROOT / 'runtimes/python' / ('Lib/site-packages' if IS_WINDOWS else 'lib/python3.11/site-packages') / 'nao_core'
|
|
env.update({
|
|
'JAVA_HOME': str(JAVA.parent.parent),
|
|
'PATH': os.pathsep.join([str(PYTHON.parent), str(JAVA.parent), str(MYSQL.parent), env.get('PATH', '')]),
|
|
'PYTHONUTF8': '1', 'PYTHONIOENCODING': 'utf-8', 'PYTHONDONTWRITEBYTECODE': '1',
|
|
'QUALITY_PROJECT_ROOT': str(ROOT), 'QUALITY_ASSISTANT_PYTHON': str(PYTHON),
|
|
'QUALITY_ASSISTANT_PACKAGE': str(package),
|
|
'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{value["mysqlPort"]}/{value.get("mysqlDatabase", "quality_platform")}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true',
|
|
'MYSQL_USER': 'quality_app', 'MYSQL_PASSWORD': value['mysqlPassword'],
|
|
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': '' if value.get('restoredFromMigration') else value['adminUsername'],
|
|
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': '' if value.get('restoredFromMigration') else value['adminPassword'],
|
|
'QUALITY_ARCHIVE_ROOT': str(ROOT / value.get('archiveDirectory', 'runtimedata/archive')), 'QUALITY_ALLOWED_ROOTS': str(DATA / 'sources'),
|
|
'QUALITY_ANALYSIS_RUNTIME_QUERY_PORT': str(value['backendPort']),
|
|
'PORT': str(value['backendPort']), 'SERVER_ADDRESS': '127.0.0.1',
|
|
'QUALITY_DATA_WORKFLOW_TEST_MODE': 'false',
|
|
'MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS': 'UNCONFIRMED',
|
|
})
|
|
env['QUALITY_DATA_WORKFLOW_TEST_MODE'] = str(value.get('dataWorkflowTestMode', False)).lower()
|
|
env['QUALITY_ANALYSIS_AUTONOMOUS_ENABLED'] = str(value.get('analysisAutonomousEnabled', False)).lower()
|
|
env['QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED'] = str(value.get('analysisProtocolV1Enabled', False)).lower()
|
|
env['QUALITY_UPDATE_ENABLED'] = 'true' if IS_WINDOWS else 'false'
|
|
# Do not accidentally inherit a development or source-machine AI command.
|
|
env.pop('QUALITY_ANALYSIS_NAO_COMMAND', None)
|
|
env.pop('QUALITY_ANALYSIS_NAO_SCRIPT', None)
|
|
env.pop('PYTHONHOME', None)
|
|
env.pop('PYTHONPATH', None)
|
|
if not IS_WINDOWS:
|
|
env['LD_LIBRARY_PATH'] = os.pathsep.join([str(ROOT / 'runtimes/linux-libs'),
|
|
str(ROOT / 'runtimes/mysql/lib'),
|
|
str(ROOT / 'runtimes/mysql/lib/private')])
|
|
return env
|
|
|
|
|
|
def record(process, command):
|
|
observed = psutil.Process(process.pid)
|
|
return {'pid': process.pid, 'created': observed.create_time(),
|
|
'executable': str(Path(observed.exe()).resolve()), 'args': [str(a) for a in command[1:]]}
|
|
|
|
|
|
def matches(info):
|
|
"""Return True for this process, False for gone/replaced, None if unknown."""
|
|
if not isinstance(info, dict):
|
|
return None
|
|
try:
|
|
pid = info['pid']
|
|
created = info['created']
|
|
executable = info['executable']
|
|
arguments = info['args']
|
|
if (type(pid) is not int or pid <= 0 or type(created) not in (int, float)
|
|
or not math.isfinite(created) or not isinstance(executable, str)
|
|
or not isinstance(arguments, list) or not all(isinstance(item, str) for item in arguments)):
|
|
return None
|
|
except (KeyError, ValueError, TypeError):
|
|
return None
|
|
try:
|
|
process = psutil.Process(pid)
|
|
except psutil.NoSuchProcess:
|
|
return False
|
|
except Exception:
|
|
return None
|
|
try:
|
|
if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE:
|
|
return False
|
|
if abs(process.create_time() - created) >= .01:
|
|
return False
|
|
if os.path.normcase(str(Path(process.exe()).resolve())) != os.path.normcase(executable):
|
|
return False
|
|
if process.cmdline()[1:] != arguments:
|
|
return False
|
|
return True
|
|
except psutil.NoSuchProcess:
|
|
return False
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def registered_alive(records):
|
|
"""Keep both confirmed-live and uninspectable registered process records."""
|
|
if not isinstance(records, list):
|
|
return [], True
|
|
alive, unknown = [], False
|
|
for info in records:
|
|
status = matches(info)
|
|
if status is True:
|
|
alive.append(info)
|
|
elif status is None:
|
|
alive.append(info)
|
|
unknown = True
|
|
return alive, unknown
|
|
|
|
|
|
def retain_registered_children(state, records, unresolved=False):
|
|
survivors, unknown = registered_alive(records)
|
|
state['children'] = survivors
|
|
unresolved = unresolved or unknown
|
|
if unresolved:
|
|
state['unknownChildren'] = True
|
|
return survivors, unresolved
|
|
|
|
|
|
def registered_named(records, name):
|
|
confirmed, unknown = [], []
|
|
for info in records:
|
|
if not isinstance(info, dict) or info.get('name') != name:
|
|
continue
|
|
status = matches(info)
|
|
if status is True:
|
|
confirmed.append(info)
|
|
elif status is None:
|
|
unknown.append(info)
|
|
return confirmed, unknown
|
|
|
|
|
|
def snapshot_process(process, name):
|
|
try:
|
|
result = record(process, process.cmdline())
|
|
result['name'] = name
|
|
return result
|
|
except (psutil.Error, OSError):
|
|
return None
|
|
|
|
|
|
def process_object_matches(process, created):
|
|
try:
|
|
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
|
|
and abs(process.create_time() - created) < .01)
|
|
except psutil.NoSuchProcess:
|
|
return False
|
|
except Exception:
|
|
# Permission failures and unexpected inspection errors are unknown,
|
|
# not evidence that the observed child has exited.
|
|
return None
|
|
|
|
|
|
def terminate_observed(processes, timeout):
|
|
"""Stop only process objects captured below a verified service child."""
|
|
unverified = False
|
|
for process, created, info in processes:
|
|
observed = process_object_matches(process, created)
|
|
if observed is None:
|
|
if info is None:
|
|
unverified = True
|
|
elif observed is True:
|
|
with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
|
|
process.terminate()
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
states = [process_object_matches(process, created) for process, created, _info in processes]
|
|
if any(observed is None for observed in states):
|
|
unverified = unverified or any(observed is None and info is None
|
|
for observed, (_process, _created, info) in zip(states, processes))
|
|
break
|
|
if not any(states):
|
|
break
|
|
time.sleep(.1)
|
|
for process, created, info in processes:
|
|
observed = process_object_matches(process, created)
|
|
if observed is None:
|
|
if info is None:
|
|
unverified = True
|
|
elif observed is True:
|
|
with contextlib.suppress(psutil.NoSuchProcess, psutil.AccessDenied):
|
|
process.kill()
|
|
deadline = time.monotonic() + 10
|
|
while time.monotonic() < deadline:
|
|
states = [process_object_matches(process, created) for process, created, _info in processes]
|
|
if any(observed is None for observed in states):
|
|
unverified = unverified or any(observed is None and info is None
|
|
for observed, (_process, _created, info) in zip(states, processes))
|
|
break
|
|
if not any(states):
|
|
break
|
|
time.sleep(.1)
|
|
survivors = []
|
|
for process, created, info in processes:
|
|
observed = process_object_matches(process, created)
|
|
if observed is None:
|
|
if info is None:
|
|
unverified = True
|
|
else:
|
|
survivors.append(info)
|
|
elif observed is True:
|
|
identity = matches(info) if info is not None else None
|
|
if info is None:
|
|
unverified = True
|
|
elif identity is None:
|
|
survivors.append(info)
|
|
elif identity is False:
|
|
survivors.append(info)
|
|
unverified = True
|
|
else:
|
|
survivors.append(info)
|
|
return survivors, unverified
|
|
|
|
|
|
def read_state():
|
|
try:
|
|
return json.loads(STATE.read_text(encoding='utf-8'))
|
|
except FileNotFoundError:
|
|
return {}
|
|
|
|
|
|
def wait_for_stop(supervisor, request, timeout_seconds=STOP_TIMEOUT_SECONDS, interval=.5):
|
|
request_sent = False
|
|
deadline = time.monotonic() + timeout_seconds
|
|
while time.monotonic() < deadline:
|
|
supervisor_status = matches(supervisor)
|
|
if supervisor_status is None:
|
|
raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。')
|
|
if supervisor_status is True:
|
|
if not request_sent:
|
|
atomic(STOP, request)
|
|
request_sent = True
|
|
time.sleep(interval)
|
|
continue
|
|
|
|
state = read_state()
|
|
if 'supervisor' in state:
|
|
current_supervisor = state['supervisor']
|
|
current_status = matches(current_supervisor)
|
|
if current_status is None:
|
|
raise RuntimeError('无法确认服务监督进程身份,未发送停止请求。')
|
|
if current_status is True:
|
|
if current_supervisor != supervisor:
|
|
raise RuntimeError('服务状态已变化,未停止任何不明进程。')
|
|
if not request_sent:
|
|
atomic(STOP, request)
|
|
request_sent = True
|
|
time.sleep(interval)
|
|
continue
|
|
survivors, unknown_children = registered_alive(state.get('children', []))
|
|
if unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
|
|
raise RuntimeError('服务监督程序已退出,但仍有未确认停止的已记录进程;未结束任何不明进程。')
|
|
if survivors:
|
|
time.sleep(interval)
|
|
continue
|
|
print('服务已停止。')
|
|
return
|
|
raise RuntimeError('服务仍在安全关闭,请稍后检查状态。')
|
|
|
|
|
|
def free_port(port, bind='127.0.0.1'):
|
|
# A recently closed TCP connection may remain in TIME_WAIT. Reject a live
|
|
# listener, but allow a clean restart without waiting for that state to expire.
|
|
try:
|
|
connection = socket.create_connection(('127.0.0.1', port), timeout=.3)
|
|
except OSError:
|
|
pass
|
|
else:
|
|
connection.close()
|
|
raise RuntimeError(f'端口 {port} 已被其他服务使用。')
|
|
with socket.socket() as test:
|
|
test.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
test.bind((bind, port))
|
|
|
|
|
|
def mysql_command(value, binary=MYSQL):
|
|
return [str(binary), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
|
|
'--port=' + str(value['mysqlPort']), '--user=root', '--connect-timeout=3']
|
|
|
|
|
|
def sql(value, query):
|
|
env = environment(value)
|
|
env['MYSQL_PWD'] = value['rootPassword']
|
|
return subprocess.run(mysql_command(value) + ['--batch', '--skip-column-names'], input=query,
|
|
text=True, encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
env=env, timeout=10)
|
|
|
|
|
|
def manifest_name(name):
|
|
if (not isinstance(name, str) or not name or '\\' in name
|
|
or PurePosixPath(name).is_absolute()
|
|
or any(part in ('', '.', '..') for part in name.split('/'))):
|
|
raise RuntimeError('文件清单包含无效路径。')
|
|
return PurePosixPath(name)
|
|
|
|
|
|
def manifest_path(name):
|
|
parts = manifest_name(name).parts
|
|
path = ROOT.joinpath(*parts)
|
|
root = ROOT.resolve()
|
|
if not path.resolve(strict=False).is_relative_to(root):
|
|
raise RuntimeError('文件路径异常,已停止校验:' + name)
|
|
parent = path.parent
|
|
while parent != ROOT and parent != parent.parent:
|
|
if parent.is_symlink():
|
|
raise RuntimeError('文件路径异常,已停止校验:' + name)
|
|
parent = parent.parent
|
|
return path
|
|
|
|
|
|
def cache_source(name):
|
|
"""Return only timestamp-based bytecode sources from the bundled Python."""
|
|
if not isinstance(name, str) or '\\' in name:
|
|
return None
|
|
path = PurePosixPath(name)
|
|
if path.parts[:2] != ('runtimes', 'python') or any(part in ('.', '..') for part in name.split('/')):
|
|
return None
|
|
if path.parent.name != '__pycache__':
|
|
return None
|
|
match = re.fullmatch(r'(.+)\.cpython-\d+(?:\.opt-\d+)?\.pyc', path.name)
|
|
return (path.parent.parent / (match[1] + '.py')).as_posix() if match else None
|
|
|
|
|
|
def file_digest(path):
|
|
with path.open('rb') as stream:
|
|
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
|
|
|
|
def regular_manifest_file(name):
|
|
path = manifest_path(name)
|
|
if path.is_symlink():
|
|
raise RuntimeError('文件路径异常,已停止校验:' + name)
|
|
return path
|
|
|
|
|
|
def program_file_path(name):
|
|
"""Mirror update_manager.program_path without importing code during a read-only check."""
|
|
path = manifest_name(name)
|
|
return (name == 'app/backend.jar' or name.startswith('app/web/')
|
|
or name == 'backend/src/main/resources/analysis-runtime-policy.json'
|
|
or name.startswith('scripts/') and path.suffix == '.py'
|
|
or name.startswith('ai/nao-quality/')
|
|
or name.startswith('deploy/offline/') and path.suffix in ('.py', '.ps1', '.md'))
|
|
|
|
|
|
def verify_manifest_files(manifest, program_only=False):
|
|
entries = manifest.get('files')
|
|
if not isinstance(entries, list):
|
|
raise RuntimeError('文件清单无效。')
|
|
by_name = {}
|
|
for entry in entries:
|
|
if not isinstance(entry, dict) or not isinstance(entry.get('path'), str):
|
|
raise RuntimeError('文件清单无效。')
|
|
name = entry['path']
|
|
manifest_name(name)
|
|
if name in by_name:
|
|
raise RuntimeError('文件清单包含重复路径。')
|
|
by_name[name] = entry
|
|
|
|
ignored_caches = set()
|
|
if not program_only:
|
|
# Timestamp-based bytecode varies between Windows copies. Ignore only
|
|
# cache entries whose exact source is present and matches its manifest
|
|
# digest; ordinary Python source files are still hashed below.
|
|
for name in by_name:
|
|
source = cache_source(name)
|
|
if source is None:
|
|
continue
|
|
source_entry = by_name.get(source)
|
|
expected = source_entry.get('sha256') if isinstance(source_entry, dict) else None
|
|
source_path = regular_manifest_file(source)
|
|
if (not isinstance(expected, str) or not source_path.is_file()
|
|
or file_digest(source_path) != expected):
|
|
raise RuntimeError('对应源码缺失或校验失败,已停止校验:' + source)
|
|
cache_path = regular_manifest_file(name)
|
|
if cache_path.exists() and not cache_path.is_file():
|
|
raise RuntimeError('缓存文件路径异常,已停止校验:' + name)
|
|
ignored_caches.add(name)
|
|
|
|
for name, entry in by_name.items():
|
|
if program_only and not program_file_path(name):
|
|
continue
|
|
if name in ignored_caches:
|
|
continue
|
|
path = manifest_path(name)
|
|
expected_link = entry.get('symlink')
|
|
if expected_link is not None:
|
|
valid = path.is_symlink() and os.readlink(path) == expected_link
|
|
else:
|
|
if path.is_symlink():
|
|
valid = False
|
|
elif path.is_file():
|
|
with path.open('rb') as stream:
|
|
valid = hashlib.file_digest(stream, 'sha256').hexdigest() == entry.get('sha256')
|
|
else:
|
|
valid = False
|
|
if not valid:
|
|
raise RuntimeError('文件缺失或校验失败:' + name)
|
|
|
|
|
|
def verify_runtime(check_hashes=True, program_only=False):
|
|
manifest = json.loads((ROOT / 'manifest.json').read_text(encoding='utf-8'))
|
|
expected_os = 'windows' if IS_WINDOWS else platform.system().lower()
|
|
if manifest['platform'] != expected_os or platform.machine().lower() not in ('amd64', 'x86_64'):
|
|
raise RuntimeError('此包仅用于 ' + manifest['platform'] + ' 的 Intel/AMD 64 位电脑。')
|
|
if check_hashes:
|
|
verify_manifest_files(manifest, program_only=program_only)
|
|
value = configuration(create=False)
|
|
env = environment(value)
|
|
if program_only:
|
|
application_probe = subprocess.run([str(PYTHON), '-c',
|
|
'import sys;sys.path.insert(0,sys.argv[1]);import nao_quality_runtime as r;'
|
|
'import nao_native_run,nao_quality_runtime_mcp,nao_visual_recovery;'
|
|
'assert len(r._config_fingerprint("https://offline.invalid/v1","offline-check","offline-check"))==64',
|
|
str(ROOT / 'scripts')], capture_output=True, env=env, timeout=45)
|
|
if application_probe.returncode:
|
|
raise RuntimeError('AI 应用脚本或策略资源检查失败。')
|
|
return value
|
|
for command in ([JAVA, '-version'], [MYSQLD, '--no-defaults', '--version']):
|
|
result = subprocess.run([str(a) for a in command], capture_output=True, env=env, timeout=30)
|
|
if result.returncode:
|
|
raise RuntimeError('运行环境检查失败:' + Path(command[0]).name + '。请查看安装说明中的系统要求。')
|
|
probe = subprocess.run([str(PYTHON), '-c', 'import duckdb,yaml,fastapi,psutil,nao_core;import sys;assert sys.version_info[:3]==(3,11,15)'],
|
|
capture_output=True, env=env, timeout=45)
|
|
if probe.returncode:
|
|
raise RuntimeError('AI Python 运行环境检查失败。')
|
|
application_probe = subprocess.run([str(PYTHON), '-c',
|
|
'import sys;sys.path.insert(0,sys.argv[1]);import nao_quality_runtime as r;'
|
|
'import nao_native_run,nao_quality_runtime_mcp,nao_visual_recovery;'
|
|
'assert len(r._config_fingerprint("https://offline.invalid/v1","offline-check","offline-check"))==64',
|
|
str(ROOT / 'scripts')], capture_output=True, env=env, timeout=45)
|
|
if application_probe.returncode:
|
|
raise RuntimeError('AI 应用脚本或策略资源检查失败。')
|
|
package = Path(env['QUALITY_ASSISTANT_PACKAGE'])
|
|
native = subprocess.run([str(package / 'bin' / ('nao-chat-server' + EXE)), '--help'],
|
|
capture_output=True, env=env, timeout=30)
|
|
if native.returncode:
|
|
raise RuntimeError('AI 原生服务无法运行,请检查系统版本和 CPU 支持。')
|
|
return value
|
|
|
|
|
|
def verify(check_hashes=True):
|
|
# Keep the historical full verification entry point and its configuration
|
|
# initialization behavior for the installer and migration recovery tools.
|
|
verify_runtime(check_hashes=check_hashes)
|
|
return configuration()
|
|
|
|
|
|
def addresses(value):
|
|
ips = set()
|
|
if value['webBind'] == '0.0.0.0':
|
|
for entries in psutil.net_if_addrs().values():
|
|
for entry in entries:
|
|
if entry.family == socket.AF_INET and not entry.address.startswith(('127.', '169.254.')):
|
|
ips.add(entry.address)
|
|
return ['http://127.0.0.1:' + str(value['webPort'])] + [f'http://{ip}:{value["webPort"]}' for ip in sorted(ips)]
|
|
|
|
|
|
def write_login(value):
|
|
path = DATA / '登录信息.txt'
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
with os.fdopen(fd, 'w', encoding='utf-8') as stream:
|
|
if value.get('restoredFromMigration'):
|
|
stream.write('质量中心\n' + '\n'.join(addresses(value)) + '\n业务数据已迁移,请使用原有账号和密码登录。\n')
|
|
else:
|
|
stream.write('质量中心\n' + '\n'.join(addresses(value)) + '\n用户名:' + value['adminUsername']
|
|
+ '\n初始密码:' + value['adminPassword'] + '\n登录后请在个人账户修改密码。此文件仅记录初始密码,请勿分享。\n')
|
|
|
|
|
|
def serve(database_only=False):
|
|
lock = os.open(PRIVATE / 'supervisor.lock', os.O_RDWR | os.O_CREAT, 0o600)
|
|
try:
|
|
host.fcntl.flock(lock, host.fcntl.LOCK_EX | host.fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
raise RuntimeError('此目录的服务已经启动。') from None
|
|
children = []
|
|
value = configuration()
|
|
state = {'supervisor': record(psutil.Process(), [sys.executable, *sys.argv]),
|
|
'token': secrets.token_hex(24), 'ready': False, 'databaseOnly': database_only, 'children': children}
|
|
stopped = threading.Event()
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
signal.signal(sig, lambda *_: stopped.set())
|
|
|
|
def stopping():
|
|
if STOP.exists():
|
|
try:
|
|
if json.loads(STOP.read_text(encoding='utf-8')).get('token') == state['token']:
|
|
stopped.set()
|
|
except (OSError, ValueError):
|
|
pass
|
|
return stopped.is_set()
|
|
|
|
def spawn(name, command):
|
|
with (DATA / 'logs' / (name + '.log')).open('ab') as log:
|
|
child = subprocess.Popen([str(a) for a in command], cwd=ROOT, env=environment(value),
|
|
stdin=subprocess.DEVNULL, stdout=log, stderr=log)
|
|
info = record(child, command)
|
|
info['name'] = name
|
|
children.append(info)
|
|
atomic(STATE, state)
|
|
return child
|
|
|
|
def wait_ready(child, check, seconds):
|
|
end = time.monotonic() + seconds
|
|
while time.monotonic() < end:
|
|
if stopping():
|
|
raise RuntimeError('启动已停止。')
|
|
if child.poll() is not None:
|
|
raise RuntimeError('服务启动失败,请查看 runtimedata/logs。')
|
|
try:
|
|
if check():
|
|
return
|
|
except (OSError, ValueError, subprocess.TimeoutExpired):
|
|
pass
|
|
time.sleep(.5)
|
|
raise RuntimeError('服务启动超时,请查看 runtimedata/logs。')
|
|
|
|
try:
|
|
for key in (('mysqlPort',) if database_only else ('mysqlPort', 'backendPort', 'webPort')):
|
|
free_port(value[key], value['webBind'] if key == 'webPort' else '127.0.0.1')
|
|
atomic(STATE, state)
|
|
db = DATA / 'mysql'
|
|
if not (db / 'auto.cnf').exists():
|
|
if db.exists() and any(db.iterdir()):
|
|
raise RuntimeError('数据库目录含有未完成的初始化文件,已保留原目录;请查看初始化日志。')
|
|
db.mkdir(exist_ok=True)
|
|
with (DATA / 'logs/mysql-initialize.log').open('ab') as log:
|
|
result = subprocess.run([str(MYSQLD), '--no-defaults', '--initialize-insecure',
|
|
'--basedir=' + str(MYSQLD.parent.parent), '--datadir=' + str(db),
|
|
'--lower-case-table-names=1'], env=environment(value),
|
|
stdout=log, stderr=log, timeout=180)
|
|
if result.returncode:
|
|
raise RuntimeError('数据库初始化失败,请查看 runtimedata/logs/mysql-initialize.log。')
|
|
bootstrap = PRIVATE / 'bootstrap.sql'
|
|
fd = os.open(bootstrap, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
with os.fdopen(fd, 'w', encoding='utf-8') as stream:
|
|
stream.write(f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{value['rootPassword']}';\n"
|
|
f"CREATE USER IF NOT EXISTS 'root'@'127.0.0.1' IDENTIFIED BY '{value['rootPassword']}';\n"
|
|
"GRANT ALL PRIVILEGES ON *.* TO 'root'@'127.0.0.1' WITH GRANT OPTION;\n"
|
|
f"CREATE DATABASE IF NOT EXISTS `{value['mysqlDatabase']}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;\n"
|
|
f"CREATE USER IF NOT EXISTS 'quality_app'@'127.0.0.1' IDENTIFIED BY '{value['mysqlPassword']}';\n"
|
|
f"GRANT ALL PRIVILEGES ON `{value['mysqlDatabase']}`.* TO 'quality_app'@'127.0.0.1';\n")
|
|
mysql_args = [MYSQLD, '--no-defaults', '--basedir=' + str(MYSQLD.parent.parent), '--datadir=' + str(db),
|
|
'--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1', '--mysqlx=OFF',
|
|
'--lower-case-table-names=1', '--skip-name-resolve', '--skip-log-bin',
|
|
'--max-connections=60', '--innodb-buffer-pool-size=' + ('1024M' if database_only else '256M'), '--init-file=' + bootstrap.as_posix()]
|
|
if database_only:
|
|
# The restored database is unpublished until validation and a clean
|
|
# MySQL shutdown finish. Normal startup restores durable defaults.
|
|
mysql_args += ['--innodb-redo-log-capacity=1G', '--innodb-flush-log-at-trx-commit=2']
|
|
if not IS_WINDOWS:
|
|
mysql_args.append('--socket=' + str(PRIVATE / 'mysql.sock'))
|
|
database = spawn('mysql', mysql_args)
|
|
wait_ready(database, lambda: sql(value, 'SELECT 1;').returncode == 0, 120)
|
|
# The server does not reread this file after startup.
|
|
bootstrap.unlink(missing_ok=True)
|
|
if database_only:
|
|
state['ready'] = True
|
|
atomic(STATE, state)
|
|
while not stopping():
|
|
if database.poll() is not None:
|
|
raise RuntimeError('恢复数据库服务已退出,请查看日志。')
|
|
time.sleep(.25)
|
|
return
|
|
backend = spawn('backend', [JAVA, '-Djava.awt.headless=true', '-Xms128m', '-Xmx2048m', '-jar', ROOT / 'app/backend.jar'])
|
|
|
|
def backend_ready():
|
|
with OPENER.open(f'http://127.0.0.1:{value["backendPort"]}/actuator/health', timeout=3) as response:
|
|
return json.load(response).get('status') == 'UP'
|
|
|
|
wait_ready(backend, backend_ready, 900)
|
|
server = make_server(ROOT / 'app/web', value['webBind'], value['webPort'], value['backendPort'])
|
|
server.timeout = .5
|
|
state['ready'] = True
|
|
write_login(value)
|
|
atomic(STATE, state)
|
|
print('质量中心已启动:' + ' '.join(addresses(value)), flush=True)
|
|
try:
|
|
while not stopping():
|
|
if database.poll() is not None or backend.poll() is not None:
|
|
raise RuntimeError('后台服务已退出,请查看 runtimedata/logs。')
|
|
server.handle_request()
|
|
finally:
|
|
server.server_close()
|
|
finally:
|
|
state['ready'] = False
|
|
atomic(STATE, state)
|
|
cleanup_failures, extra_survivors = [], []
|
|
unknown_children = False
|
|
# Only signal the backend and descendants observed beneath its verified
|
|
# PID/start-time/executable/arguments identity.
|
|
for info in reversed(children):
|
|
if not isinstance(info, dict) or info.get('name') != 'backend':
|
|
continue
|
|
backend_status = matches(info)
|
|
if backend_status is None:
|
|
extra_survivors.append(info)
|
|
continue
|
|
if backend_status is not True:
|
|
continue
|
|
try:
|
|
process = psutil.Process(info['pid'])
|
|
except psutil.NoSuchProcess:
|
|
continue
|
|
except Exception:
|
|
extra_survivors.append(info)
|
|
continue
|
|
observed = []
|
|
try:
|
|
descendants = process.children(recursive=True)
|
|
except psutil.NoSuchProcess:
|
|
descendants = []
|
|
except psutil.Error:
|
|
descendants = []
|
|
unknown_children = True
|
|
for child in descendants:
|
|
try:
|
|
created = child.create_time()
|
|
observed.append((child, created, snapshot_process(child, 'backend-descendant')))
|
|
except (psutil.Error, OSError):
|
|
unknown_children = True
|
|
observed.append((process, info['created'], info))
|
|
survivors, unverified = terminate_observed(observed, 35)
|
|
extra_survivors.extend(item for item in survivors if item.get('name') == 'backend-descendant')
|
|
unknown_children = unknown_children or unverified
|
|
|
|
try:
|
|
stopped = subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'],
|
|
env=environment(value), cwd=ROOT, capture_output=True, timeout=25)
|
|
if stopped.returncode:
|
|
cleanup_failures.append('analysisProcesses')
|
|
except (OSError, subprocess.SubprocessError):
|
|
cleanup_failures.append('analysisProcesses')
|
|
mysql_infos, unknown_mysql = registered_named(children, 'mysql')
|
|
extra_survivors.extend(unknown_mysql)
|
|
mysql_infos = [info for info in mysql_infos if matches(info) is True]
|
|
if mysql_infos:
|
|
env = environment(value)
|
|
env['MYSQL_PWD'] = value['rootPassword']
|
|
try:
|
|
shutdown = subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env,
|
|
capture_output=True, timeout=60)
|
|
if shutdown.returncode:
|
|
cleanup_failures.append('databaseShutdown')
|
|
except (OSError, subprocess.SubprocessError):
|
|
cleanup_failures.append('databaseShutdown')
|
|
# mysqladmin can return before InnoDB has finished closing files.
|
|
for info in mysql_infos:
|
|
mysql_status = matches(info)
|
|
if mysql_status is None:
|
|
extra_survivors.append(info)
|
|
elif mysql_status is True:
|
|
try:
|
|
psutil.Process(info['pid']).wait(60)
|
|
except (psutil.NoSuchProcess, psutil.TimeoutExpired):
|
|
pass
|
|
except Exception:
|
|
extra_survivors.append(info)
|
|
unique_records, seen_records = [], set()
|
|
for info in [*children, *extra_survivors]:
|
|
if not isinstance(info, dict):
|
|
unknown_children = True
|
|
continue
|
|
key = (info.get('pid'), info.get('created'))
|
|
if key not in seen_records:
|
|
seen_records.add(key)
|
|
unique_records.append(info)
|
|
survivors, unknown_children = retain_registered_children(state, unique_records, unknown_children)
|
|
if cleanup_failures:
|
|
state['cleanupFailures'] = cleanup_failures
|
|
if survivors or unknown_children or cleanup_failures:
|
|
atomic(STATE, state)
|
|
else:
|
|
STATE.unlink(missing_ok=True)
|
|
STOP.unlink(missing_ok=True)
|
|
(PRIVATE / 'bootstrap.sql').unlink(missing_ok=True)
|
|
os.close(lock)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'verify-runtime', 'serve'))
|
|
parser.add_argument('--lan', action='store_true')
|
|
parser.add_argument('--program-only', action='store_true')
|
|
parser.add_argument('--database-only', action='store_true', help=argparse.SUPPRESS)
|
|
args = parser.parse_args()
|
|
if args.program_only and args.action != 'verify-runtime':
|
|
parser.error('--program-only 只能与 verify-runtime 一起使用。')
|
|
if args.action == 'verify-runtime':
|
|
verify_runtime(program_only=args.program_only)
|
|
print('程序文件与 AI 应用脚本检查通过。' if args.program_only else
|
|
'文件完整性、Java、MySQL、AI Python 与 AI 原生程序检查通过。')
|
|
return
|
|
if args.action == 'start':
|
|
initial_state = read_state()
|
|
if 'supervisor' in initial_state and matches(initial_state['supervisor']) is None:
|
|
raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。')
|
|
_children, unknown_children = registered_alive(initial_state.get('children', []))
|
|
if unknown_children or initial_state.get('unknownChildren') or initial_state.get('cleanupFailures'):
|
|
raise RuntimeError('发现未确认停止的已记录进程,未启动任何服务。')
|
|
if args.action in ('start', 'serve', 'verify'):
|
|
private_directory(PRIVATE)
|
|
private_directory(DATA)
|
|
(DATA / 'logs').mkdir(exist_ok=True)
|
|
(DATA / 'sources').mkdir(exist_ok=True)
|
|
(DATA / 'archive').mkdir(exist_ok=True)
|
|
if args.action == 'serve':
|
|
return serve(args.database_only)
|
|
updater = ROOT / 'deploy/offline/update_manager.py'
|
|
if args.action == 'start' and IS_WINDOWS and updater.is_file():
|
|
recovered = subprocess.run([str(PYTHON), str(updater), 'recover'], cwd=ROOT,
|
|
env=environment(configuration()), capture_output=True, text=True,
|
|
encoding='utf-8')
|
|
if recovered.returncode:
|
|
try:
|
|
message = json.loads(recovered.stdout).get('error')
|
|
except (ValueError, AttributeError):
|
|
message = None
|
|
raise RuntimeError(message or '程序更新正在进行或尚未安全恢复,请查看更新记录。')
|
|
state = read_state()
|
|
running = matches(state['supervisor']) if 'supervisor' in state else False
|
|
if args.action == 'status':
|
|
if running is None:
|
|
print('无法确认服务状态')
|
|
return
|
|
print('已启动' if running and state.get('ready') else '正在启动或停止' if running else '已停止')
|
|
return
|
|
if running is None:
|
|
raise RuntimeError('无法确认服务监督进程身份,未停止或启动任何服务。')
|
|
if args.action == 'stop':
|
|
if not running:
|
|
children, unknown_children = registered_alive(state.get('children', []))
|
|
if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
|
|
raise RuntimeError('发现上次退出后仍有未确认停止的进程,请根据日志检查;未结束任何不明进程。')
|
|
print('服务已停止。')
|
|
return
|
|
wait_for_stop(state['supervisor'], {'token': state['token']})
|
|
return
|
|
if running:
|
|
if args.lan and configuration()['webBind'] != '0.0.0.0':
|
|
raise RuntimeError('请先停止服务,再使用 --lan 启动。')
|
|
print('服务已启动。' if state.get('ready') else '服务正在启动,请稍候。')
|
|
for url in addresses(configuration()):
|
|
print(url)
|
|
return
|
|
children, unknown_children = registered_alive(state.get('children', []))
|
|
if children or unknown_children or state.get('unknownChildren') or state.get('cleanupFailures'):
|
|
raise RuntimeError('发现上次退出后的服务进程,原数据已保留;请检查日志。')
|
|
value = verify()
|
|
if args.action == 'verify':
|
|
print('文件完整性、Java、MySQL、AI Python 与 AI 原生程序检查通过。')
|
|
return
|
|
if args.lan and value['webBind'] != '0.0.0.0':
|
|
value['webBind'] = '0.0.0.0'
|
|
atomic(CONFIG, value)
|
|
with (DATA / 'logs/supervisor.log').open('ab') as log:
|
|
options = {'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP} if IS_WINDOWS else {'start_new_session': True}
|
|
process = subprocess.Popen([str(PYTHON), str(Path(__file__).resolve()), 'serve'], cwd=ROOT,
|
|
env=environment(value), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **options)
|
|
for _ in range(2000):
|
|
state = read_state()
|
|
if state.get('ready') and state.get('supervisor', {}).get('pid') == process.pid and matches(state['supervisor']):
|
|
for url in addresses(value):
|
|
print(url)
|
|
print('管理员初始登录信息:' + str(DATA / '登录信息.txt'))
|
|
if IS_WINDOWS:
|
|
os.startfile(addresses(value)[0])
|
|
return
|
|
if process.poll() is not None:
|
|
raise RuntimeError('启动未完成,请查看 runtimedata/logs/supervisor.log。')
|
|
time.sleep(.5)
|
|
raise RuntimeError('服务仍在启动,请检查状态与日志。')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (RuntimeError, OSError, subprocess.SubprocessError) as error:
|
|
print(str(error), file=sys.stderr)
|
|
raise SystemExit(1)
|