完善:同月多文件导入与当前用户离线迁移更新

This commit is contained in:
2026-11-04 11:35:24 +08:00
parent 147b56d0fd
commit f04d4e96e3
89 changed files with 6634 additions and 201 deletions
+486
View File
@@ -0,0 +1,486 @@
#!/usr/bin/env python3
"""Offline, unprivileged service supervisor for the standalone distribution."""
from __future__ import annotations
import argparse
import contextlib
import hashlib
import json
import os
from pathlib import Path
import platform
import re
import secrets
import signal
import socket
import subprocess
import sys
import threading
import time
from urllib.request import ProxyHandler, build_opener
import psutil
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import runtime_platform as host
from gateway import make_server
PRIVATE = ROOT / '.runtime'
DATA = ROOT / 'runtimedata'
CONFIG = PRIVATE / 'deployment.json'
STATE = PRIVATE / 'service.json'
STOP = PRIVATE / 'stop-request.json'
IS_WINDOWS = os.name == 'nt'
EXE = '.exe' if IS_WINDOWS else ''
PYTHON = ROOT / 'runtimes/python' / ('python.exe' if IS_WINDOWS else 'bin/python3')
JAVA = ROOT / 'runtimes/java/bin' / ('java' + EXE)
MYSQL = ROOT / 'runtimes/mysql/bin' / ('mysql' + EXE)
MYSQLD = ROOT / 'runtimes/mysql/bin' / ('mysqld' + EXE)
ADMIN = ROOT / 'runtimes/mysql/bin' / ('mysqladmin' + EXE)
OPENER = build_opener(ProxyHandler({}))
def atomic(path, value):
temporary = path.with_name(path.name + '.' + secrets.token_hex(5) + '.tmp')
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
temporary.replace(path)
def private_directory(path):
if path.is_symlink():
raise RuntimeError('运行目录不能是符号链接。')
path.mkdir(parents=True, exist_ok=True, mode=0o700)
if IS_WINDOWS:
result = subprocess.run(['whoami', '/user', '/fo', 'csv', '/nh'], capture_output=True, check=True)
sid = re.search(rb'S-1-\d+(?:-\d+)+', result.stdout)
if not sid:
raise RuntimeError('无法确定当前 Windows 用户。')
subprocess.run(['icacls', str(path), '/inheritance:r', '/grant:r',
'*' + sid.group().decode() + ':(OI)(CI)F', '*S-1-5-18:(OI)(CI)F', '/Q'],
stdout=subprocess.DEVNULL, check=True)
else:
path.chmod(0o700)
def configuration():
if CONFIG.exists():
value = json.loads(CONFIG.read_text(encoding='utf-8'))
else:
value = {'mysqlPort': 53306, 'backendPort': 5281, 'webPort': 5280,
'webBind': '0.0.0.0' if IS_WINDOWS else '127.0.0.1',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24)}
atomic(CONFIG, value)
value.setdefault('mysqlDatabase', 'quality_platform')
if not re.fullmatch(r'quality_(?:platform|restore_[a-f0-9]{12,32})', value['mysqlDatabase']):
raise RuntimeError('部署数据库名称无效。')
relative_archive = Path(value.get('archiveDirectory', 'runtimedata/archive'))
if relative_archive.is_absolute() or '..' in relative_archive.parts or not (ROOT / relative_archive).resolve().is_relative_to(DATA.resolve()):
raise RuntimeError('来源归档目录无效。')
for key in ('mysqlPort', 'backendPort', 'webPort'):
if type(value.get(key)) is not int or not 1024 <= value[key] <= 65535:
raise RuntimeError('部署端口配置无效。')
if value['backendPort'] not in (5281, 5283, 5284, 5285):
raise RuntimeError('接口端口必须为 5281、5283、5284 或 5285。')
if len({value[k] for k in ('mysqlPort', 'backendPort', 'webPort')}) != 3:
raise RuntimeError('数据库、接口和网页端口不能重复。')
if value.get('webBind') not in ('127.0.0.1', '0.0.0.0'):
raise RuntimeError('网页监听配置无效。')
for key in ('mysqlPassword', 'rootPassword', 'adminPassword'):
if not re.fullmatch(r'[A-Za-z0-9_-]{20,100}', value.get(key, '')):
raise RuntimeError('本机随机凭据配置无效,原配置已保留。')
return value
def environment(value):
env = os.environ.copy()
package = ROOT / 'runtimes/python' / ('Lib/site-packages' if IS_WINDOWS else 'lib/python3.11/site-packages') / 'nao_core'
env.update({
'JAVA_HOME': str(JAVA.parent.parent),
'PATH': os.pathsep.join([str(PYTHON.parent), str(JAVA.parent), str(MYSQL.parent), env.get('PATH', '')]),
'PYTHONUTF8': '1', 'PYTHONIOENCODING': 'utf-8', 'PYTHONDONTWRITEBYTECODE': '1',
'QUALITY_PROJECT_ROOT': str(ROOT), 'QUALITY_ASSISTANT_PYTHON': str(PYTHON),
'QUALITY_ASSISTANT_PACKAGE': str(package),
'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{value["mysqlPort"]}/{value.get("mysqlDatabase", "quality_platform")}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&forceConnectionTimeZoneToSession=true&useSSL=false&allowPublicKeyRetrieval=true',
'MYSQL_USER': 'quality_app', 'MYSQL_PASSWORD': value['mysqlPassword'],
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': '' if value.get('restoredFromMigration') else value['adminUsername'],
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': '' if value.get('restoredFromMigration') else value['adminPassword'],
'QUALITY_ARCHIVE_ROOT': str(ROOT / value.get('archiveDirectory', 'runtimedata/archive')), 'QUALITY_ALLOWED_ROOTS': str(DATA / 'sources'),
'QUALITY_ANALYSIS_RUNTIME_QUERY_PORT': str(value['backendPort']),
'PORT': str(value['backendPort']), 'SERVER_ADDRESS': '127.0.0.1',
'QUALITY_DATA_WORKFLOW_TEST_MODE': 'false',
'MYSQL_LEGACY_TIMESTAMP_OFFSET_SECONDS': 'UNCONFIRMED',
})
env['QUALITY_DATA_WORKFLOW_TEST_MODE'] = str(value.get('dataWorkflowTestMode', False)).lower()
env['QUALITY_ANALYSIS_AUTONOMOUS_ENABLED'] = str(value.get('analysisAutonomousEnabled', False)).lower()
env['QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED'] = str(value.get('analysisProtocolV1Enabled', False)).lower()
env['QUALITY_UPDATE_ENABLED'] = 'true' if IS_WINDOWS else 'false'
# Do not accidentally inherit a development or source-machine AI command.
env.pop('QUALITY_ANALYSIS_NAO_COMMAND', None)
env.pop('QUALITY_ANALYSIS_NAO_SCRIPT', None)
env.pop('PYTHONHOME', None)
env.pop('PYTHONPATH', None)
if not IS_WINDOWS:
env['LD_LIBRARY_PATH'] = os.pathsep.join([str(ROOT / 'runtimes/linux-libs'),
str(ROOT / 'runtimes/mysql/lib'),
str(ROOT / 'runtimes/mysql/lib/private')])
return env
def record(process, command):
observed = psutil.Process(process.pid)
return {'pid': process.pid, 'created': observed.create_time(),
'executable': str(Path(observed.exe()).resolve()), 'args': [str(a) for a in command[1:]]}
def matches(info):
try:
process = psutil.Process(int(info['pid']))
return (process.is_running() and process.status() != psutil.STATUS_ZOMBIE
and abs(process.create_time() - info['created']) < .01
and os.path.normcase(str(Path(process.exe()).resolve())) == os.path.normcase(info['executable'])
and process.cmdline()[1:] == info['args'])
except (psutil.Error, KeyError, ValueError, TypeError):
return False
def read_state():
try:
return json.loads(STATE.read_text(encoding='utf-8'))
except FileNotFoundError:
return {}
def free_port(port, bind='127.0.0.1'):
# A recently closed TCP connection may remain in TIME_WAIT. Reject a live
# listener, but allow a clean restart without waiting for that state to expire.
try:
connection = socket.create_connection(('127.0.0.1', port), timeout=.3)
except OSError:
pass
else:
connection.close()
raise RuntimeError(f'端口 {port} 已被其他服务使用。')
with socket.socket() as test:
test.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
test.bind((bind, port))
def mysql_command(value, binary=MYSQL):
return [str(binary), '--no-defaults', '--protocol=TCP', '--host=127.0.0.1',
'--port=' + str(value['mysqlPort']), '--user=root', '--connect-timeout=3']
def sql(value, query):
env = environment(value)
env['MYSQL_PWD'] = value['rootPassword']
return subprocess.run(mysql_command(value) + ['--batch', '--skip-column-names'], input=query,
text=True, encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.PIPE,
env=env, timeout=10)
def verify(check_hashes=True):
manifest = json.loads((ROOT / 'manifest.json').read_text(encoding='utf-8'))
expected_os = 'windows' if IS_WINDOWS else platform.system().lower()
if manifest['platform'] != expected_os or platform.machine().lower() not in ('amd64', 'x86_64'):
raise RuntimeError('此包仅用于 ' + manifest['platform'] + ' 的 Intel/AMD 64 位电脑。')
if check_hashes:
for entry in manifest['files']:
path = ROOT / entry['path']
if entry.get('symlink') is not None:
valid = path.is_symlink() and os.readlink(path) == entry['symlink']
else:
valid = path.is_file() and hashlib.file_digest(path.open('rb'), 'sha256').hexdigest() == entry['sha256']
if not valid:
raise RuntimeError('文件缺失或校验失败:' + entry['path'])
value = configuration()
env = environment(value)
for command in ([JAVA, '-version'], [MYSQLD, '--no-defaults', '--version']):
result = subprocess.run([str(a) for a in command], capture_output=True, env=env, timeout=30)
if result.returncode:
raise RuntimeError('运行环境检查失败:' + Path(command[0]).name + '。请查看安装说明中的系统要求。')
probe = subprocess.run([str(PYTHON), '-c', 'import duckdb,yaml,fastapi,psutil,nao_core;import sys;assert sys.version_info[:3]==(3,11,15)'],
capture_output=True, env=env, timeout=45)
if probe.returncode:
raise RuntimeError('AI Python 运行环境检查失败。')
application_probe = subprocess.run([str(PYTHON), '-c',
'import sys;sys.path.insert(0,sys.argv[1]);import nao_quality_runtime as r;'
'import nao_native_run,nao_quality_runtime_mcp,nao_visual_recovery;'
'assert len(r._config_fingerprint("https://offline.invalid/v1","offline-check","offline-check"))==64',
str(ROOT / 'scripts')], capture_output=True, env=env, timeout=45)
if application_probe.returncode:
raise RuntimeError('AI 应用脚本或策略资源检查失败。')
package = Path(env['QUALITY_ASSISTANT_PACKAGE'])
native = subprocess.run([str(package / 'bin' / ('nao-chat-server' + EXE)), '--help'],
capture_output=True, env=env, timeout=30)
if native.returncode:
raise RuntimeError('AI 原生服务无法运行,请检查系统版本和 CPU 支持。')
return value
def addresses(value):
ips = set()
if value['webBind'] == '0.0.0.0':
for entries in psutil.net_if_addrs().values():
for entry in entries:
if entry.family == socket.AF_INET and not entry.address.startswith(('127.', '169.254.')):
ips.add(entry.address)
return ['http://127.0.0.1:' + str(value['webPort'])] + [f'http://{ip}:{value["webPort"]}' for ip in sorted(ips)]
def write_login(value):
path = DATA / '登录信息.txt'
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as stream:
if value.get('restoredFromMigration'):
stream.write('质量中心\n' + '\n'.join(addresses(value)) + '\n业务数据已迁移,请使用原有账号和密码登录。\n')
else:
stream.write('质量中心\n' + '\n'.join(addresses(value)) + '\n用户名:' + value['adminUsername']
+ '\n初始密码:' + value['adminPassword'] + '\n登录后请在个人账户修改密码。此文件仅记录初始密码,请勿分享。\n')
def serve(database_only=False):
lock = os.open(PRIVATE / 'supervisor.lock', os.O_RDWR | os.O_CREAT, 0o600)
try:
host.fcntl.flock(lock, host.fcntl.LOCK_EX | host.fcntl.LOCK_NB)
except BlockingIOError:
raise RuntimeError('此目录的服务已经启动。') from None
children = []
value = configuration()
state = {'supervisor': record(psutil.Process(), [sys.executable, *sys.argv]),
'token': secrets.token_hex(24), 'ready': False, 'databaseOnly': database_only, 'children': children}
stopped = threading.Event()
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, lambda *_: stopped.set())
def stopping():
if STOP.exists():
try:
if json.loads(STOP.read_text(encoding='utf-8')).get('token') == state['token']:
stopped.set()
except (OSError, ValueError):
pass
return stopped.is_set()
def spawn(name, command):
with (DATA / 'logs' / (name + '.log')).open('ab') as log:
child = subprocess.Popen([str(a) for a in command], cwd=ROOT, env=environment(value),
stdin=subprocess.DEVNULL, stdout=log, stderr=log)
info = record(child, command)
info['name'] = name
children.append(info)
atomic(STATE, state)
return child
def wait_ready(child, check, seconds):
end = time.monotonic() + seconds
while time.monotonic() < end:
if stopping():
raise RuntimeError('启动已停止。')
if child.poll() is not None:
raise RuntimeError('服务启动失败,请查看 runtimedata/logs。')
try:
if check():
return
except (OSError, ValueError, subprocess.TimeoutExpired):
pass
time.sleep(.5)
raise RuntimeError('服务启动超时,请查看 runtimedata/logs。')
try:
for key in (('mysqlPort',) if database_only else ('mysqlPort', 'backendPort', 'webPort')):
free_port(value[key], value['webBind'] if key == 'webPort' else '127.0.0.1')
atomic(STATE, state)
db = DATA / 'mysql'
if not (db / 'auto.cnf').exists():
if db.exists() and any(db.iterdir()):
raise RuntimeError('数据库目录含有未完成的初始化文件,已保留原目录;请查看初始化日志。')
db.mkdir(exist_ok=True)
with (DATA / 'logs/mysql-initialize.log').open('ab') as log:
result = subprocess.run([str(MYSQLD), '--no-defaults', '--initialize-insecure',
'--basedir=' + str(MYSQLD.parent.parent), '--datadir=' + str(db),
'--lower-case-table-names=1'], env=environment(value),
stdout=log, stderr=log, timeout=180)
if result.returncode:
raise RuntimeError('数据库初始化失败,请查看 runtimedata/logs/mysql-initialize.log。')
bootstrap = PRIVATE / 'bootstrap.sql'
fd = os.open(bootstrap, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, 'w', encoding='utf-8') as stream:
stream.write(f"ALTER USER 'root'@'localhost' IDENTIFIED BY '{value['rootPassword']}';\n"
f"CREATE USER IF NOT EXISTS 'root'@'127.0.0.1' IDENTIFIED BY '{value['rootPassword']}';\n"
"GRANT ALL PRIVILEGES ON *.* TO 'root'@'127.0.0.1' WITH GRANT OPTION;\n"
f"CREATE DATABASE IF NOT EXISTS `{value['mysqlDatabase']}` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;\n"
f"CREATE USER IF NOT EXISTS 'quality_app'@'127.0.0.1' IDENTIFIED BY '{value['mysqlPassword']}';\n"
f"GRANT ALL PRIVILEGES ON `{value['mysqlDatabase']}`.* TO 'quality_app'@'127.0.0.1';\n")
mysql_args = [MYSQLD, '--no-defaults', '--basedir=' + str(MYSQLD.parent.parent), '--datadir=' + str(db),
'--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1', '--mysqlx=OFF',
'--lower-case-table-names=1', '--skip-name-resolve', '--skip-log-bin',
'--max-connections=60', '--innodb-buffer-pool-size=' + ('1024M' if database_only else '256M'), '--init-file=' + bootstrap.as_posix()]
if database_only:
# The restored database is unpublished until validation and a clean
# MySQL shutdown finish. Normal startup restores durable defaults.
mysql_args += ['--innodb-redo-log-capacity=1G', '--innodb-flush-log-at-trx-commit=2']
if not IS_WINDOWS:
mysql_args.append('--socket=' + str(PRIVATE / 'mysql.sock'))
database = spawn('mysql', mysql_args)
wait_ready(database, lambda: sql(value, 'SELECT 1;').returncode == 0, 120)
# The server does not reread this file after startup.
bootstrap.unlink(missing_ok=True)
if database_only:
state['ready'] = True
atomic(STATE, state)
while not stopping():
if database.poll() is not None:
raise RuntimeError('恢复数据库服务已退出,请查看日志。')
time.sleep(.25)
return
backend = spawn('backend', [JAVA, '-Djava.awt.headless=true', '-Xms128m', '-Xmx2048m', '-jar', ROOT / 'app/backend.jar'])
def backend_ready():
with OPENER.open(f'http://127.0.0.1:{value["backendPort"]}/actuator/health', timeout=3) as response:
return json.load(response).get('status') == 'UP'
wait_ready(backend, backend_ready, 900)
server = make_server(ROOT / 'app/web', value['webBind'], value['webPort'], value['backendPort'])
server.timeout = .5
state['ready'] = True
write_login(value)
atomic(STATE, state)
print('质量中心已启动:' + ' '.join(addresses(value)), flush=True)
try:
while not stopping():
if database.poll() is not None or backend.poll() is not None:
raise RuntimeError('后台服务已退出,请查看 runtimedata/logs。')
server.handle_request()
finally:
server.server_close()
finally:
state['ready'] = False
atomic(STATE, state)
# Stop backend first, then its isolated AI processes, then the database.
for info in reversed(children):
if info['name'] != 'backend' or not matches(info):
continue
process = psutil.Process(info['pid'])
descendants = process.children(recursive=True)
process.terminate()
try:
process.wait(35)
except psutil.TimeoutExpired:
if matches(info):
process.kill()
for child in descendants:
with contextlib.suppress(psutil.NoSuchProcess):
child.terminate()
with contextlib.suppress(OSError, subprocess.SubprocessError):
subprocess.run([str(PYTHON), str(ROOT / 'scripts/nao_quality_runtime.py'), '--stop-all'],
env=environment(value), cwd=ROOT, capture_output=True, timeout=25)
if any(info['name'] == 'mysql' and matches(info) for info in children):
env = environment(value)
env['MYSQL_PWD'] = value['rootPassword']
with contextlib.suppress(OSError, subprocess.SubprocessError):
subprocess.run(mysql_command(value, ADMIN) + ['shutdown'], env=env,
capture_output=True, timeout=60)
# mysqladmin can return before InnoDB has finished closing files.
for info in children:
if info['name'] == 'mysql' and matches(info):
with contextlib.suppress(psutil.NoSuchProcess, psutil.TimeoutExpired):
psutil.Process(info['pid']).wait(60)
survivors = [info for info in children if matches(info)]
if survivors:
state['children'] = survivors
atomic(STATE, state)
else:
STATE.unlink(missing_ok=True)
STOP.unlink(missing_ok=True)
(PRIVATE / 'bootstrap.sql').unlink(missing_ok=True)
os.close(lock)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('start', 'stop', 'status', 'verify', 'serve'))
parser.add_argument('--lan', action='store_true')
parser.add_argument('--database-only', action='store_true', help=argparse.SUPPRESS)
args = parser.parse_args()
if args.action in ('start', 'serve', 'verify'):
private_directory(PRIVATE)
private_directory(DATA)
(DATA / 'logs').mkdir(exist_ok=True)
(DATA / 'sources').mkdir(exist_ok=True)
(DATA / 'archive').mkdir(exist_ok=True)
if args.action == 'serve':
return serve(args.database_only)
updater = ROOT / 'deploy/offline/update_manager.py'
if args.action == 'start' and IS_WINDOWS and updater.is_file():
recovered = subprocess.run([str(PYTHON), str(updater), 'recover'], cwd=ROOT,
env=environment(configuration()), capture_output=True, text=True,
encoding='utf-8')
if recovered.returncode:
try:
message = json.loads(recovered.stdout).get('error')
except (ValueError, AttributeError):
message = None
raise RuntimeError(message or '程序更新正在进行或尚未安全恢复,请查看更新记录。')
state = read_state()
running = matches(state.get('supervisor', {}))
if args.action == 'status':
print('已启动' if running and state.get('ready') else '正在启动或停止' if running else '已停止')
return
if args.action == 'stop':
if not running:
if any(matches(info) for info in state.get('children', [])):
raise RuntimeError('发现上次退出后遗留的进程,请根据日志检查;未结束任何不明进程。')
print('服务已停止。')
return
atomic(STOP, {'token': state['token']})
for _ in range(240):
if not matches(state['supervisor']):
print('服务已停止。')
return
time.sleep(.5)
raise RuntimeError('服务仍在安全关闭,请稍后检查状态。')
if running:
if args.lan and configuration()['webBind'] != '0.0.0.0':
raise RuntimeError('请先停止服务,再使用 --lan 启动。')
print('服务已启动。' if state.get('ready') else '服务正在启动,请稍候。')
for url in addresses(configuration()):
print(url)
return
if any(matches(info) for info in state.get('children', [])):
raise RuntimeError('发现上次退出后的服务进程,原数据已保留;请检查日志。')
value = verify()
if args.action == 'verify':
print('文件完整性、Java、MySQL、AI Python 与 AI 原生程序检查通过。')
return
if args.lan and value['webBind'] != '0.0.0.0':
value['webBind'] = '0.0.0.0'
atomic(CONFIG, value)
with (DATA / 'logs/supervisor.log').open('ab') as log:
options = {'creationflags': subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP} if IS_WINDOWS else {'start_new_session': True}
process = subprocess.Popen([str(PYTHON), str(Path(__file__).resolve()), 'serve'], cwd=ROOT,
env=environment(value), stdin=subprocess.DEVNULL, stdout=log, stderr=log, **options)
for _ in range(2000):
state = read_state()
if state.get('ready') and state.get('supervisor', {}).get('pid') == process.pid and matches(state['supervisor']):
for url in addresses(value):
print(url)
print('管理员初始登录信息:' + str(DATA / '登录信息.txt'))
if IS_WINDOWS:
os.startfile(addresses(value)[0])
return
if process.poll() is not None:
raise RuntimeError('启动未完成,请查看 runtimedata/logs/supervisor.log。')
time.sleep(.5)
raise RuntimeError('服务仍在启动,请检查状态与日志。')
if __name__ == '__main__':
try:
main()
except (RuntimeError, OSError, subprocess.SubprocessError) as error:
print(str(error), file=sys.stderr)
raise SystemExit(1)