初始化:质量数据分析平台源码

This commit is contained in:
2026-10-22 15:58:57 +08:00
commit 1e8a655adc
280 changed files with 43922 additions and 0 deletions
+267
View File
@@ -0,0 +1,267 @@
#!/usr/bin/env python3
"""Build and run the isolated local quality platform without global installs."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import secrets
import signal
import shutil
import subprocess
import sys
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
RUNTIME = ROOT / '.runtime'
OUTPUT = ROOT / 'output'
CONFIG = RUNTIME / 'local-config.json'
def save_private(path, text):
path.parent.mkdir(parents=True, exist_ok=True)
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, 'w') as stream:
stream.write(text)
path.chmod(0o600)
def config():
if CONFIG.exists():
return json.loads(CONFIG.read_text())
value = {'mysqlPort': 53306, 'mysqlDatabase': 'quality_platform', 'mysqlUser': 'quality_app',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24),
'databaseReady': False}
save_private(CONFIG, json.dumps(value, indent=2) + '\n')
return value
def environment():
value = config()
env = os.environ.copy()
env.update({'JAVA_HOME': str(RUNTIME / 'jdk/Contents/Home'),
'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{value["mysqlPort"]}/{value["mysqlDatabase"]}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true',
'MYSQL_USER': value['mysqlUser'], 'MYSQL_PASSWORD': value['mysqlPassword'],
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': value['adminUsername'],
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': value['adminPassword'],
'QUALITY_ARCHIVE_ROOT': str(ROOT / 'runtimedata/archive'),
'QUALITY_ALLOWED_ROOTS': str(ROOT / '质量数据/系统原始数据示例'),
'QUALITY_DATA_WORKFLOW_TEST_MODE': 'true',
'QUALITY_PROJECT_ROOT': str(ROOT),
'QUALITY_DEFAULT_SOURCE_YEAR': '2026'})
env['PATH'] = str(RUNTIME / 'jdk/Contents/Home/bin') + ':' + str(RUNTIME / 'maven/bin') + ':' + env.get('PATH', '')
return env
def run(args, **options):
return subprocess.run([str(a) for a in args], cwd=ROOT, env=environment(), check=True, **options)
def alive(pid):
try:
pid = int(pid)
if pid <= 1:
return False
os.kill(pid, 0)
state = subprocess.run(['ps', '-p', str(pid), '-o', 'stat='],
text=True, capture_output=True, check=False)
return state.returncode == 0 and bool(state.stdout.strip()) and not state.stdout.strip().startswith('Z')
except (OSError, ValueError):
return False
def process_stamp(pid):
result = subprocess.run(['ps', '-p', str(int(pid)), '-o', 'lstart='],
text=True, capture_output=True, check=False)
return result.stdout.strip() if result.returncode == 0 else ''
def owned_process(name, pid):
identity = RUNTIME / (name + '.process.json')
if not identity.exists():
return False
try:
recorded = json.loads(identity.read_text())
return (recorded.get('pid') == int(pid) and recorded.get('started') == process_stamp(pid)
and os.getpgid(int(pid)) == int(pid))
except (OSError, ValueError):
return False
def start_process(name, args):
pidfile = RUNTIME / (name + '.pid')
if pidfile.exists() and alive(pidfile.read_text().strip()):
if not owned_process(name, pidfile.read_text()):
raise RuntimeError(f'{name}: an existing PID has no matching project identity; inspect before reusing it.')
return int(pidfile.read_text())
(OUTPUT / 'logs').mkdir(parents=True, exist_ok=True)
with (OUTPUT / 'logs' / (name + '.log')).open('ab') as log:
process = subprocess.Popen([str(a) for a in args], cwd=ROOT, env=environment(),
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
pidfile.write_text(str(process.pid))
save_private(RUNTIME / (name + '.process.json'), json.dumps({'pid': process.pid, 'started': process_stamp(process.pid)}))
return process.pid
def wait_http(url, seconds=90):
end = time.monotonic() + seconds
while time.monotonic() < end:
try:
with urllib.request.urlopen(url, timeout=2) as response:
if response.status == 200:
return
except (OSError, urllib.error.URLError):
time.sleep(.5)
raise RuntimeError(f'Service did not become ready: {url}. Inspect output/logs/.')
def mysql_arguments():
return [RUNTIME / 'mysql/bin/mysql', '--no-defaults', '--protocol=SOCKET',
'--socket=' + str(RUNTIME / 'mysql.sock'), '--user=root', '--batch', '--skip-column-names']
def start_mysql():
value = config()
data = RUNTIME / 'mysql-data'
binary = RUNTIME / 'mysql/bin/mysqld'
if not binary.exists():
raise RuntimeError('Run python3 scripts/setup_runtime.py first.')
if not (data / 'auto.cnf').exists():
data.mkdir(parents=True, exist_ok=True)
(OUTPUT / 'logs').mkdir(parents=True, exist_ok=True)
with (OUTPUT / 'logs/mysql-initialize.log').open('ab') as log:
run([binary, '--no-defaults', '--initialize-insecure', '--basedir=' + str(RUNTIME / 'mysql'),
'--datadir=' + str(data), '--lower-case-table-names=2'], stdout=log, stderr=log)
start_process('mysql', [binary, '--no-defaults', '--basedir=' + str(RUNTIME / 'mysql'),
'--datadir=' + str(data), '--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1',
'--socket=' + str(RUNTIME / 'mysql.sock'), '--pid-file=' + str(RUNTIME / 'mysql-server.pid'),
'--log-error=' + str(OUTPUT / 'logs/mysql-server.log'), '--mysqlx=OFF',
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M'])
client_config = RUNTIME / 'mysql-root.cnf'
save_private(client_config, '[client]\nuser=root\npassword=' + value['rootPassword'] + '\nsocket=' + str(RUNTIME / 'mysql.sock') + '\n')
for _ in range(120):
if (RUNTIME / 'mysql.sock').exists():
result = subprocess.run([str(a) for a in mysql_arguments()], input='SELECT 1;\n',
text=True, capture_output=True)
if result.returncode == 0:
break
result = subprocess.run([str(RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config),
'--batch', '-e', 'SELECT 1;'], capture_output=True)
if result.returncode == 0:
break
time.sleep(.5)
else:
raise RuntimeError('Local MySQL did not start; inspect output/logs/mysql-server.log.')
if not value['databaseReady']:
sql = f'''CREATE DATABASE IF NOT EXISTS {value['mysqlDatabase']} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;
CREATE USER IF NOT EXISTS '{value['mysqlUser']}'@'localhost' IDENTIFIED BY '{value['mysqlPassword']}';
CREATE USER IF NOT EXISTS '{value['mysqlUser']}'@'127.0.0.1' IDENTIFIED BY '{value['mysqlPassword']}';
GRANT ALL PRIVILEGES ON {value['mysqlDatabase']}.* TO '{value['mysqlUser']}'@'localhost';
GRANT ALL PRIVILEGES ON {value['mysqlDatabase']}.* TO '{value['mysqlUser']}'@'127.0.0.1';
ALTER USER 'root'@'localhost' IDENTIFIED BY '{value['rootPassword']}';
'''
authenticated = [str(RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config), '--batch']
result = subprocess.run(authenticated, input=sql, text=True, capture_output=True)
if result.returncode != 0:
result = subprocess.run([str(a) for a in mysql_arguments()], input=sql, text=True, capture_output=True)
if result.returncode != 0:
raise RuntimeError('Local database bootstrap failed. Credentials have not been printed; inspect the local database configuration.')
value['databaseReady'] = True
save_private(CONFIG, json.dumps(value, indent=2) + '\n')
save_private(ROOT / 'runtimedata/本机登录信息.txt',
'质量中心 · 本机开发账户\n地址:http://127.0.0.1:5280\n用户名:' + value['adminUsername'] +
'\n密码:' + value['adminPassword'] + '\n仅限本机使用,请勿上传或分享此文件。\n')
print('Local MySQL is ready on 127.0.0.1:' + str(value['mysqlPort']), flush=True)
def build(skip_tests=False):
args = [RUNTIME / 'maven/bin/mvn', '-f', ROOT / 'backend/pom.xml',
'-Dmaven.repo.local=' + str(RUNTIME / 'm2'), '--batch-mode', 'package']
if skip_tests:
args.append('-DskipTests')
run(args)
run(['pnpm', '--dir', ROOT / 'web', 'build'])
def start():
start_mysql()
jars = sorted((ROOT / 'backend/target').glob('*.jar'), key=lambda p: p.stat().st_mtime)
jars = [p for p in jars if not p.name.endswith('.original')]
if not jars:
raise RuntimeError('Build the backend first with python3 scripts/project.py build.')
backend_pid = start_process('backend', [RUNTIME / 'jdk/Contents/Home/bin/java', '-Xms128m', '-Xmx2048m', '-jar', backend_snapshot(jars[-1])])
# Startup can resume a complete, queued workbook import before readiness becomes UP.
wait_http('http://127.0.0.1:5281/actuator/health', seconds=900)
if not alive(backend_pid) or not owned_process('backend', backend_pid):
raise RuntimeError('The project backend exited; inspect output/logs/backend.log and local port 5281.')
frontend_pid = start_process('frontend', ['pnpm', '--dir', ROOT / 'web', 'dev', '--host', '127.0.0.1'])
wait_http('http://127.0.0.1:5280')
if not alive(frontend_pid) or not owned_process('frontend', frontend_pid):
raise RuntimeError('The project frontend exited; inspect output/logs/frontend.log and local port 5280.')
print('Quality platform ready: http://127.0.0.1:5280', flush=True)
print('Local sign-in details: runtimedata/本机登录信息.txt', flush=True)
def backend_snapshot(jar):
"""Run an immutable jar so a later build cannot replace a live classpath."""
digest = hashlib.sha256(jar.read_bytes()).hexdigest()
target = RUNTIME / 'backend-builds' / (digest + '.jar')
if not target.exists():
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_suffix('.tmp')
shutil.copyfile(jar, temporary)
temporary.chmod(0o600)
temporary.replace(target)
return target
def stop():
for name in ('frontend', 'backend', 'mysql'):
path = RUNTIME / (name + '.pid')
if not path.exists():
continue
pid = int(path.read_text())
if alive(pid):
if not owned_process(name, pid):
raise RuntimeError(f'{name}: PID identity changed; no process was stopped.')
os.killpg(pid, signal.SIGTERM)
end = time.monotonic() + 20
while alive(pid) and time.monotonic() < end:
time.sleep(.2)
if alive(pid):
raise RuntimeError(f'{name} has not shut down; inspect its log before retrying.')
path.unlink(missing_ok=True)
(RUNTIME / (name + '.process.json')).unlink(missing_ok=True)
print(name + ': stopped', flush=True)
if name == 'backend' and (ROOT / 'scripts/nao_quality_runtime.py').is_file():
run([sys.executable, ROOT / 'scripts/nao_quality_runtime.py', '--stop-all'])
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=['mysql', 'build', 'start', 'stop', 'status'])
parser.add_argument('--skip-tests', action='store_true')
args = parser.parse_args()
if args.action == 'mysql':
start_mysql()
elif args.action == 'build':
build(args.skip_tests)
elif args.action == 'start':
start()
elif args.action == 'stop':
stop()
else:
for name in ('mysql', 'backend', 'frontend'):
path = RUNTIME / (name + '.pid')
print(name + ': ' + ('running' if path.exists() and alive(path.read_text()) and owned_process(name, path.read_text()) else 'stopped'))
if __name__ == '__main__':
try:
main()
except (RuntimeError, subprocess.CalledProcessError) as error:
print(str(error), file=sys.stderr)
raise SystemExit(1)