初始化:质量数据分析平台源码

This commit is contained in:
2026-10-22 15:58:57 +08:00
commit 1e8a655adc
280 changed files with 43922 additions and 0 deletions
+251
View File
@@ -0,0 +1,251 @@
#!/usr/bin/env python3
"""Create a local backup and verify restoration into a new, isolated database."""
import argparse
from datetime import datetime, timezone
import gzip
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import tarfile
import tempfile
import time
import uuid
import project
ROOT = project.ROOT
def database_config():
value = project.config().copy()
override = os.environ.get('QUALITY_BACKUP_DATABASE')
if override:
if not re.fullmatch(r'quality_(?:verify|scale)_[A-Za-z0-9_]+', override):
raise ValueError('Backup database override is restricted to this project test databases')
value['mysqlDatabase'] = override
return value
def application_files():
"""Fixed project inputs only; archives stay private and are never served as public assets."""
paths = set()
for relative in ('backend/src', 'web/src', 'web/public', 'web/dist', 'scripts'):
folder = ROOT / relative
if folder.exists():
for path in folder.rglob('*'):
if path.is_file() and '__pycache__' not in path.parts:
paths.add(path)
for relative in ('runtime.lock.json', 'backend/pom.xml', 'web/package.json', 'web/pnpm-lock.yaml',
'.runtime/local-config.json', 'apikey.txt', '启动项目.command', '停止项目.command'):
path = ROOT / relative
if path.is_file(): paths.add(path)
paths.update((ROOT / 'backend/target').glob('*.jar'))
return sorted(paths)
def archive_application(folder):
archive = folder / 'application.tar.gz'
files = {}
with tarfile.open(archive, 'w:gz') as tar:
for path in application_files():
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
raise RuntimeError('Application backup contains a link outside the project')
relative = path.relative_to(ROOT).as_posix()
before = hashlib.sha256(path.read_bytes()).hexdigest()
tar.add(path, arcname=relative, recursive=False)
if before != hashlib.sha256(path.read_bytes()).hexdigest():
raise RuntimeError('Application files changed during backup; retry after the build completes')
files[relative] = before
archive.chmod(0o600)
return files, hashlib.sha256(archive.read_bytes()).hexdigest()
def client(sql, database=None):
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
'--batch', '--skip-column-names']
if database:
if not database.replace('_', '').isalnum(): raise ValueError('Invalid database identifier')
args.append('--database=' + database)
result = subprocess.run(args, input=sql, text=True, capture_output=True)
if result.returncode:
raise RuntimeError('Local MySQL backup/restore command failed; credentials were not printed.')
return result.stdout
def counts(database):
tables = client('SHOW TABLES;', database).splitlines()
result = {}
for table in tables:
if not table.replace('_', '').isalnum(): raise ValueError('Unexpected table identifier')
result[table] = int(client('SELECT COUNT(*) FROM `' + table + '`;', database).strip())
return result
def query_evidence(database):
"""Reproduce release membership and monthly formal input components in SQL."""
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
if not current:
return {'releaseId': None, 'months': []}
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
raise RuntimeError('Unexpected release identity in backup')
result = {'releaseId': current, 'months': []}
keys = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap',
'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap',
'bright_processing_scrap', 'bright_diffusion_scrap']
for stage in ('black', 'bright'):
pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*)," + \
"'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)"
for key in keys:
pairs += ",'" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))"
rows = client("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;", database)
result['months'].extend(json.loads(line) for line in rows.splitlines())
result['members'] = [json.loads(line) for line in client("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;", database).splitlines()]
return result
def verify_references(database, archive_root):
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database)
source_root = (ROOT / 'runtimedata/archive').resolve()
for line in rows.splitlines():
row = json.loads(line)
stored = Path(row['path']).resolve()
if not stored.is_relative_to(source_root):
raise RuntimeError('A source revision references a file outside the project archive')
file = archive_root / stored.relative_to(source_root)
if not file.is_file() or file.stat().st_size != row['size'] or hashlib.sha256(file.read_bytes()).hexdigest() != row['sha256']:
raise RuntimeError('A referenced source archive is missing or does not match its database revision')
def create():
config = database_config()
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive')
before_queries = query_evidence(config['mysqlDatabase'])
folder = ROOT / 'output/backups' / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
folder.mkdir(parents=True, exist_ok=False)
folder.chmod(0o700)
sql = folder / 'database.sql.gz'
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
'--single-transaction', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
'--skip-extended-insert', config['mysqlDatabase']]
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
with gzip.GzipFile(fileobj=output, mode='wb') as compressed:
shutil.copyfileobj(process.stdout, compressed)
process.stdout.close()
if process.wait() != 0:
raise RuntimeError('Database dump failed; no backup was marked complete.')
sql.chmod(0o600)
if query_evidence(config['mysqlDatabase']) != before_queries:
raise RuntimeError('The selected release changed during backup; retry a consistent snapshot')
# Count the consistent dump itself; live audit/task tables may change during a backup.
snapshot_counts = {}
with gzip.open(sql, 'rt') as dump:
for line in dump:
table = re.match(r'^CREATE TABLE `([^`]+)`', line)
if table: snapshot_counts[table.group(1)] = 0
row = re.match(r'^INSERT INTO `([^`]+)`', line)
if row: snapshot_counts[row.group(1)] = snapshot_counts.get(row.group(1), 0) + 1
archive = folder / 'archive.tar.gz'
files = {}
source = ROOT / 'runtimedata/archive'
with tarfile.open(archive, 'w:gz') as tar:
if source.exists():
for file in sorted(source.rglob('*')):
if file.is_symlink(): raise RuntimeError('Archive contains a symbolic link; inspect before backing up.')
if file.is_file():
relative = file.relative_to(source).as_posix()
files[relative] = hashlib.sha256(file.read_bytes()).hexdigest()
tar.add(file, arcname=relative, recursive=False)
archive.chmod(0o600)
app_files, app_hash = archive_application(folder)
manifest = {'formatVersion': 2, 'createdAt': datetime.now(timezone.utc).isoformat(), 'database': config['mysqlDatabase'],
'databaseDumpSha256': hashlib.sha256(sql.read_bytes()).hexdigest(),
'archiveSha256': hashlib.sha256(archive.read_bytes()).hexdigest(), 'files': files,
'applicationSha256': app_hash, 'applicationFiles': app_files,
'containsPrivateConfiguration': True,
'recoveryMode': 'VERIFIED_FULL_SNAPSHOT',
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
'tableCounts': snapshot_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
print('Backup created: ' + str(folder))
return folder
def verify(folder):
started = time.monotonic()
folder = folder.resolve()
if not folder.is_relative_to((ROOT / 'output/backups').resolve()):
raise ValueError('Only this project backup directory may be verified')
manifest = json.loads((folder / 'manifest.json').read_text())
sql = folder / 'database.sql.gz'
archive = folder / 'archive.tar.gz'
if hashlib.sha256(sql.read_bytes()).hexdigest() != manifest['databaseDumpSha256'] or hashlib.sha256(archive.read_bytes()).hexdigest() != manifest['archiveSha256']:
raise RuntimeError('Backup checksum mismatch; restoration stopped')
if manifest.get('applicationSha256'):
app_archive = folder / 'application.tar.gz'
if hashlib.sha256(app_archive.read_bytes()).hexdigest() != manifest['applicationSha256']:
raise RuntimeError('Application backup checksum mismatch; restoration stopped')
with tempfile.TemporaryDirectory(prefix='quality-restore-application-') as temp:
Path(temp).chmod(0o700)
with tarfile.open(app_archive) as tar:
tar.extractall(temp, filter='data')
actual_app = {p.relative_to(temp).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
for p in Path(temp).rglob('*') if p.is_file()}
if actual_app != manifest['applicationFiles']:
raise RuntimeError('Restored application and configuration checksums do not match')
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
try:
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--database=' + scratch]
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
try:
shutil.copyfileobj(data, process.stdin)
finally:
process.stdin.close()
code = process.wait()
if code != 0: raise RuntimeError('Isolated restore failed; no live database was changed')
restored = counts(scratch)
if restored != manifest['tableCounts']:
raise RuntimeError('Restored table counts do not match the backup manifest')
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
raise RuntimeError('Restored release query components or source provenance do not match')
with tempfile.TemporaryDirectory(prefix='quality-restore-files-') as temp:
with tarfile.open(archive) as tar:
tar.extractall(temp, filter='data')
actual = {p.relative_to(temp).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
for p in Path(temp).rglob('*') if p.is_file()}
if actual != manifest['files']: raise RuntimeError('Restored archive checksums do not match')
verify_references(scratch, Path(temp))
manifest['status'] = 'ISOLATED_RESTORE_VERIFIED'
manifest['restoreVerifiedAt'] = datetime.now(timezone.utc).isoformat()
manifest['restoredTables'] = len(restored)
manifest['restoredApplicationFiles'] = len(manifest.get('applicationFiles', {}))
manifest['restoredQueryEvidenceVerified'] = manifest.get('queryEvidence') is not None
manifest['restoreElapsedSeconds'] = round(time.monotonic() - started, 3)
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
print('Isolated restore verified: ' + str(len(restored)) + ' tables and ' + str(len(actual)) + ' archived files; live database unchanged.')
finally:
# Only the new scratch database created above is eligible for cleanup.
if scratch.startswith('quality_restore_'):
client('DROP DATABASE `' + scratch + '`;')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify'])
parser.add_argument('--directory', type=Path)
args = parser.parse_args()
if args.action == 'verify':
if not args.directory: parser.error('--directory is required for verify')
verify(args.directory)
else:
folder = create()
if args.action == 'create-and-verify': verify(folder)
if __name__ == '__main__':
main()
+349
View File
@@ -0,0 +1,349 @@
"""Isolated Spring AI Alibaba DataAgent trial helpers.
All services bind to loopback. Secrets live only in .runtime/dataagent-trial/private.json
with mode 0600; evidence replaces the selected profile endpoint and key.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import secrets
import signal
import shutil
import sqlite3
import subprocess
import sys
import time
from pathlib import Path
from urllib.parse import urlparse
ROOT = Path(__file__).resolve().parents[1]
TRIAL = ROOT / ".runtime/dataagent-trial"
OUT = ROOT / "output/dataagent-trial"
SOURCE = TRIAL / "source"
SQLITE = ROOT / ".runtime/dbgpt-trial/data/quality_trial.sqlite"
MYSQL = ROOT / ".runtime/mysql/bin"
MYSQLD = MYSQL / "mysqld"
MYSQLCLI = MYSQL / "mysql"
PORT = 53307
SOCKET = TRIAL / "mysql.sock"
PID = TRIAL / "mysql.pid"
PRIV = TRIAL / "private.json"
def load_private():
if not PRIV.exists():
prepare()
return json.loads(PRIV.read_text())
def prepare():
TRIAL.mkdir(parents=True, exist_ok=True)
OUT.mkdir(parents=True, exist_ok=True)
sys.path.insert(0, str(ROOT / "scripts"))
from dbgpt_trial import model_settings
base, key, model = model_settings()
if model.strip() != "mimo-v2.6-flash":
raise RuntimeError("Selected profile is not the approved MiMo model")
keyfile = ROOT / "apikey.txt"
raw = keyfile.read_bytes()
text = raw.decode("utf-16" if raw[:2] in (b"\xff\xfe", b"\xfe\xff") else "utf-8-sig")
lines = [x.strip() for x in text.splitlines() if x.strip()]
if len(lines) < 6:
raise RuntimeError("The approved two-profile API configuration was not found")
emb_lines = lines[:3]
emb_base = emb_lines[0].rstrip("/")
if emb_base.endswith("/chat/completions"):
emb_base = emb_base[:-len("/chat/completions")]
if not emb_base.endswith("/v1"):
emb_base += "/v1"
emb_key = emb_lines[1].removeprefix("Bearer ").strip()
value = {
"upstream_base": base,
"upstream_key": key,
"chat_model": model,
"embedding_base": emb_base,
"embedding_key": emb_key,
"embedding_model": "qwen3-embedding-4b-local",
"chat_local_token": secrets.token_urlsafe(30),
"embedding_local_token": secrets.token_urlsafe(30),
"db_password": secrets.token_urlsafe(30),
"mysql_user": "dataagent_admin",
"quality_password": secrets.token_urlsafe(30),
"quality_user": "quality_trial_ro",
}
PRIV.write_text(json.dumps(value))
PRIV.chmod(0o600)
client_cnf = TRIAL / "mysql-client.cnf"
client_cnf.write_text("[client]\nuser=dataagent_admin\npassword=" + value["db_password"] + "\nprotocol=SOCKET\nsocket=" + str(SOCKET) + "\n")
client_cnf.chmod(0o600)
manifest = {
"repo": "https://github.com/spring-ai-alibaba/DataAgent",
"commit": subprocess.check_output(["git", "-C", str(SOURCE), "rev-parse", "HEAD"], text=True).strip(),
"chat_model": model,
"sqlite_source_sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(),
"created_local": time.strftime("%Y-%m-%d %H:%M:%S %z"),
"security": "localhost-only; no production DB; private.json mode 0600",
}
(OUT / "trial-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
print(json.dumps({"prepared": True, "chat_model": model, "commit": manifest["commit"]}))
def mysql_cli(*args, input_text=None, root=False):
p = load_private()
if root:
cmd = [str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"]
else:
cmd = [str(MYSQLCLI), f"--defaults-extra-file={TRIAL / 'mysql-client.cnf'}", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=dataagent_admin", "--default-character-set=utf8mb4"]
cmd += list(args)
return subprocess.run(cmd, input=input_text, text=True, capture_output=True, check=True)
def start_db():
load_private()
if not PID.exists():
cmd = [str(MYSQLD), "--no-defaults", f"--basedir={ROOT / '.runtime/mysql'}", f"--datadir={TRIAL / 'datadir'}", f"--port={PORT}", "--bind-address=127.0.0.1", f"--socket={SOCKET}", f"--pid-file={PID}", f"--log-error={OUT / 'mysql.log'}", "--mysqlx=OFF", "--lower-case-table-names=2", "--max-connections=40", "--innodb-buffer-pool-size=128M"]
log = (TRIAL / "mysqld-stdout.log").open("ab")
subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
deadline = time.time() + 45
while time.time() < deadline:
if SOCKET.exists():
try:
mysql_cli("-Nse", "SELECT VERSION()", root=True)
print(json.dumps({"mysql": "ready", "bind": "127.0.0.1", "port": PORT}))
return
except Exception:
pass
time.sleep(.5)
raise RuntimeError("isolated MySQL 53307 did not become ready; inspect private log")
def sql_literal(v):
if v is None:
return "NULL"
if isinstance(v, (int, float)):
if isinstance(v, float) and (v != v or abs(v) == float("inf")):
return "NULL"
return repr(v)
s = str(v).replace("\\", "\\\\").replace("\x00", "\\0").replace("\n", "\\n").replace("\r", "\\r").replace("\x1a", "\\Z").replace("'", "\\'")
return "'" + s + "'"
def mysql_type(decl, col, keycols):
d = (decl or "").upper()
if "INT" in d:
return "BIGINT"
if col in keycols:
return "VARCHAR(255)"
if any(t in d for t in ("REAL", "FLOA", "DOUB")):
return "DOUBLE"
if "BLOB" in d:
return "LONGBLOB"
if "TEXT" in d or not d:
return "LONGTEXT"
return d
SCOPE_COLUMN_COMMENTS = {
"release_id": "official published release identifier",
"scope_key": "range key: auto_combined means domestic plus overseas automotive transport; auto_domestic means domestic only; auto_overseas means overseas only; overall means company-wide",
"scope_name": "business display name for the range",
"year": "calendar year of the official monthly metric",
"month": "month number of the official monthly metric",
"period": "official reporting month in YYYY-MM form",
"official_composite_rate": "official composite qualification rate from the published KPI service; standard meaning of qualification rate",
"composite_rate_without_hq_scrap": "adjusted composite rate excluding headquarters scrap; not the standard composite rate",
"black_rate": "black-sheet stage qualification rate",
"black_qualified_weight": "qualified black-sheet weight",
"black_scrap_weight": "scrap black-sheet weight",
"black_total_weight": "total black-sheet weight",
"bright_rate": "bright-sheet stage qualification rate",
"bright_qualified_weight": "qualified bright-sheet weight",
"bright_scrap_weight": "scrap bright-sheet weight",
"bright_total_weight": "total bright-sheet weight",
"included_rows": "number of rows included in the official monthly summary",
"raw_rows": "number of raw service rows for the official monthly summary",
"official_composite_rate_exact": "exact text representation of the official composite rate",
"evidence_file": "relative path to the source evidence file",
"api_query_json": "original query request and contract metadata as JSON",
"summary_json": "original service summary JSON",
"context_json": "source context JSON",
}
PRODUCT_COLUMN_COMMENTS = {
"scope_key": SCOPE_COLUMN_COMMENTS["scope_key"],
"scope_name": SCOPE_COLUMN_COMMENTS["scope_name"],
"year": SCOPE_COLUMN_COMMENTS["year"],
"month": SCOPE_COLUMN_COMMENTS["month"],
"period": SCOPE_COLUMN_COMMENTS["period"],
"product_key": "product/specification key for a product-level lead; product rows cannot be summed as an overall-rate contribution",
"product_name": "product/specification display name; a lead only, not a proven cause or attributable contribution",
"official_composite_rate": SCOPE_COLUMN_COMMENTS["official_composite_rate"],
"black_rate": SCOPE_COLUMN_COMMENTS["black_rate"],
"bright_rate": SCOPE_COLUMN_COMMENTS["bright_rate"],
"evidence_file": SCOPE_COLUMN_COMMENTS["evidence_file"],
"row_json": "complete original official product row JSON; product rows do not reconcile to stage denominator or overall contribution",
}
def seed_db():
start_db()
p = load_private()
admin_schema = (SOURCE / "data-agent-management/src/main/resources/sql/schema.sql").read_text()
create = f"CREATE DATABASE IF NOT EXISTS saa_data_agent CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE DATABASE IF NOT EXISTS quality_trial CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'localhost'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'localhost'; DROP USER IF EXISTS 'dataagent_trial'@'127.0.0.1'; DROP USER IF EXISTS 'dataagent_trial'@'localhost'; FLUSH PRIVILEGES;"
mysql_cli(root=True, input_text=create)
mysql_cli("saa_data_agent", input_text=admin_schema)
src = sqlite3.connect(f"file:{SQLITE}?mode=ro", uri=True)
chunks = []
manifest = {"source": str(SQLITE.relative_to(ROOT)), "sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(), "tables": {}}
for table, pk in (("official_scope_month_metrics", ("release_id", "scope_key", "year", "month")), ("official_product_month_rows", ("release_id", "scope_key", "year", "month", "row_no"))):
cols = src.execute(f'PRAGMA table_info("{table}")').fetchall()
names = [c[1] for c in cols]
comments = SCOPE_COLUMN_COMMENTS if table == "official_scope_month_metrics" else PRODUCT_COLUMN_COMMENTS
colddl = [f"`{c[1]}` {mysql_type(c[2], c[1], pk)}" + (" NOT NULL" if c[3] else "") + (" COMMENT " + sql_literal(comments[c[1]]) if c[1] in comments else "") for c in cols]
colddl.append("PRIMARY KEY (" + ",".join(f"`{c}`" for c in pk) + ")")
table_comment = ("Official released monthly quality KPI facts; one row per range and month. Composite, black-sheet, and bright-sheet values come from the service; NULL stays missing." if table == "official_scope_month_metrics" else "Official product/specification aggregate rows for leads only; do not sum these rows or claim they prove an overall contribution.")
chunks.append(f"CREATE TABLE IF NOT EXISTS quality_trial.`{table}` ({','.join(colddl)}) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT={sql_literal(table_comment)};")
rows = src.execute(f'SELECT * FROM "{table}"').fetchall()
manifest["tables"][table] = {"rows": len(rows), "columns": names, "null_cells": sum(v is None for row in rows for v in row)}
chunks.append(f"DELETE FROM quality_trial.`{table}`;")
batch_size = 200
for pos in range(0, len(rows), batch_size):
vals = ["(" + ",".join(sql_literal(v) for v in row) + ")" for row in rows[pos:pos+batch_size]]
chunks.append(f"INSERT INTO quality_trial.`{table}` ({','.join(f'`{n}`' for n in names)}) VALUES " + ",".join(vals) + ";")
src.close()
seed_path = TRIAL / "quality_seed.sql"
seed_path.write_text("\n".join(chunks))
seed_path.chmod(0o600)
subprocess.run([str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"], input=seed_path.read_text(), text=True, capture_output=True, check=True)
manifest["mysql_53307_counts"] = {}
for table in manifest["tables"]:
count = mysql_cli("-Nse", f"SELECT COUNT(*) FROM quality_trial.`{table}`", root=True).stdout.strip()
manifest["mysql_53307_counts"][table] = int(count)
(OUT / "data-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
print(json.dumps({"management_db": "saa_data_agent", "analysis_db": "quality_trial", "port": PORT, "counts": manifest["mysql_53307_counts"]}, ensure_ascii=False))
def probe_embedding():
import httpx
p = load_private()
request_body = {"model": p["embedding_model"], "input": ["汽车交通综合合格率", "国内汽车交通合格率"]}
record = {"profile": 1, "model": p["embedding_model"], "tls_verify": True, "request_body": request_body}
try:
with httpx.Client(timeout=120.0) as client:
resp = client.post(p["embedding_base"] + "/embeddings", headers={"Authorization": "Bearer " + p["embedding_key"], "Content-Type": "application/json"}, json=request_body)
body_bytes = resp.content
raw = body_bytes.decode("utf-8", errors="replace")
raw = raw.replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]")
record.update({"http_status": resp.status_code, "response_raw_redacted": raw})
try:
payload = resp.json()
vectors = [x.get("embedding") for x in payload.get("data", [])]
record["vector_count"] = len(vectors)
record["dimensions"] = sorted({len(v) for v in vectors if isinstance(v, list)})
except Exception:
record["vector_count"] = 0
record["dimensions"] = []
if resp.status_code >= 400:
raise RuntimeError("embedding request returned HTTP " + str(resp.status_code))
if not record["vector_count"] or not record["dimensions"]:
raise RuntimeError("embedding response did not contain vectors")
except Exception as exc:
record["error_type"] = type(exc).__name__
record["error_safe"] = str(exc).replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]")
(OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2))
(OUT / "embedding-probe.json").chmod(0o600)
raise
(OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2))
(OUT / "embedding-probe.json").chmod(0o600)
print(json.dumps({"http_status": record["http_status"], "vector_count": record["vector_count"], "dimensions": record["dimensions"], "tls_verify": True}))
def stop_db():
if PID.exists():
try:
mysql_cli("-Nse", "SHUTDOWN", root=True)
except Exception:
pass
print(json.dumps({"stop_requested": True, "port": PORT}))
def start_backend():
import httpx
p = load_private()
jar = SOURCE / "data-agent-management/target/spring-ai-alibaba-data-agent-management-1.0.0-SNAPSHOT.jar"
if not jar.exists():
raise RuntimeError("management executable jar has not been built")
pidfile = TRIAL / "backend.pid"
if pidfile.exists():
try:
os.kill(int(pidfile.read_text()), 0)
print(json.dumps({"backend": "already-running", "port": 5673}))
return
except Exception:
pidfile.unlink(missing_ok=True)
env = os.environ.copy()
env.update({
"SERVER_ADDRESS": "127.0.0.1",
"SERVER_PORT": "5673",
"DATA_AGENT_DATASOURCE_URL": "jdbc:mysql://127.0.0.1:53307/saa_data_agent?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowMultiQueries=true&allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=Asia/Shanghai",
"DATA_AGENT_DATASOURCE_USERNAME": p["mysql_user"],
"DATA_AGENT_DATASOURCE_PASSWORD": p["db_password"],
"DATA_AGENT_DATASOURCE_SQL_INIT": "never",
"LOGGING_LEVEL_COM_ALIBABA_CLOUD_AI_DATAAGENT_SERVICE_AIMODELCONFIG_DYNAMICMODELFACTORY": "WARN",
})
java = ROOT / ".runtime/jdk/Contents/Home/bin/java"
logpath = TRIAL / "backend.log"
log = logpath.open("ab")
logpath.chmod(0o600)
proc = subprocess.Popen([str(java), "-Xms256m", "-Xmx1536m", "-jar", str(jar)], cwd=SOURCE, env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
pidfile.write_text(str(proc.pid))
pidfile.chmod(0o600)
deadline = time.time() + 180
while time.time() < deadline:
if proc.poll() is not None:
raise RuntimeError("backend exited during startup; see .runtime/dataagent-trial/backend.log")
try:
response = httpx.get("http://127.0.0.1:5673/echo/ok", timeout=2)
if response.status_code == 200 and response.text.strip() == "ok":
print(json.dumps({"backend": "ready", "port": 5673, "bind": "127.0.0.1", "pid": proc.pid}))
return
except Exception:
pass
time.sleep(1)
raise RuntimeError("backend startup exceeded 180s; see .runtime/dataagent-trial/backend.log")
def stop_backend():
pidfile = TRIAL / "backend.pid"
if not pidfile.exists():
print(json.dumps({"backend": "not-running"}))
return
pid = int(pidfile.read_text())
try:
os.killpg(pid, signal.SIGTERM)
for _ in range(30):
try:
os.kill(pid, 0)
except ProcessLookupError:
break
time.sleep(1)
else:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
pidfile.unlink(missing_ok=True)
print(json.dumps({"backend": "stopped", "pid": pid}))
def run():
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=["prepare", "probe-embedding", "start-db", "seed-db", "stop-db", "start-backend", "stop-backend"])
args = parser.parse_args()
{"prepare": prepare, "probe-embedding": probe_embedding, "start-db": start_db, "seed-db": seed_db, "stop-db": stop_db, "start-backend": start_backend, "stop-backend": stop_backend}[args.action]()
if __name__ == "__main__":
run()
+238
View File
@@ -0,0 +1,238 @@
"""Configure upstream DataAgent only through its own API and run trial graphs."""
from __future__ import annotations
import json
import os
import re
import time
import uuid
from pathlib import Path
from urllib.parse import urlencode
import httpx
from scripts.dataagent_trial import OUT, TRIAL, load_private
API = "http://127.0.0.1:5673"
PROXY = "http://127.0.0.1:5676/v1"
CALLS = OUT / "api-calls.jsonl"
STATE = TRIAL / "trial-state.json"
def scrub(obj, p):
if isinstance(obj, dict):
return {k: ("[REDACTED]" if k.lower() in {"apikey", "password", "proxy_password"} else scrub(v, p)) for k, v in obj.items()}
if isinstance(obj, list):
return [scrub(x, p) for x in obj]
if isinstance(obj, str):
for k in ("upstream_key", "embedding_key", "upstream_base", "embedding_base", "db_password"):
if p.get(k): obj = obj.replace(p[k], "[REDACTED]")
return obj
return obj
def record(method, path, status, summary, request_safe=None):
p = load_private()
CALLS.parent.mkdir(parents=True, exist_ok=True)
item = {"time": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "method": method, "path": path, "http_status": status, "summary": summary}
if request_safe is not None:
item["request"] = request_safe
line = json.dumps(scrub(item, p), ensure_ascii=False, separators=(",", ":"))
with CALLS.open("a", encoding="utf-8") as f:
f.write(line + "\n")
CALLS.chmod(0o600)
def call(client, method, path, *, json_body=None, params=None, safe_request=None, timeout=60, log_response=True):
url = API + path
response = client.request(method, url, json=json_body, params=params, timeout=timeout)
try:
data = response.json()
except Exception:
data = None
summary = data if data is not None else {"body_preview": response.text[:500]}
record(method, path, response.status_code, summary if log_response else {"saved": bool(response.is_success)}, safe_request)
if response.status_code >= 400:
raise RuntimeError(f"API {method} {path} returned HTTP {response.status_code}")
if isinstance(data, dict) and data.get("success") is False:
msg = str(data.get("message", "API operation returned success=false"))
raise RuntimeError(scrub(msg, load_private()))
return data
def setup():
p = load_private()
state = {}
with httpx.Client() as c:
state["initial_model_check"] = call(c, "GET", "/api/model-config/check-ready")
existing = call(c, "GET", "/api/model-config/list")
existing_configs = (existing or {}).get("data") or []
for item in (
{"provider":"OpenAI-compatible MiMo trial", "apiKey":p["chat_local_token"], "baseUrl":PROXY, "modelName":p["chat_model"], "modelType":"CHAT", "temperature":0.0, "maxTokens":5000, "isActive":False, "completionsPath":"/chat/completions"},
{"provider":"OpenAI-compatible Qwen embedding trial", "apiKey":p["embedding_local_token"], "baseUrl":PROXY, "modelName":p["embedding_model"], "modelType":"EMBEDDING", "temperature":0.0, "maxTokens":5000, "isActive":False, "embeddingsPath":"/embeddings"},
):
match = next((x for x in existing_configs if x.get("modelType") == item["modelType"] and x.get("modelName") == item["modelName"]), None)
if match is None:
result = call(c, "POST", "/api/model-config/add", json_body=item, safe_request={"provider":item["provider"], "modelName":item["modelName"], "modelType":item["modelType"], "baseUrl":"local evidence proxy", "apiKey":"[REDACTED]"})
state[item["modelType"].lower()+"_add"] = result
config_list = call(c, "GET", "/api/model-config/list")
configs = (config_list or {}).get("data") or []
for kind in ("CHAT", "EMBEDDING"):
match = next((x for x in configs if x.get("modelType") == kind and x.get("modelName") == (p["chat_model"] if kind == "CHAT" else p["embedding_model"])), None)
if not match or not match.get("id"):
raise RuntimeError("could not resolve newly stored model configuration")
model_id = match["id"]
safe = {"modelType":kind,"modelName":match.get("modelName"),"id":model_id}
state[kind.lower()+"_model_id"] = model_id
call(c, "POST", f"/api/model-config/test/{model_id}", safe_request=safe)
call(c, "POST", f"/api/model-config/activate/{model_id}", safe_request=safe)
state["model_check"] = call(c, "GET", "/api/model-config/check-ready")
ds_body = {"name":"DataAgent 质量只读副本53307", "type":"mysql", "host":"127.0.0.1", "port":53307, "databaseName":"quality_trial", "username":p["quality_user"], "password":p["quality_password"], "status":"active", "description":"isolated read-only duplicate from quality trial SQLite"}
ds = call(c, "POST", "/api/datasource", json_body=ds_body, safe_request={"name":ds_body["name"],"type":"mysql","host":"127.0.0.1","port":53307,"databaseName":"quality_trial","username":p["quality_user"],"password":"[REDACTED]"})
dsid = ds.get("id")
if not dsid: raise RuntimeError("datasource creation did not return an id")
state["datasource_id"] = dsid
state["datasource_test"] = call(c, "POST", f"/api/datasource/{dsid}/test", safe_request={"id":dsid})
prompt = """你是质量中心只读数据分析助手。请使用已连接数据库的正式月度指标表和产品分组表,先理解官方字段语义再查询。汽车交通合计为scope_key=auto_combined(国内+海外),国内为auto_domestic,范围不可混用;官方常用综合合格率字段为official_composite_rate,黑片与亮片分别为black_rate、bright_rate。official_scope_month_metrics是正式月度汇总;产品分组表只用于规格线索,不能据此声称对总体变化的贡献。所有NULL原样保留,不得补0。月份仅2026-01至2026-09,产品明细仅2026-08、09。用中文回答;把查询直接支持的事实、计算推断、待验证假设明确分开;根据用户任务自主决定是否生成报告,用户明确要求简短答案或不生成报告时服从。只读查询,不修改数据。"""
agent = call(c, "POST", "/api/agent", json_body={"name":"质量中心只读试验 DataAgent","description":"基于发布指标副本的本地只读分析试跑","prompt":prompt,"category":"质量分析","tags":"汽车交通,质量指标,试验"}, safe_request={"name":"质量中心只读试验 DataAgent","description":"based on isolated read-only KPI duplicate","prompt":"business semantics and evidence policy only"})
aid = agent.get("id")
if not aid: raise RuntimeError("agent creation did not return an id")
state["agent_id"] = aid
call(c, "POST", f"/api/agent/{aid}/datasources/{dsid}", safe_request={"agent_id":aid,"datasource_id":dsid})
tables = call(c, "GET", f"/api/datasource/{dsid}/tables")
names = tables if isinstance(tables, list) else (tables or {}).get("data", [])
chosen = [x for x in names if x in {"official_scope_month_metrics", "official_product_month_rows"}]
if len(chosen) != 2:
raise RuntimeError("DataAgent saw unexpected number of quality tables")
state["tables"] = chosen
call(c, "POST", f"/api/agent/{aid}/datasources/tables", json_body={"datasourceId":dsid,"tables":chosen}, safe_request={"agent_id":aid,"datasource_id":dsid,"tables":chosen})
state["schema_init"] = call(c, "POST", f"/api/agent/{aid}/datasources/init", safe_request={"agent_id":aid,"datasource_id":dsid,"tables":chosen}, timeout=300)
(TRIAL/"trial-state.json").write_text(json.dumps(state,ensure_ascii=False,indent=2))
(TRIAL/"trial-state.json").chmod(0o600)
print(json.dumps({"setup":"complete","agent_id":state["agent_id"],"datasource_id":state["datasource_id"],"tables":state["tables"],"schema_init":state.get("schema_init")},ensure_ascii=False))
def finalize_nodes(label: str, session_id: str, nodes: list[dict], export_html: bool = False):
"""Persist the upstream Nuxt-style timeline and export its Markdown as HTML."""
node_blocks=[]
current_key=None
report_markdown=""
report_block=None
for n in nodes:
node_name=n.get("nodeName")
step_key=n.get("stepId") or f"{node_name}:{n.get('attempt',1)}"
if step_key != current_key:
node_blocks.append([])
current_key=step_key
block=node_blocks[-1]
if node_name=="ReportGeneratorNode":
if report_block is None:
report_block=dict(n)
report_block["text"]=""
report_block["textType"]="MARK_DOWN"
block.append(report_block)
if n.get("textType")=="MARK_DOWN":
report_markdown += str(n.get("text", ""))
report_block["text"] = report_markdown
continue
block.append(n)
timeline_content=json.dumps(node_blocks,ensure_ascii=False)
html_file=None
html_status=None
with httpx.Client(timeout=60) as c:
timeline_resp=call(c,"POST",f"/api/sessions/{session_id}/messages",json_body={"sessionId":session_id,"role":"assistant","content":timeline_content,"messageType":"timeline"},safe_request={"session_id":session_id,"role":"assistant","messageType":"timeline","node_blocks":len(node_blocks),"content_chars":len(timeline_content)},timeout=60,log_response=False)
if report_markdown and export_html:
html_resp=c.post(API+f"/api/sessions/{session_id}/reports/html",content=report_markdown.encode("utf-8"),headers={"Content-Type":"text/plain;charset=utf-8"},timeout=60)
html_status=html_resp.status_code
html_path=OUT/f"{label}.html"
if html_resp.status_code==200:
html_path.write_bytes(html_resp.content)
html_path.chmod(0o600)
html_file=str(html_path)
record("POST",f"/api/sessions/{session_id}/reports/html",html_resp.status_code,{"content_type":html_resp.headers.get("content-type"),"bytes":len(html_resp.content),"saved_file":html_file}, {"session_id":session_id,"markdown_chars":len(report_markdown)})
if html_resp.status_code>=400:
raise RuntimeError(f"HTML report export returned HTTP {html_resp.status_code}")
return {"report_markdown_chars":len(report_markdown),"report_html_status":html_status,"report_html_file":html_file,"timeline_saved":bool(timeline_resp),"node_blocks":len(node_blocks)}
def stream_question(label: str, query: str, session_id: str | None = None, timeout: int = 600, export_html: bool = False):
"""Issue one original DataAgent Graph request and store raw SSE evidence (credentials redacted)."""
from scripts.dataagent_trial import load_private
p = load_private()
state = json.loads((TRIAL/"trial-state.json").read_text())
aid = state["agent_id"]
started = time.time()
with httpx.Client(timeout=None) as c:
if session_id is None:
s = call(c, "POST", f"/api/agent/{aid}/sessions", json_body={"title":label}, safe_request={"agent_id":aid,"title":label})
session_id = s.get("id")
if not session_id: raise RuntimeError("chat session creation did not return id")
msg = {"sessionId":session_id,"role":"user","content":query,"messageType":"text"}
call(c, "POST", f"/api/sessions/{session_id}/messages", json_body=msg, safe_request={"session_id":session_id,"role":"user","messageType":"text","content":query})
params={"agentId":str(aid),"conversationId":session_id,"query":query,"humanFeedback":"false","rejectedPlan":"false","nl2sqlOnly":"false"}
req = c.build_request("GET", API+"/api/stream/search", params=params, headers={"Accept":"text/event-stream"}, timeout=timeout)
raw = bytearray()
try:
response = c.send(req, stream=True)
try:
response.raise_for_status()
for chunk in response.iter_raw():
raw.extend(chunk)
finally:
response.close()
except Exception as exc:
path=OUT/f"{label}.sse.redacted.txt"
safe=bytes(raw).decode("utf-8",errors="replace")
for k in ("upstream_key","embedding_key","upstream_base","embedding_base","db_password"):
if p.get(k): safe=safe.replace(p[k],"[REDACTED]")
path.write_text(safe,encoding="utf-8"); path.chmod(0o600)
record("GET","/api/stream/search",0,{"error_type":type(exc).__name__,"duration_ms":round((time.time()-started)*1000)}, {"agent_id":aid,"session_id":session_id,"query":query})
raise
text=bytes(raw).decode("utf-8",errors="replace")
for k in ("upstream_key","embedding_key","upstream_base","embedding_base","db_password"):
if p.get(k): text=text.replace(p[k],"[REDACTED]")
path=OUT/f"{label}.sse.redacted.txt"
path.write_text(text,encoding="utf-8"); path.chmod(0o600)
nodes=[]
for block in re.split(r"\n\n",text):
if not block.startswith("data:"): continue
payload=block[len("data:"):].strip()
try: nodes.append(json.loads(payload))
except Exception: pass
final_text="\n".join(str(n.get("text","")) for n in nodes if n.get("eventType")=="FINAL_ANSWER")
finalization=finalize_nodes(label,session_id,nodes,export_html=export_html)
summaries=[]
for n in nodes:
item={k:n.get(k) for k in ("nodeName","eventType","textType","error","stepId","attempt") if k in n}
if n.get("text"):
item["text_preview"]=str(n["text"])[:800]
summaries.append(item)
timeline={"label":label,"query":query,"session_id":session_id,"duration_seconds":round(time.time()-started,2),"sse_http":200,"sse_event_count":len(nodes),"final_answer":final_text,**finalization,"nodes":summaries}
(OUT/f"{label}.summary.json").write_text(json.dumps(timeline,ensure_ascii=False,indent=2)); (OUT/f"{label}.summary.json").chmod(0o600)
record("GET","/api/stream/search",200,{"sse_event_count":len(nodes),"duration_seconds":timeline["duration_seconds"],"session_id":session_id,"output_file":str(path.relative_to(OUT))},{"agent_id":aid,"session_id":session_id,"query":query})
print(json.dumps({"label":label,"session_id":session_id,"duration_seconds":timeline["duration_seconds"],"sse_event_count":len(nodes),"sse_file":str(path),"summary_file":str(OUT/f"{label}.summary.json")},ensure_ascii=False))
return timeline
def main():
import argparse
ap=argparse.ArgumentParser()
ap.add_argument("action",choices=["setup","short-report","finalize-short-report","followup","deep"])
args=ap.parse_args()
if args.action=="setup": setup()
elif args.action=="short-report": stream_question("task3-short-report","请查询2026年8月和9月汽车交通合计的正式综合合格率、黑片合格率和亮片合格率,用查询结果生成一页精简中文HTML报告,注明环比变化和数据来源。报告不超过600字。",timeout=600,export_html=True)
elif args.action=="finalize-short-report":
label="task3-short-report"; summary_path=OUT/f"{label}.summary.json"; summary=json.loads(summary_path.read_text()); raw=(OUT/f"{label}.sse.redacted.txt").read_text(); nodes=[]
for block in re.split(r"\n\n",raw):
if block.startswith("data:"):
try: nodes.append(json.loads(block[5:].strip()))
except Exception: pass
summary.update(finalize_nodes(label,summary["session_id"],nodes,export_html=True)); summary_path.write_text(json.dumps(summary,ensure_ascii=False,indent=2)); summary_path.chmod(0o600)
print(json.dumps({k:summary.get(k) for k in ("session_id","report_markdown_chars","report_html_status","report_html_file","timeline_saved","node_blocks")},ensure_ascii=False))
elif args.action=="followup":
short=json.loads((OUT/"task3-short-report.summary.json").read_text())
stream_question("task2-followup","那国内9月合格绿呢,比上月变化多大?直接简短回答,不生成报告。",session_id=short["session_id"],timeout=300)
elif args.action=="deep": stream_question("task1-deep","9月份汽车交通合格率为什么降低?请自行查询数据,研究主要因素。发现线索后继续追查,并说明哪些判断有证据、哪些还需验证。先给我分析结论,下一轮再生成报告。",timeout=600)
if __name__=="__main__":
main()
+92
View File
@@ -0,0 +1,92 @@
"""Transparent, loopback-only OpenAI-compatible evidence proxy for DataAgent."""
from __future__ import annotations
import json
import time
from pathlib import Path
import httpx
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import StreamingResponse
from scripts.dataagent_trial import OUT, load_private
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
@app.get("/healthz")
async def healthz():
return {"status": "ok", "bind": "127.0.0.1"}
def _redact(text: str, p: dict) -> str:
for value in (p.get("upstream_key"), p.get("embedding_key"), p.get("upstream_base"), p.get("embedding_base")):
if value:
text = text.replace(value, "[REDACTED]")
return text
@app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
async def forward(path: str, request: Request):
p = load_private()
incoming_auth = request.headers.get("authorization", "")
model_group = "embedding" if path.rstrip("/").endswith("embeddings") else "chat"
expected = p[model_group + "_local_token"]
if incoming_auth != "Bearer " + expected:
raise HTTPException(status_code=401, detail="invalid local trial token")
base = p["embedding_base"] if model_group == "embedding" else p["upstream_base"]
key = p["embedding_key"] if model_group == "embedding" else p["upstream_key"]
body = await request.body()
# The Java client may use chunked transfer encoding. Rebuild the outbound
# content-length from the buffered body instead of forwarding TE alongside it.
headers = {k: v for k, v in request.headers.items() if k.lower() not in {"authorization", "host", "content-length", "transfer-encoding", "connection", "accept-encoding"}}
headers["authorization"] = "Bearer " + key
headers["accept-encoding"] = "identity"
upstream_url = base.rstrip("/") + "/" + path
if request.url.query:
upstream_url += "?" + request.url.query
started = time.time()
evidence = {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
"model_group": model_group,
"method": request.method,
"path": "/v1/" + path,
"model": p["embedding_model"] if model_group == "embedding" else p["chat_model"],
"request_headers": {k: v for k, v in headers.items() if k.lower() not in {"authorization", "host"}},
"request_body_redacted": _redact(body.decode("utf-8", errors="replace"), p),
"tls_verify": True,
}
client = httpx.AsyncClient(timeout=None)
try:
req = client.build_request(request.method, upstream_url, headers=headers, content=body)
response = await client.send(req, stream=True)
response_headers = {k: v for k, v in response.headers.items() if k.lower() in {"content-type", "cache-control", "x-request-id"}}
async def chunks():
collected = bytearray()
try:
async for chunk in response.aiter_raw():
collected.extend(chunk)
yield chunk
finally:
raw = _redact(collected.decode("utf-8", errors="replace"), p)
evidence.update({"http_status": response.status_code, "duration_ms": round((time.time() - started) * 1000), "response_body_redacted": raw})
_append_evidence(evidence)
await response.aclose()
await client.aclose()
return StreamingResponse(chunks(), status_code=response.status_code, headers=response_headers)
except Exception as exc:
evidence.update({"error_type": type(exc).__name__, "duration_ms": round((time.time() - started) * 1000)})
_append_evidence(evidence)
await client.aclose()
raise HTTPException(status_code=502, detail="upstream model request failed") from None
def _append_evidence(record: dict):
p = load_private()
name = "embedding-api.jsonl" if record.get("model_group") == "embedding" else "chat-api.jsonl"
path = OUT / name
line = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
line = _redact(line, p)
with path.open("a", encoding="utf-8") as f:
f.write(line + "\n")
path.chmod(0o600)
+32
View File
@@ -0,0 +1,32 @@
"""Send real consecutive research requests to the isolated local DB-GPT SSE API."""
import argparse,json,uuid,urllib.request,time
from pathlib import Path
from dbgpt_trial import model_settings
ROOT=Path(__file__).resolve().parents[1]
OUT=ROOT/'output/dbgpt-trial'
STATE=ROOT/'.runtime/dbgpt-trial/chat-state.json'
def run(question,label,new=False):
state=json.loads(STATE.read_text()) if STATE.exists() and not new else {'conv_uid':str(uuid.uuid4())}
model=model_settings()[2]
body={'conv_uid':state['conv_uid'],'user_input':question,'chat_mode':'chat_react_agent','model_name':model,'temperature':0.1,'max_new_tokens':4000,'ext_info':{'database_name':'sqlite_quality_trial','skill_name':'quality-center'}}
STATE.write_text(json.dumps(state,ensure_ascii=False));STATE.chmod(0o600)
(OUT/(label+'-request.json')).write_text(json.dumps(body,ensure_ascii=False,indent=2))
req=urllib.request.Request('http://127.0.0.1:5670/api/v1/chat/react-agent',data=json.dumps(body,ensure_ascii=False).encode(),headers={'Content-Type':'application/json','Accept':'text/event-stream'})
events=[];begin=time.monotonic();seen=set()
with urllib.request.urlopen(req,timeout=420) as response, (OUT/(label+'.sse')).open('w') as stream:
for raw in response:
line=raw.decode('utf-8',errors='replace');stream.write(line);stream.flush()
if not line.startswith('data:'):continue
data=line[5:].strip()
if data=='[DONE]':print('DONE',label,round(time.monotonic()-begin,1),flush=True);break
try:item=json.loads(data)
except json.JSONDecodeError:continue
events.append(item)
kind=item.get('type') or item.get('event') or item.get('status') or ''
# Only event kinds are echoed; full user data stays in the local artifact.
if kind and kind not in seen:print('event',str(kind)[:100],flush=True);seen.add(kind)
(OUT/(label+'-events.json')).write_text(json.dumps(events,ensure_ascii=False,indent=2))
print('saved',len(events),'events',state['conv_uid'],flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('label');p.add_argument('question');p.add_argument('--new',action='store_true');a=p.parse_args();run(a.question,a.label,a.new)
+43
View File
@@ -0,0 +1,43 @@
"""Apply a narrow, reversible DB-GPT 0.8.2 reasoning-format compatibility fix.
Reproducer: removing a vis-thinking block with an empty string can join
`Thought: ` to the following `Action:` line, making valid tool calls unparseable.
The fix preserves a logical newline. It does not change tool choice or answers.
"""
from pathlib import Path
import hashlib,json,sys
ROOT=Path(__file__).resolve().parents[1]
TRIAL=ROOT/'.runtime/dbgpt-trial'
TARGET=TRIAL/'venv/lib/python3.11/site-packages/dbgpt/agent/util/react_parser.py'
OLD='return re.sub(pattern, "", text, flags=re.DOTALL)'
NEW='return re.sub(pattern, "\\n", text, flags=re.DOTALL)'
def apply():
source=TARGET.read_text()
if NEW in source:
print('Reasoning newline compatibility fix already present');return
if source.count(OLD)!=1:raise RuntimeError('Unexpected parser version; no files modified')
backup=TRIAL/'compat/react_parser.py.original'
backup.parent.mkdir(parents=True,exist_ok=True)
if backup.exists() and backup.read_text()!=source:raise RuntimeError('Original parser backup differs; no files modified')
backup.write_text(source)
patched=source.replace(OLD,NEW)
TARGET.write_text(patched)
(backup.parent/'patch.json').write_text(json.dumps({'package':'dbgpt','version':'0.8.2','change':'Preserve newline when removing vis-thinking blocks','original_sha256':hashlib.sha256(source.encode()).hexdigest(),'patched_sha256':hashlib.sha256(patched.encode()).hexdigest()},indent=2))
print('Applied reasoning newline compatibility fix; original preserved')
def check():
from dbgpt.agent.util.react_parser import ReActOutputParser
raw=(TRIAL/'mimo-round-raw.txt').read_text()
parsed=ReActOutputParser().parse_current_step(raw)
assert len(parsed)==1 and parsed[0].action=='sql_query', 'Recorded real response must remain an executable SQL action'
assert isinstance(parsed[0].action_input,dict) and 'sql' in parsed[0].action_input
fenced='Thought: explain\n```text\nAction: terminate\nAction Input: {"result":"bad"}\n```\nAction: sql_query\nAction Input: {"sql":"SELECT 1"}'
steps=ReActOutputParser().parse_current_step(fenced)
assert len(steps)==1 and steps[0].action=='sql_query', 'Code-fenced labels must remain inert'
print('PASS: real MiMo response and fenced-label regression checks')
if __name__=='__main__':
if len(sys.argv)>1 and sys.argv[1]=='check':check()
else:apply()
+408
View File
@@ -0,0 +1,408 @@
#!/usr/bin/env python3
"""Build a local SQLite copy of published quality metrics for a DB-GPT trial.
Reads only from the existing local metrics API via scripts/local_api.py. It does
not inspect credentials, change source data, or call any write endpoint.
"""
from __future__ import annotations
import base64
import hashlib
import json
import sqlite3
import sys
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlencode
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from local_api import Api # noqa: E402
DATA_DIR = ROOT / ".runtime" / "dbgpt-trial" / "data"
EVIDENCE_DIR = DATA_DIR / "evidence"
DB_PATH = DATA_DIR / "quality_trial.sqlite"
MANIFEST_PATH = ROOT / "output" / "dbgpt-trial" / "data-manifest.json"
CONTEXT_PATH = DATA_DIR / "business-context.md"
RELEASE_FIXTURE = ROOT / "output" / "ai-chat-refinement" / "expected-car-scope.json"
OVERALL_FIXTURE = ROOT / "output" / "ai-chat-refinement" / "metric-before.json"
RELEASE_ID = "194b0287-9044-4d80-9138-ecd3c481015c"
YEAR = 2026
MONTHS = tuple(range(1, 10))
PRODUCT_MONTHS = (8, 9)
def encoded_selection(values: list[str]) -> str:
raw = json.dumps(values, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
return "~qv1~" + base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
CAR_BOTH_WIRE = encoded_selection(["汽车交通-国内", "汽车交通-海外"])
SCOPES = (
{"key": "overall", "name": "官方总体", "group": None},
{"key": "auto_domestic", "name": "汽车交通-国内", "group": "汽车交通-国内"},
{"key": "auto_overseas", "name": "汽车交通-海外", "group": "汽车交通-海外"},
{"key": "auto_combined", "name": "汽车交通合计(国内+海外)", "group": CAR_BOTH_WIRE},
)
def write_json(path: Path, value: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8")
def number(value: object) -> float | None:
return None if value is None else float(value)
def number_text(value: object) -> str | None:
if value is None:
return None
# Api.call parses JSON numbers as Python floats; repr is the shortest
# round-trip representation of the official service value, without math.
return repr(value) if isinstance(value, float) else str(value)
def stage_value(summary: dict, stage: str, field: str) -> object:
block = summary.get(stage)
return None if block is None else block.get(field)
def coverage_value(summary: dict, stage: str) -> object:
coverage = summary.get("coverage") or {}
return coverage.get(f"{stage}Rows")
def check_fixtures() -> tuple[dict, dict]:
car = json.loads(RELEASE_FIXTURE.read_text(encoding="utf-8"))
overall = json.loads(OVERALL_FIXTURE.read_text(encoding="utf-8"))
if car.get("release", {}).get("id") != RELEASE_ID or overall.get("release", {}).get("id") != RELEASE_ID:
raise RuntimeError("现有正式结果参考文件绑定的发布版本不一致")
if car.get("context", {}).get("group") != CAR_BOTH_WIRE:
raise RuntimeError("汽车交通参考筛选值与 SelectionValues 编码不一致")
for fixture in (car, overall):
context = fixture.get("context", {})
if context.get("year") != YEAR or context.get("month") != 9:
raise RuntimeError("参考正式结果不是 2026 年 9 月")
return car, overall
def get_snapshot(api: Api, scope: dict, month: int, view: str) -> tuple[dict, dict, str]:
params: dict[str, object] = {
"year": YEAR,
"month": month,
"view": view,
"includeUnassigned": "false",
"releaseId": RELEASE_ID,
"trendGranularity": "month",
}
if scope["group"] is not None:
params["group"] = scope["group"]
query = urlencode(params)
response = api.call("/api/metrics?" + query)
context = response.get("context") or {}
release = response.get("release") or {}
if release.get("id") != RELEASE_ID or release.get("status") != "PUBLISHED":
raise RuntimeError(f"正式发布版本校验失败:{scope['key']} 2026-{month:02d}")
if context.get("year") != YEAR or context.get("month") != month or context.get("view") != view:
raise RuntimeError(f"查询期间或视图校验失败:{scope['key']} 2026-{month:02d} {view}")
if context.get("group") != scope["group"]:
raise RuntimeError(f"业务范围筛选校验失败:{scope['key']} 2026-{month:02d}")
return response, params, query
def schema(connection: sqlite3.Connection) -> None:
connection.executescript("""
PRAGMA foreign_keys=ON;
CREATE TABLE official_scope_month_metrics (
release_id TEXT NOT NULL,
release_name TEXT,
scope_key TEXT NOT NULL,
scope_name TEXT NOT NULL,
year INTEGER NOT NULL,
month INTEGER NOT NULL,
period TEXT NOT NULL,
group_filter_wire TEXT,
official_composite_rate REAL,
official_composite_rate_exact TEXT,
composite_rate_without_hq_scrap REAL,
composite_rate_without_hq_scrap_exact TEXT,
black_rate REAL,
black_rate_exact TEXT,
black_qualified_weight REAL,
black_scrap_weight REAL,
black_total_weight REAL,
bright_rate REAL,
bright_rate_exact TEXT,
bright_qualified_weight REAL,
bright_scrap_weight REAL,
bright_total_weight REAL,
bright_headquarters_scrap_weight REAL,
bright_rate_without_hq_scrap REAL,
bright_rate_without_hq_scrap_exact TEXT,
black_rows INTEGER,
bright_rows INTEGER,
included_rows INTEGER,
raw_rows INTEGER,
rule_version_id TEXT,
evidence_file TEXT NOT NULL,
api_query_json TEXT NOT NULL,
coverage_json TEXT,
summary_json TEXT NOT NULL,
context_json TEXT NOT NULL,
PRIMARY KEY (release_id, scope_key, year, month)
);
CREATE INDEX idx_scope_metrics_period ON official_scope_month_metrics(scope_key, period);
CREATE TABLE official_product_month_rows (
release_id TEXT NOT NULL,
scope_key TEXT NOT NULL,
scope_name TEXT NOT NULL,
year INTEGER NOT NULL,
month INTEGER NOT NULL,
period TEXT NOT NULL,
group_filter_wire TEXT,
row_no INTEGER NOT NULL,
product_key TEXT,
product_name TEXT,
official_composite_rate REAL,
official_composite_rate_exact TEXT,
composite_rate_without_hq_scrap REAL,
black_rate REAL,
black_qualified_weight REAL,
black_scrap_weight REAL,
black_total_weight REAL,
bright_rate REAL,
bright_qualified_weight REAL,
bright_scrap_weight REAL,
bright_total_weight REAL,
bright_headquarters_scrap_weight REAL,
bright_rate_without_hq_scrap REAL,
black_rows INTEGER,
bright_rows INTEGER,
evidence_file TEXT NOT NULL,
row_json TEXT NOT NULL,
PRIMARY KEY (release_id, scope_key, year, month, row_no)
);
CREATE INDEX idx_product_scope_period ON official_product_month_rows(scope_key, period);
CREATE INDEX idx_product_name ON official_product_month_rows(product_name);
""")
def insert_metric(connection: sqlite3.Connection, scope: dict, month: int, response: dict,
params: dict, evidence_rel: str) -> None:
release = response["release"]
context = response["context"]
summary = response["summary"]
black = summary.get("black") or {}
bright = summary.get("bright") or {}
coverage = summary.get("coverage") or response.get("coverage") or {}
rate = summary.get("compositeRate")
adj = summary.get("compositeRateWithoutHeadquartersScrap")
black_rate = black.get("rate")
bright_rate = bright.get("rate")
bright_adjusted = bright.get("rateWithoutHeadquartersScrap")
values = (
release.get("id"), release.get("name"), scope["key"], scope["name"], YEAR, month,
f"{YEAR}-{month:02d}", scope["group"], number(rate), number_text(rate), number(adj), number_text(adj),
number(black_rate), number_text(black_rate), number(black.get("qualifiedWeight")),
number(black.get("scrapWeight")), number(black.get("totalWeight")), number(bright_rate),
number_text(bright_rate), number(bright.get("qualifiedWeight")), number(bright.get("scrapWeight")),
number(bright.get("totalWeight")), number(bright.get("headquartersScrapWeight")),
number(bright_adjusted), number_text(bright_adjusted), coverage_value(summary, "black"),
coverage_value(summary, "bright"), coverage.get("includedRows"), coverage.get("rawRows"),
context.get("ruleVersionId"), evidence_rel, json.dumps(params, ensure_ascii=False, sort_keys=True),
json.dumps(coverage, ensure_ascii=False, sort_keys=True),
json.dumps(summary, ensure_ascii=False, sort_keys=True),
json.dumps(context, ensure_ascii=False, sort_keys=True),
)
connection.execute(
"INSERT INTO official_scope_month_metrics VALUES (" + ",".join("?" for _ in values) + ")", values
)
def insert_products(connection: sqlite3.Connection, scope: dict, month: int, response: dict,
evidence_rel: str) -> int:
count = 0
for row_no, row in enumerate(response.get("rows", []), start=1):
black = row.get("black") or {}
bright = row.get("bright") or {}
rate = row.get("compositeRate")
values = (
RELEASE_ID, scope["key"], scope["name"], YEAR, month, f"{YEAR}-{month:02d}",
scope["group"], row_no, row.get("key"), row.get("product") or row.get("label"),
number(rate), number_text(rate), number(row.get("compositeRateWithoutHeadquartersScrap")),
number(black.get("rate")), number(black.get("qualifiedWeight")), number(black.get("scrapWeight")),
number(black.get("totalWeight")), number(bright.get("rate")), number(bright.get("qualifiedWeight")),
number(bright.get("scrapWeight")), number(bright.get("totalWeight")),
number(bright.get("headquartersScrapWeight")), number(bright.get("rateWithoutHeadquartersScrap")),
((row.get("coverage") or {}).get("blackRows")), ((row.get("coverage") or {}).get("brightRows")),
evidence_rel, json.dumps(row, ensure_ascii=False, sort_keys=True),
)
connection.execute(
"INSERT INTO official_product_month_rows VALUES (" + ",".join("?" for _ in values) + ")", values
)
count += 1
return count
def fixture_rate(fixture: dict) -> float:
return float(fixture["summary"]["compositeRate"])
def main() -> None:
expected_car, expected_overall = check_fixtures()
DATA_DIR.mkdir(parents=True, exist_ok=True)
EVIDENCE_DIR.mkdir(parents=True, exist_ok=True)
MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True)
# Build into a fresh database file beside the destination and replace only
# this trial database once all read-only API fetches have succeeded.
temporary_db = DB_PATH.with_suffix(".sqlite.tmp")
temporary_db.unlink(missing_ok=True)
api = Api().login()
counts = {"official_scope_month_metrics": 0, "official_product_month_rows": 0}
evidence_records: list[dict] = []
sep_scope_metrics: dict[str, dict] = {}
conn = sqlite3.connect(temporary_db)
try:
schema(conn)
with conn:
for scope in SCOPES:
for month in MONTHS:
response, params, query = get_snapshot(api, scope, month, "overall")
evidence_name = f"metrics_{scope['key']}_{YEAR}-{month:02d}.json"
evidence_path = EVIDENCE_DIR / evidence_name
envelope = {"request": {"method": "GET", "path": "/api/metrics", "query": params,
"encoded_query": query}, "response": response}
write_json(evidence_path, envelope)
evidence_rel = evidence_path.relative_to(DATA_DIR).as_posix()
insert_metric(conn, scope, month, response, params, evidence_rel)
counts["official_scope_month_metrics"] += 1
evidence_records.append({"path": evidence_rel, "bytes": evidence_path.stat().st_size,
"sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest(),
"kind": "official_scope_month_metrics", "scope": scope["key"],
"period": f"{YEAR}-{month:02d}"})
if month == 9:
sep_scope_metrics[scope["key"]] = response
for month in PRODUCT_MONTHS:
response, params, query = get_snapshot(api, scope, month, "product")
evidence_name = f"products_{scope['key']}_{YEAR}-{month:02d}.json"
evidence_path = EVIDENCE_DIR / evidence_name
envelope = {"request": {"method": "GET", "path": "/api/metrics", "query": params,
"encoded_query": query}, "response": response}
write_json(evidence_path, envelope)
evidence_rel = evidence_path.relative_to(DATA_DIR).as_posix()
added = insert_products(conn, scope, month, response, evidence_rel)
counts["official_product_month_rows"] += added
evidence_records.append({"path": evidence_rel, "bytes": evidence_path.stat().st_size,
"sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest(),
"kind": "official_product_month_rows", "scope": scope["key"],
"period": f"{YEAR}-{month:02d}", "rows": added})
conn.close()
temporary_db.replace(DB_PATH)
except Exception:
conn.close()
temporary_db.unlink(missing_ok=True)
raise
with sqlite3.connect(DB_PATH) as check:
counts["official_scope_month_metrics"] = check.execute(
"SELECT COUNT(*) FROM official_scope_month_metrics").fetchone()[0]
counts["official_product_month_rows"] = check.execute(
"SELECT COUNT(*) FROM official_product_month_rows").fetchone()[0]
missing_periods = check.execute("""
SELECT scope_key, GROUP_CONCAT(period, ',') FROM official_scope_month_metrics
GROUP BY scope_key HAVING COUNT(*) != 9
""").fetchall()
if missing_periods:
raise RuntimeError(f"Jan-Sep 月度指标行数不完整:{missing_periods}")
check_car = check.execute("""
SELECT official_composite_rate FROM official_scope_month_metrics
WHERE scope_key='auto_combined' AND period='2026-09'
""").fetchone()[0]
check_overall = check.execute("""
SELECT official_composite_rate FROM official_scope_month_metrics
WHERE scope_key='overall' AND period='2026-09'
""").fetchone()[0]
trend_ranges = check.execute("""
SELECT scope_key, MIN(period), MAX(period), COUNT(*)
FROM official_scope_month_metrics GROUP BY scope_key ORDER BY scope_key
""").fetchall()
if abs(check_car - fixture_rate(expected_car)) > 1e-12:
raise RuntimeError("汽车交通合计 2026-09 综合合格率与参考正式结果不一致")
if abs(check_overall - fixture_rate(expected_overall)) > 1e-12:
raise RuntimeError("总体 2026-09 综合合格率与参考正式结果不一致")
rule_version = expected_car.get("context", {}).get("ruleVersionId")
context = f"""# DB-GPT 质量数据试验副本说明
本目录是供公司本地 AI 按需读取的 SQLite 数据副本与原始 API 证据。数据来自本机正式指标只读接口,绑定已发布版本 `{RELEASE_ID}`({expected_car['release'].get('name', '')});规则版本为 `{rule_version}`。没有改写源数据库、正式数据或计算规则。
## 指标口径与范围
- `official_composite_rate` 对应服务返回的 `summary.compositeRate`,业务名称为“综合合格率”。它是正式发布版本的服务端固定指标;数据库中的数值直接来自指标服务,没有在导入时重新计算。
- `composite_rate_without_hq_scrap` 是服务另外返回的调整口径,不能和综合合格率混称。
- 汽车交通包括“汽车交通-国内”和“汽车交通-海外”两组。合计筛选按正式 `SelectionValues` 合约编码,wire 值为 `{CAR_BOTH_WIRE}`。
- 当前副本覆盖 `{YEAR}-01` 至 `{YEAR}-09` 的总体和三种汽车交通范围月度指标;产品汇总覆盖 `{YEAR}-08`、`{YEAR}-09`。发布 ID、规则版本、筛选值和原始 API 响应均可在表和 `evidence/` 中追溯。
- 空值在 SQLite 中保留为 `NULL`;`NULL` 表示该服务值缺失或不适用,不能当作 0。
- `official_product_month_rows` 的产品分组用于找规格线索。服务说明这些分组行无法保证重新汇总为阶段分母或总体贡献;只能作为下钻线索,不能据此认定对总体率变化的贡献。
## 表说明
- `official_scope_month_metrics`:总体、汽车交通国内、海外、合计在 2026 年 1–9 月的正式月度汇总率、黑片/亮片阶段率与重量、覆盖行数、范围和证据路径。每范围每月一行。
- `official_product_month_rows`:上述范围在 2026 年 8、9 月由 `view=product` 返回的全量正式产品汇总行。每行保留产品名、正式综合率、阶段率、阶段重量、覆盖行数与完整行 JSON。
## 读取边界
公司本地 AI 可按需使用副本明细执行 SQL 探索,并应区分正式汇总指标和产品分组线索。回答中的正式 KPI 以 `official_scope_month_metrics` 的正式服务值为准;不要把产品行加总成总体贡献,也不要把空值补成 0。`evidence/` 保存原始完整响应,可用于复核字段来源。
"""
CONTEXT_PATH.write_text(context, encoding="utf-8")
manifest = {
"artifact": "DB-GPT local quality-data trial copy",
"generated_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"source": {"kind": "read-only local API", "base_url": "http://127.0.0.1:5281",
"endpoint": "/api/metrics", "credentials_written": False},
"release": {"id": RELEASE_ID, "status": "PUBLISHED",
"name": expected_car["release"].get("name"), "rule_version_id": rule_version},
"coverage": {"year": YEAR, "monthly_metrics": [f"{YEAR}-{month:02d}" for month in MONTHS],
"product_months": [f"{YEAR}-{month:02d}" for month in PRODUCT_MONTHS],
"scopes": [{"key": s["key"], "name": s["name"], "group_filter_wire": s["group"]}
for s in SCOPES]},
"files": {"sqlite": str(DB_PATH.relative_to(ROOT)),
"business_context": str(CONTEXT_PATH.relative_to(ROOT)),
"evidence_directory": str(EVIDENCE_DIR.relative_to(ROOT))},
"tables": {
"official_scope_month_metrics": {"rows": counts["official_scope_month_metrics"],
"description_zh": "正式指标服务返回的总体及汽车交通各范围月度汇总;rate不在导入时重算。",
"periods_by_scope": [{"scope_key": r[0], "first_period": r[1], "last_period": r[2], "rows": r[3]}
for r in trend_ranges]},
"official_product_month_rows": {"rows": counts["official_product_month_rows"],
"description_zh": "指标API view=product在2026年8、9月返回的全量产品汇总行;只作规格线索,不代表可回加的总体贡献。"},
},
"reference_checks": {
"overall_2026_09_official_composite_rate": {"expected": fixture_rate(expected_overall),
"sqlite": check_overall, "matched": abs(check_overall - fixture_rate(expected_overall)) <= 1e-12},
"auto_combined_2026_09_official_composite_rate": {"expected": fixture_rate(expected_car),
"sqlite": check_car, "matched": abs(check_car - fixture_rate(expected_car)) <= 1e-12,
"display_percent_2dp": round(check_car * 100, 2)},
"sqlite_query_verified": True,
"null_policy": "preserve SQL NULL; do not interpret as zero",
},
"evidence": evidence_records,
}
write_json(MANIFEST_PATH, manifest)
print(json.dumps({"database": str(DB_PATH), "manifest": str(MANIFEST_PATH),
"business_context": str(CONTEXT_PATH), "counts": counts,
"release_id": RELEASE_ID, "rule_version_id": rule_version,
"overall_2026_09_rate": check_overall,
"auto_combined_2026_09_rate": check_car,
"evidence_files": len(evidence_records)}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Import and publish the supplied, hash-verified development sample files locally."""
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
from local_api import Api
import project
ROOT = project.ROOT
api = Api().login()
sources = api.call('/api/sources/scan', {})['items']
# Restrict this reproducible bootstrap to the delivered evidence set.
evidence = json.loads((ROOT/'verification/source_hashes.json').read_text())
allowed = set()
if isinstance(evidence, list):
allowed.update(x['sha256'] for x in evidence)
if isinstance(evidence, dict):
# Evidence formats carry paths as keys or as a files collection.
candidates = evidence.get('files', evidence)
if isinstance(candidates, dict):
for key, value in candidates.items():
allowed.add(value.get('sha256') if isinstance(value, dict) else value)
elif isinstance(candidates, list):
allowed.update(x.get('sha256') for x in candidates)
selected = [s for s in sources if s['sha256'] in allowed]
if len(selected) != 15:
raise RuntimeError('Expected exactly the 15 supplied source-period files; no data was imported')
ids = sorted(s['id'] for s in selected)
mapping = [x['id'] for x in api.call('/api/mappings')['items'] if x['status'] == 'ACTIVE']
rules = [x['id'] for x in api.call('/api/rules')['items'] if x['status'] == 'ACTIVE']
key = 'provided-samples-' + hashlib.sha256(json.dumps([ids, mapping, rules]).encode()).hexdigest()
releases = api.call('/api/releases')
run = api.call('/api/import-runs', {'sourceIds': ids, 'idempotencyKey': key,
'expectedCurrentReleaseId': releases.get('currentReleaseId')})['run']
if run['status'] != 'COMPLETED' or not run.get('resultReleaseId'):
raise RuntimeError('Sample import did not produce a completed candidate; inspect the import page')
release = run['resultReleaseId']
current = api.call('/api/releases').get('currentReleaseId')
if current != release:
api.call('/api/releases/'+release+'/publish', {'expectedCurrentReleaseId':current,
'idempotencyKey':key+'-publish', 'reason':'本机开发:已提供正常样本的独立基线验证与页面联调'})
summary = {'verifiedAt': datetime.now(timezone.utc).isoformat(), 'environment':'LOCAL_DEVELOPMENT_ONLY',
'sourcePeriods':len(selected), 'processedRows':run['processed'], 'releaseId':release,
'validation':run.get('validation'), 'status':'IMPORTED_AND_PUBLISHED'}
output = ROOT/'verification/local-import.json'
output.write_text(json.dumps(summary, ensure_ascii=False, indent=2)+'\n')
print('Imported and published locally:',len(selected),'source periods,',run['processed'],'rows')
print('Evidence:',output)
+36
View File
@@ -0,0 +1,36 @@
"""Local-only API client; keeps credentials and cookies out of output."""
import http.cookiejar
import json
from urllib.request import Request, build_opener, HTTPCookieProcessor
from urllib.error import HTTPError
import project
class Api:
def __init__(self):
self.opener = build_opener(HTTPCookieProcessor(http.cookiejar.CookieJar()))
self.csrf = None
def call(self, path, payload=None, method=None):
if not path.startswith('/api/'):
raise ValueError('Only local project API paths are supported')
verb = method or ('POST' if payload is not None else 'GET')
headers = {'Accept': 'application/json'}
if payload is not None:
headers['Content-Type'] = 'application/json'
if self.csrf:
headers['X-XSRF-TOKEN'] = self.csrf
request = Request('http://127.0.0.1:5281'+path, method=verb, headers=headers,
data=json.dumps(payload, ensure_ascii=False).encode() if payload is not None else None)
try:
with self.opener.open(request, timeout=180) as response:
data = response.read()
return json.loads(data) if data else {}
except HTTPError as error:
raise RuntimeError(f'HTTP {error.code} at {verb} {path}') from None
def login(self):
value = project.config()
self.csrf = self.call('/api/auth/csrf')['token']
self.call('/api/auth/login', {'username': value['adminUsername'], 'password': value['adminPassword']})
self.csrf = self.call('/api/auth/csrf')['token']
return self
+25
View File
@@ -0,0 +1,25 @@
"""Read the UI-managed, private NAO model configuration with legacy bootstrap."""
import json
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
SETTINGS_FILE = ROOT / '.runtime/nao-model-config.json'
def model_settings():
if not SETTINGS_FILE.exists():
from dbgpt_trial import model_settings as legacy_settings
return legacy_settings()
if SETTINGS_FILE.is_symlink() or SETTINGS_FILE.parent.is_symlink() or SETTINGS_FILE.stat().st_size > 32768:
raise ValueError('Invalid private model configuration')
value = json.loads(SETTINGS_FILE.read_text(encoding='utf-8'))
base, key, model = (value.get(name) for name in ('baseUrl', 'apiKey', 'model'))
if not all(isinstance(item, str) and item.strip() for item in (base, key, model)):
raise ValueError('Incomplete model configuration')
u = urlsplit(base)
if u.scheme not in ('http', 'https') or not u.hostname or u.username or u.password or u.query or u.fragment:
raise ValueError('Invalid model endpoint')
if len(key) > 4096 or any(c.isspace() or ord(c) < 32 for c in key):
raise ValueError('Invalid model credential')
return base.rstrip('/'), key, model
+172
View File
@@ -0,0 +1,172 @@
"""Install the quality investigation context into the existing local nao runtime.
Uses upstream nao extension points: project rules, system prompt, model settings,
and a stdio MCP. The packaged engine, source workbooks and business DB stay intact.
"""
import argparse
import ast
import hashlib
import json
import os
import signal
import shutil
import socket
import subprocess
import sys
import time
from datetime import datetime
from pathlib import Path
from urllib.request import urlopen
import nao_trial
ROOT = Path(__file__).resolve().parents[1]
TEMPLATES = ROOT / "ai/nao-quality"
ACTIVE = nao_trial.PROJECT
STATE = nao_trial.TRIAL / "quality-context.json"
OUT = ROOT / "output/nao-quality"
def _digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def configure():
for name in ("nao_quality_tools.py", "nao_quality_mcp.py"):
path = ROOT / "scripts" / name
ast.parse(path.read_text())
_, _, model = nao_trial.model_settings()
stamp = datetime.now().astimezone().strftime("%Y%m%d-%H%M%S-%f")
backup = nao_trial.TRIAL / "context-backups" / stamp
backup.mkdir(parents=True, mode=0o700)
ACTIVE.mkdir(parents=True, exist_ok=True)
files = ["RULES.md", "nao_config.yaml", "agent/prompts/system.md", "agent/mcps/mcp.json"]
for relative in files:
old = ACTIVE / relative
if old.exists():
target = backup / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(old, target)
# Old baseline warehouse context is preserved outside the active project.
# Deep investigation uses the live, version-bound quality API MCP instead.
old_databases = ACTIVE / "databases"
if old_databases.exists():
shutil.move(str(old_databases), str(backup / "databases"))
# Upstream deliberately reuses discovered specs across restarts. Move only
# our generated server directory aside so startup discovers the live schema.
old_quality_specs = ACTIVE / "agent/mcps/quality"
if old_quality_specs.exists():
target = backup / "agent/mcps/quality"
target.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(old_quality_specs), str(target))
for relative in ("RULES.md", "agent/prompts/system.md"):
target = ACTIVE / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(TEMPLATES / relative, target)
(ACTIVE / "nao_config.yaml").write_text(
"project_name: quality-investigation\n"
"databases: []\n"
"llm:\n"
f" annotation_model: openaiCompatible:{model}\n"
" providers:\n"
" - provider: openaiCompatible\n"
" api_key: \"{{ env('NAO_MODEL_KEY') }}\"\n"
" base_url: \"{{ env('NAO_MODEL_BASE') }}\"\n"
" models:\n"
f" - id: {json.dumps(model)}\n"
" default: true\n"
" settings:\n"
" maxOutputTokens: 8192\n"
" temperature: 0.2\n"
" reasoningEffort: low\n"
)
old_mcp = backup / "agent/mcps/mcp.json"
mcps = json.loads(old_mcp.read_text()) if old_mcp.exists() else {"mcpServers": {}}
mcps.setdefault("mcpServers", {})["quality"] = {
"transport": "stdio",
"command": sys.executable,
"args": [str(ROOT / "scripts/nao_quality_mcp.py")],
}
mcp_path = ACTIVE / "agent/mcps/mcp.json"
mcp_path.parent.mkdir(parents=True, exist_ok=True)
mcp_path.write_text(json.dumps(mcps, ensure_ascii=False, indent=2))
mcp_path.chmod(0o600)
state = {
"mode": "QUALITY_API_MCP",
"model": model,
"backup": str(backup),
"configuredAt": datetime.now().astimezone().isoformat(),
"sourceHashes": {relative: _digest(TEMPLATES / relative)
for relative in ("RULES.md", "agent/prompts/system.md")},
"upstreamCoreModified": False,
"qualityApi": "http://127.0.0.1:5281",
"naoUi": "http://127.0.0.1:5674",
"accessBoundary": "Local maintenance-account project only, not a shared multi-user integration",
}
STATE.write_text(json.dumps(state, ensure_ascii=False, indent=2))
STATE.chmod(0o600)
OUT.mkdir(parents=True, exist_ok=True)
(OUT / "environment.json").write_text(json.dumps(state, ensure_ascii=False, indent=2))
print("Quality investigation context installed; prior context backed up")
def stop():
state_path = nao_trial.TRIAL / "pids.json"
previous = json.loads(state_path.read_text()) if state_path.exists() else {}
nao_trial.stop()
deadline = time.monotonic() + 8
while time.monotonic() < deadline:
alive = []
for pid in previous.values():
result = subprocess.run(["ps", "-p", str(pid), "-o", "stat="], capture_output=True, text=True)
if result.returncode == 0 and not result.stdout.strip().startswith("Z"):
alive.append(pid)
if not alive:
return
time.sleep(0.1)
for pid in alive:
result = subprocess.run(["ps", "-p", str(pid), "-o", "args="], capture_output=True, text=True)
if result.returncode:
continue
if str(nao_trial.TRIAL) not in result.stdout or os.getpgid(pid) != pid:
raise RuntimeError("Local nao process identity changed; force stop refused")
os.killpg(pid, signal.SIGKILL)
print("Stopped verified local nao process groups after graceful shutdown timeout")
def activate():
# Validate first so a missing adapter never takes the working trial offline.
for name in ("nao_quality_tools.py", "nao_quality_mcp.py"):
ast.parse((ROOT / "scripts" / name).read_text())
stop()
deadline = time.monotonic() + 15
for port in (5674, 5675):
while True:
with socket.socket() as connection:
connection.settimeout(0.3)
occupied = connection.connect_ex(("127.0.0.1", port)) == 0
if not occupied:
break
if time.monotonic() > deadline:
raise RuntimeError(f"Prior local nao port {port} is still occupied; restart was not attempted")
time.sleep(0.15)
configure()
nao_trial.start()
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
try:
with urlopen("http://127.0.0.1:5674", timeout=2) as response:
if response.status == 200:
print("Quality investigation ready at http://127.0.0.1:5674")
return
except OSError:
time.sleep(0.3)
raise RuntimeError("nao did not become ready; preserved backup is recorded in quality-context.json")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=("configure", "activate", "start", "stop"))
args = parser.parse_args()
{"configure": configure, "activate": activate, "start": nao_trial.start,
"stop": stop}[args.action]()
+143
View File
@@ -0,0 +1,143 @@
"""Run real nao quality conversations, with evidence and an active cancellation budget."""
import argparse
import base64
import json
import re
import threading
import time
from pathlib import Path
from urllib.error import HTTPError
from urllib.parse import quote
from urllib.request import Request
from nao_trial import ROOT, model_settings
from nao_trial_chat import BASE, opener
OUT = ROOT / "output/nao-quality"
def run(label, question, chat_id=None, timeout=240):
if not re.fullmatch(r"[a-zA-Z0-9_-]{1,80}", label):
raise ValueError("Use a short alphanumeric evidence label")
OUT.mkdir(parents=True, exist_ok=True)
client = opener()
model = model_settings()[2]
body = {"message": {"text": question},
"model": {"provider": "openaiCompatible", "modelId": model},
"timezone": "Asia/Shanghai"}
if chat_id:
body["chatId"] = chat_id
(OUT / (label + "-request.json")).write_text(json.dumps(body, ensure_ascii=False, indent=2))
state = {"chatId": chat_id, "budgetExpired": False, "stopRequest": None}
events, text_blocks, tools, tool_outputs = [], {}, [], {}
finished = threading.Event()
response_ref = []
def cancel_at_budget():
if finished.wait(timeout):
return
state["budgetExpired"] = True
if state.get("chatId"):
request = Request(BASE + "/api/trpc/chat.stop", method="POST",
data=json.dumps({"json": {"chatId": state["chatId"]}}).encode(),
headers={"Content-Type": "application/json", "Origin": BASE})
try:
with client.open(request, timeout=15) as response:
state["stopRequest"] = {"httpStatus": response.status}
except HTTPError as error:
state["stopRequest"] = {"httpStatus": error.code}
except OSError as error:
state["stopRequest"] = {"errorType": type(error).__name__}
if response_ref:
response_ref[0].close()
timer = threading.Thread(target=cancel_at_budget, daemon=True)
started = time.monotonic()
timer.start()
error_type = None
model_base, model_key, _ = model_settings()
request = Request(BASE + "/api/agent/", data=json.dumps(body, ensure_ascii=False).encode(),
headers={"Content-Type": "application/json", "Origin": BASE,
"Accept": "text/event-stream"})
try:
with client.open(request, timeout=60) as response, (OUT / (label + ".sse")).open("w") as stream:
response_ref.append(response)
for raw in response:
line = raw.decode("utf-8", errors="replace").replace(model_key, "[REDACTED_KEY]")
line = line.replace(model_base, "[REDACTED_MODEL_ENDPOINT]")
stream.write(line)
stream.flush()
if not line.startswith("data:"):
continue
raw_data = line[5:].strip()
if raw_data == "[DONE]":
break
try:
event = json.loads(raw_data)
except json.JSONDecodeError:
continue
events.append(event)
kind = event.get("type")
if kind == "data-newChat":
state["chatId"] = event.get("data", {}).get("id") or state["chatId"]
elif kind == "text-delta":
block_id = event.get("id", "text")
text_blocks[block_id] = text_blocks.get(block_id, "") + event.get("delta", "")
elif kind == "tool-input-available":
tools.append({"toolCallId": event.get("toolCallId"), "tool": event.get("toolName"),
"input": event.get("input")})
print("tool", event.get("toolName"), flush=True)
elif kind == "tool-output-available":
tool_outputs[event.get("toolCallId")] = event.get("output")
except (OSError, ValueError, AttributeError) as error:
error_type = type(error).__name__
finally:
finished.set()
timer.join(timeout=1)
for tool in tools:
tool["output"] = tool_outputs.get(tool["toolCallId"])
texts = [text.strip() for text in text_blocks.values() if text.strip()]
final_text = texts[-1] if texts else ""
finish_events = [event for event in events if event.get("type") == "finish"]
truncated = any(event.get("finishReason") == "length" for event in finish_events)
error_events = [event for event in events if event.get("type") in ("error", "tool-output-error")]
summary = {
"label": label, "question": question, "model": model, "chatId": state["chatId"],
"durationSeconds": round(time.monotonic() - started, 2),
"status": "budget_cancelled" if state["budgetExpired"] else
"truncated" if truncated else
"finished" if finish_events and final_text and not any(e.get("type") == "error" for e in error_events)
else "incomplete",
"finishReceived": bool(finish_events), "finishReasons": [event.get("finishReason") for event in finish_events], "errors": error_events,
"transportErrorType": error_type, "stopRequest": state["stopRequest"],
"toolCalls": tools, "eventCount": len(events), "answer": final_text,
}
(OUT / (label + "-events.json")).write_text(json.dumps(events, ensure_ascii=False, indent=2))
(OUT / (label + "-run.json")).write_text(json.dumps(summary, ensure_ascii=False, indent=2))
(OUT / (label + "-answer.md")).write_text(final_text)
(OUT / (label + "-all-text.md")).write_text("\n\n---\n\n".join(texts))
for tool in tools:
result = tool.get("output")
if tool["tool"] != "story" or not isinstance(result, dict) or not result.get("success"):
continue
slug = result.get("id")
(OUT / (label + "-story.md")).write_text(result.get("code", ""))
data = {"json": {"chatId": state["chatId"], "storySlug": slug, "format": "html"}}
request = Request(BASE + "/api/trpc/story.download?input=" + quote(json.dumps(data)),
headers={"Origin": BASE})
with client.open(request, timeout=30) as response:
export = json.load(response)["result"]["data"]["json"]
(OUT / (label + "-report.html")).write_bytes(base64.b64decode(export["data"]))
print(json.dumps({key: summary[key] for key in
("label", "chatId", "durationSeconds", "status", "eventCount")}, ensure_ascii=False))
return summary
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("label")
parser.add_argument("question")
parser.add_argument("--chat-id")
parser.add_argument("--timeout", type=int, default=240)
args = parser.parse_args()
run(args.label, args.question, args.chat_id, args.timeout)
+138
View File
@@ -0,0 +1,138 @@
"""Minimal stdio MCP server exposing the local read-only quality_data tool."""
from __future__ import annotations
import json
import logging
import os
import sys
from pathlib import Path
from typing import Any
from nao_quality_tools import QualityToolError, QualityTools, tool_definition
ROOT = Path(__file__).resolve().parents[1]
LOG_DIR = ROOT / "output" / "nao-quality" / "tools"
SUPPORTED_PROTOCOLS = ("2025-03-26", "2024-11-05")
def _error(code: int, message: str, request_id: Any = None) -> dict[str, Any]:
return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}}
def _tool_result(value: dict[str, Any], *, is_error: bool = False) -> dict[str, Any]:
encoded = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
return {
"content": [{"type": "text", "text": encoded}],
"structuredContent": value,
"isError": is_error,
}
class MCPServer:
def __init__(self, tools: QualityTools | None = None, logger: logging.Logger | None = None) -> None:
self.tools = tools or QualityTools()
self.logger = logger or logging.getLogger("nao_quality_mcp")
self.initialized = False
self.protocol_version: str | None = None
def handle(self, request: Any) -> dict[str, Any] | None:
if not isinstance(request, dict) or request.get("jsonrpc") != "2.0" or not isinstance(request.get("method"), str):
request_id = request.get("id") if isinstance(request, dict) else None
return _error(-32600, "无效 JSON-RPC 请求。", request_id)
method = request["method"]
request_id = request.get("id")
is_notification = "id" not in request or method.startswith("notifications/")
params = request.get("params") or {}
if not isinstance(params, dict):
if is_notification:
return None
return _error(-32602, "params 必须是对象。", request_id)
if method == "initialize":
version = params.get("protocolVersion")
# MCP initialization negotiates the protocol version in the response.
# If this client asks for a newer revision, advertise the newest revision
# implemented here and let the client decide whether it can continue.
selected_version = version if version in SUPPORTED_PROTOCOLS else SUPPORTED_PROTOCOLS[0]
self.protocol_version = selected_version
self.initialized = True
return {
"jsonrpc": "2.0", "id": request_id,
"result": {
"protocolVersion": selected_version,
"capabilities": {"tools": {"listChanged": False}},
"serverInfo": {"name": "quality", "version": "1.0.0"},
"instructions": "quality_data 只读查询本机质量中心正式 HTTP API;沿用返回的 releaseId,并引用 evidenceId。",
},
}
if method == "notifications/initialized":
self.initialized = True
return None
if method == "ping":
return None if is_notification else {"jsonrpc": "2.0", "id": request_id, "result": {}}
if method == "tools/list":
return None if is_notification else {
"jsonrpc": "2.0", "id": request_id,
"result": {"tools": [tool_definition()]},
}
if method == "tools/call":
if is_notification:
return None
name = params.get("name")
arguments = params.get("arguments", {})
if name != "quality_data":
return _error(-32602, "未知工具;当前只提供 quality_data。", request_id)
if not isinstance(arguments, dict):
return _error(-32602, "quality_data arguments 必须是对象。", request_id)
mode = arguments.get("mode")
self.logger.info("tool call mode=%s", mode if isinstance(mode, str) else "invalid")
try:
result = self.tools.quality_data(arguments)
return {"jsonrpc": "2.0", "id": request_id, "result": _tool_result(result)}
except QualityToolError as error:
self.logger.warning("tool rejected mode=%s message=%s", mode, str(error)[:400])
return {"jsonrpc": "2.0", "id": request_id,
"result": _tool_result({"error": {"code": "invalid_or_unavailable", "message": str(error)}}, is_error=True)}
except Exception:
# Do not log request arguments, credentials, API bodies, or stack locals.
self.logger.error("tool failed mode=%s; inspect local service and safe API status", mode)
return {"jsonrpc": "2.0", "id": request_id,
"result": _tool_result({"error": {"code": "api_failure", "message": "质量中心正式 API 查询未完成;请检查本机服务/登录权限后重试。没有返回空数据成功。"}}, is_error=True)}
if is_notification:
return None
return _error(-32601, f"不支持的 MCP 方法:{method}。", request_id)
def _configure_logger() -> logging.Logger:
LOG_DIR.mkdir(parents=True, exist_ok=True)
path = LOG_DIR / "nao_quality_mcp.log"
handler = logging.FileHandler(path, encoding="utf-8")
os.chmod(path, 0o600)
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
logger = logging.getLogger("nao_quality_mcp")
logger.setLevel(logging.INFO)
logger.handlers.clear()
logger.addHandler(handler)
logger.propagate = False
return logger
def serve(stdin: Any = sys.stdin, stdout: Any = sys.stdout, tools: QualityTools | None = None) -> None:
server = MCPServer(tools=tools, logger=_configure_logger())
for raw_line in stdin:
line = raw_line.strip()
if not line:
continue
try:
request = json.loads(line)
except json.JSONDecodeError:
response = _error(-32700, "无法解析 JSON。")
else:
response = server.handle(request)
if response is not None:
stdout.write(json.dumps(response, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + "\n")
stdout.flush()
if __name__ == "__main__":
serve()
File diff suppressed because it is too large Load Diff
+82
View File
@@ -0,0 +1,82 @@
"""NAO stdio MCP shim backed by per-actor delegated quality API access."""
from __future__ import annotations
import os
import sys
import json
import logging
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from nao_quality_runtime import DelegatedApi, _uuid # noqa: E402
from nao_quality_tools import QualityTools # noqa: E402
import nao_quality_mcp # noqa: E402
def _paths() -> tuple[Path, Path, Path, str, str, Path]:
users_root = Path(os.environ.get("QUALITY_AGENT_USERS_ROOT", "")).resolve()
access_path = Path(os.environ.get("QUALITY_AGENT_ACCESS_FILE", ""))
evidence_dir = Path(os.environ.get("QUALITY_AGENT_EVIDENCE_DIR", ""))
actor_id = _uuid(os.environ.get("QUALITY_AGENT_ACTOR_ID"), "actorId")
session_id = _uuid(os.environ.get("QUALITY_AGENT_SESSION_ID"), "sessionId")
session_state = Path(os.environ.get("QUALITY_AGENT_SESSION_STATE_FILE", ""))
actor_root = users_root / actor_id
if users_root != (ROOT / ".runtime" / "nao-users").resolve():
raise ValueError("质量MCP运行目录无效。")
if access_path.is_symlink() or access_path.absolute() != actor_root / "access.json":
raise ValueError("质量MCP访问凭据路径无效。")
expected_evidence = actor_root / session_id / "evidence"
if evidence_dir.is_symlink() or evidence_dir.resolve() != expected_evidence.resolve():
raise ValueError("质量MCP证据目录路径无效。")
if session_state.is_symlink() or session_state.absolute() != actor_root / "private" / "current-session.json":
raise ValueError("质量MCP会话状态路径无效。")
evidence_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(evidence_dir, 0o700)
return access_path, evidence_dir, session_state, actor_id, session_id, users_root
def main() -> int:
access_path, evidence_dir, session_state, actor_id, _session_id, users_root = _paths()
adapter = DelegatedApi(access_path, runtime_root=users_root, expected_actor_id=actor_id)
class SessionScopedQualityTools(QualityTools):
def quality_data(self, arguments):
try:
current = json.loads(session_state.read_text(encoding="utf-8"))
current_session = _uuid(current.get("sessionId"), "sessionId")
except Exception:
raise ValueError("当前质量分析会话不可用。") from None
if not isinstance(current, dict) or current.get("actorId") != actor_id:
raise ValueError("当前质量分析 actor 不匹配。")
target = users_root / actor_id / current_session / "evidence"
if target.is_symlink() or target.resolve().parent.parent != (users_root / actor_id).resolve():
raise ValueError("质量证据目录路径无效。")
target.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(target, 0o700)
self._evidence_dir = target
return super().quality_data(arguments)
tools = SessionScopedQualityTools(api_factory=lambda: adapter, evidence_dir=evidence_dir)
# The upstream serve loop stays intact; protocol details go to stdout and
# no shared log file can mix evidence from different actors or sessions.
def quiet_logger():
logger = logging.getLogger("nao_quality_mcp")
logger.handlers.clear()
logger.addHandler(logging.NullHandler())
logger.propagate = False
return logger
nao_quality_mcp._configure_logger = quiet_logger
nao_quality_mcp.serve(tools=tools)
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except Exception:
# MCP stderr is consumed by NAO and must never contain request bodies,
# access tokens or stack frames. The upstream server's own safe MCP
# errors are still returned through the JSON-RPC protocol.
raise SystemExit("quality MCP unavailable")
File diff suppressed because it is too large Load Diff
+97
View File
@@ -0,0 +1,97 @@
"""Isolated nao trial using the upstream packaged servers and existing model profile."""
import argparse,json,os,secrets,signal,subprocess,sys
from pathlib import Path
from dbgpt_trial import model_settings
ROOT=Path(__file__).resolve().parents[1]
TRIAL=ROOT/'.runtime/nao-trial'
PROJECT=TRIAL/'project'
SITE=TRIAL/'venv/lib/python3.11/site-packages/nao_core'
BIN=SITE/'bin'
def environment():
base,key,model=model_settings()
state=TRIAL/'private.json'
if not state.exists():
state.write_text(json.dumps({'auth_secret':secrets.token_urlsafe(40),'password':secrets.token_urlsafe(24),'email':'quality-trial@localhost.invalid'}));state.chmod(0o600)
private=json.loads(state.read_text())
e=os.environ.copy()
e.update(NAO_MODEL_BASE=base,NAO_MODEL_KEY=key,NAO_MODEL_NAME=model,NAO_DEFAULT_PROJECT_PATH=str(PROJECT),
BETTER_AUTH_URL='http://127.0.0.1:5674',BETTER_AUTH_SECRET=private['auth_secret'],
DB_URI='sqlite:'+str(TRIAL/'metadata.sqlite'),FASTAPI_PORT='5675',PORT='5675',MODE='prod',
NAO_CORE_VERSION='0.3.26',POSTHOG_DISABLED='true',DO_NOT_TRACK='1')
return e
def prepare():
import sqlite3,duckdb
PROJECT.mkdir(parents=True,exist_ok=True)
src=sqlite3.connect('file:'+str(ROOT/'.runtime/dbgpt-trial/data/quality_trial.sqlite')+'?mode=ro',uri=True)
target=PROJECT/'quality_trial.duckdb'
if not target.exists():
db=duckdb.connect(str(target))
for (table,) in src.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"):
cols=src.execute('PRAGMA table_info("'+table+'")').fetchall()
kinds={'TEXT':'VARCHAR','INTEGER':'BIGINT','REAL':'DOUBLE','BLOB':'BLOB','NUMERIC':'DOUBLE'}
db.execute('CREATE TABLE "'+table+'" ('+','.join('"'+c[1]+'" '+kinds.get(c[2].upper(),'VARCHAR') for c in cols)+')')
rows=src.execute('SELECT * FROM "'+table+'"').fetchall()
if rows:db.executemany('INSERT INTO "'+table+'" VALUES ('+','.join('?' for _ in cols)+')',rows)
db.close()
rule=(ROOT/'.runtime/dbgpt-trial/home/skills/quality-center/SKILL.md').read_text()
rule=rule.split('---',2)[-1].replace('sqlite_quality_trial','quality_trial')
(PROJECT/'RULES.md').write_text('# 质量中心本地试验\n\n请默认使用中文。数据库为DuckDB质量副本,使用相应SQL语法。\n'+rule)
cfg='''project_name: quality-trial
databases:
- name: quality_trial
type: duckdb
path: ./quality_trial.duckdb
templates: [columns, preview]
llm:
annotation_model: openaiCompatible:mimo-v2.6-flash
providers:
- provider: openaiCompatible
api_key: "{{ env('NAO_MODEL_KEY') }}"
base_url: "{{ env('NAO_MODEL_BASE') }}"
models:
- id: mimo-v2.6-flash
default: true
'''
(PROJECT/'nao_config.yaml').write_text(cfg)
print('Prepared isolated DuckDB copy and business context')
def run_cli(args):
return subprocess.run([str(TRIAL/'venv/bin/nao'),*args],cwd=PROJECT,env=environment()).returncode
def start():
state=TRIAL/'pids.json'
if state.exists():
alive=[]
for pid in json.loads(state.read_text()).values():
try:os.kill(pid,0);alive.append(pid)
except ProcessLookupError:pass
if len(alive)==2:print('nao trial is already running');return
if alive:raise RuntimeError('A trial server is still running; stop it before starting again')
env=environment();pids={}
for name,args,cwd in [('fastapi',[str(TRIAL/'venv/bin/python'),str(BIN/'fastapi/main.py')],PROJECT),('server',[str(BIN/'nao-chat-server'),'serve','--port','5674','--host','127.0.0.1'],BIN)]:
fd=os.open(TRIAL/(name+'.log'),os.O_CREAT|os.O_APPEND|os.O_WRONLY,0o600)
with os.fdopen(fd,'ab') as log:
p=subprocess.Popen(args,cwd=cwd,env=env,stdin=subprocess.DEVNULL,stdout=log,stderr=log,start_new_session=True)
pids[name]=p.pid
(TRIAL/'pids.json').write_text(json.dumps(pids))
print('nao trial started: http://127.0.0.1:5674; model='+model_settings()[2])
def stop():
p=TRIAL/'pids.json'
if not p.exists():return
for name,pid in json.loads(p.read_text()).items():
r=subprocess.run(['ps','-p',str(pid),'-o','args='],capture_output=True,text=True,errors='replace')
if r.returncode:continue
if str(TRIAL) not in r.stdout or os.getpgid(pid)!=pid:raise RuntimeError('Process identity mismatch')
os.killpg(pid,signal.SIGTERM)
p.unlink();print('nao trial stopped')
if __name__=='__main__':
a=argparse.ArgumentParser();a.add_argument('action',choices=['prepare','sync','start','stop']);v=a.parse_args()
if v.action=='prepare':prepare()
elif v.action=='sync':sys.exit(run_cli(['sync','--provider','databases']))
elif v.action=='start':start()
else:stop()
+53
View File
@@ -0,0 +1,53 @@
"""Exercise the actual upstream nao HTTP agent and preserve its SSE evidence."""
import argparse,http.cookiejar,json,time,urllib.request,urllib.error
from pathlib import Path
from nao_trial import TRIAL,ROOT,model_settings
BASE='http://127.0.0.1:5674'
OUT=ROOT/'output/nao-trial'
STATE=TRIAL/'chat-state.json'
def opener():
jar=http.cookiejar.LWPCookieJar(str(TRIAL/'cookies.txt'))
if Path(jar.filename).exists():jar.load(ignore_discard=True)
client=urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar))
p=json.loads((TRIAL/'private.json').read_text())
for endpoint in ('sign-in/email','sign-up/email'):
body={k:p[k] for k in ('email','password')};body['name']='Quality Trial'
req=urllib.request.Request(BASE+'/api/auth/'+endpoint,data=json.dumps(body).encode(),headers={'Content-Type':'application/json','Origin':BASE})
try:
with client.open(req,timeout=20) as r: result=json.load(r)
jar.save(ignore_discard=True);Path(jar.filename).chmod(0o600)
return client
except urllib.error.HTTPError as e:
if e.code not in (400,401,403,404,422):raise
raise RuntimeError('Local nao authentication failed')
def run(label,question,new=False,chat_id=None):
client=opener();state={'chatId':chat_id} if chat_id else json.loads(STATE.read_text()) if STATE.exists() and not new else {}
model=model_settings()[2]
body={'message':{'text':question},'model':{'provider':'openaiCompatible','modelId':model},'timezone':'Asia/Shanghai'}
if state.get('chatId'):body['chatId']=state['chatId']
(OUT/(label+'-request.json')).write_text(json.dumps(body,ensure_ascii=False,indent=2))
req=urllib.request.Request(BASE+'/api/agent/',data=json.dumps(body,ensure_ascii=False).encode(),headers={'Content-Type':'application/json','Origin':BASE,'Accept':'text/event-stream'})
begin=time.monotonic();events=[];seen=set();status='done'
with client.open(req,timeout=120) as response,(OUT/(label+'.sse')).open('w') as stream:
for raw in response:
line=raw.decode('utf-8',errors='replace');stream.write(line);stream.flush()
if not line.startswith('data:'):continue
data=line[5:].strip()
if data=='[DONE]':break
try:e=json.loads(data)
except json.JSONDecodeError:continue
events.append(e);kind=e.get('type','')
if kind=='data-newChat' and e.get('data',{}).get('id'):
state['chatId']=e['data']['id'];STATE.write_text(json.dumps(state));STATE.chmod(0o600)
if kind not in seen:print('event',kind,flush=True);seen.add(kind)
if kind=='tool-input-available':print('tool',e.get('toolName'),flush=True)
if time.monotonic()-begin>600:status='stopped_at_ten_minute_budget';break
(OUT/(label+'-events.json')).write_text(json.dumps(events,ensure_ascii=False,indent=2))
(OUT/(label+'-run.json')).write_text(json.dumps({'status':status,'elapsed_seconds':round(time.monotonic()-begin,1),'model':model,**state},indent=2))
print('saved',len(events),'events;',state,'elapsed',round(time.monotonic()-begin,1),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('label');p.add_argument('question');p.add_argument('--new',action='store_true');p.add_argument('--chat-id');a=p.parse_args();run(a.label,a.question,a.new,a.chat_id)
+267
View File
@@ -0,0 +1,267 @@
#!/usr/bin/env python3
"""Build and run the isolated local quality platform without global installs."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import secrets
import signal
import shutil
import subprocess
import sys
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
RUNTIME = ROOT / '.runtime'
OUTPUT = ROOT / 'output'
CONFIG = RUNTIME / 'local-config.json'
def save_private(path, text):
path.parent.mkdir(parents=True, exist_ok=True)
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, 'w') as stream:
stream.write(text)
path.chmod(0o600)
def config():
if CONFIG.exists():
return json.loads(CONFIG.read_text())
value = {'mysqlPort': 53306, 'mysqlDatabase': 'quality_platform', 'mysqlUser': 'quality_app',
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24),
'databaseReady': False}
save_private(CONFIG, json.dumps(value, indent=2) + '\n')
return value
def environment():
value = config()
env = os.environ.copy()
env.update({'JAVA_HOME': str(RUNTIME / 'jdk/Contents/Home'),
'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{value["mysqlPort"]}/{value["mysqlDatabase"]}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true',
'MYSQL_USER': value['mysqlUser'], 'MYSQL_PASSWORD': value['mysqlPassword'],
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': value['adminUsername'],
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': value['adminPassword'],
'QUALITY_ARCHIVE_ROOT': str(ROOT / 'runtimedata/archive'),
'QUALITY_ALLOWED_ROOTS': str(ROOT / '质量数据/系统原始数据示例'),
'QUALITY_DATA_WORKFLOW_TEST_MODE': 'true',
'QUALITY_PROJECT_ROOT': str(ROOT),
'QUALITY_DEFAULT_SOURCE_YEAR': '2026'})
env['PATH'] = str(RUNTIME / 'jdk/Contents/Home/bin') + ':' + str(RUNTIME / 'maven/bin') + ':' + env.get('PATH', '')
return env
def run(args, **options):
return subprocess.run([str(a) for a in args], cwd=ROOT, env=environment(), check=True, **options)
def alive(pid):
try:
pid = int(pid)
if pid <= 1:
return False
os.kill(pid, 0)
state = subprocess.run(['ps', '-p', str(pid), '-o', 'stat='],
text=True, capture_output=True, check=False)
return state.returncode == 0 and bool(state.stdout.strip()) and not state.stdout.strip().startswith('Z')
except (OSError, ValueError):
return False
def process_stamp(pid):
result = subprocess.run(['ps', '-p', str(int(pid)), '-o', 'lstart='],
text=True, capture_output=True, check=False)
return result.stdout.strip() if result.returncode == 0 else ''
def owned_process(name, pid):
identity = RUNTIME / (name + '.process.json')
if not identity.exists():
return False
try:
recorded = json.loads(identity.read_text())
return (recorded.get('pid') == int(pid) and recorded.get('started') == process_stamp(pid)
and os.getpgid(int(pid)) == int(pid))
except (OSError, ValueError):
return False
def start_process(name, args):
pidfile = RUNTIME / (name + '.pid')
if pidfile.exists() and alive(pidfile.read_text().strip()):
if not owned_process(name, pidfile.read_text()):
raise RuntimeError(f'{name}: an existing PID has no matching project identity; inspect before reusing it.')
return int(pidfile.read_text())
(OUTPUT / 'logs').mkdir(parents=True, exist_ok=True)
with (OUTPUT / 'logs' / (name + '.log')).open('ab') as log:
process = subprocess.Popen([str(a) for a in args], cwd=ROOT, env=environment(),
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
pidfile.write_text(str(process.pid))
save_private(RUNTIME / (name + '.process.json'), json.dumps({'pid': process.pid, 'started': process_stamp(process.pid)}))
return process.pid
def wait_http(url, seconds=90):
end = time.monotonic() + seconds
while time.monotonic() < end:
try:
with urllib.request.urlopen(url, timeout=2) as response:
if response.status == 200:
return
except (OSError, urllib.error.URLError):
time.sleep(.5)
raise RuntimeError(f'Service did not become ready: {url}. Inspect output/logs/.')
def mysql_arguments():
return [RUNTIME / 'mysql/bin/mysql', '--no-defaults', '--protocol=SOCKET',
'--socket=' + str(RUNTIME / 'mysql.sock'), '--user=root', '--batch', '--skip-column-names']
def start_mysql():
value = config()
data = RUNTIME / 'mysql-data'
binary = RUNTIME / 'mysql/bin/mysqld'
if not binary.exists():
raise RuntimeError('Run python3 scripts/setup_runtime.py first.')
if not (data / 'auto.cnf').exists():
data.mkdir(parents=True, exist_ok=True)
(OUTPUT / 'logs').mkdir(parents=True, exist_ok=True)
with (OUTPUT / 'logs/mysql-initialize.log').open('ab') as log:
run([binary, '--no-defaults', '--initialize-insecure', '--basedir=' + str(RUNTIME / 'mysql'),
'--datadir=' + str(data), '--lower-case-table-names=2'], stdout=log, stderr=log)
start_process('mysql', [binary, '--no-defaults', '--basedir=' + str(RUNTIME / 'mysql'),
'--datadir=' + str(data), '--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1',
'--socket=' + str(RUNTIME / 'mysql.sock'), '--pid-file=' + str(RUNTIME / 'mysql-server.pid'),
'--log-error=' + str(OUTPUT / 'logs/mysql-server.log'), '--mysqlx=OFF',
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M'])
client_config = RUNTIME / 'mysql-root.cnf'
save_private(client_config, '[client]\nuser=root\npassword=' + value['rootPassword'] + '\nsocket=' + str(RUNTIME / 'mysql.sock') + '\n')
for _ in range(120):
if (RUNTIME / 'mysql.sock').exists():
result = subprocess.run([str(a) for a in mysql_arguments()], input='SELECT 1;\n',
text=True, capture_output=True)
if result.returncode == 0:
break
result = subprocess.run([str(RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config),
'--batch', '-e', 'SELECT 1;'], capture_output=True)
if result.returncode == 0:
break
time.sleep(.5)
else:
raise RuntimeError('Local MySQL did not start; inspect output/logs/mysql-server.log.')
if not value['databaseReady']:
sql = f'''CREATE DATABASE IF NOT EXISTS {value['mysqlDatabase']} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;
CREATE USER IF NOT EXISTS '{value['mysqlUser']}'@'localhost' IDENTIFIED BY '{value['mysqlPassword']}';
CREATE USER IF NOT EXISTS '{value['mysqlUser']}'@'127.0.0.1' IDENTIFIED BY '{value['mysqlPassword']}';
GRANT ALL PRIVILEGES ON {value['mysqlDatabase']}.* TO '{value['mysqlUser']}'@'localhost';
GRANT ALL PRIVILEGES ON {value['mysqlDatabase']}.* TO '{value['mysqlUser']}'@'127.0.0.1';
ALTER USER 'root'@'localhost' IDENTIFIED BY '{value['rootPassword']}';
'''
authenticated = [str(RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config), '--batch']
result = subprocess.run(authenticated, input=sql, text=True, capture_output=True)
if result.returncode != 0:
result = subprocess.run([str(a) for a in mysql_arguments()], input=sql, text=True, capture_output=True)
if result.returncode != 0:
raise RuntimeError('Local database bootstrap failed. Credentials have not been printed; inspect the local database configuration.')
value['databaseReady'] = True
save_private(CONFIG, json.dumps(value, indent=2) + '\n')
save_private(ROOT / 'runtimedata/本机登录信息.txt',
'质量中心 · 本机开发账户\n地址:http://127.0.0.1:5280\n用户名:' + value['adminUsername'] +
'\n密码:' + value['adminPassword'] + '\n仅限本机使用,请勿上传或分享此文件。\n')
print('Local MySQL is ready on 127.0.0.1:' + str(value['mysqlPort']), flush=True)
def build(skip_tests=False):
args = [RUNTIME / 'maven/bin/mvn', '-f', ROOT / 'backend/pom.xml',
'-Dmaven.repo.local=' + str(RUNTIME / 'm2'), '--batch-mode', 'package']
if skip_tests:
args.append('-DskipTests')
run(args)
run(['pnpm', '--dir', ROOT / 'web', 'build'])
def start():
start_mysql()
jars = sorted((ROOT / 'backend/target').glob('*.jar'), key=lambda p: p.stat().st_mtime)
jars = [p for p in jars if not p.name.endswith('.original')]
if not jars:
raise RuntimeError('Build the backend first with python3 scripts/project.py build.')
backend_pid = start_process('backend', [RUNTIME / 'jdk/Contents/Home/bin/java', '-Xms128m', '-Xmx2048m', '-jar', backend_snapshot(jars[-1])])
# Startup can resume a complete, queued workbook import before readiness becomes UP.
wait_http('http://127.0.0.1:5281/actuator/health', seconds=900)
if not alive(backend_pid) or not owned_process('backend', backend_pid):
raise RuntimeError('The project backend exited; inspect output/logs/backend.log and local port 5281.')
frontend_pid = start_process('frontend', ['pnpm', '--dir', ROOT / 'web', 'dev', '--host', '127.0.0.1'])
wait_http('http://127.0.0.1:5280')
if not alive(frontend_pid) or not owned_process('frontend', frontend_pid):
raise RuntimeError('The project frontend exited; inspect output/logs/frontend.log and local port 5280.')
print('Quality platform ready: http://127.0.0.1:5280', flush=True)
print('Local sign-in details: runtimedata/本机登录信息.txt', flush=True)
def backend_snapshot(jar):
"""Run an immutable jar so a later build cannot replace a live classpath."""
digest = hashlib.sha256(jar.read_bytes()).hexdigest()
target = RUNTIME / 'backend-builds' / (digest + '.jar')
if not target.exists():
target.parent.mkdir(parents=True, exist_ok=True)
temporary = target.with_suffix('.tmp')
shutil.copyfile(jar, temporary)
temporary.chmod(0o600)
temporary.replace(target)
return target
def stop():
for name in ('frontend', 'backend', 'mysql'):
path = RUNTIME / (name + '.pid')
if not path.exists():
continue
pid = int(path.read_text())
if alive(pid):
if not owned_process(name, pid):
raise RuntimeError(f'{name}: PID identity changed; no process was stopped.')
os.killpg(pid, signal.SIGTERM)
end = time.monotonic() + 20
while alive(pid) and time.monotonic() < end:
time.sleep(.2)
if alive(pid):
raise RuntimeError(f'{name} has not shut down; inspect its log before retrying.')
path.unlink(missing_ok=True)
(RUNTIME / (name + '.process.json')).unlink(missing_ok=True)
print(name + ': stopped', flush=True)
if name == 'backend' and (ROOT / 'scripts/nao_quality_runtime.py').is_file():
run([sys.executable, ROOT / 'scripts/nao_quality_runtime.py', '--stop-all'])
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=['mysql', 'build', 'start', 'stop', 'status'])
parser.add_argument('--skip-tests', action='store_true')
args = parser.parse_args()
if args.action == 'mysql':
start_mysql()
elif args.action == 'build':
build(args.skip_tests)
elif args.action == 'start':
start()
elif args.action == 'stop':
stop()
else:
for name in ('mysql', 'backend', 'frontend'):
path = RUNTIME / (name + '.pid')
print(name + ': ' + ('running' if path.exists() and alive(path.read_text()) and owned_process(name, path.read_text()) else 'stopped'))
if __name__ == '__main__':
try:
main()
except (RuntimeError, subprocess.CalledProcessError) as error:
print(str(error), file=sys.stderr)
raise SystemExit(1)
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Install checksum-pinned, project-local runtimes from official distributions."""
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import hashlib
import json
import platform
import shutil
import subprocess
import tarfile
ROOT = Path(__file__).resolve().parents[1]
RUNTIME = ROOT / '.runtime'
def install(name, item):
target = RUNTIME / name
marker = target / '.quality-runtime-version'
if marker.exists() and marker.read_text().strip() == item['version']:
print(f'{name}: already installed ({item["version"]})', flush=True)
return
if target.exists():
raise RuntimeError(f'{target} already exists without the expected version; preserve it and inspect manually')
archive = RUNTIME / 'downloads' / (name + '.tar.gz')
algorithm = next(a for a in ('sha512', 'sha256', 'md5') if a in item)
if not archive.exists() or hashlib.file_digest(archive.open('rb'), algorithm).hexdigest() != item[algorithm]:
print(f'{name}: downloading official distribution', flush=True)
partial = archive.with_suffix('.partial')
subprocess.run(['curl', '--fail', '--location', '--silent', '--show-error', '--retry', '2',
'--connect-timeout', '30', '--max-time', '900', item['url'], '-o', str(partial)], check=True)
with partial.open('rb') as stream:
actual = hashlib.file_digest(stream, algorithm).hexdigest()
if actual != item[algorithm]:
raise RuntimeError(f'{name}: checksum mismatch; installation stopped')
partial.replace(archive)
stage = RUNTIME / 'staging' / name
stage.mkdir(parents=True, exist_ok=True)
with tarfile.open(archive) as tar:
top_names = {Path(m.name).parts[0] for m in tar.getmembers() if m.name and m.name != '.'}
if len(top_names) != 1:
raise RuntimeError(f'{name}: expected one distribution root')
tar.extractall(stage, filter='data')
extracted = stage / next(iter(top_names))
shutil.move(str(extracted), target)
marker.write_text(item['version'] + '\n')
print(f'{name}: installed and checksum verified ({item["version"]})', flush=True)
def main():
if platform.system() != 'Darwin' or platform.machine() != 'arm64':
raise SystemExit('The locked local runtime packages target Apple Silicon macOS. Use a supported system JDK 21/MySQL 8.4 on other hosts.')
(RUNTIME / 'downloads').mkdir(parents=True, exist_ok=True)
lock = json.loads((ROOT / 'runtime.lock.json').read_text())
with ThreadPoolExecutor(max_workers=3) as pool:
futures = [pool.submit(install, name, item) for name, item in lock.items()]
for future in futures:
future.result()
print('All local runtimes are ready.', flush=True)
if __name__ == '__main__':
main()