初始化:质量数据分析平台源码
This commit is contained in:
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a local backup and verify restoration into a new, isolated database."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
import project
|
||||
|
||||
ROOT = project.ROOT
|
||||
|
||||
|
||||
def database_config():
|
||||
value = project.config().copy()
|
||||
override = os.environ.get('QUALITY_BACKUP_DATABASE')
|
||||
if override:
|
||||
if not re.fullmatch(r'quality_(?:verify|scale)_[A-Za-z0-9_]+', override):
|
||||
raise ValueError('Backup database override is restricted to this project test databases')
|
||||
value['mysqlDatabase'] = override
|
||||
return value
|
||||
|
||||
|
||||
def application_files():
|
||||
"""Fixed project inputs only; archives stay private and are never served as public assets."""
|
||||
paths = set()
|
||||
for relative in ('backend/src', 'web/src', 'web/public', 'web/dist', 'scripts'):
|
||||
folder = ROOT / relative
|
||||
if folder.exists():
|
||||
for path in folder.rglob('*'):
|
||||
if path.is_file() and '__pycache__' not in path.parts:
|
||||
paths.add(path)
|
||||
for relative in ('runtime.lock.json', 'backend/pom.xml', 'web/package.json', 'web/pnpm-lock.yaml',
|
||||
'.runtime/local-config.json', 'apikey.txt', '启动项目.command', '停止项目.command'):
|
||||
path = ROOT / relative
|
||||
if path.is_file(): paths.add(path)
|
||||
paths.update((ROOT / 'backend/target').glob('*.jar'))
|
||||
return sorted(paths)
|
||||
|
||||
|
||||
def archive_application(folder):
|
||||
archive = folder / 'application.tar.gz'
|
||||
files = {}
|
||||
with tarfile.open(archive, 'w:gz') as tar:
|
||||
for path in application_files():
|
||||
if path.is_symlink() or not path.resolve().is_relative_to(ROOT.resolve()):
|
||||
raise RuntimeError('Application backup contains a link outside the project')
|
||||
relative = path.relative_to(ROOT).as_posix()
|
||||
before = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
tar.add(path, arcname=relative, recursive=False)
|
||||
if before != hashlib.sha256(path.read_bytes()).hexdigest():
|
||||
raise RuntimeError('Application files changed during backup; retry after the build completes')
|
||||
files[relative] = before
|
||||
archive.chmod(0o600)
|
||||
return files, hashlib.sha256(archive.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def client(sql, database=None):
|
||||
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
||||
'--batch', '--skip-column-names']
|
||||
if database:
|
||||
if not database.replace('_', '').isalnum(): raise ValueError('Invalid database identifier')
|
||||
args.append('--database=' + database)
|
||||
result = subprocess.run(args, input=sql, text=True, capture_output=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError('Local MySQL backup/restore command failed; credentials were not printed.')
|
||||
return result.stdout
|
||||
|
||||
|
||||
def counts(database):
|
||||
tables = client('SHOW TABLES;', database).splitlines()
|
||||
result = {}
|
||||
for table in tables:
|
||||
if not table.replace('_', '').isalnum(): raise ValueError('Unexpected table identifier')
|
||||
result[table] = int(client('SELECT COUNT(*) FROM `' + table + '`;', database).strip())
|
||||
return result
|
||||
|
||||
|
||||
def query_evidence(database):
|
||||
"""Reproduce release membership and monthly formal input components in SQL."""
|
||||
current = client("SELECT release_id FROM current_release WHERE dataset_id='quality';", database).strip()
|
||||
if not current:
|
||||
return {'releaseId': None, 'months': []}
|
||||
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
|
||||
raise RuntimeError('Unexpected release identity in backup')
|
||||
result = {'releaseId': current, 'months': []}
|
||||
keys = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap',
|
||||
'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap',
|
||||
'bright_processing_scrap', 'bright_diffusion_scrap']
|
||||
for stage in ('black', 'bright'):
|
||||
pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*)," + \
|
||||
"'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)"
|
||||
for key in keys:
|
||||
pairs += ",'" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))"
|
||||
rows = client("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;", database)
|
||||
result['months'].extend(json.loads(line) for line in rows.splitlines())
|
||||
result['members'] = [json.loads(line) for line in client("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;", database).splitlines()]
|
||||
return result
|
||||
|
||||
|
||||
def verify_references(database, archive_root):
|
||||
rows = client("SELECT JSON_OBJECT('path',archive_path,'sha256',sha256,'size',file_size) FROM source_revision;", database)
|
||||
source_root = (ROOT / 'runtimedata/archive').resolve()
|
||||
for line in rows.splitlines():
|
||||
row = json.loads(line)
|
||||
stored = Path(row['path']).resolve()
|
||||
if not stored.is_relative_to(source_root):
|
||||
raise RuntimeError('A source revision references a file outside the project archive')
|
||||
file = archive_root / stored.relative_to(source_root)
|
||||
if not file.is_file() or file.stat().st_size != row['size'] or hashlib.sha256(file.read_bytes()).hexdigest() != row['sha256']:
|
||||
raise RuntimeError('A referenced source archive is missing or does not match its database revision')
|
||||
|
||||
|
||||
def create():
|
||||
config = database_config()
|
||||
verify_references(config['mysqlDatabase'], ROOT / 'runtimedata/archive')
|
||||
before_queries = query_evidence(config['mysqlDatabase'])
|
||||
folder = ROOT / 'output/backups' / (datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + '-' + uuid.uuid4().hex[:8])
|
||||
folder.mkdir(parents=True, exist_ok=False)
|
||||
folder.chmod(0o700)
|
||||
sql = folder / 'database.sql.gz'
|
||||
command = [str(project.RUNTIME / 'mysql/bin/mysqldump'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'),
|
||||
'--single-transaction', '--no-tablespaces', '--set-gtid-purged=OFF', '--skip-add-locks',
|
||||
'--skip-extended-insert', config['mysqlDatabase']]
|
||||
with tempfile.TemporaryFile() as errors, sql.open('wb') as output:
|
||||
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
|
||||
with gzip.GzipFile(fileobj=output, mode='wb') as compressed:
|
||||
shutil.copyfileobj(process.stdout, compressed)
|
||||
process.stdout.close()
|
||||
if process.wait() != 0:
|
||||
raise RuntimeError('Database dump failed; no backup was marked complete.')
|
||||
sql.chmod(0o600)
|
||||
if query_evidence(config['mysqlDatabase']) != before_queries:
|
||||
raise RuntimeError('The selected release changed during backup; retry a consistent snapshot')
|
||||
# Count the consistent dump itself; live audit/task tables may change during a backup.
|
||||
snapshot_counts = {}
|
||||
with gzip.open(sql, 'rt') as dump:
|
||||
for line in dump:
|
||||
table = re.match(r'^CREATE TABLE `([^`]+)`', line)
|
||||
if table: snapshot_counts[table.group(1)] = 0
|
||||
row = re.match(r'^INSERT INTO `([^`]+)`', line)
|
||||
if row: snapshot_counts[row.group(1)] = snapshot_counts.get(row.group(1), 0) + 1
|
||||
archive = folder / 'archive.tar.gz'
|
||||
files = {}
|
||||
source = ROOT / 'runtimedata/archive'
|
||||
with tarfile.open(archive, 'w:gz') as tar:
|
||||
if source.exists():
|
||||
for file in sorted(source.rglob('*')):
|
||||
if file.is_symlink(): raise RuntimeError('Archive contains a symbolic link; inspect before backing up.')
|
||||
if file.is_file():
|
||||
relative = file.relative_to(source).as_posix()
|
||||
files[relative] = hashlib.sha256(file.read_bytes()).hexdigest()
|
||||
tar.add(file, arcname=relative, recursive=False)
|
||||
archive.chmod(0o600)
|
||||
app_files, app_hash = archive_application(folder)
|
||||
manifest = {'formatVersion': 2, 'createdAt': datetime.now(timezone.utc).isoformat(), 'database': config['mysqlDatabase'],
|
||||
'databaseDumpSha256': hashlib.sha256(sql.read_bytes()).hexdigest(),
|
||||
'archiveSha256': hashlib.sha256(archive.read_bytes()).hexdigest(), 'files': files,
|
||||
'applicationSha256': app_hash, 'applicationFiles': app_files,
|
||||
'containsPrivateConfiguration': True,
|
||||
'recoveryMode': 'VERIFIED_FULL_SNAPSHOT',
|
||||
'pointInTimeRecovery': {'available': False, 'reason': 'Continuous binary-log archival is not configured'},
|
||||
'tableCounts': snapshot_counts, 'queryEvidence': before_queries, 'status': 'BACKUP_CREATED_NOT_RESTORE_VERIFIED'}
|
||||
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
||||
print('Backup created: ' + str(folder))
|
||||
return folder
|
||||
|
||||
|
||||
def verify(folder):
|
||||
started = time.monotonic()
|
||||
folder = folder.resolve()
|
||||
if not folder.is_relative_to((ROOT / 'output/backups').resolve()):
|
||||
raise ValueError('Only this project backup directory may be verified')
|
||||
manifest = json.loads((folder / 'manifest.json').read_text())
|
||||
sql = folder / 'database.sql.gz'
|
||||
archive = folder / 'archive.tar.gz'
|
||||
if hashlib.sha256(sql.read_bytes()).hexdigest() != manifest['databaseDumpSha256'] or hashlib.sha256(archive.read_bytes()).hexdigest() != manifest['archiveSha256']:
|
||||
raise RuntimeError('Backup checksum mismatch; restoration stopped')
|
||||
if manifest.get('applicationSha256'):
|
||||
app_archive = folder / 'application.tar.gz'
|
||||
if hashlib.sha256(app_archive.read_bytes()).hexdigest() != manifest['applicationSha256']:
|
||||
raise RuntimeError('Application backup checksum mismatch; restoration stopped')
|
||||
with tempfile.TemporaryDirectory(prefix='quality-restore-application-') as temp:
|
||||
Path(temp).chmod(0o700)
|
||||
with tarfile.open(app_archive) as tar:
|
||||
tar.extractall(temp, filter='data')
|
||||
actual_app = {p.relative_to(temp).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in Path(temp).rglob('*') if p.is_file()}
|
||||
if actual_app != manifest['applicationFiles']:
|
||||
raise RuntimeError('Restored application and configuration checksums do not match')
|
||||
scratch = 'quality_restore_' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + '_' + uuid.uuid4().hex[:8]
|
||||
client('CREATE DATABASE `' + scratch + '` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;')
|
||||
try:
|
||||
args = [str(project.RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(project.RUNTIME / 'mysql-root.cnf'), '--database=' + scratch]
|
||||
with gzip.open(sql, 'rb') as data, tempfile.TemporaryFile() as errors:
|
||||
process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=errors)
|
||||
try:
|
||||
shutil.copyfileobj(data, process.stdin)
|
||||
finally:
|
||||
process.stdin.close()
|
||||
code = process.wait()
|
||||
if code != 0: raise RuntimeError('Isolated restore failed; no live database was changed')
|
||||
restored = counts(scratch)
|
||||
if restored != manifest['tableCounts']:
|
||||
raise RuntimeError('Restored table counts do not match the backup manifest')
|
||||
if manifest.get('queryEvidence') is not None and query_evidence(scratch) != manifest['queryEvidence']:
|
||||
raise RuntimeError('Restored release query components or source provenance do not match')
|
||||
with tempfile.TemporaryDirectory(prefix='quality-restore-files-') as temp:
|
||||
with tarfile.open(archive) as tar:
|
||||
tar.extractall(temp, filter='data')
|
||||
actual = {p.relative_to(temp).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in Path(temp).rglob('*') if p.is_file()}
|
||||
if actual != manifest['files']: raise RuntimeError('Restored archive checksums do not match')
|
||||
verify_references(scratch, Path(temp))
|
||||
manifest['status'] = 'ISOLATED_RESTORE_VERIFIED'
|
||||
manifest['restoreVerifiedAt'] = datetime.now(timezone.utc).isoformat()
|
||||
manifest['restoredTables'] = len(restored)
|
||||
manifest['restoredApplicationFiles'] = len(manifest.get('applicationFiles', {}))
|
||||
manifest['restoredQueryEvidenceVerified'] = manifest.get('queryEvidence') is not None
|
||||
manifest['restoreElapsedSeconds'] = round(time.monotonic() - started, 3)
|
||||
project.save_private(folder / 'manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2) + '\n')
|
||||
print('Isolated restore verified: ' + str(len(restored)) + ' tables and ' + str(len(actual)) + ' archived files; live database unchanged.')
|
||||
finally:
|
||||
# Only the new scratch database created above is eligible for cleanup.
|
||||
if scratch.startswith('quality_restore_'):
|
||||
client('DROP DATABASE `' + scratch + '`;')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=['create', 'verify', 'create-and-verify'])
|
||||
parser.add_argument('--directory', type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.action == 'verify':
|
||||
if not args.directory: parser.error('--directory is required for verify')
|
||||
verify(args.directory)
|
||||
else:
|
||||
folder = create()
|
||||
if args.action == 'create-and-verify': verify(folder)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,349 @@
|
||||
"""Isolated Spring AI Alibaba DataAgent trial helpers.
|
||||
|
||||
All services bind to loopback. Secrets live only in .runtime/dataagent-trial/private.json
|
||||
with mode 0600; evidence replaces the selected profile endpoint and key.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import signal
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TRIAL = ROOT / ".runtime/dataagent-trial"
|
||||
OUT = ROOT / "output/dataagent-trial"
|
||||
SOURCE = TRIAL / "source"
|
||||
SQLITE = ROOT / ".runtime/dbgpt-trial/data/quality_trial.sqlite"
|
||||
MYSQL = ROOT / ".runtime/mysql/bin"
|
||||
MYSQLD = MYSQL / "mysqld"
|
||||
MYSQLCLI = MYSQL / "mysql"
|
||||
PORT = 53307
|
||||
SOCKET = TRIAL / "mysql.sock"
|
||||
PID = TRIAL / "mysql.pid"
|
||||
PRIV = TRIAL / "private.json"
|
||||
|
||||
|
||||
def load_private():
|
||||
if not PRIV.exists():
|
||||
prepare()
|
||||
return json.loads(PRIV.read_text())
|
||||
|
||||
|
||||
def prepare():
|
||||
TRIAL.mkdir(parents=True, exist_ok=True)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from dbgpt_trial import model_settings
|
||||
|
||||
base, key, model = model_settings()
|
||||
if model.strip() != "mimo-v2.6-flash":
|
||||
raise RuntimeError("Selected profile is not the approved MiMo model")
|
||||
keyfile = ROOT / "apikey.txt"
|
||||
raw = keyfile.read_bytes()
|
||||
text = raw.decode("utf-16" if raw[:2] in (b"\xff\xfe", b"\xfe\xff") else "utf-8-sig")
|
||||
lines = [x.strip() for x in text.splitlines() if x.strip()]
|
||||
if len(lines) < 6:
|
||||
raise RuntimeError("The approved two-profile API configuration was not found")
|
||||
emb_lines = lines[:3]
|
||||
emb_base = emb_lines[0].rstrip("/")
|
||||
if emb_base.endswith("/chat/completions"):
|
||||
emb_base = emb_base[:-len("/chat/completions")]
|
||||
if not emb_base.endswith("/v1"):
|
||||
emb_base += "/v1"
|
||||
emb_key = emb_lines[1].removeprefix("Bearer ").strip()
|
||||
value = {
|
||||
"upstream_base": base,
|
||||
"upstream_key": key,
|
||||
"chat_model": model,
|
||||
"embedding_base": emb_base,
|
||||
"embedding_key": emb_key,
|
||||
"embedding_model": "qwen3-embedding-4b-local",
|
||||
"chat_local_token": secrets.token_urlsafe(30),
|
||||
"embedding_local_token": secrets.token_urlsafe(30),
|
||||
"db_password": secrets.token_urlsafe(30),
|
||||
"mysql_user": "dataagent_admin",
|
||||
"quality_password": secrets.token_urlsafe(30),
|
||||
"quality_user": "quality_trial_ro",
|
||||
}
|
||||
PRIV.write_text(json.dumps(value))
|
||||
PRIV.chmod(0o600)
|
||||
client_cnf = TRIAL / "mysql-client.cnf"
|
||||
client_cnf.write_text("[client]\nuser=dataagent_admin\npassword=" + value["db_password"] + "\nprotocol=SOCKET\nsocket=" + str(SOCKET) + "\n")
|
||||
client_cnf.chmod(0o600)
|
||||
manifest = {
|
||||
"repo": "https://github.com/spring-ai-alibaba/DataAgent",
|
||||
"commit": subprocess.check_output(["git", "-C", str(SOURCE), "rev-parse", "HEAD"], text=True).strip(),
|
||||
"chat_model": model,
|
||||
"sqlite_source_sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(),
|
||||
"created_local": time.strftime("%Y-%m-%d %H:%M:%S %z"),
|
||||
"security": "localhost-only; no production DB; private.json mode 0600",
|
||||
}
|
||||
(OUT / "trial-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
print(json.dumps({"prepared": True, "chat_model": model, "commit": manifest["commit"]}))
|
||||
|
||||
|
||||
def mysql_cli(*args, input_text=None, root=False):
|
||||
p = load_private()
|
||||
if root:
|
||||
cmd = [str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"]
|
||||
else:
|
||||
cmd = [str(MYSQLCLI), f"--defaults-extra-file={TRIAL / 'mysql-client.cnf'}", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=dataagent_admin", "--default-character-set=utf8mb4"]
|
||||
cmd += list(args)
|
||||
return subprocess.run(cmd, input=input_text, text=True, capture_output=True, check=True)
|
||||
|
||||
|
||||
def start_db():
|
||||
load_private()
|
||||
if not PID.exists():
|
||||
cmd = [str(MYSQLD), "--no-defaults", f"--basedir={ROOT / '.runtime/mysql'}", f"--datadir={TRIAL / 'datadir'}", f"--port={PORT}", "--bind-address=127.0.0.1", f"--socket={SOCKET}", f"--pid-file={PID}", f"--log-error={OUT / 'mysql.log'}", "--mysqlx=OFF", "--lower-case-table-names=2", "--max-connections=40", "--innodb-buffer-pool-size=128M"]
|
||||
log = (TRIAL / "mysqld-stdout.log").open("ab")
|
||||
subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
deadline = time.time() + 45
|
||||
while time.time() < deadline:
|
||||
if SOCKET.exists():
|
||||
try:
|
||||
mysql_cli("-Nse", "SELECT VERSION()", root=True)
|
||||
print(json.dumps({"mysql": "ready", "bind": "127.0.0.1", "port": PORT}))
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(.5)
|
||||
raise RuntimeError("isolated MySQL 53307 did not become ready; inspect private log")
|
||||
|
||||
|
||||
def sql_literal(v):
|
||||
if v is None:
|
||||
return "NULL"
|
||||
if isinstance(v, (int, float)):
|
||||
if isinstance(v, float) and (v != v or abs(v) == float("inf")):
|
||||
return "NULL"
|
||||
return repr(v)
|
||||
s = str(v).replace("\\", "\\\\").replace("\x00", "\\0").replace("\n", "\\n").replace("\r", "\\r").replace("\x1a", "\\Z").replace("'", "\\'")
|
||||
return "'" + s + "'"
|
||||
|
||||
|
||||
def mysql_type(decl, col, keycols):
|
||||
d = (decl or "").upper()
|
||||
if "INT" in d:
|
||||
return "BIGINT"
|
||||
if col in keycols:
|
||||
return "VARCHAR(255)"
|
||||
if any(t in d for t in ("REAL", "FLOA", "DOUB")):
|
||||
return "DOUBLE"
|
||||
if "BLOB" in d:
|
||||
return "LONGBLOB"
|
||||
if "TEXT" in d or not d:
|
||||
return "LONGTEXT"
|
||||
return d
|
||||
|
||||
|
||||
SCOPE_COLUMN_COMMENTS = {
|
||||
"release_id": "official published release identifier",
|
||||
"scope_key": "range key: auto_combined means domestic plus overseas automotive transport; auto_domestic means domestic only; auto_overseas means overseas only; overall means company-wide",
|
||||
"scope_name": "business display name for the range",
|
||||
"year": "calendar year of the official monthly metric",
|
||||
"month": "month number of the official monthly metric",
|
||||
"period": "official reporting month in YYYY-MM form",
|
||||
"official_composite_rate": "official composite qualification rate from the published KPI service; standard meaning of qualification rate",
|
||||
"composite_rate_without_hq_scrap": "adjusted composite rate excluding headquarters scrap; not the standard composite rate",
|
||||
"black_rate": "black-sheet stage qualification rate",
|
||||
"black_qualified_weight": "qualified black-sheet weight",
|
||||
"black_scrap_weight": "scrap black-sheet weight",
|
||||
"black_total_weight": "total black-sheet weight",
|
||||
"bright_rate": "bright-sheet stage qualification rate",
|
||||
"bright_qualified_weight": "qualified bright-sheet weight",
|
||||
"bright_scrap_weight": "scrap bright-sheet weight",
|
||||
"bright_total_weight": "total bright-sheet weight",
|
||||
"included_rows": "number of rows included in the official monthly summary",
|
||||
"raw_rows": "number of raw service rows for the official monthly summary",
|
||||
"official_composite_rate_exact": "exact text representation of the official composite rate",
|
||||
"evidence_file": "relative path to the source evidence file",
|
||||
"api_query_json": "original query request and contract metadata as JSON",
|
||||
"summary_json": "original service summary JSON",
|
||||
"context_json": "source context JSON",
|
||||
}
|
||||
PRODUCT_COLUMN_COMMENTS = {
|
||||
"scope_key": SCOPE_COLUMN_COMMENTS["scope_key"],
|
||||
"scope_name": SCOPE_COLUMN_COMMENTS["scope_name"],
|
||||
"year": SCOPE_COLUMN_COMMENTS["year"],
|
||||
"month": SCOPE_COLUMN_COMMENTS["month"],
|
||||
"period": SCOPE_COLUMN_COMMENTS["period"],
|
||||
"product_key": "product/specification key for a product-level lead; product rows cannot be summed as an overall-rate contribution",
|
||||
"product_name": "product/specification display name; a lead only, not a proven cause or attributable contribution",
|
||||
"official_composite_rate": SCOPE_COLUMN_COMMENTS["official_composite_rate"],
|
||||
"black_rate": SCOPE_COLUMN_COMMENTS["black_rate"],
|
||||
"bright_rate": SCOPE_COLUMN_COMMENTS["bright_rate"],
|
||||
"evidence_file": SCOPE_COLUMN_COMMENTS["evidence_file"],
|
||||
"row_json": "complete original official product row JSON; product rows do not reconcile to stage denominator or overall contribution",
|
||||
}
|
||||
|
||||
|
||||
def seed_db():
|
||||
start_db()
|
||||
p = load_private()
|
||||
admin_schema = (SOURCE / "data-agent-management/src/main/resources/sql/schema.sql").read_text()
|
||||
create = f"CREATE DATABASE IF NOT EXISTS saa_data_agent CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE DATABASE IF NOT EXISTS quality_trial CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'localhost'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'localhost'; DROP USER IF EXISTS 'dataagent_trial'@'127.0.0.1'; DROP USER IF EXISTS 'dataagent_trial'@'localhost'; FLUSH PRIVILEGES;"
|
||||
mysql_cli(root=True, input_text=create)
|
||||
mysql_cli("saa_data_agent", input_text=admin_schema)
|
||||
src = sqlite3.connect(f"file:{SQLITE}?mode=ro", uri=True)
|
||||
chunks = []
|
||||
manifest = {"source": str(SQLITE.relative_to(ROOT)), "sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(), "tables": {}}
|
||||
for table, pk in (("official_scope_month_metrics", ("release_id", "scope_key", "year", "month")), ("official_product_month_rows", ("release_id", "scope_key", "year", "month", "row_no"))):
|
||||
cols = src.execute(f'PRAGMA table_info("{table}")').fetchall()
|
||||
names = [c[1] for c in cols]
|
||||
comments = SCOPE_COLUMN_COMMENTS if table == "official_scope_month_metrics" else PRODUCT_COLUMN_COMMENTS
|
||||
colddl = [f"`{c[1]}` {mysql_type(c[2], c[1], pk)}" + (" NOT NULL" if c[3] else "") + (" COMMENT " + sql_literal(comments[c[1]]) if c[1] in comments else "") for c in cols]
|
||||
colddl.append("PRIMARY KEY (" + ",".join(f"`{c}`" for c in pk) + ")")
|
||||
table_comment = ("Official released monthly quality KPI facts; one row per range and month. Composite, black-sheet, and bright-sheet values come from the service; NULL stays missing." if table == "official_scope_month_metrics" else "Official product/specification aggregate rows for leads only; do not sum these rows or claim they prove an overall contribution.")
|
||||
chunks.append(f"CREATE TABLE IF NOT EXISTS quality_trial.`{table}` ({','.join(colddl)}) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT={sql_literal(table_comment)};")
|
||||
rows = src.execute(f'SELECT * FROM "{table}"').fetchall()
|
||||
manifest["tables"][table] = {"rows": len(rows), "columns": names, "null_cells": sum(v is None for row in rows for v in row)}
|
||||
chunks.append(f"DELETE FROM quality_trial.`{table}`;")
|
||||
batch_size = 200
|
||||
for pos in range(0, len(rows), batch_size):
|
||||
vals = ["(" + ",".join(sql_literal(v) for v in row) + ")" for row in rows[pos:pos+batch_size]]
|
||||
chunks.append(f"INSERT INTO quality_trial.`{table}` ({','.join(f'`{n}`' for n in names)}) VALUES " + ",".join(vals) + ";")
|
||||
src.close()
|
||||
seed_path = TRIAL / "quality_seed.sql"
|
||||
seed_path.write_text("\n".join(chunks))
|
||||
seed_path.chmod(0o600)
|
||||
subprocess.run([str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"], input=seed_path.read_text(), text=True, capture_output=True, check=True)
|
||||
manifest["mysql_53307_counts"] = {}
|
||||
for table in manifest["tables"]:
|
||||
count = mysql_cli("-Nse", f"SELECT COUNT(*) FROM quality_trial.`{table}`", root=True).stdout.strip()
|
||||
manifest["mysql_53307_counts"][table] = int(count)
|
||||
(OUT / "data-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
print(json.dumps({"management_db": "saa_data_agent", "analysis_db": "quality_trial", "port": PORT, "counts": manifest["mysql_53307_counts"]}, ensure_ascii=False))
|
||||
|
||||
|
||||
def probe_embedding():
|
||||
import httpx
|
||||
p = load_private()
|
||||
request_body = {"model": p["embedding_model"], "input": ["汽车交通综合合格率", "国内汽车交通合格率"]}
|
||||
record = {"profile": 1, "model": p["embedding_model"], "tls_verify": True, "request_body": request_body}
|
||||
try:
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
resp = client.post(p["embedding_base"] + "/embeddings", headers={"Authorization": "Bearer " + p["embedding_key"], "Content-Type": "application/json"}, json=request_body)
|
||||
body_bytes = resp.content
|
||||
raw = body_bytes.decode("utf-8", errors="replace")
|
||||
raw = raw.replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]")
|
||||
record.update({"http_status": resp.status_code, "response_raw_redacted": raw})
|
||||
try:
|
||||
payload = resp.json()
|
||||
vectors = [x.get("embedding") for x in payload.get("data", [])]
|
||||
record["vector_count"] = len(vectors)
|
||||
record["dimensions"] = sorted({len(v) for v in vectors if isinstance(v, list)})
|
||||
except Exception:
|
||||
record["vector_count"] = 0
|
||||
record["dimensions"] = []
|
||||
if resp.status_code >= 400:
|
||||
raise RuntimeError("embedding request returned HTTP " + str(resp.status_code))
|
||||
if not record["vector_count"] or not record["dimensions"]:
|
||||
raise RuntimeError("embedding response did not contain vectors")
|
||||
except Exception as exc:
|
||||
record["error_type"] = type(exc).__name__
|
||||
record["error_safe"] = str(exc).replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]")
|
||||
(OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2))
|
||||
(OUT / "embedding-probe.json").chmod(0o600)
|
||||
raise
|
||||
(OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2))
|
||||
(OUT / "embedding-probe.json").chmod(0o600)
|
||||
print(json.dumps({"http_status": record["http_status"], "vector_count": record["vector_count"], "dimensions": record["dimensions"], "tls_verify": True}))
|
||||
|
||||
|
||||
def stop_db():
|
||||
if PID.exists():
|
||||
try:
|
||||
mysql_cli("-Nse", "SHUTDOWN", root=True)
|
||||
except Exception:
|
||||
pass
|
||||
print(json.dumps({"stop_requested": True, "port": PORT}))
|
||||
|
||||
|
||||
def start_backend():
|
||||
import httpx
|
||||
p = load_private()
|
||||
jar = SOURCE / "data-agent-management/target/spring-ai-alibaba-data-agent-management-1.0.0-SNAPSHOT.jar"
|
||||
if not jar.exists():
|
||||
raise RuntimeError("management executable jar has not been built")
|
||||
pidfile = TRIAL / "backend.pid"
|
||||
if pidfile.exists():
|
||||
try:
|
||||
os.kill(int(pidfile.read_text()), 0)
|
||||
print(json.dumps({"backend": "already-running", "port": 5673}))
|
||||
return
|
||||
except Exception:
|
||||
pidfile.unlink(missing_ok=True)
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"SERVER_ADDRESS": "127.0.0.1",
|
||||
"SERVER_PORT": "5673",
|
||||
"DATA_AGENT_DATASOURCE_URL": "jdbc:mysql://127.0.0.1:53307/saa_data_agent?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowMultiQueries=true&allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=Asia/Shanghai",
|
||||
"DATA_AGENT_DATASOURCE_USERNAME": p["mysql_user"],
|
||||
"DATA_AGENT_DATASOURCE_PASSWORD": p["db_password"],
|
||||
"DATA_AGENT_DATASOURCE_SQL_INIT": "never",
|
||||
"LOGGING_LEVEL_COM_ALIBABA_CLOUD_AI_DATAAGENT_SERVICE_AIMODELCONFIG_DYNAMICMODELFACTORY": "WARN",
|
||||
})
|
||||
java = ROOT / ".runtime/jdk/Contents/Home/bin/java"
|
||||
logpath = TRIAL / "backend.log"
|
||||
log = logpath.open("ab")
|
||||
logpath.chmod(0o600)
|
||||
proc = subprocess.Popen([str(java), "-Xms256m", "-Xmx1536m", "-jar", str(jar)], cwd=SOURCE, env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
pidfile.write_text(str(proc.pid))
|
||||
pidfile.chmod(0o600)
|
||||
deadline = time.time() + 180
|
||||
while time.time() < deadline:
|
||||
if proc.poll() is not None:
|
||||
raise RuntimeError("backend exited during startup; see .runtime/dataagent-trial/backend.log")
|
||||
try:
|
||||
response = httpx.get("http://127.0.0.1:5673/echo/ok", timeout=2)
|
||||
if response.status_code == 200 and response.text.strip() == "ok":
|
||||
print(json.dumps({"backend": "ready", "port": 5673, "bind": "127.0.0.1", "pid": proc.pid}))
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise RuntimeError("backend startup exceeded 180s; see .runtime/dataagent-trial/backend.log")
|
||||
|
||||
|
||||
def stop_backend():
|
||||
pidfile = TRIAL / "backend.pid"
|
||||
if not pidfile.exists():
|
||||
print(json.dumps({"backend": "not-running"}))
|
||||
return
|
||||
pid = int(pidfile.read_text())
|
||||
try:
|
||||
os.killpg(pid, signal.SIGTERM)
|
||||
for _ in range(30):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
pidfile.unlink(missing_ok=True)
|
||||
print(json.dumps({"backend": "stopped", "pid": pid}))
|
||||
|
||||
|
||||
def run():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=["prepare", "probe-embedding", "start-db", "seed-db", "stop-db", "start-backend", "stop-backend"])
|
||||
args = parser.parse_args()
|
||||
{"prepare": prepare, "probe-embedding": probe_embedding, "start-db": start_db, "seed-db": seed_db, "stop-db": stop_db, "start-backend": start_backend, "stop-backend": stop_backend}[args.action]()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Configure upstream DataAgent only through its own API and run trial graphs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.dataagent_trial import OUT, TRIAL, load_private
|
||||
|
||||
API = "http://127.0.0.1:5673"
|
||||
PROXY = "http://127.0.0.1:5676/v1"
|
||||
CALLS = OUT / "api-calls.jsonl"
|
||||
STATE = TRIAL / "trial-state.json"
|
||||
|
||||
|
||||
def scrub(obj, p):
|
||||
if isinstance(obj, dict):
|
||||
return {k: ("[REDACTED]" if k.lower() in {"apikey", "password", "proxy_password"} else scrub(v, p)) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [scrub(x, p) for x in obj]
|
||||
if isinstance(obj, str):
|
||||
for k in ("upstream_key", "embedding_key", "upstream_base", "embedding_base", "db_password"):
|
||||
if p.get(k): obj = obj.replace(p[k], "[REDACTED]")
|
||||
return obj
|
||||
return obj
|
||||
|
||||
|
||||
def record(method, path, status, summary, request_safe=None):
|
||||
p = load_private()
|
||||
CALLS.parent.mkdir(parents=True, exist_ok=True)
|
||||
item = {"time": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "method": method, "path": path, "http_status": status, "summary": summary}
|
||||
if request_safe is not None:
|
||||
item["request"] = request_safe
|
||||
line = json.dumps(scrub(item, p), ensure_ascii=False, separators=(",", ":"))
|
||||
with CALLS.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
CALLS.chmod(0o600)
|
||||
|
||||
|
||||
def call(client, method, path, *, json_body=None, params=None, safe_request=None, timeout=60, log_response=True):
|
||||
url = API + path
|
||||
response = client.request(method, url, json=json_body, params=params, timeout=timeout)
|
||||
try:
|
||||
data = response.json()
|
||||
except Exception:
|
||||
data = None
|
||||
summary = data if data is not None else {"body_preview": response.text[:500]}
|
||||
record(method, path, response.status_code, summary if log_response else {"saved": bool(response.is_success)}, safe_request)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"API {method} {path} returned HTTP {response.status_code}")
|
||||
if isinstance(data, dict) and data.get("success") is False:
|
||||
msg = str(data.get("message", "API operation returned success=false"))
|
||||
raise RuntimeError(scrub(msg, load_private()))
|
||||
return data
|
||||
|
||||
|
||||
def setup():
|
||||
p = load_private()
|
||||
state = {}
|
||||
with httpx.Client() as c:
|
||||
state["initial_model_check"] = call(c, "GET", "/api/model-config/check-ready")
|
||||
existing = call(c, "GET", "/api/model-config/list")
|
||||
existing_configs = (existing or {}).get("data") or []
|
||||
for item in (
|
||||
{"provider":"OpenAI-compatible MiMo trial", "apiKey":p["chat_local_token"], "baseUrl":PROXY, "modelName":p["chat_model"], "modelType":"CHAT", "temperature":0.0, "maxTokens":5000, "isActive":False, "completionsPath":"/chat/completions"},
|
||||
{"provider":"OpenAI-compatible Qwen embedding trial", "apiKey":p["embedding_local_token"], "baseUrl":PROXY, "modelName":p["embedding_model"], "modelType":"EMBEDDING", "temperature":0.0, "maxTokens":5000, "isActive":False, "embeddingsPath":"/embeddings"},
|
||||
):
|
||||
match = next((x for x in existing_configs if x.get("modelType") == item["modelType"] and x.get("modelName") == item["modelName"]), None)
|
||||
if match is None:
|
||||
result = call(c, "POST", "/api/model-config/add", json_body=item, safe_request={"provider":item["provider"], "modelName":item["modelName"], "modelType":item["modelType"], "baseUrl":"local evidence proxy", "apiKey":"[REDACTED]"})
|
||||
state[item["modelType"].lower()+"_add"] = result
|
||||
config_list = call(c, "GET", "/api/model-config/list")
|
||||
configs = (config_list or {}).get("data") or []
|
||||
for kind in ("CHAT", "EMBEDDING"):
|
||||
match = next((x for x in configs if x.get("modelType") == kind and x.get("modelName") == (p["chat_model"] if kind == "CHAT" else p["embedding_model"])), None)
|
||||
if not match or not match.get("id"):
|
||||
raise RuntimeError("could not resolve newly stored model configuration")
|
||||
model_id = match["id"]
|
||||
safe = {"modelType":kind,"modelName":match.get("modelName"),"id":model_id}
|
||||
state[kind.lower()+"_model_id"] = model_id
|
||||
call(c, "POST", f"/api/model-config/test/{model_id}", safe_request=safe)
|
||||
call(c, "POST", f"/api/model-config/activate/{model_id}", safe_request=safe)
|
||||
state["model_check"] = call(c, "GET", "/api/model-config/check-ready")
|
||||
ds_body = {"name":"DataAgent 质量只读副本53307", "type":"mysql", "host":"127.0.0.1", "port":53307, "databaseName":"quality_trial", "username":p["quality_user"], "password":p["quality_password"], "status":"active", "description":"isolated read-only duplicate from quality trial SQLite"}
|
||||
ds = call(c, "POST", "/api/datasource", json_body=ds_body, safe_request={"name":ds_body["name"],"type":"mysql","host":"127.0.0.1","port":53307,"databaseName":"quality_trial","username":p["quality_user"],"password":"[REDACTED]"})
|
||||
dsid = ds.get("id")
|
||||
if not dsid: raise RuntimeError("datasource creation did not return an id")
|
||||
state["datasource_id"] = dsid
|
||||
state["datasource_test"] = call(c, "POST", f"/api/datasource/{dsid}/test", safe_request={"id":dsid})
|
||||
prompt = """你是质量中心只读数据分析助手。请使用已连接数据库的正式月度指标表和产品分组表,先理解官方字段语义再查询。汽车交通合计为scope_key=auto_combined(国内+海外),国内为auto_domestic,范围不可混用;官方常用综合合格率字段为official_composite_rate,黑片与亮片分别为black_rate、bright_rate。official_scope_month_metrics是正式月度汇总;产品分组表只用于规格线索,不能据此声称对总体变化的贡献。所有NULL原样保留,不得补0。月份仅2026-01至2026-09,产品明细仅2026-08、09。用中文回答;把查询直接支持的事实、计算推断、待验证假设明确分开;根据用户任务自主决定是否生成报告,用户明确要求简短答案或不生成报告时服从。只读查询,不修改数据。"""
|
||||
agent = call(c, "POST", "/api/agent", json_body={"name":"质量中心只读试验 DataAgent","description":"基于发布指标副本的本地只读分析试跑","prompt":prompt,"category":"质量分析","tags":"汽车交通,质量指标,试验"}, safe_request={"name":"质量中心只读试验 DataAgent","description":"based on isolated read-only KPI duplicate","prompt":"business semantics and evidence policy only"})
|
||||
aid = agent.get("id")
|
||||
if not aid: raise RuntimeError("agent creation did not return an id")
|
||||
state["agent_id"] = aid
|
||||
call(c, "POST", f"/api/agent/{aid}/datasources/{dsid}", safe_request={"agent_id":aid,"datasource_id":dsid})
|
||||
tables = call(c, "GET", f"/api/datasource/{dsid}/tables")
|
||||
names = tables if isinstance(tables, list) else (tables or {}).get("data", [])
|
||||
chosen = [x for x in names if x in {"official_scope_month_metrics", "official_product_month_rows"}]
|
||||
if len(chosen) != 2:
|
||||
raise RuntimeError("DataAgent saw unexpected number of quality tables")
|
||||
state["tables"] = chosen
|
||||
call(c, "POST", f"/api/agent/{aid}/datasources/tables", json_body={"datasourceId":dsid,"tables":chosen}, safe_request={"agent_id":aid,"datasource_id":dsid,"tables":chosen})
|
||||
state["schema_init"] = call(c, "POST", f"/api/agent/{aid}/datasources/init", safe_request={"agent_id":aid,"datasource_id":dsid,"tables":chosen}, timeout=300)
|
||||
(TRIAL/"trial-state.json").write_text(json.dumps(state,ensure_ascii=False,indent=2))
|
||||
(TRIAL/"trial-state.json").chmod(0o600)
|
||||
print(json.dumps({"setup":"complete","agent_id":state["agent_id"],"datasource_id":state["datasource_id"],"tables":state["tables"],"schema_init":state.get("schema_init")},ensure_ascii=False))
|
||||
|
||||
|
||||
def finalize_nodes(label: str, session_id: str, nodes: list[dict], export_html: bool = False):
|
||||
"""Persist the upstream Nuxt-style timeline and export its Markdown as HTML."""
|
||||
node_blocks=[]
|
||||
current_key=None
|
||||
report_markdown=""
|
||||
report_block=None
|
||||
for n in nodes:
|
||||
node_name=n.get("nodeName")
|
||||
step_key=n.get("stepId") or f"{node_name}:{n.get('attempt',1)}"
|
||||
if step_key != current_key:
|
||||
node_blocks.append([])
|
||||
current_key=step_key
|
||||
block=node_blocks[-1]
|
||||
if node_name=="ReportGeneratorNode":
|
||||
if report_block is None:
|
||||
report_block=dict(n)
|
||||
report_block["text"]=""
|
||||
report_block["textType"]="MARK_DOWN"
|
||||
block.append(report_block)
|
||||
if n.get("textType")=="MARK_DOWN":
|
||||
report_markdown += str(n.get("text", ""))
|
||||
report_block["text"] = report_markdown
|
||||
continue
|
||||
block.append(n)
|
||||
timeline_content=json.dumps(node_blocks,ensure_ascii=False)
|
||||
html_file=None
|
||||
html_status=None
|
||||
with httpx.Client(timeout=60) as c:
|
||||
timeline_resp=call(c,"POST",f"/api/sessions/{session_id}/messages",json_body={"sessionId":session_id,"role":"assistant","content":timeline_content,"messageType":"timeline"},safe_request={"session_id":session_id,"role":"assistant","messageType":"timeline","node_blocks":len(node_blocks),"content_chars":len(timeline_content)},timeout=60,log_response=False)
|
||||
if report_markdown and export_html:
|
||||
html_resp=c.post(API+f"/api/sessions/{session_id}/reports/html",content=report_markdown.encode("utf-8"),headers={"Content-Type":"text/plain;charset=utf-8"},timeout=60)
|
||||
html_status=html_resp.status_code
|
||||
html_path=OUT/f"{label}.html"
|
||||
if html_resp.status_code==200:
|
||||
html_path.write_bytes(html_resp.content)
|
||||
html_path.chmod(0o600)
|
||||
html_file=str(html_path)
|
||||
record("POST",f"/api/sessions/{session_id}/reports/html",html_resp.status_code,{"content_type":html_resp.headers.get("content-type"),"bytes":len(html_resp.content),"saved_file":html_file}, {"session_id":session_id,"markdown_chars":len(report_markdown)})
|
||||
if html_resp.status_code>=400:
|
||||
raise RuntimeError(f"HTML report export returned HTTP {html_resp.status_code}")
|
||||
return {"report_markdown_chars":len(report_markdown),"report_html_status":html_status,"report_html_file":html_file,"timeline_saved":bool(timeline_resp),"node_blocks":len(node_blocks)}
|
||||
|
||||
|
||||
def stream_question(label: str, query: str, session_id: str | None = None, timeout: int = 600, export_html: bool = False):
|
||||
"""Issue one original DataAgent Graph request and store raw SSE evidence (credentials redacted)."""
|
||||
from scripts.dataagent_trial import load_private
|
||||
p = load_private()
|
||||
state = json.loads((TRIAL/"trial-state.json").read_text())
|
||||
aid = state["agent_id"]
|
||||
started = time.time()
|
||||
with httpx.Client(timeout=None) as c:
|
||||
if session_id is None:
|
||||
s = call(c, "POST", f"/api/agent/{aid}/sessions", json_body={"title":label}, safe_request={"agent_id":aid,"title":label})
|
||||
session_id = s.get("id")
|
||||
if not session_id: raise RuntimeError("chat session creation did not return id")
|
||||
msg = {"sessionId":session_id,"role":"user","content":query,"messageType":"text"}
|
||||
call(c, "POST", f"/api/sessions/{session_id}/messages", json_body=msg, safe_request={"session_id":session_id,"role":"user","messageType":"text","content":query})
|
||||
params={"agentId":str(aid),"conversationId":session_id,"query":query,"humanFeedback":"false","rejectedPlan":"false","nl2sqlOnly":"false"}
|
||||
req = c.build_request("GET", API+"/api/stream/search", params=params, headers={"Accept":"text/event-stream"}, timeout=timeout)
|
||||
raw = bytearray()
|
||||
try:
|
||||
response = c.send(req, stream=True)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
for chunk in response.iter_raw():
|
||||
raw.extend(chunk)
|
||||
finally:
|
||||
response.close()
|
||||
except Exception as exc:
|
||||
path=OUT/f"{label}.sse.redacted.txt"
|
||||
safe=bytes(raw).decode("utf-8",errors="replace")
|
||||
for k in ("upstream_key","embedding_key","upstream_base","embedding_base","db_password"):
|
||||
if p.get(k): safe=safe.replace(p[k],"[REDACTED]")
|
||||
path.write_text(safe,encoding="utf-8"); path.chmod(0o600)
|
||||
record("GET","/api/stream/search",0,{"error_type":type(exc).__name__,"duration_ms":round((time.time()-started)*1000)}, {"agent_id":aid,"session_id":session_id,"query":query})
|
||||
raise
|
||||
text=bytes(raw).decode("utf-8",errors="replace")
|
||||
for k in ("upstream_key","embedding_key","upstream_base","embedding_base","db_password"):
|
||||
if p.get(k): text=text.replace(p[k],"[REDACTED]")
|
||||
path=OUT/f"{label}.sse.redacted.txt"
|
||||
path.write_text(text,encoding="utf-8"); path.chmod(0o600)
|
||||
nodes=[]
|
||||
for block in re.split(r"\n\n",text):
|
||||
if not block.startswith("data:"): continue
|
||||
payload=block[len("data:"):].strip()
|
||||
try: nodes.append(json.loads(payload))
|
||||
except Exception: pass
|
||||
final_text="\n".join(str(n.get("text","")) for n in nodes if n.get("eventType")=="FINAL_ANSWER")
|
||||
finalization=finalize_nodes(label,session_id,nodes,export_html=export_html)
|
||||
summaries=[]
|
||||
for n in nodes:
|
||||
item={k:n.get(k) for k in ("nodeName","eventType","textType","error","stepId","attempt") if k in n}
|
||||
if n.get("text"):
|
||||
item["text_preview"]=str(n["text"])[:800]
|
||||
summaries.append(item)
|
||||
timeline={"label":label,"query":query,"session_id":session_id,"duration_seconds":round(time.time()-started,2),"sse_http":200,"sse_event_count":len(nodes),"final_answer":final_text,**finalization,"nodes":summaries}
|
||||
(OUT/f"{label}.summary.json").write_text(json.dumps(timeline,ensure_ascii=False,indent=2)); (OUT/f"{label}.summary.json").chmod(0o600)
|
||||
record("GET","/api/stream/search",200,{"sse_event_count":len(nodes),"duration_seconds":timeline["duration_seconds"],"session_id":session_id,"output_file":str(path.relative_to(OUT))},{"agent_id":aid,"session_id":session_id,"query":query})
|
||||
print(json.dumps({"label":label,"session_id":session_id,"duration_seconds":timeline["duration_seconds"],"sse_event_count":len(nodes),"sse_file":str(path),"summary_file":str(OUT/f"{label}.summary.json")},ensure_ascii=False))
|
||||
return timeline
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
ap=argparse.ArgumentParser()
|
||||
ap.add_argument("action",choices=["setup","short-report","finalize-short-report","followup","deep"])
|
||||
args=ap.parse_args()
|
||||
if args.action=="setup": setup()
|
||||
elif args.action=="short-report": stream_question("task3-short-report","请查询2026年8月和9月汽车交通合计的正式综合合格率、黑片合格率和亮片合格率,用查询结果生成一页精简中文HTML报告,注明环比变化和数据来源。报告不超过600字。",timeout=600,export_html=True)
|
||||
elif args.action=="finalize-short-report":
|
||||
label="task3-short-report"; summary_path=OUT/f"{label}.summary.json"; summary=json.loads(summary_path.read_text()); raw=(OUT/f"{label}.sse.redacted.txt").read_text(); nodes=[]
|
||||
for block in re.split(r"\n\n",raw):
|
||||
if block.startswith("data:"):
|
||||
try: nodes.append(json.loads(block[5:].strip()))
|
||||
except Exception: pass
|
||||
summary.update(finalize_nodes(label,summary["session_id"],nodes,export_html=True)); summary_path.write_text(json.dumps(summary,ensure_ascii=False,indent=2)); summary_path.chmod(0o600)
|
||||
print(json.dumps({k:summary.get(k) for k in ("session_id","report_markdown_chars","report_html_status","report_html_file","timeline_saved","node_blocks")},ensure_ascii=False))
|
||||
elif args.action=="followup":
|
||||
short=json.loads((OUT/"task3-short-report.summary.json").read_text())
|
||||
stream_question("task2-followup","那国内9月合格绿呢,比上月变化多大?直接简短回答,不生成报告。",session_id=short["session_id"],timeout=300)
|
||||
elif args.action=="deep": stream_question("task1-deep","9月份汽车交通合格率为什么降低?请自行查询数据,研究主要因素。发现线索后继续追查,并说明哪些判断有证据、哪些还需验证。先给我分析结论,下一轮再生成报告。",timeout=600)
|
||||
|
||||
if __name__=="__main__":
|
||||
main()
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Transparent, loopback-only OpenAI-compatible evidence proxy for DataAgent."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from scripts.dataagent_trial import OUT, load_private
|
||||
|
||||
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
async def healthz():
|
||||
return {"status": "ok", "bind": "127.0.0.1"}
|
||||
|
||||
|
||||
def _redact(text: str, p: dict) -> str:
|
||||
for value in (p.get("upstream_key"), p.get("embedding_key"), p.get("upstream_base"), p.get("embedding_base")):
|
||||
if value:
|
||||
text = text.replace(value, "[REDACTED]")
|
||||
return text
|
||||
|
||||
|
||||
@app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])
|
||||
async def forward(path: str, request: Request):
|
||||
p = load_private()
|
||||
incoming_auth = request.headers.get("authorization", "")
|
||||
model_group = "embedding" if path.rstrip("/").endswith("embeddings") else "chat"
|
||||
expected = p[model_group + "_local_token"]
|
||||
if incoming_auth != "Bearer " + expected:
|
||||
raise HTTPException(status_code=401, detail="invalid local trial token")
|
||||
base = p["embedding_base"] if model_group == "embedding" else p["upstream_base"]
|
||||
key = p["embedding_key"] if model_group == "embedding" else p["upstream_key"]
|
||||
body = await request.body()
|
||||
# The Java client may use chunked transfer encoding. Rebuild the outbound
|
||||
# content-length from the buffered body instead of forwarding TE alongside it.
|
||||
headers = {k: v for k, v in request.headers.items() if k.lower() not in {"authorization", "host", "content-length", "transfer-encoding", "connection", "accept-encoding"}}
|
||||
headers["authorization"] = "Bearer " + key
|
||||
headers["accept-encoding"] = "identity"
|
||||
upstream_url = base.rstrip("/") + "/" + path
|
||||
if request.url.query:
|
||||
upstream_url += "?" + request.url.query
|
||||
started = time.time()
|
||||
evidence = {
|
||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"model_group": model_group,
|
||||
"method": request.method,
|
||||
"path": "/v1/" + path,
|
||||
"model": p["embedding_model"] if model_group == "embedding" else p["chat_model"],
|
||||
"request_headers": {k: v for k, v in headers.items() if k.lower() not in {"authorization", "host"}},
|
||||
"request_body_redacted": _redact(body.decode("utf-8", errors="replace"), p),
|
||||
"tls_verify": True,
|
||||
}
|
||||
client = httpx.AsyncClient(timeout=None)
|
||||
try:
|
||||
req = client.build_request(request.method, upstream_url, headers=headers, content=body)
|
||||
response = await client.send(req, stream=True)
|
||||
response_headers = {k: v for k, v in response.headers.items() if k.lower() in {"content-type", "cache-control", "x-request-id"}}
|
||||
async def chunks():
|
||||
collected = bytearray()
|
||||
try:
|
||||
async for chunk in response.aiter_raw():
|
||||
collected.extend(chunk)
|
||||
yield chunk
|
||||
finally:
|
||||
raw = _redact(collected.decode("utf-8", errors="replace"), p)
|
||||
evidence.update({"http_status": response.status_code, "duration_ms": round((time.time() - started) * 1000), "response_body_redacted": raw})
|
||||
_append_evidence(evidence)
|
||||
await response.aclose()
|
||||
await client.aclose()
|
||||
return StreamingResponse(chunks(), status_code=response.status_code, headers=response_headers)
|
||||
except Exception as exc:
|
||||
evidence.update({"error_type": type(exc).__name__, "duration_ms": round((time.time() - started) * 1000)})
|
||||
_append_evidence(evidence)
|
||||
await client.aclose()
|
||||
raise HTTPException(status_code=502, detail="upstream model request failed") from None
|
||||
|
||||
|
||||
def _append_evidence(record: dict):
|
||||
p = load_private()
|
||||
name = "embedding-api.jsonl" if record.get("model_group") == "embedding" else "chat-api.jsonl"
|
||||
path = OUT / name
|
||||
line = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||||
line = _redact(line, p)
|
||||
with path.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
path.chmod(0o600)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Send real consecutive research requests to the isolated local DB-GPT SSE API."""
|
||||
import argparse,json,uuid,urllib.request,time
|
||||
from pathlib import Path
|
||||
from dbgpt_trial import model_settings
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
OUT=ROOT/'output/dbgpt-trial'
|
||||
STATE=ROOT/'.runtime/dbgpt-trial/chat-state.json'
|
||||
|
||||
def run(question,label,new=False):
|
||||
state=json.loads(STATE.read_text()) if STATE.exists() and not new else {'conv_uid':str(uuid.uuid4())}
|
||||
model=model_settings()[2]
|
||||
body={'conv_uid':state['conv_uid'],'user_input':question,'chat_mode':'chat_react_agent','model_name':model,'temperature':0.1,'max_new_tokens':4000,'ext_info':{'database_name':'sqlite_quality_trial','skill_name':'quality-center'}}
|
||||
STATE.write_text(json.dumps(state,ensure_ascii=False));STATE.chmod(0o600)
|
||||
(OUT/(label+'-request.json')).write_text(json.dumps(body,ensure_ascii=False,indent=2))
|
||||
req=urllib.request.Request('http://127.0.0.1:5670/api/v1/chat/react-agent',data=json.dumps(body,ensure_ascii=False).encode(),headers={'Content-Type':'application/json','Accept':'text/event-stream'})
|
||||
events=[];begin=time.monotonic();seen=set()
|
||||
with urllib.request.urlopen(req,timeout=420) as response, (OUT/(label+'.sse')).open('w') as stream:
|
||||
for raw in response:
|
||||
line=raw.decode('utf-8',errors='replace');stream.write(line);stream.flush()
|
||||
if not line.startswith('data:'):continue
|
||||
data=line[5:].strip()
|
||||
if data=='[DONE]':print('DONE',label,round(time.monotonic()-begin,1),flush=True);break
|
||||
try:item=json.loads(data)
|
||||
except json.JSONDecodeError:continue
|
||||
events.append(item)
|
||||
kind=item.get('type') or item.get('event') or item.get('status') or ''
|
||||
# Only event kinds are echoed; full user data stays in the local artifact.
|
||||
if kind and kind not in seen:print('event',str(kind)[:100],flush=True);seen.add(kind)
|
||||
(OUT/(label+'-events.json')).write_text(json.dumps(events,ensure_ascii=False,indent=2))
|
||||
print('saved',len(events),'events',state['conv_uid'],flush=True)
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('label');p.add_argument('question');p.add_argument('--new',action='store_true');a=p.parse_args();run(a.question,a.label,a.new)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Apply a narrow, reversible DB-GPT 0.8.2 reasoning-format compatibility fix.
|
||||
|
||||
Reproducer: removing a vis-thinking block with an empty string can join
|
||||
`Thought: ` to the following `Action:` line, making valid tool calls unparseable.
|
||||
The fix preserves a logical newline. It does not change tool choice or answers.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import hashlib,json,sys
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
TRIAL=ROOT/'.runtime/dbgpt-trial'
|
||||
TARGET=TRIAL/'venv/lib/python3.11/site-packages/dbgpt/agent/util/react_parser.py'
|
||||
OLD='return re.sub(pattern, "", text, flags=re.DOTALL)'
|
||||
NEW='return re.sub(pattern, "\\n", text, flags=re.DOTALL)'
|
||||
|
||||
def apply():
|
||||
source=TARGET.read_text()
|
||||
if NEW in source:
|
||||
print('Reasoning newline compatibility fix already present');return
|
||||
if source.count(OLD)!=1:raise RuntimeError('Unexpected parser version; no files modified')
|
||||
backup=TRIAL/'compat/react_parser.py.original'
|
||||
backup.parent.mkdir(parents=True,exist_ok=True)
|
||||
if backup.exists() and backup.read_text()!=source:raise RuntimeError('Original parser backup differs; no files modified')
|
||||
backup.write_text(source)
|
||||
patched=source.replace(OLD,NEW)
|
||||
TARGET.write_text(patched)
|
||||
(backup.parent/'patch.json').write_text(json.dumps({'package':'dbgpt','version':'0.8.2','change':'Preserve newline when removing vis-thinking blocks','original_sha256':hashlib.sha256(source.encode()).hexdigest(),'patched_sha256':hashlib.sha256(patched.encode()).hexdigest()},indent=2))
|
||||
print('Applied reasoning newline compatibility fix; original preserved')
|
||||
|
||||
def check():
|
||||
from dbgpt.agent.util.react_parser import ReActOutputParser
|
||||
raw=(TRIAL/'mimo-round-raw.txt').read_text()
|
||||
parsed=ReActOutputParser().parse_current_step(raw)
|
||||
assert len(parsed)==1 and parsed[0].action=='sql_query', 'Recorded real response must remain an executable SQL action'
|
||||
assert isinstance(parsed[0].action_input,dict) and 'sql' in parsed[0].action_input
|
||||
fenced='Thought: explain\n```text\nAction: terminate\nAction Input: {"result":"bad"}\n```\nAction: sql_query\nAction Input: {"sql":"SELECT 1"}'
|
||||
steps=ReActOutputParser().parse_current_step(fenced)
|
||||
assert len(steps)==1 and steps[0].action=='sql_query', 'Code-fenced labels must remain inert'
|
||||
print('PASS: real MiMo response and fenced-label regression checks')
|
||||
|
||||
if __name__=='__main__':
|
||||
if len(sys.argv)>1 and sys.argv[1]=='check':check()
|
||||
else:apply()
|
||||
@@ -0,0 +1,408 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a local SQLite copy of published quality metrics for a DB-GPT trial.
|
||||
|
||||
Reads only from the existing local metrics API via scripts/local_api.py. It does
|
||||
not inspect credentials, change source data, or call any write endpoint.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from local_api import Api # noqa: E402
|
||||
|
||||
DATA_DIR = ROOT / ".runtime" / "dbgpt-trial" / "data"
|
||||
EVIDENCE_DIR = DATA_DIR / "evidence"
|
||||
DB_PATH = DATA_DIR / "quality_trial.sqlite"
|
||||
MANIFEST_PATH = ROOT / "output" / "dbgpt-trial" / "data-manifest.json"
|
||||
CONTEXT_PATH = DATA_DIR / "business-context.md"
|
||||
RELEASE_FIXTURE = ROOT / "output" / "ai-chat-refinement" / "expected-car-scope.json"
|
||||
OVERALL_FIXTURE = ROOT / "output" / "ai-chat-refinement" / "metric-before.json"
|
||||
|
||||
RELEASE_ID = "194b0287-9044-4d80-9138-ecd3c481015c"
|
||||
YEAR = 2026
|
||||
MONTHS = tuple(range(1, 10))
|
||||
PRODUCT_MONTHS = (8, 9)
|
||||
|
||||
|
||||
def encoded_selection(values: list[str]) -> str:
|
||||
raw = json.dumps(values, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
return "~qv1~" + base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
CAR_BOTH_WIRE = encoded_selection(["汽车交通-国内", "汽车交通-海外"])
|
||||
SCOPES = (
|
||||
{"key": "overall", "name": "官方总体", "group": None},
|
||||
{"key": "auto_domestic", "name": "汽车交通-国内", "group": "汽车交通-国内"},
|
||||
{"key": "auto_overseas", "name": "汽车交通-海外", "group": "汽车交通-海外"},
|
||||
{"key": "auto_combined", "name": "汽车交通合计(国内+海外)", "group": CAR_BOTH_WIRE},
|
||||
)
|
||||
|
||||
|
||||
def write_json(path: Path, value: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def number(value: object) -> float | None:
|
||||
return None if value is None else float(value)
|
||||
|
||||
|
||||
def number_text(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
# Api.call parses JSON numbers as Python floats; repr is the shortest
|
||||
# round-trip representation of the official service value, without math.
|
||||
return repr(value) if isinstance(value, float) else str(value)
|
||||
|
||||
|
||||
def stage_value(summary: dict, stage: str, field: str) -> object:
|
||||
block = summary.get(stage)
|
||||
return None if block is None else block.get(field)
|
||||
|
||||
|
||||
def coverage_value(summary: dict, stage: str) -> object:
|
||||
coverage = summary.get("coverage") or {}
|
||||
return coverage.get(f"{stage}Rows")
|
||||
|
||||
|
||||
def check_fixtures() -> tuple[dict, dict]:
|
||||
car = json.loads(RELEASE_FIXTURE.read_text(encoding="utf-8"))
|
||||
overall = json.loads(OVERALL_FIXTURE.read_text(encoding="utf-8"))
|
||||
if car.get("release", {}).get("id") != RELEASE_ID or overall.get("release", {}).get("id") != RELEASE_ID:
|
||||
raise RuntimeError("现有正式结果参考文件绑定的发布版本不一致")
|
||||
if car.get("context", {}).get("group") != CAR_BOTH_WIRE:
|
||||
raise RuntimeError("汽车交通参考筛选值与 SelectionValues 编码不一致")
|
||||
for fixture in (car, overall):
|
||||
context = fixture.get("context", {})
|
||||
if context.get("year") != YEAR or context.get("month") != 9:
|
||||
raise RuntimeError("参考正式结果不是 2026 年 9 月")
|
||||
return car, overall
|
||||
|
||||
|
||||
def get_snapshot(api: Api, scope: dict, month: int, view: str) -> tuple[dict, dict, str]:
|
||||
params: dict[str, object] = {
|
||||
"year": YEAR,
|
||||
"month": month,
|
||||
"view": view,
|
||||
"includeUnassigned": "false",
|
||||
"releaseId": RELEASE_ID,
|
||||
"trendGranularity": "month",
|
||||
}
|
||||
if scope["group"] is not None:
|
||||
params["group"] = scope["group"]
|
||||
query = urlencode(params)
|
||||
response = api.call("/api/metrics?" + query)
|
||||
context = response.get("context") or {}
|
||||
release = response.get("release") or {}
|
||||
if release.get("id") != RELEASE_ID or release.get("status") != "PUBLISHED":
|
||||
raise RuntimeError(f"正式发布版本校验失败:{scope['key']} 2026-{month:02d}")
|
||||
if context.get("year") != YEAR or context.get("month") != month or context.get("view") != view:
|
||||
raise RuntimeError(f"查询期间或视图校验失败:{scope['key']} 2026-{month:02d} {view}")
|
||||
if context.get("group") != scope["group"]:
|
||||
raise RuntimeError(f"业务范围筛选校验失败:{scope['key']} 2026-{month:02d}")
|
||||
return response, params, query
|
||||
|
||||
|
||||
def schema(connection: sqlite3.Connection) -> None:
|
||||
connection.executescript("""
|
||||
PRAGMA foreign_keys=ON;
|
||||
CREATE TABLE official_scope_month_metrics (
|
||||
release_id TEXT NOT NULL,
|
||||
release_name TEXT,
|
||||
scope_key TEXT NOT NULL,
|
||||
scope_name TEXT NOT NULL,
|
||||
year INTEGER NOT NULL,
|
||||
month INTEGER NOT NULL,
|
||||
period TEXT NOT NULL,
|
||||
group_filter_wire TEXT,
|
||||
official_composite_rate REAL,
|
||||
official_composite_rate_exact TEXT,
|
||||
composite_rate_without_hq_scrap REAL,
|
||||
composite_rate_without_hq_scrap_exact TEXT,
|
||||
black_rate REAL,
|
||||
black_rate_exact TEXT,
|
||||
black_qualified_weight REAL,
|
||||
black_scrap_weight REAL,
|
||||
black_total_weight REAL,
|
||||
bright_rate REAL,
|
||||
bright_rate_exact TEXT,
|
||||
bright_qualified_weight REAL,
|
||||
bright_scrap_weight REAL,
|
||||
bright_total_weight REAL,
|
||||
bright_headquarters_scrap_weight REAL,
|
||||
bright_rate_without_hq_scrap REAL,
|
||||
bright_rate_without_hq_scrap_exact TEXT,
|
||||
black_rows INTEGER,
|
||||
bright_rows INTEGER,
|
||||
included_rows INTEGER,
|
||||
raw_rows INTEGER,
|
||||
rule_version_id TEXT,
|
||||
evidence_file TEXT NOT NULL,
|
||||
api_query_json TEXT NOT NULL,
|
||||
coverage_json TEXT,
|
||||
summary_json TEXT NOT NULL,
|
||||
context_json TEXT NOT NULL,
|
||||
PRIMARY KEY (release_id, scope_key, year, month)
|
||||
);
|
||||
CREATE INDEX idx_scope_metrics_period ON official_scope_month_metrics(scope_key, period);
|
||||
|
||||
CREATE TABLE official_product_month_rows (
|
||||
release_id TEXT NOT NULL,
|
||||
scope_key TEXT NOT NULL,
|
||||
scope_name TEXT NOT NULL,
|
||||
year INTEGER NOT NULL,
|
||||
month INTEGER NOT NULL,
|
||||
period TEXT NOT NULL,
|
||||
group_filter_wire TEXT,
|
||||
row_no INTEGER NOT NULL,
|
||||
product_key TEXT,
|
||||
product_name TEXT,
|
||||
official_composite_rate REAL,
|
||||
official_composite_rate_exact TEXT,
|
||||
composite_rate_without_hq_scrap REAL,
|
||||
black_rate REAL,
|
||||
black_qualified_weight REAL,
|
||||
black_scrap_weight REAL,
|
||||
black_total_weight REAL,
|
||||
bright_rate REAL,
|
||||
bright_qualified_weight REAL,
|
||||
bright_scrap_weight REAL,
|
||||
bright_total_weight REAL,
|
||||
bright_headquarters_scrap_weight REAL,
|
||||
bright_rate_without_hq_scrap REAL,
|
||||
black_rows INTEGER,
|
||||
bright_rows INTEGER,
|
||||
evidence_file TEXT NOT NULL,
|
||||
row_json TEXT NOT NULL,
|
||||
PRIMARY KEY (release_id, scope_key, year, month, row_no)
|
||||
);
|
||||
CREATE INDEX idx_product_scope_period ON official_product_month_rows(scope_key, period);
|
||||
CREATE INDEX idx_product_name ON official_product_month_rows(product_name);
|
||||
""")
|
||||
|
||||
|
||||
def insert_metric(connection: sqlite3.Connection, scope: dict, month: int, response: dict,
|
||||
params: dict, evidence_rel: str) -> None:
|
||||
release = response["release"]
|
||||
context = response["context"]
|
||||
summary = response["summary"]
|
||||
black = summary.get("black") or {}
|
||||
bright = summary.get("bright") or {}
|
||||
coverage = summary.get("coverage") or response.get("coverage") or {}
|
||||
rate = summary.get("compositeRate")
|
||||
adj = summary.get("compositeRateWithoutHeadquartersScrap")
|
||||
black_rate = black.get("rate")
|
||||
bright_rate = bright.get("rate")
|
||||
bright_adjusted = bright.get("rateWithoutHeadquartersScrap")
|
||||
values = (
|
||||
release.get("id"), release.get("name"), scope["key"], scope["name"], YEAR, month,
|
||||
f"{YEAR}-{month:02d}", scope["group"], number(rate), number_text(rate), number(adj), number_text(adj),
|
||||
number(black_rate), number_text(black_rate), number(black.get("qualifiedWeight")),
|
||||
number(black.get("scrapWeight")), number(black.get("totalWeight")), number(bright_rate),
|
||||
number_text(bright_rate), number(bright.get("qualifiedWeight")), number(bright.get("scrapWeight")),
|
||||
number(bright.get("totalWeight")), number(bright.get("headquartersScrapWeight")),
|
||||
number(bright_adjusted), number_text(bright_adjusted), coverage_value(summary, "black"),
|
||||
coverage_value(summary, "bright"), coverage.get("includedRows"), coverage.get("rawRows"),
|
||||
context.get("ruleVersionId"), evidence_rel, json.dumps(params, ensure_ascii=False, sort_keys=True),
|
||||
json.dumps(coverage, ensure_ascii=False, sort_keys=True),
|
||||
json.dumps(summary, ensure_ascii=False, sort_keys=True),
|
||||
json.dumps(context, ensure_ascii=False, sort_keys=True),
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO official_scope_month_metrics VALUES (" + ",".join("?" for _ in values) + ")", values
|
||||
)
|
||||
|
||||
|
||||
def insert_products(connection: sqlite3.Connection, scope: dict, month: int, response: dict,
|
||||
evidence_rel: str) -> int:
|
||||
count = 0
|
||||
for row_no, row in enumerate(response.get("rows", []), start=1):
|
||||
black = row.get("black") or {}
|
||||
bright = row.get("bright") or {}
|
||||
rate = row.get("compositeRate")
|
||||
values = (
|
||||
RELEASE_ID, scope["key"], scope["name"], YEAR, month, f"{YEAR}-{month:02d}",
|
||||
scope["group"], row_no, row.get("key"), row.get("product") or row.get("label"),
|
||||
number(rate), number_text(rate), number(row.get("compositeRateWithoutHeadquartersScrap")),
|
||||
number(black.get("rate")), number(black.get("qualifiedWeight")), number(black.get("scrapWeight")),
|
||||
number(black.get("totalWeight")), number(bright.get("rate")), number(bright.get("qualifiedWeight")),
|
||||
number(bright.get("scrapWeight")), number(bright.get("totalWeight")),
|
||||
number(bright.get("headquartersScrapWeight")), number(bright.get("rateWithoutHeadquartersScrap")),
|
||||
((row.get("coverage") or {}).get("blackRows")), ((row.get("coverage") or {}).get("brightRows")),
|
||||
evidence_rel, json.dumps(row, ensure_ascii=False, sort_keys=True),
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO official_product_month_rows VALUES (" + ",".join("?" for _ in values) + ")", values
|
||||
)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def fixture_rate(fixture: dict) -> float:
|
||||
return float(fixture["summary"]["compositeRate"])
|
||||
|
||||
|
||||
def main() -> None:
|
||||
expected_car, expected_overall = check_fixtures()
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
EVIDENCE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Build into a fresh database file beside the destination and replace only
|
||||
# this trial database once all read-only API fetches have succeeded.
|
||||
temporary_db = DB_PATH.with_suffix(".sqlite.tmp")
|
||||
temporary_db.unlink(missing_ok=True)
|
||||
api = Api().login()
|
||||
counts = {"official_scope_month_metrics": 0, "official_product_month_rows": 0}
|
||||
evidence_records: list[dict] = []
|
||||
sep_scope_metrics: dict[str, dict] = {}
|
||||
conn = sqlite3.connect(temporary_db)
|
||||
try:
|
||||
schema(conn)
|
||||
with conn:
|
||||
for scope in SCOPES:
|
||||
for month in MONTHS:
|
||||
response, params, query = get_snapshot(api, scope, month, "overall")
|
||||
evidence_name = f"metrics_{scope['key']}_{YEAR}-{month:02d}.json"
|
||||
evidence_path = EVIDENCE_DIR / evidence_name
|
||||
envelope = {"request": {"method": "GET", "path": "/api/metrics", "query": params,
|
||||
"encoded_query": query}, "response": response}
|
||||
write_json(evidence_path, envelope)
|
||||
evidence_rel = evidence_path.relative_to(DATA_DIR).as_posix()
|
||||
insert_metric(conn, scope, month, response, params, evidence_rel)
|
||||
counts["official_scope_month_metrics"] += 1
|
||||
evidence_records.append({"path": evidence_rel, "bytes": evidence_path.stat().st_size,
|
||||
"sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest(),
|
||||
"kind": "official_scope_month_metrics", "scope": scope["key"],
|
||||
"period": f"{YEAR}-{month:02d}"})
|
||||
if month == 9:
|
||||
sep_scope_metrics[scope["key"]] = response
|
||||
|
||||
for month in PRODUCT_MONTHS:
|
||||
response, params, query = get_snapshot(api, scope, month, "product")
|
||||
evidence_name = f"products_{scope['key']}_{YEAR}-{month:02d}.json"
|
||||
evidence_path = EVIDENCE_DIR / evidence_name
|
||||
envelope = {"request": {"method": "GET", "path": "/api/metrics", "query": params,
|
||||
"encoded_query": query}, "response": response}
|
||||
write_json(evidence_path, envelope)
|
||||
evidence_rel = evidence_path.relative_to(DATA_DIR).as_posix()
|
||||
added = insert_products(conn, scope, month, response, evidence_rel)
|
||||
counts["official_product_month_rows"] += added
|
||||
evidence_records.append({"path": evidence_rel, "bytes": evidence_path.stat().st_size,
|
||||
"sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest(),
|
||||
"kind": "official_product_month_rows", "scope": scope["key"],
|
||||
"period": f"{YEAR}-{month:02d}", "rows": added})
|
||||
conn.close()
|
||||
temporary_db.replace(DB_PATH)
|
||||
except Exception:
|
||||
conn.close()
|
||||
temporary_db.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
with sqlite3.connect(DB_PATH) as check:
|
||||
counts["official_scope_month_metrics"] = check.execute(
|
||||
"SELECT COUNT(*) FROM official_scope_month_metrics").fetchone()[0]
|
||||
counts["official_product_month_rows"] = check.execute(
|
||||
"SELECT COUNT(*) FROM official_product_month_rows").fetchone()[0]
|
||||
missing_periods = check.execute("""
|
||||
SELECT scope_key, GROUP_CONCAT(period, ',') FROM official_scope_month_metrics
|
||||
GROUP BY scope_key HAVING COUNT(*) != 9
|
||||
""").fetchall()
|
||||
if missing_periods:
|
||||
raise RuntimeError(f"Jan-Sep 月度指标行数不完整:{missing_periods}")
|
||||
check_car = check.execute("""
|
||||
SELECT official_composite_rate FROM official_scope_month_metrics
|
||||
WHERE scope_key='auto_combined' AND period='2026-09'
|
||||
""").fetchone()[0]
|
||||
check_overall = check.execute("""
|
||||
SELECT official_composite_rate FROM official_scope_month_metrics
|
||||
WHERE scope_key='overall' AND period='2026-09'
|
||||
""").fetchone()[0]
|
||||
trend_ranges = check.execute("""
|
||||
SELECT scope_key, MIN(period), MAX(period), COUNT(*)
|
||||
FROM official_scope_month_metrics GROUP BY scope_key ORDER BY scope_key
|
||||
""").fetchall()
|
||||
|
||||
if abs(check_car - fixture_rate(expected_car)) > 1e-12:
|
||||
raise RuntimeError("汽车交通合计 2026-09 综合合格率与参考正式结果不一致")
|
||||
if abs(check_overall - fixture_rate(expected_overall)) > 1e-12:
|
||||
raise RuntimeError("总体 2026-09 综合合格率与参考正式结果不一致")
|
||||
|
||||
rule_version = expected_car.get("context", {}).get("ruleVersionId")
|
||||
context = f"""# DB-GPT 质量数据试验副本说明
|
||||
|
||||
本目录是供公司本地 AI 按需读取的 SQLite 数据副本与原始 API 证据。数据来自本机正式指标只读接口,绑定已发布版本 `{RELEASE_ID}`({expected_car['release'].get('name', '')});规则版本为 `{rule_version}`。没有改写源数据库、正式数据或计算规则。
|
||||
|
||||
## 指标口径与范围
|
||||
|
||||
- `official_composite_rate` 对应服务返回的 `summary.compositeRate`,业务名称为“综合合格率”。它是正式发布版本的服务端固定指标;数据库中的数值直接来自指标服务,没有在导入时重新计算。
|
||||
- `composite_rate_without_hq_scrap` 是服务另外返回的调整口径,不能和综合合格率混称。
|
||||
- 汽车交通包括“汽车交通-国内”和“汽车交通-海外”两组。合计筛选按正式 `SelectionValues` 合约编码,wire 值为 `{CAR_BOTH_WIRE}`。
|
||||
- 当前副本覆盖 `{YEAR}-01` 至 `{YEAR}-09` 的总体和三种汽车交通范围月度指标;产品汇总覆盖 `{YEAR}-08`、`{YEAR}-09`。发布 ID、规则版本、筛选值和原始 API 响应均可在表和 `evidence/` 中追溯。
|
||||
- 空值在 SQLite 中保留为 `NULL`;`NULL` 表示该服务值缺失或不适用,不能当作 0。
|
||||
- `official_product_month_rows` 的产品分组用于找规格线索。服务说明这些分组行无法保证重新汇总为阶段分母或总体贡献;只能作为下钻线索,不能据此认定对总体率变化的贡献。
|
||||
|
||||
## 表说明
|
||||
|
||||
- `official_scope_month_metrics`:总体、汽车交通国内、海外、合计在 2026 年 1–9 月的正式月度汇总率、黑片/亮片阶段率与重量、覆盖行数、范围和证据路径。每范围每月一行。
|
||||
- `official_product_month_rows`:上述范围在 2026 年 8、9 月由 `view=product` 返回的全量正式产品汇总行。每行保留产品名、正式综合率、阶段率、阶段重量、覆盖行数与完整行 JSON。
|
||||
|
||||
## 读取边界
|
||||
|
||||
公司本地 AI 可按需使用副本明细执行 SQL 探索,并应区分正式汇总指标和产品分组线索。回答中的正式 KPI 以 `official_scope_month_metrics` 的正式服务值为准;不要把产品行加总成总体贡献,也不要把空值补成 0。`evidence/` 保存原始完整响应,可用于复核字段来源。
|
||||
"""
|
||||
CONTEXT_PATH.write_text(context, encoding="utf-8")
|
||||
|
||||
manifest = {
|
||||
"artifact": "DB-GPT local quality-data trial copy",
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"source": {"kind": "read-only local API", "base_url": "http://127.0.0.1:5281",
|
||||
"endpoint": "/api/metrics", "credentials_written": False},
|
||||
"release": {"id": RELEASE_ID, "status": "PUBLISHED",
|
||||
"name": expected_car["release"].get("name"), "rule_version_id": rule_version},
|
||||
"coverage": {"year": YEAR, "monthly_metrics": [f"{YEAR}-{month:02d}" for month in MONTHS],
|
||||
"product_months": [f"{YEAR}-{month:02d}" for month in PRODUCT_MONTHS],
|
||||
"scopes": [{"key": s["key"], "name": s["name"], "group_filter_wire": s["group"]}
|
||||
for s in SCOPES]},
|
||||
"files": {"sqlite": str(DB_PATH.relative_to(ROOT)),
|
||||
"business_context": str(CONTEXT_PATH.relative_to(ROOT)),
|
||||
"evidence_directory": str(EVIDENCE_DIR.relative_to(ROOT))},
|
||||
"tables": {
|
||||
"official_scope_month_metrics": {"rows": counts["official_scope_month_metrics"],
|
||||
"description_zh": "正式指标服务返回的总体及汽车交通各范围月度汇总;rate不在导入时重算。",
|
||||
"periods_by_scope": [{"scope_key": r[0], "first_period": r[1], "last_period": r[2], "rows": r[3]}
|
||||
for r in trend_ranges]},
|
||||
"official_product_month_rows": {"rows": counts["official_product_month_rows"],
|
||||
"description_zh": "指标API view=product在2026年8、9月返回的全量产品汇总行;只作规格线索,不代表可回加的总体贡献。"},
|
||||
},
|
||||
"reference_checks": {
|
||||
"overall_2026_09_official_composite_rate": {"expected": fixture_rate(expected_overall),
|
||||
"sqlite": check_overall, "matched": abs(check_overall - fixture_rate(expected_overall)) <= 1e-12},
|
||||
"auto_combined_2026_09_official_composite_rate": {"expected": fixture_rate(expected_car),
|
||||
"sqlite": check_car, "matched": abs(check_car - fixture_rate(expected_car)) <= 1e-12,
|
||||
"display_percent_2dp": round(check_car * 100, 2)},
|
||||
"sqlite_query_verified": True,
|
||||
"null_policy": "preserve SQL NULL; do not interpret as zero",
|
||||
},
|
||||
"evidence": evidence_records,
|
||||
}
|
||||
write_json(MANIFEST_PATH, manifest)
|
||||
print(json.dumps({"database": str(DB_PATH), "manifest": str(MANIFEST_PATH),
|
||||
"business_context": str(CONTEXT_PATH), "counts": counts,
|
||||
"release_id": RELEASE_ID, "rule_version_id": rule_version,
|
||||
"overall_2026_09_rate": check_overall,
|
||||
"auto_combined_2026_09_rate": check_car,
|
||||
"evidence_files": len(evidence_records)}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Import and publish the supplied, hash-verified development sample files locally."""
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from local_api import Api
|
||||
import project
|
||||
|
||||
ROOT = project.ROOT
|
||||
api = Api().login()
|
||||
sources = api.call('/api/sources/scan', {})['items']
|
||||
# Restrict this reproducible bootstrap to the delivered evidence set.
|
||||
evidence = json.loads((ROOT/'verification/source_hashes.json').read_text())
|
||||
allowed = set()
|
||||
if isinstance(evidence, list):
|
||||
allowed.update(x['sha256'] for x in evidence)
|
||||
if isinstance(evidence, dict):
|
||||
# Evidence formats carry paths as keys or as a files collection.
|
||||
candidates = evidence.get('files', evidence)
|
||||
if isinstance(candidates, dict):
|
||||
for key, value in candidates.items():
|
||||
allowed.add(value.get('sha256') if isinstance(value, dict) else value)
|
||||
elif isinstance(candidates, list):
|
||||
allowed.update(x.get('sha256') for x in candidates)
|
||||
selected = [s for s in sources if s['sha256'] in allowed]
|
||||
if len(selected) != 15:
|
||||
raise RuntimeError('Expected exactly the 15 supplied source-period files; no data was imported')
|
||||
ids = sorted(s['id'] for s in selected)
|
||||
mapping = [x['id'] for x in api.call('/api/mappings')['items'] if x['status'] == 'ACTIVE']
|
||||
rules = [x['id'] for x in api.call('/api/rules')['items'] if x['status'] == 'ACTIVE']
|
||||
key = 'provided-samples-' + hashlib.sha256(json.dumps([ids, mapping, rules]).encode()).hexdigest()
|
||||
releases = api.call('/api/releases')
|
||||
run = api.call('/api/import-runs', {'sourceIds': ids, 'idempotencyKey': key,
|
||||
'expectedCurrentReleaseId': releases.get('currentReleaseId')})['run']
|
||||
if run['status'] != 'COMPLETED' or not run.get('resultReleaseId'):
|
||||
raise RuntimeError('Sample import did not produce a completed candidate; inspect the import page')
|
||||
release = run['resultReleaseId']
|
||||
current = api.call('/api/releases').get('currentReleaseId')
|
||||
if current != release:
|
||||
api.call('/api/releases/'+release+'/publish', {'expectedCurrentReleaseId':current,
|
||||
'idempotencyKey':key+'-publish', 'reason':'本机开发:已提供正常样本的独立基线验证与页面联调'})
|
||||
summary = {'verifiedAt': datetime.now(timezone.utc).isoformat(), 'environment':'LOCAL_DEVELOPMENT_ONLY',
|
||||
'sourcePeriods':len(selected), 'processedRows':run['processed'], 'releaseId':release,
|
||||
'validation':run.get('validation'), 'status':'IMPORTED_AND_PUBLISHED'}
|
||||
output = ROOT/'verification/local-import.json'
|
||||
output.write_text(json.dumps(summary, ensure_ascii=False, indent=2)+'\n')
|
||||
print('Imported and published locally:',len(selected),'source periods,',run['processed'],'rows')
|
||||
print('Evidence:',output)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Local-only API client; keeps credentials and cookies out of output."""
|
||||
import http.cookiejar
|
||||
import json
|
||||
from urllib.request import Request, build_opener, HTTPCookieProcessor
|
||||
from urllib.error import HTTPError
|
||||
import project
|
||||
|
||||
class Api:
|
||||
def __init__(self):
|
||||
self.opener = build_opener(HTTPCookieProcessor(http.cookiejar.CookieJar()))
|
||||
self.csrf = None
|
||||
|
||||
def call(self, path, payload=None, method=None):
|
||||
if not path.startswith('/api/'):
|
||||
raise ValueError('Only local project API paths are supported')
|
||||
verb = method or ('POST' if payload is not None else 'GET')
|
||||
headers = {'Accept': 'application/json'}
|
||||
if payload is not None:
|
||||
headers['Content-Type'] = 'application/json'
|
||||
if self.csrf:
|
||||
headers['X-XSRF-TOKEN'] = self.csrf
|
||||
request = Request('http://127.0.0.1:5281'+path, method=verb, headers=headers,
|
||||
data=json.dumps(payload, ensure_ascii=False).encode() if payload is not None else None)
|
||||
try:
|
||||
with self.opener.open(request, timeout=180) as response:
|
||||
data = response.read()
|
||||
return json.loads(data) if data else {}
|
||||
except HTTPError as error:
|
||||
raise RuntimeError(f'HTTP {error.code} at {verb} {path}') from None
|
||||
|
||||
def login(self):
|
||||
value = project.config()
|
||||
self.csrf = self.call('/api/auth/csrf')['token']
|
||||
self.call('/api/auth/login', {'username': value['adminUsername'], 'password': value['adminPassword']})
|
||||
self.csrf = self.call('/api/auth/csrf')['token']
|
||||
return self
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Read the UI-managed, private NAO model configuration with legacy bootstrap."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SETTINGS_FILE = ROOT / '.runtime/nao-model-config.json'
|
||||
|
||||
|
||||
def model_settings():
|
||||
if not SETTINGS_FILE.exists():
|
||||
from dbgpt_trial import model_settings as legacy_settings
|
||||
return legacy_settings()
|
||||
if SETTINGS_FILE.is_symlink() or SETTINGS_FILE.parent.is_symlink() or SETTINGS_FILE.stat().st_size > 32768:
|
||||
raise ValueError('Invalid private model configuration')
|
||||
value = json.loads(SETTINGS_FILE.read_text(encoding='utf-8'))
|
||||
base, key, model = (value.get(name) for name in ('baseUrl', 'apiKey', 'model'))
|
||||
if not all(isinstance(item, str) and item.strip() for item in (base, key, model)):
|
||||
raise ValueError('Incomplete model configuration')
|
||||
u = urlsplit(base)
|
||||
if u.scheme not in ('http', 'https') or not u.hostname or u.username or u.password or u.query or u.fragment:
|
||||
raise ValueError('Invalid model endpoint')
|
||||
if len(key) > 4096 or any(c.isspace() or ord(c) < 32 for c in key):
|
||||
raise ValueError('Invalid model credential')
|
||||
return base.rstrip('/'), key, model
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Install the quality investigation context into the existing local nao runtime.
|
||||
|
||||
Uses upstream nao extension points: project rules, system prompt, model settings,
|
||||
and a stdio MCP. The packaged engine, source workbooks and business DB stay intact.
|
||||
"""
|
||||
import argparse
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.request import urlopen
|
||||
|
||||
import nao_trial
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TEMPLATES = ROOT / "ai/nao-quality"
|
||||
ACTIVE = nao_trial.PROJECT
|
||||
STATE = nao_trial.TRIAL / "quality-context.json"
|
||||
OUT = ROOT / "output/nao-quality"
|
||||
|
||||
|
||||
def _digest(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def configure():
|
||||
for name in ("nao_quality_tools.py", "nao_quality_mcp.py"):
|
||||
path = ROOT / "scripts" / name
|
||||
ast.parse(path.read_text())
|
||||
_, _, model = nao_trial.model_settings()
|
||||
stamp = datetime.now().astimezone().strftime("%Y%m%d-%H%M%S-%f")
|
||||
backup = nao_trial.TRIAL / "context-backups" / stamp
|
||||
backup.mkdir(parents=True, mode=0o700)
|
||||
ACTIVE.mkdir(parents=True, exist_ok=True)
|
||||
files = ["RULES.md", "nao_config.yaml", "agent/prompts/system.md", "agent/mcps/mcp.json"]
|
||||
for relative in files:
|
||||
old = ACTIVE / relative
|
||||
if old.exists():
|
||||
target = backup / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(old, target)
|
||||
# Old baseline warehouse context is preserved outside the active project.
|
||||
# Deep investigation uses the live, version-bound quality API MCP instead.
|
||||
old_databases = ACTIVE / "databases"
|
||||
if old_databases.exists():
|
||||
shutil.move(str(old_databases), str(backup / "databases"))
|
||||
# Upstream deliberately reuses discovered specs across restarts. Move only
|
||||
# our generated server directory aside so startup discovers the live schema.
|
||||
old_quality_specs = ACTIVE / "agent/mcps/quality"
|
||||
if old_quality_specs.exists():
|
||||
target = backup / "agent/mcps/quality"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(old_quality_specs), str(target))
|
||||
for relative in ("RULES.md", "agent/prompts/system.md"):
|
||||
target = ACTIVE / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(TEMPLATES / relative, target)
|
||||
(ACTIVE / "nao_config.yaml").write_text(
|
||||
"project_name: quality-investigation\n"
|
||||
"databases: []\n"
|
||||
"llm:\n"
|
||||
f" annotation_model: openaiCompatible:{model}\n"
|
||||
" providers:\n"
|
||||
" - provider: openaiCompatible\n"
|
||||
" api_key: \"{{ env('NAO_MODEL_KEY') }}\"\n"
|
||||
" base_url: \"{{ env('NAO_MODEL_BASE') }}\"\n"
|
||||
" models:\n"
|
||||
f" - id: {json.dumps(model)}\n"
|
||||
" default: true\n"
|
||||
" settings:\n"
|
||||
" maxOutputTokens: 8192\n"
|
||||
" temperature: 0.2\n"
|
||||
" reasoningEffort: low\n"
|
||||
)
|
||||
old_mcp = backup / "agent/mcps/mcp.json"
|
||||
mcps = json.loads(old_mcp.read_text()) if old_mcp.exists() else {"mcpServers": {}}
|
||||
mcps.setdefault("mcpServers", {})["quality"] = {
|
||||
"transport": "stdio",
|
||||
"command": sys.executable,
|
||||
"args": [str(ROOT / "scripts/nao_quality_mcp.py")],
|
||||
}
|
||||
mcp_path = ACTIVE / "agent/mcps/mcp.json"
|
||||
mcp_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
mcp_path.write_text(json.dumps(mcps, ensure_ascii=False, indent=2))
|
||||
mcp_path.chmod(0o600)
|
||||
state = {
|
||||
"mode": "QUALITY_API_MCP",
|
||||
"model": model,
|
||||
"backup": str(backup),
|
||||
"configuredAt": datetime.now().astimezone().isoformat(),
|
||||
"sourceHashes": {relative: _digest(TEMPLATES / relative)
|
||||
for relative in ("RULES.md", "agent/prompts/system.md")},
|
||||
"upstreamCoreModified": False,
|
||||
"qualityApi": "http://127.0.0.1:5281",
|
||||
"naoUi": "http://127.0.0.1:5674",
|
||||
"accessBoundary": "Local maintenance-account project only, not a shared multi-user integration",
|
||||
}
|
||||
STATE.write_text(json.dumps(state, ensure_ascii=False, indent=2))
|
||||
STATE.chmod(0o600)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
(OUT / "environment.json").write_text(json.dumps(state, ensure_ascii=False, indent=2))
|
||||
print("Quality investigation context installed; prior context backed up")
|
||||
|
||||
|
||||
def stop():
|
||||
state_path = nao_trial.TRIAL / "pids.json"
|
||||
previous = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||
nao_trial.stop()
|
||||
deadline = time.monotonic() + 8
|
||||
while time.monotonic() < deadline:
|
||||
alive = []
|
||||
for pid in previous.values():
|
||||
result = subprocess.run(["ps", "-p", str(pid), "-o", "stat="], capture_output=True, text=True)
|
||||
if result.returncode == 0 and not result.stdout.strip().startswith("Z"):
|
||||
alive.append(pid)
|
||||
if not alive:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
for pid in alive:
|
||||
result = subprocess.run(["ps", "-p", str(pid), "-o", "args="], capture_output=True, text=True)
|
||||
if result.returncode:
|
||||
continue
|
||||
if str(nao_trial.TRIAL) not in result.stdout or os.getpgid(pid) != pid:
|
||||
raise RuntimeError("Local nao process identity changed; force stop refused")
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
print("Stopped verified local nao process groups after graceful shutdown timeout")
|
||||
|
||||
|
||||
def activate():
|
||||
# Validate first so a missing adapter never takes the working trial offline.
|
||||
for name in ("nao_quality_tools.py", "nao_quality_mcp.py"):
|
||||
ast.parse((ROOT / "scripts" / name).read_text())
|
||||
stop()
|
||||
deadline = time.monotonic() + 15
|
||||
for port in (5674, 5675):
|
||||
while True:
|
||||
with socket.socket() as connection:
|
||||
connection.settimeout(0.3)
|
||||
occupied = connection.connect_ex(("127.0.0.1", port)) == 0
|
||||
if not occupied:
|
||||
break
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError(f"Prior local nao port {port} is still occupied; restart was not attempted")
|
||||
time.sleep(0.15)
|
||||
configure()
|
||||
nao_trial.start()
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urlopen("http://127.0.0.1:5674", timeout=2) as response:
|
||||
if response.status == 200:
|
||||
print("Quality investigation ready at http://127.0.0.1:5674")
|
||||
return
|
||||
except OSError:
|
||||
time.sleep(0.3)
|
||||
raise RuntimeError("nao did not become ready; preserved backup is recorded in quality-context.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=("configure", "activate", "start", "stop"))
|
||||
args = parser.parse_args()
|
||||
{"configure": configure, "activate": activate, "start": nao_trial.start,
|
||||
"stop": stop}[args.action]()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Run real nao quality conversations, with evidence and an active cancellation budget."""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request
|
||||
|
||||
from nao_trial import ROOT, model_settings
|
||||
from nao_trial_chat import BASE, opener
|
||||
|
||||
OUT = ROOT / "output/nao-quality"
|
||||
|
||||
|
||||
def run(label, question, chat_id=None, timeout=240):
|
||||
if not re.fullmatch(r"[a-zA-Z0-9_-]{1,80}", label):
|
||||
raise ValueError("Use a short alphanumeric evidence label")
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
client = opener()
|
||||
model = model_settings()[2]
|
||||
body = {"message": {"text": question},
|
||||
"model": {"provider": "openaiCompatible", "modelId": model},
|
||||
"timezone": "Asia/Shanghai"}
|
||||
if chat_id:
|
||||
body["chatId"] = chat_id
|
||||
(OUT / (label + "-request.json")).write_text(json.dumps(body, ensure_ascii=False, indent=2))
|
||||
state = {"chatId": chat_id, "budgetExpired": False, "stopRequest": None}
|
||||
events, text_blocks, tools, tool_outputs = [], {}, [], {}
|
||||
finished = threading.Event()
|
||||
response_ref = []
|
||||
|
||||
def cancel_at_budget():
|
||||
if finished.wait(timeout):
|
||||
return
|
||||
state["budgetExpired"] = True
|
||||
if state.get("chatId"):
|
||||
request = Request(BASE + "/api/trpc/chat.stop", method="POST",
|
||||
data=json.dumps({"json": {"chatId": state["chatId"]}}).encode(),
|
||||
headers={"Content-Type": "application/json", "Origin": BASE})
|
||||
try:
|
||||
with client.open(request, timeout=15) as response:
|
||||
state["stopRequest"] = {"httpStatus": response.status}
|
||||
except HTTPError as error:
|
||||
state["stopRequest"] = {"httpStatus": error.code}
|
||||
except OSError as error:
|
||||
state["stopRequest"] = {"errorType": type(error).__name__}
|
||||
if response_ref:
|
||||
response_ref[0].close()
|
||||
|
||||
timer = threading.Thread(target=cancel_at_budget, daemon=True)
|
||||
started = time.monotonic()
|
||||
timer.start()
|
||||
error_type = None
|
||||
model_base, model_key, _ = model_settings()
|
||||
request = Request(BASE + "/api/agent/", data=json.dumps(body, ensure_ascii=False).encode(),
|
||||
headers={"Content-Type": "application/json", "Origin": BASE,
|
||||
"Accept": "text/event-stream"})
|
||||
try:
|
||||
with client.open(request, timeout=60) as response, (OUT / (label + ".sse")).open("w") as stream:
|
||||
response_ref.append(response)
|
||||
for raw in response:
|
||||
line = raw.decode("utf-8", errors="replace").replace(model_key, "[REDACTED_KEY]")
|
||||
line = line.replace(model_base, "[REDACTED_MODEL_ENDPOINT]")
|
||||
stream.write(line)
|
||||
stream.flush()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
raw_data = line[5:].strip()
|
||||
if raw_data == "[DONE]":
|
||||
break
|
||||
try:
|
||||
event = json.loads(raw_data)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
events.append(event)
|
||||
kind = event.get("type")
|
||||
if kind == "data-newChat":
|
||||
state["chatId"] = event.get("data", {}).get("id") or state["chatId"]
|
||||
elif kind == "text-delta":
|
||||
block_id = event.get("id", "text")
|
||||
text_blocks[block_id] = text_blocks.get(block_id, "") + event.get("delta", "")
|
||||
elif kind == "tool-input-available":
|
||||
tools.append({"toolCallId": event.get("toolCallId"), "tool": event.get("toolName"),
|
||||
"input": event.get("input")})
|
||||
print("tool", event.get("toolName"), flush=True)
|
||||
elif kind == "tool-output-available":
|
||||
tool_outputs[event.get("toolCallId")] = event.get("output")
|
||||
except (OSError, ValueError, AttributeError) as error:
|
||||
error_type = type(error).__name__
|
||||
finally:
|
||||
finished.set()
|
||||
timer.join(timeout=1)
|
||||
for tool in tools:
|
||||
tool["output"] = tool_outputs.get(tool["toolCallId"])
|
||||
texts = [text.strip() for text in text_blocks.values() if text.strip()]
|
||||
final_text = texts[-1] if texts else ""
|
||||
finish_events = [event for event in events if event.get("type") == "finish"]
|
||||
truncated = any(event.get("finishReason") == "length" for event in finish_events)
|
||||
error_events = [event for event in events if event.get("type") in ("error", "tool-output-error")]
|
||||
summary = {
|
||||
"label": label, "question": question, "model": model, "chatId": state["chatId"],
|
||||
"durationSeconds": round(time.monotonic() - started, 2),
|
||||
"status": "budget_cancelled" if state["budgetExpired"] else
|
||||
"truncated" if truncated else
|
||||
"finished" if finish_events and final_text and not any(e.get("type") == "error" for e in error_events)
|
||||
else "incomplete",
|
||||
"finishReceived": bool(finish_events), "finishReasons": [event.get("finishReason") for event in finish_events], "errors": error_events,
|
||||
"transportErrorType": error_type, "stopRequest": state["stopRequest"],
|
||||
"toolCalls": tools, "eventCount": len(events), "answer": final_text,
|
||||
}
|
||||
(OUT / (label + "-events.json")).write_text(json.dumps(events, ensure_ascii=False, indent=2))
|
||||
(OUT / (label + "-run.json")).write_text(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
(OUT / (label + "-answer.md")).write_text(final_text)
|
||||
(OUT / (label + "-all-text.md")).write_text("\n\n---\n\n".join(texts))
|
||||
for tool in tools:
|
||||
result = tool.get("output")
|
||||
if tool["tool"] != "story" or not isinstance(result, dict) or not result.get("success"):
|
||||
continue
|
||||
slug = result.get("id")
|
||||
(OUT / (label + "-story.md")).write_text(result.get("code", ""))
|
||||
data = {"json": {"chatId": state["chatId"], "storySlug": slug, "format": "html"}}
|
||||
request = Request(BASE + "/api/trpc/story.download?input=" + quote(json.dumps(data)),
|
||||
headers={"Origin": BASE})
|
||||
with client.open(request, timeout=30) as response:
|
||||
export = json.load(response)["result"]["data"]["json"]
|
||||
(OUT / (label + "-report.html")).write_bytes(base64.b64decode(export["data"]))
|
||||
print(json.dumps({key: summary[key] for key in
|
||||
("label", "chatId", "durationSeconds", "status", "eventCount")}, ensure_ascii=False))
|
||||
return summary
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("label")
|
||||
parser.add_argument("question")
|
||||
parser.add_argument("--chat-id")
|
||||
parser.add_argument("--timeout", type=int, default=240)
|
||||
args = parser.parse_args()
|
||||
run(args.label, args.question, args.chat_id, args.timeout)
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Minimal stdio MCP server exposing the local read-only quality_data tool."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from nao_quality_tools import QualityToolError, QualityTools, tool_definition
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LOG_DIR = ROOT / "output" / "nao-quality" / "tools"
|
||||
SUPPORTED_PROTOCOLS = ("2025-03-26", "2024-11-05")
|
||||
|
||||
|
||||
def _error(code: int, message: str, request_id: Any = None) -> dict[str, Any]:
|
||||
return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}}
|
||||
|
||||
|
||||
def _tool_result(value: dict[str, Any], *, is_error: bool = False) -> dict[str, Any]:
|
||||
encoded = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
|
||||
return {
|
||||
"content": [{"type": "text", "text": encoded}],
|
||||
"structuredContent": value,
|
||||
"isError": is_error,
|
||||
}
|
||||
|
||||
|
||||
class MCPServer:
|
||||
def __init__(self, tools: QualityTools | None = None, logger: logging.Logger | None = None) -> None:
|
||||
self.tools = tools or QualityTools()
|
||||
self.logger = logger or logging.getLogger("nao_quality_mcp")
|
||||
self.initialized = False
|
||||
self.protocol_version: str | None = None
|
||||
|
||||
def handle(self, request: Any) -> dict[str, Any] | None:
|
||||
if not isinstance(request, dict) or request.get("jsonrpc") != "2.0" or not isinstance(request.get("method"), str):
|
||||
request_id = request.get("id") if isinstance(request, dict) else None
|
||||
return _error(-32600, "无效 JSON-RPC 请求。", request_id)
|
||||
method = request["method"]
|
||||
request_id = request.get("id")
|
||||
is_notification = "id" not in request or method.startswith("notifications/")
|
||||
params = request.get("params") or {}
|
||||
if not isinstance(params, dict):
|
||||
if is_notification:
|
||||
return None
|
||||
return _error(-32602, "params 必须是对象。", request_id)
|
||||
|
||||
if method == "initialize":
|
||||
version = params.get("protocolVersion")
|
||||
# MCP initialization negotiates the protocol version in the response.
|
||||
# If this client asks for a newer revision, advertise the newest revision
|
||||
# implemented here and let the client decide whether it can continue.
|
||||
selected_version = version if version in SUPPORTED_PROTOCOLS else SUPPORTED_PROTOCOLS[0]
|
||||
self.protocol_version = selected_version
|
||||
self.initialized = True
|
||||
return {
|
||||
"jsonrpc": "2.0", "id": request_id,
|
||||
"result": {
|
||||
"protocolVersion": selected_version,
|
||||
"capabilities": {"tools": {"listChanged": False}},
|
||||
"serverInfo": {"name": "quality", "version": "1.0.0"},
|
||||
"instructions": "quality_data 只读查询本机质量中心正式 HTTP API;沿用返回的 releaseId,并引用 evidenceId。",
|
||||
},
|
||||
}
|
||||
if method == "notifications/initialized":
|
||||
self.initialized = True
|
||||
return None
|
||||
if method == "ping":
|
||||
return None if is_notification else {"jsonrpc": "2.0", "id": request_id, "result": {}}
|
||||
if method == "tools/list":
|
||||
return None if is_notification else {
|
||||
"jsonrpc": "2.0", "id": request_id,
|
||||
"result": {"tools": [tool_definition()]},
|
||||
}
|
||||
if method == "tools/call":
|
||||
if is_notification:
|
||||
return None
|
||||
name = params.get("name")
|
||||
arguments = params.get("arguments", {})
|
||||
if name != "quality_data":
|
||||
return _error(-32602, "未知工具;当前只提供 quality_data。", request_id)
|
||||
if not isinstance(arguments, dict):
|
||||
return _error(-32602, "quality_data arguments 必须是对象。", request_id)
|
||||
mode = arguments.get("mode")
|
||||
self.logger.info("tool call mode=%s", mode if isinstance(mode, str) else "invalid")
|
||||
try:
|
||||
result = self.tools.quality_data(arguments)
|
||||
return {"jsonrpc": "2.0", "id": request_id, "result": _tool_result(result)}
|
||||
except QualityToolError as error:
|
||||
self.logger.warning("tool rejected mode=%s message=%s", mode, str(error)[:400])
|
||||
return {"jsonrpc": "2.0", "id": request_id,
|
||||
"result": _tool_result({"error": {"code": "invalid_or_unavailable", "message": str(error)}}, is_error=True)}
|
||||
except Exception:
|
||||
# Do not log request arguments, credentials, API bodies, or stack locals.
|
||||
self.logger.error("tool failed mode=%s; inspect local service and safe API status", mode)
|
||||
return {"jsonrpc": "2.0", "id": request_id,
|
||||
"result": _tool_result({"error": {"code": "api_failure", "message": "质量中心正式 API 查询未完成;请检查本机服务/登录权限后重试。没有返回空数据成功。"}}, is_error=True)}
|
||||
if is_notification:
|
||||
return None
|
||||
return _error(-32601, f"不支持的 MCP 方法:{method}。", request_id)
|
||||
|
||||
|
||||
def _configure_logger() -> logging.Logger:
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
path = LOG_DIR / "nao_quality_mcp.log"
|
||||
handler = logging.FileHandler(path, encoding="utf-8")
|
||||
os.chmod(path, 0o600)
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
logger = logging.getLogger("nao_quality_mcp")
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.handlers.clear()
|
||||
logger.addHandler(handler)
|
||||
logger.propagate = False
|
||||
return logger
|
||||
|
||||
|
||||
def serve(stdin: Any = sys.stdin, stdout: Any = sys.stdout, tools: QualityTools | None = None) -> None:
|
||||
server = MCPServer(tools=tools, logger=_configure_logger())
|
||||
for raw_line in stdin:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
request = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
response = _error(-32700, "无法解析 JSON。")
|
||||
else:
|
||||
response = server.handle(request)
|
||||
if response is not None:
|
||||
stdout.write(json.dumps(response, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + "\n")
|
||||
stdout.flush()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
serve()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,82 @@
|
||||
"""NAO stdio MCP shim backed by per-actor delegated quality API access."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from nao_quality_runtime import DelegatedApi, _uuid # noqa: E402
|
||||
from nao_quality_tools import QualityTools # noqa: E402
|
||||
import nao_quality_mcp # noqa: E402
|
||||
|
||||
|
||||
def _paths() -> tuple[Path, Path, Path, str, str, Path]:
|
||||
users_root = Path(os.environ.get("QUALITY_AGENT_USERS_ROOT", "")).resolve()
|
||||
access_path = Path(os.environ.get("QUALITY_AGENT_ACCESS_FILE", ""))
|
||||
evidence_dir = Path(os.environ.get("QUALITY_AGENT_EVIDENCE_DIR", ""))
|
||||
actor_id = _uuid(os.environ.get("QUALITY_AGENT_ACTOR_ID"), "actorId")
|
||||
session_id = _uuid(os.environ.get("QUALITY_AGENT_SESSION_ID"), "sessionId")
|
||||
session_state = Path(os.environ.get("QUALITY_AGENT_SESSION_STATE_FILE", ""))
|
||||
actor_root = users_root / actor_id
|
||||
if users_root != (ROOT / ".runtime" / "nao-users").resolve():
|
||||
raise ValueError("质量MCP运行目录无效。")
|
||||
if access_path.is_symlink() or access_path.absolute() != actor_root / "access.json":
|
||||
raise ValueError("质量MCP访问凭据路径无效。")
|
||||
expected_evidence = actor_root / session_id / "evidence"
|
||||
if evidence_dir.is_symlink() or evidence_dir.resolve() != expected_evidence.resolve():
|
||||
raise ValueError("质量MCP证据目录路径无效。")
|
||||
if session_state.is_symlink() or session_state.absolute() != actor_root / "private" / "current-session.json":
|
||||
raise ValueError("质量MCP会话状态路径无效。")
|
||||
evidence_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
os.chmod(evidence_dir, 0o700)
|
||||
return access_path, evidence_dir, session_state, actor_id, session_id, users_root
|
||||
|
||||
|
||||
def main() -> int:
|
||||
access_path, evidence_dir, session_state, actor_id, _session_id, users_root = _paths()
|
||||
adapter = DelegatedApi(access_path, runtime_root=users_root, expected_actor_id=actor_id)
|
||||
class SessionScopedQualityTools(QualityTools):
|
||||
def quality_data(self, arguments):
|
||||
try:
|
||||
current = json.loads(session_state.read_text(encoding="utf-8"))
|
||||
current_session = _uuid(current.get("sessionId"), "sessionId")
|
||||
except Exception:
|
||||
raise ValueError("当前质量分析会话不可用。") from None
|
||||
if not isinstance(current, dict) or current.get("actorId") != actor_id:
|
||||
raise ValueError("当前质量分析 actor 不匹配。")
|
||||
target = users_root / actor_id / current_session / "evidence"
|
||||
if target.is_symlink() or target.resolve().parent.parent != (users_root / actor_id).resolve():
|
||||
raise ValueError("质量证据目录路径无效。")
|
||||
target.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
os.chmod(target, 0o700)
|
||||
self._evidence_dir = target
|
||||
return super().quality_data(arguments)
|
||||
|
||||
tools = SessionScopedQualityTools(api_factory=lambda: adapter, evidence_dir=evidence_dir)
|
||||
# The upstream serve loop stays intact; protocol details go to stdout and
|
||||
# no shared log file can mix evidence from different actors or sessions.
|
||||
def quiet_logger():
|
||||
logger = logging.getLogger("nao_quality_mcp")
|
||||
logger.handlers.clear()
|
||||
logger.addHandler(logging.NullHandler())
|
||||
logger.propagate = False
|
||||
return logger
|
||||
|
||||
nao_quality_mcp._configure_logger = quiet_logger
|
||||
nao_quality_mcp.serve(tools=tools)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception:
|
||||
# MCP stderr is consumed by NAO and must never contain request bodies,
|
||||
# access tokens or stack frames. The upstream server's own safe MCP
|
||||
# errors are still returned through the JSON-RPC protocol.
|
||||
raise SystemExit("quality MCP unavailable")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,97 @@
|
||||
"""Isolated nao trial using the upstream packaged servers and existing model profile."""
|
||||
import argparse,json,os,secrets,signal,subprocess,sys
|
||||
from pathlib import Path
|
||||
from dbgpt_trial import model_settings
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
TRIAL=ROOT/'.runtime/nao-trial'
|
||||
PROJECT=TRIAL/'project'
|
||||
SITE=TRIAL/'venv/lib/python3.11/site-packages/nao_core'
|
||||
BIN=SITE/'bin'
|
||||
|
||||
def environment():
|
||||
base,key,model=model_settings()
|
||||
state=TRIAL/'private.json'
|
||||
if not state.exists():
|
||||
state.write_text(json.dumps({'auth_secret':secrets.token_urlsafe(40),'password':secrets.token_urlsafe(24),'email':'quality-trial@localhost.invalid'}));state.chmod(0o600)
|
||||
private=json.loads(state.read_text())
|
||||
e=os.environ.copy()
|
||||
e.update(NAO_MODEL_BASE=base,NAO_MODEL_KEY=key,NAO_MODEL_NAME=model,NAO_DEFAULT_PROJECT_PATH=str(PROJECT),
|
||||
BETTER_AUTH_URL='http://127.0.0.1:5674',BETTER_AUTH_SECRET=private['auth_secret'],
|
||||
DB_URI='sqlite:'+str(TRIAL/'metadata.sqlite'),FASTAPI_PORT='5675',PORT='5675',MODE='prod',
|
||||
NAO_CORE_VERSION='0.3.26',POSTHOG_DISABLED='true',DO_NOT_TRACK='1')
|
||||
return e
|
||||
|
||||
def prepare():
|
||||
import sqlite3,duckdb
|
||||
PROJECT.mkdir(parents=True,exist_ok=True)
|
||||
src=sqlite3.connect('file:'+str(ROOT/'.runtime/dbgpt-trial/data/quality_trial.sqlite')+'?mode=ro',uri=True)
|
||||
target=PROJECT/'quality_trial.duckdb'
|
||||
if not target.exists():
|
||||
db=duckdb.connect(str(target))
|
||||
for (table,) in src.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"):
|
||||
cols=src.execute('PRAGMA table_info("'+table+'")').fetchall()
|
||||
kinds={'TEXT':'VARCHAR','INTEGER':'BIGINT','REAL':'DOUBLE','BLOB':'BLOB','NUMERIC':'DOUBLE'}
|
||||
db.execute('CREATE TABLE "'+table+'" ('+','.join('"'+c[1]+'" '+kinds.get(c[2].upper(),'VARCHAR') for c in cols)+')')
|
||||
rows=src.execute('SELECT * FROM "'+table+'"').fetchall()
|
||||
if rows:db.executemany('INSERT INTO "'+table+'" VALUES ('+','.join('?' for _ in cols)+')',rows)
|
||||
db.close()
|
||||
rule=(ROOT/'.runtime/dbgpt-trial/home/skills/quality-center/SKILL.md').read_text()
|
||||
rule=rule.split('---',2)[-1].replace('sqlite_quality_trial','quality_trial')
|
||||
(PROJECT/'RULES.md').write_text('# 质量中心本地试验\n\n请默认使用中文。数据库为DuckDB质量副本,使用相应SQL语法。\n'+rule)
|
||||
cfg='''project_name: quality-trial
|
||||
databases:
|
||||
- name: quality_trial
|
||||
type: duckdb
|
||||
path: ./quality_trial.duckdb
|
||||
templates: [columns, preview]
|
||||
llm:
|
||||
annotation_model: openaiCompatible:mimo-v2.6-flash
|
||||
providers:
|
||||
- provider: openaiCompatible
|
||||
api_key: "{{ env('NAO_MODEL_KEY') }}"
|
||||
base_url: "{{ env('NAO_MODEL_BASE') }}"
|
||||
models:
|
||||
- id: mimo-v2.6-flash
|
||||
default: true
|
||||
'''
|
||||
(PROJECT/'nao_config.yaml').write_text(cfg)
|
||||
print('Prepared isolated DuckDB copy and business context')
|
||||
|
||||
def run_cli(args):
|
||||
return subprocess.run([str(TRIAL/'venv/bin/nao'),*args],cwd=PROJECT,env=environment()).returncode
|
||||
|
||||
def start():
|
||||
state=TRIAL/'pids.json'
|
||||
if state.exists():
|
||||
alive=[]
|
||||
for pid in json.loads(state.read_text()).values():
|
||||
try:os.kill(pid,0);alive.append(pid)
|
||||
except ProcessLookupError:pass
|
||||
if len(alive)==2:print('nao trial is already running');return
|
||||
if alive:raise RuntimeError('A trial server is still running; stop it before starting again')
|
||||
env=environment();pids={}
|
||||
for name,args,cwd in [('fastapi',[str(TRIAL/'venv/bin/python'),str(BIN/'fastapi/main.py')],PROJECT),('server',[str(BIN/'nao-chat-server'),'serve','--port','5674','--host','127.0.0.1'],BIN)]:
|
||||
fd=os.open(TRIAL/(name+'.log'),os.O_CREAT|os.O_APPEND|os.O_WRONLY,0o600)
|
||||
with os.fdopen(fd,'ab') as log:
|
||||
p=subprocess.Popen(args,cwd=cwd,env=env,stdin=subprocess.DEVNULL,stdout=log,stderr=log,start_new_session=True)
|
||||
pids[name]=p.pid
|
||||
(TRIAL/'pids.json').write_text(json.dumps(pids))
|
||||
print('nao trial started: http://127.0.0.1:5674; model='+model_settings()[2])
|
||||
|
||||
def stop():
|
||||
p=TRIAL/'pids.json'
|
||||
if not p.exists():return
|
||||
for name,pid in json.loads(p.read_text()).items():
|
||||
r=subprocess.run(['ps','-p',str(pid),'-o','args='],capture_output=True,text=True,errors='replace')
|
||||
if r.returncode:continue
|
||||
if str(TRIAL) not in r.stdout or os.getpgid(pid)!=pid:raise RuntimeError('Process identity mismatch')
|
||||
os.killpg(pid,signal.SIGTERM)
|
||||
p.unlink();print('nao trial stopped')
|
||||
|
||||
if __name__=='__main__':
|
||||
a=argparse.ArgumentParser();a.add_argument('action',choices=['prepare','sync','start','stop']);v=a.parse_args()
|
||||
if v.action=='prepare':prepare()
|
||||
elif v.action=='sync':sys.exit(run_cli(['sync','--provider','databases']))
|
||||
elif v.action=='start':start()
|
||||
else:stop()
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Exercise the actual upstream nao HTTP agent and preserve its SSE evidence."""
|
||||
import argparse,http.cookiejar,json,time,urllib.request,urllib.error
|
||||
from pathlib import Path
|
||||
from nao_trial import TRIAL,ROOT,model_settings
|
||||
|
||||
BASE='http://127.0.0.1:5674'
|
||||
OUT=ROOT/'output/nao-trial'
|
||||
STATE=TRIAL/'chat-state.json'
|
||||
|
||||
def opener():
|
||||
jar=http.cookiejar.LWPCookieJar(str(TRIAL/'cookies.txt'))
|
||||
if Path(jar.filename).exists():jar.load(ignore_discard=True)
|
||||
client=urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar))
|
||||
p=json.loads((TRIAL/'private.json').read_text())
|
||||
for endpoint in ('sign-in/email','sign-up/email'):
|
||||
body={k:p[k] for k in ('email','password')};body['name']='Quality Trial'
|
||||
req=urllib.request.Request(BASE+'/api/auth/'+endpoint,data=json.dumps(body).encode(),headers={'Content-Type':'application/json','Origin':BASE})
|
||||
try:
|
||||
with client.open(req,timeout=20) as r: result=json.load(r)
|
||||
jar.save(ignore_discard=True);Path(jar.filename).chmod(0o600)
|
||||
return client
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code not in (400,401,403,404,422):raise
|
||||
raise RuntimeError('Local nao authentication failed')
|
||||
|
||||
def run(label,question,new=False,chat_id=None):
|
||||
client=opener();state={'chatId':chat_id} if chat_id else json.loads(STATE.read_text()) if STATE.exists() and not new else {}
|
||||
model=model_settings()[2]
|
||||
body={'message':{'text':question},'model':{'provider':'openaiCompatible','modelId':model},'timezone':'Asia/Shanghai'}
|
||||
if state.get('chatId'):body['chatId']=state['chatId']
|
||||
(OUT/(label+'-request.json')).write_text(json.dumps(body,ensure_ascii=False,indent=2))
|
||||
req=urllib.request.Request(BASE+'/api/agent/',data=json.dumps(body,ensure_ascii=False).encode(),headers={'Content-Type':'application/json','Origin':BASE,'Accept':'text/event-stream'})
|
||||
begin=time.monotonic();events=[];seen=set();status='done'
|
||||
with client.open(req,timeout=120) as response,(OUT/(label+'.sse')).open('w') as stream:
|
||||
for raw in response:
|
||||
line=raw.decode('utf-8',errors='replace');stream.write(line);stream.flush()
|
||||
if not line.startswith('data:'):continue
|
||||
data=line[5:].strip()
|
||||
if data=='[DONE]':break
|
||||
try:e=json.loads(data)
|
||||
except json.JSONDecodeError:continue
|
||||
events.append(e);kind=e.get('type','')
|
||||
if kind=='data-newChat' and e.get('data',{}).get('id'):
|
||||
state['chatId']=e['data']['id'];STATE.write_text(json.dumps(state));STATE.chmod(0o600)
|
||||
if kind not in seen:print('event',kind,flush=True);seen.add(kind)
|
||||
if kind=='tool-input-available':print('tool',e.get('toolName'),flush=True)
|
||||
if time.monotonic()-begin>600:status='stopped_at_ten_minute_budget';break
|
||||
(OUT/(label+'-events.json')).write_text(json.dumps(events,ensure_ascii=False,indent=2))
|
||||
(OUT/(label+'-run.json')).write_text(json.dumps({'status':status,'elapsed_seconds':round(time.monotonic()-begin,1),'model':model,**state},indent=2))
|
||||
print('saved',len(events),'events;',state,'elapsed',round(time.monotonic()-begin,1),flush=True)
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('label');p.add_argument('question');p.add_argument('--new',action='store_true');p.add_argument('--chat-id');a=p.parse_args();run(a.label,a.question,a.new,a.chat_id)
|
||||
@@ -0,0 +1,267 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and run the isolated local quality platform without global installs."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
RUNTIME = ROOT / '.runtime'
|
||||
OUTPUT = ROOT / 'output'
|
||||
CONFIG = RUNTIME / 'local-config.json'
|
||||
|
||||
|
||||
def save_private(path, text):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, 'w') as stream:
|
||||
stream.write(text)
|
||||
path.chmod(0o600)
|
||||
|
||||
|
||||
def config():
|
||||
if CONFIG.exists():
|
||||
return json.loads(CONFIG.read_text())
|
||||
value = {'mysqlPort': 53306, 'mysqlDatabase': 'quality_platform', 'mysqlUser': 'quality_app',
|
||||
'mysqlPassword': secrets.token_urlsafe(32), 'rootPassword': secrets.token_urlsafe(32),
|
||||
'adminUsername': 'admin', 'adminPassword': secrets.token_urlsafe(24),
|
||||
'databaseReady': False}
|
||||
save_private(CONFIG, json.dumps(value, indent=2) + '\n')
|
||||
return value
|
||||
|
||||
|
||||
def environment():
|
||||
value = config()
|
||||
env = os.environ.copy()
|
||||
env.update({'JAVA_HOME': str(RUNTIME / 'jdk/Contents/Home'),
|
||||
'MYSQL_URL': f'jdbc:mysql://127.0.0.1:{value["mysqlPort"]}/{value["mysqlDatabase"]}?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true',
|
||||
'MYSQL_USER': value['mysqlUser'], 'MYSQL_PASSWORD': value['mysqlPassword'],
|
||||
'QUALITY_BOOTSTRAP_ADMIN_USERNAME': value['adminUsername'],
|
||||
'QUALITY_BOOTSTRAP_ADMIN_PASSWORD': value['adminPassword'],
|
||||
'QUALITY_ARCHIVE_ROOT': str(ROOT / 'runtimedata/archive'),
|
||||
'QUALITY_ALLOWED_ROOTS': str(ROOT / '质量数据/系统原始数据示例'),
|
||||
'QUALITY_DATA_WORKFLOW_TEST_MODE': 'true',
|
||||
'QUALITY_PROJECT_ROOT': str(ROOT),
|
||||
'QUALITY_DEFAULT_SOURCE_YEAR': '2026'})
|
||||
env['PATH'] = str(RUNTIME / 'jdk/Contents/Home/bin') + ':' + str(RUNTIME / 'maven/bin') + ':' + env.get('PATH', '')
|
||||
return env
|
||||
|
||||
|
||||
def run(args, **options):
|
||||
return subprocess.run([str(a) for a in args], cwd=ROOT, env=environment(), check=True, **options)
|
||||
|
||||
|
||||
def alive(pid):
|
||||
try:
|
||||
pid = int(pid)
|
||||
if pid <= 1:
|
||||
return False
|
||||
os.kill(pid, 0)
|
||||
state = subprocess.run(['ps', '-p', str(pid), '-o', 'stat='],
|
||||
text=True, capture_output=True, check=False)
|
||||
return state.returncode == 0 and bool(state.stdout.strip()) and not state.stdout.strip().startswith('Z')
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def process_stamp(pid):
|
||||
result = subprocess.run(['ps', '-p', str(int(pid)), '-o', 'lstart='],
|
||||
text=True, capture_output=True, check=False)
|
||||
return result.stdout.strip() if result.returncode == 0 else ''
|
||||
|
||||
|
||||
def owned_process(name, pid):
|
||||
identity = RUNTIME / (name + '.process.json')
|
||||
if not identity.exists():
|
||||
return False
|
||||
try:
|
||||
recorded = json.loads(identity.read_text())
|
||||
return (recorded.get('pid') == int(pid) and recorded.get('started') == process_stamp(pid)
|
||||
and os.getpgid(int(pid)) == int(pid))
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def start_process(name, args):
|
||||
pidfile = RUNTIME / (name + '.pid')
|
||||
if pidfile.exists() and alive(pidfile.read_text().strip()):
|
||||
if not owned_process(name, pidfile.read_text()):
|
||||
raise RuntimeError(f'{name}: an existing PID has no matching project identity; inspect before reusing it.')
|
||||
return int(pidfile.read_text())
|
||||
(OUTPUT / 'logs').mkdir(parents=True, exist_ok=True)
|
||||
with (OUTPUT / 'logs' / (name + '.log')).open('ab') as log:
|
||||
process = subprocess.Popen([str(a) for a in args], cwd=ROOT, env=environment(),
|
||||
stdin=subprocess.DEVNULL, stdout=log, stderr=log, start_new_session=True)
|
||||
pidfile.write_text(str(process.pid))
|
||||
save_private(RUNTIME / (name + '.process.json'), json.dumps({'pid': process.pid, 'started': process_stamp(process.pid)}))
|
||||
return process.pid
|
||||
|
||||
|
||||
def wait_http(url, seconds=90):
|
||||
end = time.monotonic() + seconds
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except (OSError, urllib.error.URLError):
|
||||
time.sleep(.5)
|
||||
raise RuntimeError(f'Service did not become ready: {url}. Inspect output/logs/.')
|
||||
|
||||
|
||||
def mysql_arguments():
|
||||
return [RUNTIME / 'mysql/bin/mysql', '--no-defaults', '--protocol=SOCKET',
|
||||
'--socket=' + str(RUNTIME / 'mysql.sock'), '--user=root', '--batch', '--skip-column-names']
|
||||
|
||||
|
||||
def start_mysql():
|
||||
value = config()
|
||||
data = RUNTIME / 'mysql-data'
|
||||
binary = RUNTIME / 'mysql/bin/mysqld'
|
||||
if not binary.exists():
|
||||
raise RuntimeError('Run python3 scripts/setup_runtime.py first.')
|
||||
if not (data / 'auto.cnf').exists():
|
||||
data.mkdir(parents=True, exist_ok=True)
|
||||
(OUTPUT / 'logs').mkdir(parents=True, exist_ok=True)
|
||||
with (OUTPUT / 'logs/mysql-initialize.log').open('ab') as log:
|
||||
run([binary, '--no-defaults', '--initialize-insecure', '--basedir=' + str(RUNTIME / 'mysql'),
|
||||
'--datadir=' + str(data), '--lower-case-table-names=2'], stdout=log, stderr=log)
|
||||
start_process('mysql', [binary, '--no-defaults', '--basedir=' + str(RUNTIME / 'mysql'),
|
||||
'--datadir=' + str(data), '--port=' + str(value['mysqlPort']), '--bind-address=127.0.0.1',
|
||||
'--socket=' + str(RUNTIME / 'mysql.sock'), '--pid-file=' + str(RUNTIME / 'mysql-server.pid'),
|
||||
'--log-error=' + str(OUTPUT / 'logs/mysql-server.log'), '--mysqlx=OFF',
|
||||
'--lower-case-table-names=2', '--max-connections=60', '--innodb-buffer-pool-size=256M'])
|
||||
client_config = RUNTIME / 'mysql-root.cnf'
|
||||
save_private(client_config, '[client]\nuser=root\npassword=' + value['rootPassword'] + '\nsocket=' + str(RUNTIME / 'mysql.sock') + '\n')
|
||||
for _ in range(120):
|
||||
if (RUNTIME / 'mysql.sock').exists():
|
||||
result = subprocess.run([str(a) for a in mysql_arguments()], input='SELECT 1;\n',
|
||||
text=True, capture_output=True)
|
||||
if result.returncode == 0:
|
||||
break
|
||||
result = subprocess.run([str(RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config),
|
||||
'--batch', '-e', 'SELECT 1;'], capture_output=True)
|
||||
if result.returncode == 0:
|
||||
break
|
||||
time.sleep(.5)
|
||||
else:
|
||||
raise RuntimeError('Local MySQL did not start; inspect output/logs/mysql-server.log.')
|
||||
if not value['databaseReady']:
|
||||
sql = f'''CREATE DATABASE IF NOT EXISTS {value['mysqlDatabase']} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;
|
||||
CREATE USER IF NOT EXISTS '{value['mysqlUser']}'@'localhost' IDENTIFIED BY '{value['mysqlPassword']}';
|
||||
CREATE USER IF NOT EXISTS '{value['mysqlUser']}'@'127.0.0.1' IDENTIFIED BY '{value['mysqlPassword']}';
|
||||
GRANT ALL PRIVILEGES ON {value['mysqlDatabase']}.* TO '{value['mysqlUser']}'@'localhost';
|
||||
GRANT ALL PRIVILEGES ON {value['mysqlDatabase']}.* TO '{value['mysqlUser']}'@'127.0.0.1';
|
||||
ALTER USER 'root'@'localhost' IDENTIFIED BY '{value['rootPassword']}';
|
||||
'''
|
||||
authenticated = [str(RUNTIME / 'mysql/bin/mysql'), '--defaults-file=' + str(client_config), '--batch']
|
||||
result = subprocess.run(authenticated, input=sql, text=True, capture_output=True)
|
||||
if result.returncode != 0:
|
||||
result = subprocess.run([str(a) for a in mysql_arguments()], input=sql, text=True, capture_output=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError('Local database bootstrap failed. Credentials have not been printed; inspect the local database configuration.')
|
||||
value['databaseReady'] = True
|
||||
save_private(CONFIG, json.dumps(value, indent=2) + '\n')
|
||||
save_private(ROOT / 'runtimedata/本机登录信息.txt',
|
||||
'质量中心 · 本机开发账户\n地址:http://127.0.0.1:5280\n用户名:' + value['adminUsername'] +
|
||||
'\n密码:' + value['adminPassword'] + '\n仅限本机使用,请勿上传或分享此文件。\n')
|
||||
print('Local MySQL is ready on 127.0.0.1:' + str(value['mysqlPort']), flush=True)
|
||||
|
||||
|
||||
def build(skip_tests=False):
|
||||
args = [RUNTIME / 'maven/bin/mvn', '-f', ROOT / 'backend/pom.xml',
|
||||
'-Dmaven.repo.local=' + str(RUNTIME / 'm2'), '--batch-mode', 'package']
|
||||
if skip_tests:
|
||||
args.append('-DskipTests')
|
||||
run(args)
|
||||
run(['pnpm', '--dir', ROOT / 'web', 'build'])
|
||||
|
||||
|
||||
def start():
|
||||
start_mysql()
|
||||
jars = sorted((ROOT / 'backend/target').glob('*.jar'), key=lambda p: p.stat().st_mtime)
|
||||
jars = [p for p in jars if not p.name.endswith('.original')]
|
||||
if not jars:
|
||||
raise RuntimeError('Build the backend first with python3 scripts/project.py build.')
|
||||
backend_pid = start_process('backend', [RUNTIME / 'jdk/Contents/Home/bin/java', '-Xms128m', '-Xmx2048m', '-jar', backend_snapshot(jars[-1])])
|
||||
# Startup can resume a complete, queued workbook import before readiness becomes UP.
|
||||
wait_http('http://127.0.0.1:5281/actuator/health', seconds=900)
|
||||
if not alive(backend_pid) or not owned_process('backend', backend_pid):
|
||||
raise RuntimeError('The project backend exited; inspect output/logs/backend.log and local port 5281.')
|
||||
frontend_pid = start_process('frontend', ['pnpm', '--dir', ROOT / 'web', 'dev', '--host', '127.0.0.1'])
|
||||
wait_http('http://127.0.0.1:5280')
|
||||
if not alive(frontend_pid) or not owned_process('frontend', frontend_pid):
|
||||
raise RuntimeError('The project frontend exited; inspect output/logs/frontend.log and local port 5280.')
|
||||
print('Quality platform ready: http://127.0.0.1:5280', flush=True)
|
||||
print('Local sign-in details: runtimedata/本机登录信息.txt', flush=True)
|
||||
|
||||
|
||||
def backend_snapshot(jar):
|
||||
"""Run an immutable jar so a later build cannot replace a live classpath."""
|
||||
digest = hashlib.sha256(jar.read_bytes()).hexdigest()
|
||||
target = RUNTIME / 'backend-builds' / (digest + '.jar')
|
||||
if not target.exists():
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = target.with_suffix('.tmp')
|
||||
shutil.copyfile(jar, temporary)
|
||||
temporary.chmod(0o600)
|
||||
temporary.replace(target)
|
||||
return target
|
||||
|
||||
|
||||
def stop():
|
||||
for name in ('frontend', 'backend', 'mysql'):
|
||||
path = RUNTIME / (name + '.pid')
|
||||
if not path.exists():
|
||||
continue
|
||||
pid = int(path.read_text())
|
||||
if alive(pid):
|
||||
if not owned_process(name, pid):
|
||||
raise RuntimeError(f'{name}: PID identity changed; no process was stopped.')
|
||||
os.killpg(pid, signal.SIGTERM)
|
||||
end = time.monotonic() + 20
|
||||
while alive(pid) and time.monotonic() < end:
|
||||
time.sleep(.2)
|
||||
if alive(pid):
|
||||
raise RuntimeError(f'{name} has not shut down; inspect its log before retrying.')
|
||||
path.unlink(missing_ok=True)
|
||||
(RUNTIME / (name + '.process.json')).unlink(missing_ok=True)
|
||||
print(name + ': stopped', flush=True)
|
||||
if name == 'backend' and (ROOT / 'scripts/nao_quality_runtime.py').is_file():
|
||||
run([sys.executable, ROOT / 'scripts/nao_quality_runtime.py', '--stop-all'])
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=['mysql', 'build', 'start', 'stop', 'status'])
|
||||
parser.add_argument('--skip-tests', action='store_true')
|
||||
args = parser.parse_args()
|
||||
if args.action == 'mysql':
|
||||
start_mysql()
|
||||
elif args.action == 'build':
|
||||
build(args.skip_tests)
|
||||
elif args.action == 'start':
|
||||
start()
|
||||
elif args.action == 'stop':
|
||||
stop()
|
||||
else:
|
||||
for name in ('mysql', 'backend', 'frontend'):
|
||||
path = RUNTIME / (name + '.pid')
|
||||
print(name + ': ' + ('running' if path.exists() and alive(path.read_text()) and owned_process(name, path.read_text()) else 'stopped'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (RuntimeError, subprocess.CalledProcessError) as error:
|
||||
print(str(error), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install checksum-pinned, project-local runtimes from official distributions."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import json
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
RUNTIME = ROOT / '.runtime'
|
||||
|
||||
|
||||
def install(name, item):
|
||||
target = RUNTIME / name
|
||||
marker = target / '.quality-runtime-version'
|
||||
if marker.exists() and marker.read_text().strip() == item['version']:
|
||||
print(f'{name}: already installed ({item["version"]})', flush=True)
|
||||
return
|
||||
if target.exists():
|
||||
raise RuntimeError(f'{target} already exists without the expected version; preserve it and inspect manually')
|
||||
archive = RUNTIME / 'downloads' / (name + '.tar.gz')
|
||||
algorithm = next(a for a in ('sha512', 'sha256', 'md5') if a in item)
|
||||
if not archive.exists() or hashlib.file_digest(archive.open('rb'), algorithm).hexdigest() != item[algorithm]:
|
||||
print(f'{name}: downloading official distribution', flush=True)
|
||||
partial = archive.with_suffix('.partial')
|
||||
subprocess.run(['curl', '--fail', '--location', '--silent', '--show-error', '--retry', '2',
|
||||
'--connect-timeout', '30', '--max-time', '900', item['url'], '-o', str(partial)], check=True)
|
||||
with partial.open('rb') as stream:
|
||||
actual = hashlib.file_digest(stream, algorithm).hexdigest()
|
||||
if actual != item[algorithm]:
|
||||
raise RuntimeError(f'{name}: checksum mismatch; installation stopped')
|
||||
partial.replace(archive)
|
||||
stage = RUNTIME / 'staging' / name
|
||||
stage.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(archive) as tar:
|
||||
top_names = {Path(m.name).parts[0] for m in tar.getmembers() if m.name and m.name != '.'}
|
||||
if len(top_names) != 1:
|
||||
raise RuntimeError(f'{name}: expected one distribution root')
|
||||
tar.extractall(stage, filter='data')
|
||||
extracted = stage / next(iter(top_names))
|
||||
shutil.move(str(extracted), target)
|
||||
marker.write_text(item['version'] + '\n')
|
||||
print(f'{name}: installed and checksum verified ({item["version"]})', flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
if platform.system() != 'Darwin' or platform.machine() != 'arm64':
|
||||
raise SystemExit('The locked local runtime packages target Apple Silicon macOS. Use a supported system JDK 21/MySQL 8.4 on other hosts.')
|
||||
(RUNTIME / 'downloads').mkdir(parents=True, exist_ok=True)
|
||||
lock = json.loads((ROOT / 'runtime.lock.json').read_text())
|
||||
with ThreadPoolExecutor(max_workers=3) as pool:
|
||||
futures = [pool.submit(install, name, item) for name, item in lock.items()]
|
||||
for future in futures:
|
||||
future.result()
|
||||
print('All local runtimes are ready.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user