350 lines
18 KiB
Python
350 lines
18 KiB
Python
"""Isolated Spring AI Alibaba DataAgent trial helpers.
|
|
|
|
All services bind to loopback. Secrets live only in .runtime/dataagent-trial/private.json
|
|
with mode 0600; evidence replaces the selected profile endpoint and key.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import secrets
|
|
import signal
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
TRIAL = ROOT / ".runtime/dataagent-trial"
|
|
OUT = ROOT / "output/dataagent-trial"
|
|
SOURCE = TRIAL / "source"
|
|
SQLITE = ROOT / ".runtime/dbgpt-trial/data/quality_trial.sqlite"
|
|
MYSQL = ROOT / ".runtime/mysql/bin"
|
|
MYSQLD = MYSQL / "mysqld"
|
|
MYSQLCLI = MYSQL / "mysql"
|
|
PORT = 53307
|
|
SOCKET = TRIAL / "mysql.sock"
|
|
PID = TRIAL / "mysql.pid"
|
|
PRIV = TRIAL / "private.json"
|
|
|
|
|
|
def load_private():
|
|
if not PRIV.exists():
|
|
prepare()
|
|
return json.loads(PRIV.read_text())
|
|
|
|
|
|
def prepare():
|
|
TRIAL.mkdir(parents=True, exist_ok=True)
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from dbgpt_trial import model_settings
|
|
|
|
base, key, model = model_settings()
|
|
if model.strip() != "mimo-v2.6-flash":
|
|
raise RuntimeError("Selected profile is not the approved MiMo model")
|
|
keyfile = ROOT / "apikey.txt"
|
|
raw = keyfile.read_bytes()
|
|
text = raw.decode("utf-16" if raw[:2] in (b"\xff\xfe", b"\xfe\xff") else "utf-8-sig")
|
|
lines = [x.strip() for x in text.splitlines() if x.strip()]
|
|
if len(lines) < 6:
|
|
raise RuntimeError("The approved two-profile API configuration was not found")
|
|
emb_lines = lines[:3]
|
|
emb_base = emb_lines[0].rstrip("/")
|
|
if emb_base.endswith("/chat/completions"):
|
|
emb_base = emb_base[:-len("/chat/completions")]
|
|
if not emb_base.endswith("/v1"):
|
|
emb_base += "/v1"
|
|
emb_key = emb_lines[1].removeprefix("Bearer ").strip()
|
|
value = {
|
|
"upstream_base": base,
|
|
"upstream_key": key,
|
|
"chat_model": model,
|
|
"embedding_base": emb_base,
|
|
"embedding_key": emb_key,
|
|
"embedding_model": "qwen3-embedding-4b-local",
|
|
"chat_local_token": secrets.token_urlsafe(30),
|
|
"embedding_local_token": secrets.token_urlsafe(30),
|
|
"db_password": secrets.token_urlsafe(30),
|
|
"mysql_user": "dataagent_admin",
|
|
"quality_password": secrets.token_urlsafe(30),
|
|
"quality_user": "quality_trial_ro",
|
|
}
|
|
PRIV.write_text(json.dumps(value))
|
|
PRIV.chmod(0o600)
|
|
client_cnf = TRIAL / "mysql-client.cnf"
|
|
client_cnf.write_text("[client]\nuser=dataagent_admin\npassword=" + value["db_password"] + "\nprotocol=SOCKET\nsocket=" + str(SOCKET) + "\n")
|
|
client_cnf.chmod(0o600)
|
|
manifest = {
|
|
"repo": "https://github.com/spring-ai-alibaba/DataAgent",
|
|
"commit": subprocess.check_output(["git", "-C", str(SOURCE), "rev-parse", "HEAD"], text=True).strip(),
|
|
"chat_model": model,
|
|
"sqlite_source_sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(),
|
|
"created_local": time.strftime("%Y-%m-%d %H:%M:%S %z"),
|
|
"security": "localhost-only; no production DB; private.json mode 0600",
|
|
}
|
|
(OUT / "trial-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
|
print(json.dumps({"prepared": True, "chat_model": model, "commit": manifest["commit"]}))
|
|
|
|
|
|
def mysql_cli(*args, input_text=None, root=False):
|
|
p = load_private()
|
|
if root:
|
|
cmd = [str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"]
|
|
else:
|
|
cmd = [str(MYSQLCLI), f"--defaults-extra-file={TRIAL / 'mysql-client.cnf'}", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=dataagent_admin", "--default-character-set=utf8mb4"]
|
|
cmd += list(args)
|
|
return subprocess.run(cmd, input=input_text, text=True, capture_output=True, check=True)
|
|
|
|
|
|
def start_db():
|
|
load_private()
|
|
if not PID.exists():
|
|
cmd = [str(MYSQLD), "--no-defaults", f"--basedir={ROOT / '.runtime/mysql'}", f"--datadir={TRIAL / 'datadir'}", f"--port={PORT}", "--bind-address=127.0.0.1", f"--socket={SOCKET}", f"--pid-file={PID}", f"--log-error={OUT / 'mysql.log'}", "--mysqlx=OFF", "--lower-case-table-names=2", "--max-connections=40", "--innodb-buffer-pool-size=128M"]
|
|
log = (TRIAL / "mysqld-stdout.log").open("ab")
|
|
subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
|
deadline = time.time() + 45
|
|
while time.time() < deadline:
|
|
if SOCKET.exists():
|
|
try:
|
|
mysql_cli("-Nse", "SELECT VERSION()", root=True)
|
|
print(json.dumps({"mysql": "ready", "bind": "127.0.0.1", "port": PORT}))
|
|
return
|
|
except Exception:
|
|
pass
|
|
time.sleep(.5)
|
|
raise RuntimeError("isolated MySQL 53307 did not become ready; inspect private log")
|
|
|
|
|
|
def sql_literal(v):
|
|
if v is None:
|
|
return "NULL"
|
|
if isinstance(v, (int, float)):
|
|
if isinstance(v, float) and (v != v or abs(v) == float("inf")):
|
|
return "NULL"
|
|
return repr(v)
|
|
s = str(v).replace("\\", "\\\\").replace("\x00", "\\0").replace("\n", "\\n").replace("\r", "\\r").replace("\x1a", "\\Z").replace("'", "\\'")
|
|
return "'" + s + "'"
|
|
|
|
|
|
def mysql_type(decl, col, keycols):
|
|
d = (decl or "").upper()
|
|
if "INT" in d:
|
|
return "BIGINT"
|
|
if col in keycols:
|
|
return "VARCHAR(255)"
|
|
if any(t in d for t in ("REAL", "FLOA", "DOUB")):
|
|
return "DOUBLE"
|
|
if "BLOB" in d:
|
|
return "LONGBLOB"
|
|
if "TEXT" in d or not d:
|
|
return "LONGTEXT"
|
|
return d
|
|
|
|
|
|
SCOPE_COLUMN_COMMENTS = {
|
|
"release_id": "official published release identifier",
|
|
"scope_key": "range key: auto_combined means domestic plus overseas automotive transport; auto_domestic means domestic only; auto_overseas means overseas only; overall means company-wide",
|
|
"scope_name": "business display name for the range",
|
|
"year": "calendar year of the official monthly metric",
|
|
"month": "month number of the official monthly metric",
|
|
"period": "official reporting month in YYYY-MM form",
|
|
"official_composite_rate": "official composite qualification rate from the published KPI service; standard meaning of qualification rate",
|
|
"composite_rate_without_hq_scrap": "adjusted composite rate excluding headquarters scrap; not the standard composite rate",
|
|
"black_rate": "black-sheet stage qualification rate",
|
|
"black_qualified_weight": "qualified black-sheet weight",
|
|
"black_scrap_weight": "scrap black-sheet weight",
|
|
"black_total_weight": "total black-sheet weight",
|
|
"bright_rate": "bright-sheet stage qualification rate",
|
|
"bright_qualified_weight": "qualified bright-sheet weight",
|
|
"bright_scrap_weight": "scrap bright-sheet weight",
|
|
"bright_total_weight": "total bright-sheet weight",
|
|
"included_rows": "number of rows included in the official monthly summary",
|
|
"raw_rows": "number of raw service rows for the official monthly summary",
|
|
"official_composite_rate_exact": "exact text representation of the official composite rate",
|
|
"evidence_file": "relative path to the source evidence file",
|
|
"api_query_json": "original query request and contract metadata as JSON",
|
|
"summary_json": "original service summary JSON",
|
|
"context_json": "source context JSON",
|
|
}
|
|
PRODUCT_COLUMN_COMMENTS = {
|
|
"scope_key": SCOPE_COLUMN_COMMENTS["scope_key"],
|
|
"scope_name": SCOPE_COLUMN_COMMENTS["scope_name"],
|
|
"year": SCOPE_COLUMN_COMMENTS["year"],
|
|
"month": SCOPE_COLUMN_COMMENTS["month"],
|
|
"period": SCOPE_COLUMN_COMMENTS["period"],
|
|
"product_key": "product/specification key for a product-level lead; product rows cannot be summed as an overall-rate contribution",
|
|
"product_name": "product/specification display name; a lead only, not a proven cause or attributable contribution",
|
|
"official_composite_rate": SCOPE_COLUMN_COMMENTS["official_composite_rate"],
|
|
"black_rate": SCOPE_COLUMN_COMMENTS["black_rate"],
|
|
"bright_rate": SCOPE_COLUMN_COMMENTS["bright_rate"],
|
|
"evidence_file": SCOPE_COLUMN_COMMENTS["evidence_file"],
|
|
"row_json": "complete original official product row JSON; product rows do not reconcile to stage denominator or overall contribution",
|
|
}
|
|
|
|
|
|
def seed_db():
|
|
start_db()
|
|
p = load_private()
|
|
admin_schema = (SOURCE / "data-agent-management/src/main/resources/sql/schema.sql").read_text()
|
|
create = f"CREATE DATABASE IF NOT EXISTS saa_data_agent CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE DATABASE IF NOT EXISTS quality_trial CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'localhost'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'localhost'; DROP USER IF EXISTS 'dataagent_trial'@'127.0.0.1'; DROP USER IF EXISTS 'dataagent_trial'@'localhost'; FLUSH PRIVILEGES;"
|
|
mysql_cli(root=True, input_text=create)
|
|
mysql_cli("saa_data_agent", input_text=admin_schema)
|
|
src = sqlite3.connect(f"file:{SQLITE}?mode=ro", uri=True)
|
|
chunks = []
|
|
manifest = {"source": str(SQLITE.relative_to(ROOT)), "sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(), "tables": {}}
|
|
for table, pk in (("official_scope_month_metrics", ("release_id", "scope_key", "year", "month")), ("official_product_month_rows", ("release_id", "scope_key", "year", "month", "row_no"))):
|
|
cols = src.execute(f'PRAGMA table_info("{table}")').fetchall()
|
|
names = [c[1] for c in cols]
|
|
comments = SCOPE_COLUMN_COMMENTS if table == "official_scope_month_metrics" else PRODUCT_COLUMN_COMMENTS
|
|
colddl = [f"`{c[1]}` {mysql_type(c[2], c[1], pk)}" + (" NOT NULL" if c[3] else "") + (" COMMENT " + sql_literal(comments[c[1]]) if c[1] in comments else "") for c in cols]
|
|
colddl.append("PRIMARY KEY (" + ",".join(f"`{c}`" for c in pk) + ")")
|
|
table_comment = ("Official released monthly quality KPI facts; one row per range and month. Composite, black-sheet, and bright-sheet values come from the service; NULL stays missing." if table == "official_scope_month_metrics" else "Official product/specification aggregate rows for leads only; do not sum these rows or claim they prove an overall contribution.")
|
|
chunks.append(f"CREATE TABLE IF NOT EXISTS quality_trial.`{table}` ({','.join(colddl)}) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT={sql_literal(table_comment)};")
|
|
rows = src.execute(f'SELECT * FROM "{table}"').fetchall()
|
|
manifest["tables"][table] = {"rows": len(rows), "columns": names, "null_cells": sum(v is None for row in rows for v in row)}
|
|
chunks.append(f"DELETE FROM quality_trial.`{table}`;")
|
|
batch_size = 200
|
|
for pos in range(0, len(rows), batch_size):
|
|
vals = ["(" + ",".join(sql_literal(v) for v in row) + ")" for row in rows[pos:pos+batch_size]]
|
|
chunks.append(f"INSERT INTO quality_trial.`{table}` ({','.join(f'`{n}`' for n in names)}) VALUES " + ",".join(vals) + ";")
|
|
src.close()
|
|
seed_path = TRIAL / "quality_seed.sql"
|
|
seed_path.write_text("\n".join(chunks))
|
|
seed_path.chmod(0o600)
|
|
subprocess.run([str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"], input=seed_path.read_text(), text=True, capture_output=True, check=True)
|
|
manifest["mysql_53307_counts"] = {}
|
|
for table in manifest["tables"]:
|
|
count = mysql_cli("-Nse", f"SELECT COUNT(*) FROM quality_trial.`{table}`", root=True).stdout.strip()
|
|
manifest["mysql_53307_counts"][table] = int(count)
|
|
(OUT / "data-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
|
print(json.dumps({"management_db": "saa_data_agent", "analysis_db": "quality_trial", "port": PORT, "counts": manifest["mysql_53307_counts"]}, ensure_ascii=False))
|
|
|
|
|
|
def probe_embedding():
|
|
import httpx
|
|
p = load_private()
|
|
request_body = {"model": p["embedding_model"], "input": ["汽车交通综合合格率", "国内汽车交通合格率"]}
|
|
record = {"profile": 1, "model": p["embedding_model"], "tls_verify": True, "request_body": request_body}
|
|
try:
|
|
with httpx.Client(timeout=120.0) as client:
|
|
resp = client.post(p["embedding_base"] + "/embeddings", headers={"Authorization": "Bearer " + p["embedding_key"], "Content-Type": "application/json"}, json=request_body)
|
|
body_bytes = resp.content
|
|
raw = body_bytes.decode("utf-8", errors="replace")
|
|
raw = raw.replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]")
|
|
record.update({"http_status": resp.status_code, "response_raw_redacted": raw})
|
|
try:
|
|
payload = resp.json()
|
|
vectors = [x.get("embedding") for x in payload.get("data", [])]
|
|
record["vector_count"] = len(vectors)
|
|
record["dimensions"] = sorted({len(v) for v in vectors if isinstance(v, list)})
|
|
except Exception:
|
|
record["vector_count"] = 0
|
|
record["dimensions"] = []
|
|
if resp.status_code >= 400:
|
|
raise RuntimeError("embedding request returned HTTP " + str(resp.status_code))
|
|
if not record["vector_count"] or not record["dimensions"]:
|
|
raise RuntimeError("embedding response did not contain vectors")
|
|
except Exception as exc:
|
|
record["error_type"] = type(exc).__name__
|
|
record["error_safe"] = str(exc).replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]")
|
|
(OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2))
|
|
(OUT / "embedding-probe.json").chmod(0o600)
|
|
raise
|
|
(OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2))
|
|
(OUT / "embedding-probe.json").chmod(0o600)
|
|
print(json.dumps({"http_status": record["http_status"], "vector_count": record["vector_count"], "dimensions": record["dimensions"], "tls_verify": True}))
|
|
|
|
|
|
def stop_db():
|
|
if PID.exists():
|
|
try:
|
|
mysql_cli("-Nse", "SHUTDOWN", root=True)
|
|
except Exception:
|
|
pass
|
|
print(json.dumps({"stop_requested": True, "port": PORT}))
|
|
|
|
|
|
def start_backend():
|
|
import httpx
|
|
p = load_private()
|
|
jar = SOURCE / "data-agent-management/target/spring-ai-alibaba-data-agent-management-1.0.0-SNAPSHOT.jar"
|
|
if not jar.exists():
|
|
raise RuntimeError("management executable jar has not been built")
|
|
pidfile = TRIAL / "backend.pid"
|
|
if pidfile.exists():
|
|
try:
|
|
os.kill(int(pidfile.read_text()), 0)
|
|
print(json.dumps({"backend": "already-running", "port": 5673}))
|
|
return
|
|
except Exception:
|
|
pidfile.unlink(missing_ok=True)
|
|
env = os.environ.copy()
|
|
env.update({
|
|
"SERVER_ADDRESS": "127.0.0.1",
|
|
"SERVER_PORT": "5673",
|
|
"DATA_AGENT_DATASOURCE_URL": "jdbc:mysql://127.0.0.1:53307/saa_data_agent?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowMultiQueries=true&allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=Asia/Shanghai",
|
|
"DATA_AGENT_DATASOURCE_USERNAME": p["mysql_user"],
|
|
"DATA_AGENT_DATASOURCE_PASSWORD": p["db_password"],
|
|
"DATA_AGENT_DATASOURCE_SQL_INIT": "never",
|
|
"LOGGING_LEVEL_COM_ALIBABA_CLOUD_AI_DATAAGENT_SERVICE_AIMODELCONFIG_DYNAMICMODELFACTORY": "WARN",
|
|
})
|
|
java = ROOT / ".runtime/jdk/Contents/Home/bin/java"
|
|
logpath = TRIAL / "backend.log"
|
|
log = logpath.open("ab")
|
|
logpath.chmod(0o600)
|
|
proc = subprocess.Popen([str(java), "-Xms256m", "-Xmx1536m", "-jar", str(jar)], cwd=SOURCE, env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
|
pidfile.write_text(str(proc.pid))
|
|
pidfile.chmod(0o600)
|
|
deadline = time.time() + 180
|
|
while time.time() < deadline:
|
|
if proc.poll() is not None:
|
|
raise RuntimeError("backend exited during startup; see .runtime/dataagent-trial/backend.log")
|
|
try:
|
|
response = httpx.get("http://127.0.0.1:5673/echo/ok", timeout=2)
|
|
if response.status_code == 200 and response.text.strip() == "ok":
|
|
print(json.dumps({"backend": "ready", "port": 5673, "bind": "127.0.0.1", "pid": proc.pid}))
|
|
return
|
|
except Exception:
|
|
pass
|
|
time.sleep(1)
|
|
raise RuntimeError("backend startup exceeded 180s; see .runtime/dataagent-trial/backend.log")
|
|
|
|
|
|
def stop_backend():
|
|
pidfile = TRIAL / "backend.pid"
|
|
if not pidfile.exists():
|
|
print(json.dumps({"backend": "not-running"}))
|
|
return
|
|
pid = int(pidfile.read_text())
|
|
try:
|
|
os.killpg(pid, signal.SIGTERM)
|
|
for _ in range(30):
|
|
try:
|
|
os.kill(pid, 0)
|
|
except ProcessLookupError:
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
os.killpg(pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
pidfile.unlink(missing_ok=True)
|
|
print(json.dumps({"backend": "stopped", "pid": pid}))
|
|
|
|
|
|
def run():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("action", choices=["prepare", "probe-embedding", "start-db", "seed-db", "stop-db", "start-backend", "stop-backend"])
|
|
args = parser.parse_args()
|
|
{"prepare": prepare, "probe-embedding": probe_embedding, "start-db": start_db, "seed-db": seed_db, "stop-db": stop_db, "start-backend": start_backend, "stop-backend": stop_backend}[args.action]()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run()
|