"""Isolated Spring AI Alibaba DataAgent trial helpers. All services bind to loopback. Secrets live only in .runtime/dataagent-trial/private.json with mode 0600; evidence replaces the selected profile endpoint and key. """ from __future__ import annotations import argparse import hashlib import json import os import secrets import signal import shutil import sqlite3 import subprocess import sys import time from pathlib import Path from urllib.parse import urlparse ROOT = Path(__file__).resolve().parents[1] TRIAL = ROOT / ".runtime/dataagent-trial" OUT = ROOT / "output/dataagent-trial" SOURCE = TRIAL / "source" SQLITE = ROOT / ".runtime/dbgpt-trial/data/quality_trial.sqlite" MYSQL = ROOT / ".runtime/mysql/bin" MYSQLD = MYSQL / "mysqld" MYSQLCLI = MYSQL / "mysql" PORT = 53307 SOCKET = TRIAL / "mysql.sock" PID = TRIAL / "mysql.pid" PRIV = TRIAL / "private.json" def load_private(): if not PRIV.exists(): prepare() return json.loads(PRIV.read_text()) def prepare(): TRIAL.mkdir(parents=True, exist_ok=True) OUT.mkdir(parents=True, exist_ok=True) sys.path.insert(0, str(ROOT / "scripts")) from dbgpt_trial import model_settings base, key, model = model_settings() if model.strip() != "mimo-v2.6-flash": raise RuntimeError("Selected profile is not the approved MiMo model") keyfile = ROOT / "apikey.txt" raw = keyfile.read_bytes() text = raw.decode("utf-16" if raw[:2] in (b"\xff\xfe", b"\xfe\xff") else "utf-8-sig") lines = [x.strip() for x in text.splitlines() if x.strip()] if len(lines) < 6: raise RuntimeError("The approved two-profile API configuration was not found") emb_lines = lines[:3] emb_base = emb_lines[0].rstrip("/") if emb_base.endswith("/chat/completions"): emb_base = emb_base[:-len("/chat/completions")] if not emb_base.endswith("/v1"): emb_base += "/v1" emb_key = emb_lines[1].removeprefix("Bearer ").strip() value = { "upstream_base": base, "upstream_key": key, "chat_model": model, "embedding_base": emb_base, "embedding_key": emb_key, "embedding_model": "qwen3-embedding-4b-local", "chat_local_token": secrets.token_urlsafe(30), "embedding_local_token": secrets.token_urlsafe(30), "db_password": secrets.token_urlsafe(30), "mysql_user": "dataagent_admin", "quality_password": secrets.token_urlsafe(30), "quality_user": "quality_trial_ro", } PRIV.write_text(json.dumps(value)) PRIV.chmod(0o600) client_cnf = TRIAL / "mysql-client.cnf" client_cnf.write_text("[client]\nuser=dataagent_admin\npassword=" + value["db_password"] + "\nprotocol=SOCKET\nsocket=" + str(SOCKET) + "\n") client_cnf.chmod(0o600) manifest = { "repo": "https://github.com/spring-ai-alibaba/DataAgent", "commit": subprocess.check_output(["git", "-C", str(SOURCE), "rev-parse", "HEAD"], text=True).strip(), "chat_model": model, "sqlite_source_sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(), "created_local": time.strftime("%Y-%m-%d %H:%M:%S %z"), "security": "localhost-only; no production DB; private.json mode 0600", } (OUT / "trial-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) print(json.dumps({"prepared": True, "chat_model": model, "commit": manifest["commit"]})) def mysql_cli(*args, input_text=None, root=False): p = load_private() if root: cmd = [str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"] else: cmd = [str(MYSQLCLI), f"--defaults-extra-file={TRIAL / 'mysql-client.cnf'}", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=dataagent_admin", "--default-character-set=utf8mb4"] cmd += list(args) return subprocess.run(cmd, input=input_text, text=True, capture_output=True, check=True) def start_db(): load_private() if not PID.exists(): cmd = [str(MYSQLD), "--no-defaults", f"--basedir={ROOT / '.runtime/mysql'}", f"--datadir={TRIAL / 'datadir'}", f"--port={PORT}", "--bind-address=127.0.0.1", f"--socket={SOCKET}", f"--pid-file={PID}", f"--log-error={OUT / 'mysql.log'}", "--mysqlx=OFF", "--lower-case-table-names=2", "--max-connections=40", "--innodb-buffer-pool-size=128M"] log = (TRIAL / "mysqld-stdout.log").open("ab") subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) deadline = time.time() + 45 while time.time() < deadline: if SOCKET.exists(): try: mysql_cli("-Nse", "SELECT VERSION()", root=True) print(json.dumps({"mysql": "ready", "bind": "127.0.0.1", "port": PORT})) return except Exception: pass time.sleep(.5) raise RuntimeError("isolated MySQL 53307 did not become ready; inspect private log") def sql_literal(v): if v is None: return "NULL" if isinstance(v, (int, float)): if isinstance(v, float) and (v != v or abs(v) == float("inf")): return "NULL" return repr(v) s = str(v).replace("\\", "\\\\").replace("\x00", "\\0").replace("\n", "\\n").replace("\r", "\\r").replace("\x1a", "\\Z").replace("'", "\\'") return "'" + s + "'" def mysql_type(decl, col, keycols): d = (decl or "").upper() if "INT" in d: return "BIGINT" if col in keycols: return "VARCHAR(255)" if any(t in d for t in ("REAL", "FLOA", "DOUB")): return "DOUBLE" if "BLOB" in d: return "LONGBLOB" if "TEXT" in d or not d: return "LONGTEXT" return d SCOPE_COLUMN_COMMENTS = { "release_id": "official published release identifier", "scope_key": "range key: auto_combined means domestic plus overseas automotive transport; auto_domestic means domestic only; auto_overseas means overseas only; overall means company-wide", "scope_name": "business display name for the range", "year": "calendar year of the official monthly metric", "month": "month number of the official monthly metric", "period": "official reporting month in YYYY-MM form", "official_composite_rate": "official composite qualification rate from the published KPI service; standard meaning of qualification rate", "composite_rate_without_hq_scrap": "adjusted composite rate excluding headquarters scrap; not the standard composite rate", "black_rate": "black-sheet stage qualification rate", "black_qualified_weight": "qualified black-sheet weight", "black_scrap_weight": "scrap black-sheet weight", "black_total_weight": "total black-sheet weight", "bright_rate": "bright-sheet stage qualification rate", "bright_qualified_weight": "qualified bright-sheet weight", "bright_scrap_weight": "scrap bright-sheet weight", "bright_total_weight": "total bright-sheet weight", "included_rows": "number of rows included in the official monthly summary", "raw_rows": "number of raw service rows for the official monthly summary", "official_composite_rate_exact": "exact text representation of the official composite rate", "evidence_file": "relative path to the source evidence file", "api_query_json": "original query request and contract metadata as JSON", "summary_json": "original service summary JSON", "context_json": "source context JSON", } PRODUCT_COLUMN_COMMENTS = { "scope_key": SCOPE_COLUMN_COMMENTS["scope_key"], "scope_name": SCOPE_COLUMN_COMMENTS["scope_name"], "year": SCOPE_COLUMN_COMMENTS["year"], "month": SCOPE_COLUMN_COMMENTS["month"], "period": SCOPE_COLUMN_COMMENTS["period"], "product_key": "product/specification key for a product-level lead; product rows cannot be summed as an overall-rate contribution", "product_name": "product/specification display name; a lead only, not a proven cause or attributable contribution", "official_composite_rate": SCOPE_COLUMN_COMMENTS["official_composite_rate"], "black_rate": SCOPE_COLUMN_COMMENTS["black_rate"], "bright_rate": SCOPE_COLUMN_COMMENTS["bright_rate"], "evidence_file": SCOPE_COLUMN_COMMENTS["evidence_file"], "row_json": "complete original official product row JSON; product rows do not reconcile to stage denominator or overall contribution", } def seed_db(): start_db() p = load_private() admin_schema = (SOURCE / "data-agent-management/src/main/resources/sql/schema.sql").read_text() create = f"CREATE DATABASE IF NOT EXISTS saa_data_agent CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE DATABASE IF NOT EXISTS quality_trial CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'localhost'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'localhost'; DROP USER IF EXISTS 'dataagent_trial'@'127.0.0.1'; DROP USER IF EXISTS 'dataagent_trial'@'localhost'; FLUSH PRIVILEGES;" mysql_cli(root=True, input_text=create) mysql_cli("saa_data_agent", input_text=admin_schema) src = sqlite3.connect(f"file:{SQLITE}?mode=ro", uri=True) chunks = [] manifest = {"source": str(SQLITE.relative_to(ROOT)), "sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(), "tables": {}} for table, pk in (("official_scope_month_metrics", ("release_id", "scope_key", "year", "month")), ("official_product_month_rows", ("release_id", "scope_key", "year", "month", "row_no"))): cols = src.execute(f'PRAGMA table_info("{table}")').fetchall() names = [c[1] for c in cols] comments = SCOPE_COLUMN_COMMENTS if table == "official_scope_month_metrics" else PRODUCT_COLUMN_COMMENTS colddl = [f"`{c[1]}` {mysql_type(c[2], c[1], pk)}" + (" NOT NULL" if c[3] else "") + (" COMMENT " + sql_literal(comments[c[1]]) if c[1] in comments else "") for c in cols] colddl.append("PRIMARY KEY (" + ",".join(f"`{c}`" for c in pk) + ")") table_comment = ("Official released monthly quality KPI facts; one row per range and month. Composite, black-sheet, and bright-sheet values come from the service; NULL stays missing." if table == "official_scope_month_metrics" else "Official product/specification aggregate rows for leads only; do not sum these rows or claim they prove an overall contribution.") chunks.append(f"CREATE TABLE IF NOT EXISTS quality_trial.`{table}` ({','.join(colddl)}) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT={sql_literal(table_comment)};") rows = src.execute(f'SELECT * FROM "{table}"').fetchall() manifest["tables"][table] = {"rows": len(rows), "columns": names, "null_cells": sum(v is None for row in rows for v in row)} chunks.append(f"DELETE FROM quality_trial.`{table}`;") batch_size = 200 for pos in range(0, len(rows), batch_size): vals = ["(" + ",".join(sql_literal(v) for v in row) + ")" for row in rows[pos:pos+batch_size]] chunks.append(f"INSERT INTO quality_trial.`{table}` ({','.join(f'`{n}`' for n in names)}) VALUES " + ",".join(vals) + ";") src.close() seed_path = TRIAL / "quality_seed.sql" seed_path.write_text("\n".join(chunks)) seed_path.chmod(0o600) subprocess.run([str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"], input=seed_path.read_text(), text=True, capture_output=True, check=True) manifest["mysql_53307_counts"] = {} for table in manifest["tables"]: count = mysql_cli("-Nse", f"SELECT COUNT(*) FROM quality_trial.`{table}`", root=True).stdout.strip() manifest["mysql_53307_counts"][table] = int(count) (OUT / "data-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) print(json.dumps({"management_db": "saa_data_agent", "analysis_db": "quality_trial", "port": PORT, "counts": manifest["mysql_53307_counts"]}, ensure_ascii=False)) def probe_embedding(): import httpx p = load_private() request_body = {"model": p["embedding_model"], "input": ["汽车交通综合合格率", "国内汽车交通合格率"]} record = {"profile": 1, "model": p["embedding_model"], "tls_verify": True, "request_body": request_body} try: with httpx.Client(timeout=120.0) as client: resp = client.post(p["embedding_base"] + "/embeddings", headers={"Authorization": "Bearer " + p["embedding_key"], "Content-Type": "application/json"}, json=request_body) body_bytes = resp.content raw = body_bytes.decode("utf-8", errors="replace") raw = raw.replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]") record.update({"http_status": resp.status_code, "response_raw_redacted": raw}) try: payload = resp.json() vectors = [x.get("embedding") for x in payload.get("data", [])] record["vector_count"] = len(vectors) record["dimensions"] = sorted({len(v) for v in vectors if isinstance(v, list)}) except Exception: record["vector_count"] = 0 record["dimensions"] = [] if resp.status_code >= 400: raise RuntimeError("embedding request returned HTTP " + str(resp.status_code)) if not record["vector_count"] or not record["dimensions"]: raise RuntimeError("embedding response did not contain vectors") except Exception as exc: record["error_type"] = type(exc).__name__ record["error_safe"] = str(exc).replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]") (OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2)) (OUT / "embedding-probe.json").chmod(0o600) raise (OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2)) (OUT / "embedding-probe.json").chmod(0o600) print(json.dumps({"http_status": record["http_status"], "vector_count": record["vector_count"], "dimensions": record["dimensions"], "tls_verify": True})) def stop_db(): if PID.exists(): try: mysql_cli("-Nse", "SHUTDOWN", root=True) except Exception: pass print(json.dumps({"stop_requested": True, "port": PORT})) def start_backend(): import httpx p = load_private() jar = SOURCE / "data-agent-management/target/spring-ai-alibaba-data-agent-management-1.0.0-SNAPSHOT.jar" if not jar.exists(): raise RuntimeError("management executable jar has not been built") pidfile = TRIAL / "backend.pid" if pidfile.exists(): try: os.kill(int(pidfile.read_text()), 0) print(json.dumps({"backend": "already-running", "port": 5673})) return except Exception: pidfile.unlink(missing_ok=True) env = os.environ.copy() env.update({ "SERVER_ADDRESS": "127.0.0.1", "SERVER_PORT": "5673", "DATA_AGENT_DATASOURCE_URL": "jdbc:mysql://127.0.0.1:53307/saa_data_agent?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowMultiQueries=true&allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=Asia/Shanghai", "DATA_AGENT_DATASOURCE_USERNAME": p["mysql_user"], "DATA_AGENT_DATASOURCE_PASSWORD": p["db_password"], "DATA_AGENT_DATASOURCE_SQL_INIT": "never", "LOGGING_LEVEL_COM_ALIBABA_CLOUD_AI_DATAAGENT_SERVICE_AIMODELCONFIG_DYNAMICMODELFACTORY": "WARN", }) java = ROOT / ".runtime/jdk/Contents/Home/bin/java" logpath = TRIAL / "backend.log" log = logpath.open("ab") logpath.chmod(0o600) proc = subprocess.Popen([str(java), "-Xms256m", "-Xmx1536m", "-jar", str(jar)], cwd=SOURCE, env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) pidfile.write_text(str(proc.pid)) pidfile.chmod(0o600) deadline = time.time() + 180 while time.time() < deadline: if proc.poll() is not None: raise RuntimeError("backend exited during startup; see .runtime/dataagent-trial/backend.log") try: response = httpx.get("http://127.0.0.1:5673/echo/ok", timeout=2) if response.status_code == 200 and response.text.strip() == "ok": print(json.dumps({"backend": "ready", "port": 5673, "bind": "127.0.0.1", "pid": proc.pid})) return except Exception: pass time.sleep(1) raise RuntimeError("backend startup exceeded 180s; see .runtime/dataagent-trial/backend.log") def stop_backend(): pidfile = TRIAL / "backend.pid" if not pidfile.exists(): print(json.dumps({"backend": "not-running"})) return pid = int(pidfile.read_text()) try: os.killpg(pid, signal.SIGTERM) for _ in range(30): try: os.kill(pid, 0) except ProcessLookupError: break time.sleep(1) else: os.killpg(pid, signal.SIGKILL) except ProcessLookupError: pass pidfile.unlink(missing_ok=True) print(json.dumps({"backend": "stopped", "pid": pid})) def run(): parser = argparse.ArgumentParser() parser.add_argument("action", choices=["prepare", "probe-embedding", "start-db", "seed-db", "stop-db", "start-backend", "stop-backend"]) args = parser.parse_args() {"prepare": prepare, "probe-embedding": probe_embedding, "start-db": start_db, "seed-db": seed_db, "stop-db": stop_db, "start-backend": start_backend, "stop-backend": stop_backend}[args.action]() if __name__ == "__main__": run()