初始化:质量数据分析平台源码
This commit is contained in:
@@ -0,0 +1,349 @@
|
||||
"""Isolated Spring AI Alibaba DataAgent trial helpers.
|
||||
|
||||
All services bind to loopback. Secrets live only in .runtime/dataagent-trial/private.json
|
||||
with mode 0600; evidence replaces the selected profile endpoint and key.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import signal
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TRIAL = ROOT / ".runtime/dataagent-trial"
|
||||
OUT = ROOT / "output/dataagent-trial"
|
||||
SOURCE = TRIAL / "source"
|
||||
SQLITE = ROOT / ".runtime/dbgpt-trial/data/quality_trial.sqlite"
|
||||
MYSQL = ROOT / ".runtime/mysql/bin"
|
||||
MYSQLD = MYSQL / "mysqld"
|
||||
MYSQLCLI = MYSQL / "mysql"
|
||||
PORT = 53307
|
||||
SOCKET = TRIAL / "mysql.sock"
|
||||
PID = TRIAL / "mysql.pid"
|
||||
PRIV = TRIAL / "private.json"
|
||||
|
||||
|
||||
def load_private():
|
||||
if not PRIV.exists():
|
||||
prepare()
|
||||
return json.loads(PRIV.read_text())
|
||||
|
||||
|
||||
def prepare():
|
||||
TRIAL.mkdir(parents=True, exist_ok=True)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from dbgpt_trial import model_settings
|
||||
|
||||
base, key, model = model_settings()
|
||||
if model.strip() != "mimo-v2.6-flash":
|
||||
raise RuntimeError("Selected profile is not the approved MiMo model")
|
||||
keyfile = ROOT / "apikey.txt"
|
||||
raw = keyfile.read_bytes()
|
||||
text = raw.decode("utf-16" if raw[:2] in (b"\xff\xfe", b"\xfe\xff") else "utf-8-sig")
|
||||
lines = [x.strip() for x in text.splitlines() if x.strip()]
|
||||
if len(lines) < 6:
|
||||
raise RuntimeError("The approved two-profile API configuration was not found")
|
||||
emb_lines = lines[:3]
|
||||
emb_base = emb_lines[0].rstrip("/")
|
||||
if emb_base.endswith("/chat/completions"):
|
||||
emb_base = emb_base[:-len("/chat/completions")]
|
||||
if not emb_base.endswith("/v1"):
|
||||
emb_base += "/v1"
|
||||
emb_key = emb_lines[1].removeprefix("Bearer ").strip()
|
||||
value = {
|
||||
"upstream_base": base,
|
||||
"upstream_key": key,
|
||||
"chat_model": model,
|
||||
"embedding_base": emb_base,
|
||||
"embedding_key": emb_key,
|
||||
"embedding_model": "qwen3-embedding-4b-local",
|
||||
"chat_local_token": secrets.token_urlsafe(30),
|
||||
"embedding_local_token": secrets.token_urlsafe(30),
|
||||
"db_password": secrets.token_urlsafe(30),
|
||||
"mysql_user": "dataagent_admin",
|
||||
"quality_password": secrets.token_urlsafe(30),
|
||||
"quality_user": "quality_trial_ro",
|
||||
}
|
||||
PRIV.write_text(json.dumps(value))
|
||||
PRIV.chmod(0o600)
|
||||
client_cnf = TRIAL / "mysql-client.cnf"
|
||||
client_cnf.write_text("[client]\nuser=dataagent_admin\npassword=" + value["db_password"] + "\nprotocol=SOCKET\nsocket=" + str(SOCKET) + "\n")
|
||||
client_cnf.chmod(0o600)
|
||||
manifest = {
|
||||
"repo": "https://github.com/spring-ai-alibaba/DataAgent",
|
||||
"commit": subprocess.check_output(["git", "-C", str(SOURCE), "rev-parse", "HEAD"], text=True).strip(),
|
||||
"chat_model": model,
|
||||
"sqlite_source_sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(),
|
||||
"created_local": time.strftime("%Y-%m-%d %H:%M:%S %z"),
|
||||
"security": "localhost-only; no production DB; private.json mode 0600",
|
||||
}
|
||||
(OUT / "trial-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
print(json.dumps({"prepared": True, "chat_model": model, "commit": manifest["commit"]}))
|
||||
|
||||
|
||||
def mysql_cli(*args, input_text=None, root=False):
|
||||
p = load_private()
|
||||
if root:
|
||||
cmd = [str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"]
|
||||
else:
|
||||
cmd = [str(MYSQLCLI), f"--defaults-extra-file={TRIAL / 'mysql-client.cnf'}", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=dataagent_admin", "--default-character-set=utf8mb4"]
|
||||
cmd += list(args)
|
||||
return subprocess.run(cmd, input=input_text, text=True, capture_output=True, check=True)
|
||||
|
||||
|
||||
def start_db():
|
||||
load_private()
|
||||
if not PID.exists():
|
||||
cmd = [str(MYSQLD), "--no-defaults", f"--basedir={ROOT / '.runtime/mysql'}", f"--datadir={TRIAL / 'datadir'}", f"--port={PORT}", "--bind-address=127.0.0.1", f"--socket={SOCKET}", f"--pid-file={PID}", f"--log-error={OUT / 'mysql.log'}", "--mysqlx=OFF", "--lower-case-table-names=2", "--max-connections=40", "--innodb-buffer-pool-size=128M"]
|
||||
log = (TRIAL / "mysqld-stdout.log").open("ab")
|
||||
subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
deadline = time.time() + 45
|
||||
while time.time() < deadline:
|
||||
if SOCKET.exists():
|
||||
try:
|
||||
mysql_cli("-Nse", "SELECT VERSION()", root=True)
|
||||
print(json.dumps({"mysql": "ready", "bind": "127.0.0.1", "port": PORT}))
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(.5)
|
||||
raise RuntimeError("isolated MySQL 53307 did not become ready; inspect private log")
|
||||
|
||||
|
||||
def sql_literal(v):
|
||||
if v is None:
|
||||
return "NULL"
|
||||
if isinstance(v, (int, float)):
|
||||
if isinstance(v, float) and (v != v or abs(v) == float("inf")):
|
||||
return "NULL"
|
||||
return repr(v)
|
||||
s = str(v).replace("\\", "\\\\").replace("\x00", "\\0").replace("\n", "\\n").replace("\r", "\\r").replace("\x1a", "\\Z").replace("'", "\\'")
|
||||
return "'" + s + "'"
|
||||
|
||||
|
||||
def mysql_type(decl, col, keycols):
|
||||
d = (decl or "").upper()
|
||||
if "INT" in d:
|
||||
return "BIGINT"
|
||||
if col in keycols:
|
||||
return "VARCHAR(255)"
|
||||
if any(t in d for t in ("REAL", "FLOA", "DOUB")):
|
||||
return "DOUBLE"
|
||||
if "BLOB" in d:
|
||||
return "LONGBLOB"
|
||||
if "TEXT" in d or not d:
|
||||
return "LONGTEXT"
|
||||
return d
|
||||
|
||||
|
||||
SCOPE_COLUMN_COMMENTS = {
|
||||
"release_id": "official published release identifier",
|
||||
"scope_key": "range key: auto_combined means domestic plus overseas automotive transport; auto_domestic means domestic only; auto_overseas means overseas only; overall means company-wide",
|
||||
"scope_name": "business display name for the range",
|
||||
"year": "calendar year of the official monthly metric",
|
||||
"month": "month number of the official monthly metric",
|
||||
"period": "official reporting month in YYYY-MM form",
|
||||
"official_composite_rate": "official composite qualification rate from the published KPI service; standard meaning of qualification rate",
|
||||
"composite_rate_without_hq_scrap": "adjusted composite rate excluding headquarters scrap; not the standard composite rate",
|
||||
"black_rate": "black-sheet stage qualification rate",
|
||||
"black_qualified_weight": "qualified black-sheet weight",
|
||||
"black_scrap_weight": "scrap black-sheet weight",
|
||||
"black_total_weight": "total black-sheet weight",
|
||||
"bright_rate": "bright-sheet stage qualification rate",
|
||||
"bright_qualified_weight": "qualified bright-sheet weight",
|
||||
"bright_scrap_weight": "scrap bright-sheet weight",
|
||||
"bright_total_weight": "total bright-sheet weight",
|
||||
"included_rows": "number of rows included in the official monthly summary",
|
||||
"raw_rows": "number of raw service rows for the official monthly summary",
|
||||
"official_composite_rate_exact": "exact text representation of the official composite rate",
|
||||
"evidence_file": "relative path to the source evidence file",
|
||||
"api_query_json": "original query request and contract metadata as JSON",
|
||||
"summary_json": "original service summary JSON",
|
||||
"context_json": "source context JSON",
|
||||
}
|
||||
PRODUCT_COLUMN_COMMENTS = {
|
||||
"scope_key": SCOPE_COLUMN_COMMENTS["scope_key"],
|
||||
"scope_name": SCOPE_COLUMN_COMMENTS["scope_name"],
|
||||
"year": SCOPE_COLUMN_COMMENTS["year"],
|
||||
"month": SCOPE_COLUMN_COMMENTS["month"],
|
||||
"period": SCOPE_COLUMN_COMMENTS["period"],
|
||||
"product_key": "product/specification key for a product-level lead; product rows cannot be summed as an overall-rate contribution",
|
||||
"product_name": "product/specification display name; a lead only, not a proven cause or attributable contribution",
|
||||
"official_composite_rate": SCOPE_COLUMN_COMMENTS["official_composite_rate"],
|
||||
"black_rate": SCOPE_COLUMN_COMMENTS["black_rate"],
|
||||
"bright_rate": SCOPE_COLUMN_COMMENTS["bright_rate"],
|
||||
"evidence_file": SCOPE_COLUMN_COMMENTS["evidence_file"],
|
||||
"row_json": "complete original official product row JSON; product rows do not reconcile to stage denominator or overall contribution",
|
||||
}
|
||||
|
||||
|
||||
def seed_db():
|
||||
start_db()
|
||||
p = load_private()
|
||||
admin_schema = (SOURCE / "data-agent-management/src/main/resources/sql/schema.sql").read_text()
|
||||
create = f"CREATE DATABASE IF NOT EXISTS saa_data_agent CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE DATABASE IF NOT EXISTS quality_trial CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'127.0.0.1' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; ALTER USER '{p['mysql_user']}'@'localhost' IDENTIFIED BY '{p['db_password']}'; GRANT ALL PRIVILEGES ON saa_data_agent.* TO '{p['mysql_user']}'@'localhost'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'127.0.0.1' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'127.0.0.1'; CREATE USER IF NOT EXISTS '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; ALTER USER '{p['quality_user']}'@'localhost' IDENTIFIED BY '{p['quality_password']}'; GRANT SELECT ON quality_trial.* TO '{p['quality_user']}'@'localhost'; DROP USER IF EXISTS 'dataagent_trial'@'127.0.0.1'; DROP USER IF EXISTS 'dataagent_trial'@'localhost'; FLUSH PRIVILEGES;"
|
||||
mysql_cli(root=True, input_text=create)
|
||||
mysql_cli("saa_data_agent", input_text=admin_schema)
|
||||
src = sqlite3.connect(f"file:{SQLITE}?mode=ro", uri=True)
|
||||
chunks = []
|
||||
manifest = {"source": str(SQLITE.relative_to(ROOT)), "sha256": hashlib.sha256(SQLITE.read_bytes()).hexdigest(), "tables": {}}
|
||||
for table, pk in (("official_scope_month_metrics", ("release_id", "scope_key", "year", "month")), ("official_product_month_rows", ("release_id", "scope_key", "year", "month", "row_no"))):
|
||||
cols = src.execute(f'PRAGMA table_info("{table}")').fetchall()
|
||||
names = [c[1] for c in cols]
|
||||
comments = SCOPE_COLUMN_COMMENTS if table == "official_scope_month_metrics" else PRODUCT_COLUMN_COMMENTS
|
||||
colddl = [f"`{c[1]}` {mysql_type(c[2], c[1], pk)}" + (" NOT NULL" if c[3] else "") + (" COMMENT " + sql_literal(comments[c[1]]) if c[1] in comments else "") for c in cols]
|
||||
colddl.append("PRIMARY KEY (" + ",".join(f"`{c}`" for c in pk) + ")")
|
||||
table_comment = ("Official released monthly quality KPI facts; one row per range and month. Composite, black-sheet, and bright-sheet values come from the service; NULL stays missing." if table == "official_scope_month_metrics" else "Official product/specification aggregate rows for leads only; do not sum these rows or claim they prove an overall contribution.")
|
||||
chunks.append(f"CREATE TABLE IF NOT EXISTS quality_trial.`{table}` ({','.join(colddl)}) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT={sql_literal(table_comment)};")
|
||||
rows = src.execute(f'SELECT * FROM "{table}"').fetchall()
|
||||
manifest["tables"][table] = {"rows": len(rows), "columns": names, "null_cells": sum(v is None for row in rows for v in row)}
|
||||
chunks.append(f"DELETE FROM quality_trial.`{table}`;")
|
||||
batch_size = 200
|
||||
for pos in range(0, len(rows), batch_size):
|
||||
vals = ["(" + ",".join(sql_literal(v) for v in row) + ")" for row in rows[pos:pos+batch_size]]
|
||||
chunks.append(f"INSERT INTO quality_trial.`{table}` ({','.join(f'`{n}`' for n in names)}) VALUES " + ",".join(vals) + ";")
|
||||
src.close()
|
||||
seed_path = TRIAL / "quality_seed.sql"
|
||||
seed_path.write_text("\n".join(chunks))
|
||||
seed_path.chmod(0o600)
|
||||
subprocess.run([str(MYSQLCLI), "--no-defaults", "--protocol=SOCKET", f"--socket={SOCKET}", "--user=root", "--default-character-set=utf8mb4"], input=seed_path.read_text(), text=True, capture_output=True, check=True)
|
||||
manifest["mysql_53307_counts"] = {}
|
||||
for table in manifest["tables"]:
|
||||
count = mysql_cli("-Nse", f"SELECT COUNT(*) FROM quality_trial.`{table}`", root=True).stdout.strip()
|
||||
manifest["mysql_53307_counts"][table] = int(count)
|
||||
(OUT / "data-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
print(json.dumps({"management_db": "saa_data_agent", "analysis_db": "quality_trial", "port": PORT, "counts": manifest["mysql_53307_counts"]}, ensure_ascii=False))
|
||||
|
||||
|
||||
def probe_embedding():
|
||||
import httpx
|
||||
p = load_private()
|
||||
request_body = {"model": p["embedding_model"], "input": ["汽车交通综合合格率", "国内汽车交通合格率"]}
|
||||
record = {"profile": 1, "model": p["embedding_model"], "tls_verify": True, "request_body": request_body}
|
||||
try:
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
resp = client.post(p["embedding_base"] + "/embeddings", headers={"Authorization": "Bearer " + p["embedding_key"], "Content-Type": "application/json"}, json=request_body)
|
||||
body_bytes = resp.content
|
||||
raw = body_bytes.decode("utf-8", errors="replace")
|
||||
raw = raw.replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]")
|
||||
record.update({"http_status": resp.status_code, "response_raw_redacted": raw})
|
||||
try:
|
||||
payload = resp.json()
|
||||
vectors = [x.get("embedding") for x in payload.get("data", [])]
|
||||
record["vector_count"] = len(vectors)
|
||||
record["dimensions"] = sorted({len(v) for v in vectors if isinstance(v, list)})
|
||||
except Exception:
|
||||
record["vector_count"] = 0
|
||||
record["dimensions"] = []
|
||||
if resp.status_code >= 400:
|
||||
raise RuntimeError("embedding request returned HTTP " + str(resp.status_code))
|
||||
if not record["vector_count"] or not record["dimensions"]:
|
||||
raise RuntimeError("embedding response did not contain vectors")
|
||||
except Exception as exc:
|
||||
record["error_type"] = type(exc).__name__
|
||||
record["error_safe"] = str(exc).replace(p["embedding_key"], "[REDACTED_KEY]").replace(p["embedding_base"], "[REDACTED_ENDPOINT]")
|
||||
(OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2))
|
||||
(OUT / "embedding-probe.json").chmod(0o600)
|
||||
raise
|
||||
(OUT / "embedding-probe.json").write_text(json.dumps(record, ensure_ascii=False, indent=2))
|
||||
(OUT / "embedding-probe.json").chmod(0o600)
|
||||
print(json.dumps({"http_status": record["http_status"], "vector_count": record["vector_count"], "dimensions": record["dimensions"], "tls_verify": True}))
|
||||
|
||||
|
||||
def stop_db():
|
||||
if PID.exists():
|
||||
try:
|
||||
mysql_cli("-Nse", "SHUTDOWN", root=True)
|
||||
except Exception:
|
||||
pass
|
||||
print(json.dumps({"stop_requested": True, "port": PORT}))
|
||||
|
||||
|
||||
def start_backend():
|
||||
import httpx
|
||||
p = load_private()
|
||||
jar = SOURCE / "data-agent-management/target/spring-ai-alibaba-data-agent-management-1.0.0-SNAPSHOT.jar"
|
||||
if not jar.exists():
|
||||
raise RuntimeError("management executable jar has not been built")
|
||||
pidfile = TRIAL / "backend.pid"
|
||||
if pidfile.exists():
|
||||
try:
|
||||
os.kill(int(pidfile.read_text()), 0)
|
||||
print(json.dumps({"backend": "already-running", "port": 5673}))
|
||||
return
|
||||
except Exception:
|
||||
pidfile.unlink(missing_ok=True)
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"SERVER_ADDRESS": "127.0.0.1",
|
||||
"SERVER_PORT": "5673",
|
||||
"DATA_AGENT_DATASOURCE_URL": "jdbc:mysql://127.0.0.1:53307/saa_data_agent?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowMultiQueries=true&allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=Asia/Shanghai",
|
||||
"DATA_AGENT_DATASOURCE_USERNAME": p["mysql_user"],
|
||||
"DATA_AGENT_DATASOURCE_PASSWORD": p["db_password"],
|
||||
"DATA_AGENT_DATASOURCE_SQL_INIT": "never",
|
||||
"LOGGING_LEVEL_COM_ALIBABA_CLOUD_AI_DATAAGENT_SERVICE_AIMODELCONFIG_DYNAMICMODELFACTORY": "WARN",
|
||||
})
|
||||
java = ROOT / ".runtime/jdk/Contents/Home/bin/java"
|
||||
logpath = TRIAL / "backend.log"
|
||||
log = logpath.open("ab")
|
||||
logpath.chmod(0o600)
|
||||
proc = subprocess.Popen([str(java), "-Xms256m", "-Xmx1536m", "-jar", str(jar)], cwd=SOURCE, env=env, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
pidfile.write_text(str(proc.pid))
|
||||
pidfile.chmod(0o600)
|
||||
deadline = time.time() + 180
|
||||
while time.time() < deadline:
|
||||
if proc.poll() is not None:
|
||||
raise RuntimeError("backend exited during startup; see .runtime/dataagent-trial/backend.log")
|
||||
try:
|
||||
response = httpx.get("http://127.0.0.1:5673/echo/ok", timeout=2)
|
||||
if response.status_code == 200 and response.text.strip() == "ok":
|
||||
print(json.dumps({"backend": "ready", "port": 5673, "bind": "127.0.0.1", "pid": proc.pid}))
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise RuntimeError("backend startup exceeded 180s; see .runtime/dataagent-trial/backend.log")
|
||||
|
||||
|
||||
def stop_backend():
|
||||
pidfile = TRIAL / "backend.pid"
|
||||
if not pidfile.exists():
|
||||
print(json.dumps({"backend": "not-running"}))
|
||||
return
|
||||
pid = int(pidfile.read_text())
|
||||
try:
|
||||
os.killpg(pid, signal.SIGTERM)
|
||||
for _ in range(30):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
pidfile.unlink(missing_ok=True)
|
||||
print(json.dumps({"backend": "stopped", "pid": pid}))
|
||||
|
||||
|
||||
def run():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=["prepare", "probe-embedding", "start-db", "seed-db", "stop-db", "start-backend", "stop-backend"])
|
||||
args = parser.parse_args()
|
||||
{"prepare": prepare, "probe-embedding": probe_embedding, "start-db": start_db, "seed-db": seed_db, "stop-db": stop_db, "start-backend": start_backend, "stop-backend": stop_backend}[args.action]()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
Reference in New Issue
Block a user