初始化:质量数据分析平台源码
This commit is contained in:
@@ -0,0 +1,238 @@
|
||||
"""Configure upstream DataAgent only through its own API and run trial graphs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.dataagent_trial import OUT, TRIAL, load_private
|
||||
|
||||
API = "http://127.0.0.1:5673"
|
||||
PROXY = "http://127.0.0.1:5676/v1"
|
||||
CALLS = OUT / "api-calls.jsonl"
|
||||
STATE = TRIAL / "trial-state.json"
|
||||
|
||||
|
||||
def scrub(obj, p):
|
||||
if isinstance(obj, dict):
|
||||
return {k: ("[REDACTED]" if k.lower() in {"apikey", "password", "proxy_password"} else scrub(v, p)) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [scrub(x, p) for x in obj]
|
||||
if isinstance(obj, str):
|
||||
for k in ("upstream_key", "embedding_key", "upstream_base", "embedding_base", "db_password"):
|
||||
if p.get(k): obj = obj.replace(p[k], "[REDACTED]")
|
||||
return obj
|
||||
return obj
|
||||
|
||||
|
||||
def record(method, path, status, summary, request_safe=None):
|
||||
p = load_private()
|
||||
CALLS.parent.mkdir(parents=True, exist_ok=True)
|
||||
item = {"time": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "method": method, "path": path, "http_status": status, "summary": summary}
|
||||
if request_safe is not None:
|
||||
item["request"] = request_safe
|
||||
line = json.dumps(scrub(item, p), ensure_ascii=False, separators=(",", ":"))
|
||||
with CALLS.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
CALLS.chmod(0o600)
|
||||
|
||||
|
||||
def call(client, method, path, *, json_body=None, params=None, safe_request=None, timeout=60, log_response=True):
|
||||
url = API + path
|
||||
response = client.request(method, url, json=json_body, params=params, timeout=timeout)
|
||||
try:
|
||||
data = response.json()
|
||||
except Exception:
|
||||
data = None
|
||||
summary = data if data is not None else {"body_preview": response.text[:500]}
|
||||
record(method, path, response.status_code, summary if log_response else {"saved": bool(response.is_success)}, safe_request)
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"API {method} {path} returned HTTP {response.status_code}")
|
||||
if isinstance(data, dict) and data.get("success") is False:
|
||||
msg = str(data.get("message", "API operation returned success=false"))
|
||||
raise RuntimeError(scrub(msg, load_private()))
|
||||
return data
|
||||
|
||||
|
||||
def setup():
|
||||
p = load_private()
|
||||
state = {}
|
||||
with httpx.Client() as c:
|
||||
state["initial_model_check"] = call(c, "GET", "/api/model-config/check-ready")
|
||||
existing = call(c, "GET", "/api/model-config/list")
|
||||
existing_configs = (existing or {}).get("data") or []
|
||||
for item in (
|
||||
{"provider":"OpenAI-compatible MiMo trial", "apiKey":p["chat_local_token"], "baseUrl":PROXY, "modelName":p["chat_model"], "modelType":"CHAT", "temperature":0.0, "maxTokens":5000, "isActive":False, "completionsPath":"/chat/completions"},
|
||||
{"provider":"OpenAI-compatible Qwen embedding trial", "apiKey":p["embedding_local_token"], "baseUrl":PROXY, "modelName":p["embedding_model"], "modelType":"EMBEDDING", "temperature":0.0, "maxTokens":5000, "isActive":False, "embeddingsPath":"/embeddings"},
|
||||
):
|
||||
match = next((x for x in existing_configs if x.get("modelType") == item["modelType"] and x.get("modelName") == item["modelName"]), None)
|
||||
if match is None:
|
||||
result = call(c, "POST", "/api/model-config/add", json_body=item, safe_request={"provider":item["provider"], "modelName":item["modelName"], "modelType":item["modelType"], "baseUrl":"local evidence proxy", "apiKey":"[REDACTED]"})
|
||||
state[item["modelType"].lower()+"_add"] = result
|
||||
config_list = call(c, "GET", "/api/model-config/list")
|
||||
configs = (config_list or {}).get("data") or []
|
||||
for kind in ("CHAT", "EMBEDDING"):
|
||||
match = next((x for x in configs if x.get("modelType") == kind and x.get("modelName") == (p["chat_model"] if kind == "CHAT" else p["embedding_model"])), None)
|
||||
if not match or not match.get("id"):
|
||||
raise RuntimeError("could not resolve newly stored model configuration")
|
||||
model_id = match["id"]
|
||||
safe = {"modelType":kind,"modelName":match.get("modelName"),"id":model_id}
|
||||
state[kind.lower()+"_model_id"] = model_id
|
||||
call(c, "POST", f"/api/model-config/test/{model_id}", safe_request=safe)
|
||||
call(c, "POST", f"/api/model-config/activate/{model_id}", safe_request=safe)
|
||||
state["model_check"] = call(c, "GET", "/api/model-config/check-ready")
|
||||
ds_body = {"name":"DataAgent 质量只读副本53307", "type":"mysql", "host":"127.0.0.1", "port":53307, "databaseName":"quality_trial", "username":p["quality_user"], "password":p["quality_password"], "status":"active", "description":"isolated read-only duplicate from quality trial SQLite"}
|
||||
ds = call(c, "POST", "/api/datasource", json_body=ds_body, safe_request={"name":ds_body["name"],"type":"mysql","host":"127.0.0.1","port":53307,"databaseName":"quality_trial","username":p["quality_user"],"password":"[REDACTED]"})
|
||||
dsid = ds.get("id")
|
||||
if not dsid: raise RuntimeError("datasource creation did not return an id")
|
||||
state["datasource_id"] = dsid
|
||||
state["datasource_test"] = call(c, "POST", f"/api/datasource/{dsid}/test", safe_request={"id":dsid})
|
||||
prompt = """你是质量中心只读数据分析助手。请使用已连接数据库的正式月度指标表和产品分组表,先理解官方字段语义再查询。汽车交通合计为scope_key=auto_combined(国内+海外),国内为auto_domestic,范围不可混用;官方常用综合合格率字段为official_composite_rate,黑片与亮片分别为black_rate、bright_rate。official_scope_month_metrics是正式月度汇总;产品分组表只用于规格线索,不能据此声称对总体变化的贡献。所有NULL原样保留,不得补0。月份仅2026-01至2026-09,产品明细仅2026-08、09。用中文回答;把查询直接支持的事实、计算推断、待验证假设明确分开;根据用户任务自主决定是否生成报告,用户明确要求简短答案或不生成报告时服从。只读查询,不修改数据。"""
|
||||
agent = call(c, "POST", "/api/agent", json_body={"name":"质量中心只读试验 DataAgent","description":"基于发布指标副本的本地只读分析试跑","prompt":prompt,"category":"质量分析","tags":"汽车交通,质量指标,试验"}, safe_request={"name":"质量中心只读试验 DataAgent","description":"based on isolated read-only KPI duplicate","prompt":"business semantics and evidence policy only"})
|
||||
aid = agent.get("id")
|
||||
if not aid: raise RuntimeError("agent creation did not return an id")
|
||||
state["agent_id"] = aid
|
||||
call(c, "POST", f"/api/agent/{aid}/datasources/{dsid}", safe_request={"agent_id":aid,"datasource_id":dsid})
|
||||
tables = call(c, "GET", f"/api/datasource/{dsid}/tables")
|
||||
names = tables if isinstance(tables, list) else (tables or {}).get("data", [])
|
||||
chosen = [x for x in names if x in {"official_scope_month_metrics", "official_product_month_rows"}]
|
||||
if len(chosen) != 2:
|
||||
raise RuntimeError("DataAgent saw unexpected number of quality tables")
|
||||
state["tables"] = chosen
|
||||
call(c, "POST", f"/api/agent/{aid}/datasources/tables", json_body={"datasourceId":dsid,"tables":chosen}, safe_request={"agent_id":aid,"datasource_id":dsid,"tables":chosen})
|
||||
state["schema_init"] = call(c, "POST", f"/api/agent/{aid}/datasources/init", safe_request={"agent_id":aid,"datasource_id":dsid,"tables":chosen}, timeout=300)
|
||||
(TRIAL/"trial-state.json").write_text(json.dumps(state,ensure_ascii=False,indent=2))
|
||||
(TRIAL/"trial-state.json").chmod(0o600)
|
||||
print(json.dumps({"setup":"complete","agent_id":state["agent_id"],"datasource_id":state["datasource_id"],"tables":state["tables"],"schema_init":state.get("schema_init")},ensure_ascii=False))
|
||||
|
||||
|
||||
def finalize_nodes(label: str, session_id: str, nodes: list[dict], export_html: bool = False):
|
||||
"""Persist the upstream Nuxt-style timeline and export its Markdown as HTML."""
|
||||
node_blocks=[]
|
||||
current_key=None
|
||||
report_markdown=""
|
||||
report_block=None
|
||||
for n in nodes:
|
||||
node_name=n.get("nodeName")
|
||||
step_key=n.get("stepId") or f"{node_name}:{n.get('attempt',1)}"
|
||||
if step_key != current_key:
|
||||
node_blocks.append([])
|
||||
current_key=step_key
|
||||
block=node_blocks[-1]
|
||||
if node_name=="ReportGeneratorNode":
|
||||
if report_block is None:
|
||||
report_block=dict(n)
|
||||
report_block["text"]=""
|
||||
report_block["textType"]="MARK_DOWN"
|
||||
block.append(report_block)
|
||||
if n.get("textType")=="MARK_DOWN":
|
||||
report_markdown += str(n.get("text", ""))
|
||||
report_block["text"] = report_markdown
|
||||
continue
|
||||
block.append(n)
|
||||
timeline_content=json.dumps(node_blocks,ensure_ascii=False)
|
||||
html_file=None
|
||||
html_status=None
|
||||
with httpx.Client(timeout=60) as c:
|
||||
timeline_resp=call(c,"POST",f"/api/sessions/{session_id}/messages",json_body={"sessionId":session_id,"role":"assistant","content":timeline_content,"messageType":"timeline"},safe_request={"session_id":session_id,"role":"assistant","messageType":"timeline","node_blocks":len(node_blocks),"content_chars":len(timeline_content)},timeout=60,log_response=False)
|
||||
if report_markdown and export_html:
|
||||
html_resp=c.post(API+f"/api/sessions/{session_id}/reports/html",content=report_markdown.encode("utf-8"),headers={"Content-Type":"text/plain;charset=utf-8"},timeout=60)
|
||||
html_status=html_resp.status_code
|
||||
html_path=OUT/f"{label}.html"
|
||||
if html_resp.status_code==200:
|
||||
html_path.write_bytes(html_resp.content)
|
||||
html_path.chmod(0o600)
|
||||
html_file=str(html_path)
|
||||
record("POST",f"/api/sessions/{session_id}/reports/html",html_resp.status_code,{"content_type":html_resp.headers.get("content-type"),"bytes":len(html_resp.content),"saved_file":html_file}, {"session_id":session_id,"markdown_chars":len(report_markdown)})
|
||||
if html_resp.status_code>=400:
|
||||
raise RuntimeError(f"HTML report export returned HTTP {html_resp.status_code}")
|
||||
return {"report_markdown_chars":len(report_markdown),"report_html_status":html_status,"report_html_file":html_file,"timeline_saved":bool(timeline_resp),"node_blocks":len(node_blocks)}
|
||||
|
||||
|
||||
def stream_question(label: str, query: str, session_id: str | None = None, timeout: int = 600, export_html: bool = False):
|
||||
"""Issue one original DataAgent Graph request and store raw SSE evidence (credentials redacted)."""
|
||||
from scripts.dataagent_trial import load_private
|
||||
p = load_private()
|
||||
state = json.loads((TRIAL/"trial-state.json").read_text())
|
||||
aid = state["agent_id"]
|
||||
started = time.time()
|
||||
with httpx.Client(timeout=None) as c:
|
||||
if session_id is None:
|
||||
s = call(c, "POST", f"/api/agent/{aid}/sessions", json_body={"title":label}, safe_request={"agent_id":aid,"title":label})
|
||||
session_id = s.get("id")
|
||||
if not session_id: raise RuntimeError("chat session creation did not return id")
|
||||
msg = {"sessionId":session_id,"role":"user","content":query,"messageType":"text"}
|
||||
call(c, "POST", f"/api/sessions/{session_id}/messages", json_body=msg, safe_request={"session_id":session_id,"role":"user","messageType":"text","content":query})
|
||||
params={"agentId":str(aid),"conversationId":session_id,"query":query,"humanFeedback":"false","rejectedPlan":"false","nl2sqlOnly":"false"}
|
||||
req = c.build_request("GET", API+"/api/stream/search", params=params, headers={"Accept":"text/event-stream"}, timeout=timeout)
|
||||
raw = bytearray()
|
||||
try:
|
||||
response = c.send(req, stream=True)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
for chunk in response.iter_raw():
|
||||
raw.extend(chunk)
|
||||
finally:
|
||||
response.close()
|
||||
except Exception as exc:
|
||||
path=OUT/f"{label}.sse.redacted.txt"
|
||||
safe=bytes(raw).decode("utf-8",errors="replace")
|
||||
for k in ("upstream_key","embedding_key","upstream_base","embedding_base","db_password"):
|
||||
if p.get(k): safe=safe.replace(p[k],"[REDACTED]")
|
||||
path.write_text(safe,encoding="utf-8"); path.chmod(0o600)
|
||||
record("GET","/api/stream/search",0,{"error_type":type(exc).__name__,"duration_ms":round((time.time()-started)*1000)}, {"agent_id":aid,"session_id":session_id,"query":query})
|
||||
raise
|
||||
text=bytes(raw).decode("utf-8",errors="replace")
|
||||
for k in ("upstream_key","embedding_key","upstream_base","embedding_base","db_password"):
|
||||
if p.get(k): text=text.replace(p[k],"[REDACTED]")
|
||||
path=OUT/f"{label}.sse.redacted.txt"
|
||||
path.write_text(text,encoding="utf-8"); path.chmod(0o600)
|
||||
nodes=[]
|
||||
for block in re.split(r"\n\n",text):
|
||||
if not block.startswith("data:"): continue
|
||||
payload=block[len("data:"):].strip()
|
||||
try: nodes.append(json.loads(payload))
|
||||
except Exception: pass
|
||||
final_text="\n".join(str(n.get("text","")) for n in nodes if n.get("eventType")=="FINAL_ANSWER")
|
||||
finalization=finalize_nodes(label,session_id,nodes,export_html=export_html)
|
||||
summaries=[]
|
||||
for n in nodes:
|
||||
item={k:n.get(k) for k in ("nodeName","eventType","textType","error","stepId","attempt") if k in n}
|
||||
if n.get("text"):
|
||||
item["text_preview"]=str(n["text"])[:800]
|
||||
summaries.append(item)
|
||||
timeline={"label":label,"query":query,"session_id":session_id,"duration_seconds":round(time.time()-started,2),"sse_http":200,"sse_event_count":len(nodes),"final_answer":final_text,**finalization,"nodes":summaries}
|
||||
(OUT/f"{label}.summary.json").write_text(json.dumps(timeline,ensure_ascii=False,indent=2)); (OUT/f"{label}.summary.json").chmod(0o600)
|
||||
record("GET","/api/stream/search",200,{"sse_event_count":len(nodes),"duration_seconds":timeline["duration_seconds"],"session_id":session_id,"output_file":str(path.relative_to(OUT))},{"agent_id":aid,"session_id":session_id,"query":query})
|
||||
print(json.dumps({"label":label,"session_id":session_id,"duration_seconds":timeline["duration_seconds"],"sse_event_count":len(nodes),"sse_file":str(path),"summary_file":str(OUT/f"{label}.summary.json")},ensure_ascii=False))
|
||||
return timeline
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
ap=argparse.ArgumentParser()
|
||||
ap.add_argument("action",choices=["setup","short-report","finalize-short-report","followup","deep"])
|
||||
args=ap.parse_args()
|
||||
if args.action=="setup": setup()
|
||||
elif args.action=="short-report": stream_question("task3-short-report","请查询2026年8月和9月汽车交通合计的正式综合合格率、黑片合格率和亮片合格率,用查询结果生成一页精简中文HTML报告,注明环比变化和数据来源。报告不超过600字。",timeout=600,export_html=True)
|
||||
elif args.action=="finalize-short-report":
|
||||
label="task3-short-report"; summary_path=OUT/f"{label}.summary.json"; summary=json.loads(summary_path.read_text()); raw=(OUT/f"{label}.sse.redacted.txt").read_text(); nodes=[]
|
||||
for block in re.split(r"\n\n",raw):
|
||||
if block.startswith("data:"):
|
||||
try: nodes.append(json.loads(block[5:].strip()))
|
||||
except Exception: pass
|
||||
summary.update(finalize_nodes(label,summary["session_id"],nodes,export_html=True)); summary_path.write_text(json.dumps(summary,ensure_ascii=False,indent=2)); summary_path.chmod(0o600)
|
||||
print(json.dumps({k:summary.get(k) for k in ("session_id","report_markdown_chars","report_html_status","report_html_file","timeline_saved","node_blocks")},ensure_ascii=False))
|
||||
elif args.action=="followup":
|
||||
short=json.loads((OUT/"task3-short-report.summary.json").read_text())
|
||||
stream_question("task2-followup","那国内9月合格绿呢,比上月变化多大?直接简短回答,不生成报告。",session_id=short["session_id"],timeout=300)
|
||||
elif args.action=="deep": stream_question("task1-deep","9月份汽车交通合格率为什么降低?请自行查询数据,研究主要因素。发现线索后继续追查,并说明哪些判断有证据、哪些还需验证。先给我分析结论,下一轮再生成报告。",timeout=600)
|
||||
|
||||
if __name__=="__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user