239 lines
16 KiB
Python
239 lines
16 KiB
Python
"""Configure upstream DataAgent only through its own API and run trial graphs."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.parse import urlencode
|
|
|
|
import httpx
|
|
|
|
from scripts.dataagent_trial import OUT, TRIAL, load_private
|
|
|
|
API = "http://127.0.0.1:5673"
|
|
PROXY = "http://127.0.0.1:5676/v1"
|
|
CALLS = OUT / "api-calls.jsonl"
|
|
STATE = TRIAL / "trial-state.json"
|
|
|
|
|
|
def scrub(obj, p):
|
|
if isinstance(obj, dict):
|
|
return {k: ("[REDACTED]" if k.lower() in {"apikey", "password", "proxy_password"} else scrub(v, p)) for k, v in obj.items()}
|
|
if isinstance(obj, list):
|
|
return [scrub(x, p) for x in obj]
|
|
if isinstance(obj, str):
|
|
for k in ("upstream_key", "embedding_key", "upstream_base", "embedding_base", "db_password"):
|
|
if p.get(k): obj = obj.replace(p[k], "[REDACTED]")
|
|
return obj
|
|
return obj
|
|
|
|
|
|
def record(method, path, status, summary, request_safe=None):
|
|
p = load_private()
|
|
CALLS.parent.mkdir(parents=True, exist_ok=True)
|
|
item = {"time": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "method": method, "path": path, "http_status": status, "summary": summary}
|
|
if request_safe is not None:
|
|
item["request"] = request_safe
|
|
line = json.dumps(scrub(item, p), ensure_ascii=False, separators=(",", ":"))
|
|
with CALLS.open("a", encoding="utf-8") as f:
|
|
f.write(line + "\n")
|
|
CALLS.chmod(0o600)
|
|
|
|
|
|
def call(client, method, path, *, json_body=None, params=None, safe_request=None, timeout=60, log_response=True):
|
|
url = API + path
|
|
response = client.request(method, url, json=json_body, params=params, timeout=timeout)
|
|
try:
|
|
data = response.json()
|
|
except Exception:
|
|
data = None
|
|
summary = data if data is not None else {"body_preview": response.text[:500]}
|
|
record(method, path, response.status_code, summary if log_response else {"saved": bool(response.is_success)}, safe_request)
|
|
if response.status_code >= 400:
|
|
raise RuntimeError(f"API {method} {path} returned HTTP {response.status_code}")
|
|
if isinstance(data, dict) and data.get("success") is False:
|
|
msg = str(data.get("message", "API operation returned success=false"))
|
|
raise RuntimeError(scrub(msg, load_private()))
|
|
return data
|
|
|
|
|
|
def setup():
|
|
p = load_private()
|
|
state = {}
|
|
with httpx.Client() as c:
|
|
state["initial_model_check"] = call(c, "GET", "/api/model-config/check-ready")
|
|
existing = call(c, "GET", "/api/model-config/list")
|
|
existing_configs = (existing or {}).get("data") or []
|
|
for item in (
|
|
{"provider":"OpenAI-compatible MiMo trial", "apiKey":p["chat_local_token"], "baseUrl":PROXY, "modelName":p["chat_model"], "modelType":"CHAT", "temperature":0.0, "maxTokens":5000, "isActive":False, "completionsPath":"/chat/completions"},
|
|
{"provider":"OpenAI-compatible Qwen embedding trial", "apiKey":p["embedding_local_token"], "baseUrl":PROXY, "modelName":p["embedding_model"], "modelType":"EMBEDDING", "temperature":0.0, "maxTokens":5000, "isActive":False, "embeddingsPath":"/embeddings"},
|
|
):
|
|
match = next((x for x in existing_configs if x.get("modelType") == item["modelType"] and x.get("modelName") == item["modelName"]), None)
|
|
if match is None:
|
|
result = call(c, "POST", "/api/model-config/add", json_body=item, safe_request={"provider":item["provider"], "modelName":item["modelName"], "modelType":item["modelType"], "baseUrl":"local evidence proxy", "apiKey":"[REDACTED]"})
|
|
state[item["modelType"].lower()+"_add"] = result
|
|
config_list = call(c, "GET", "/api/model-config/list")
|
|
configs = (config_list or {}).get("data") or []
|
|
for kind in ("CHAT", "EMBEDDING"):
|
|
match = next((x for x in configs if x.get("modelType") == kind and x.get("modelName") == (p["chat_model"] if kind == "CHAT" else p["embedding_model"])), None)
|
|
if not match or not match.get("id"):
|
|
raise RuntimeError("could not resolve newly stored model configuration")
|
|
model_id = match["id"]
|
|
safe = {"modelType":kind,"modelName":match.get("modelName"),"id":model_id}
|
|
state[kind.lower()+"_model_id"] = model_id
|
|
call(c, "POST", f"/api/model-config/test/{model_id}", safe_request=safe)
|
|
call(c, "POST", f"/api/model-config/activate/{model_id}", safe_request=safe)
|
|
state["model_check"] = call(c, "GET", "/api/model-config/check-ready")
|
|
ds_body = {"name":"DataAgent 质量只读副本53307", "type":"mysql", "host":"127.0.0.1", "port":53307, "databaseName":"quality_trial", "username":p["quality_user"], "password":p["quality_password"], "status":"active", "description":"isolated read-only duplicate from quality trial SQLite"}
|
|
ds = call(c, "POST", "/api/datasource", json_body=ds_body, safe_request={"name":ds_body["name"],"type":"mysql","host":"127.0.0.1","port":53307,"databaseName":"quality_trial","username":p["quality_user"],"password":"[REDACTED]"})
|
|
dsid = ds.get("id")
|
|
if not dsid: raise RuntimeError("datasource creation did not return an id")
|
|
state["datasource_id"] = dsid
|
|
state["datasource_test"] = call(c, "POST", f"/api/datasource/{dsid}/test", safe_request={"id":dsid})
|
|
prompt = """你是质量中心只读数据分析助手。请使用已连接数据库的正式月度指标表和产品分组表,先理解官方字段语义再查询。汽车交通合计为scope_key=auto_combined(国内+海外),国内为auto_domestic,范围不可混用;官方常用综合合格率字段为official_composite_rate,黑片与亮片分别为black_rate、bright_rate。official_scope_month_metrics是正式月度汇总;产品分组表只用于规格线索,不能据此声称对总体变化的贡献。所有NULL原样保留,不得补0。月份仅2026-01至2026-09,产品明细仅2026-08、09。用中文回答;把查询直接支持的事实、计算推断、待验证假设明确分开;根据用户任务自主决定是否生成报告,用户明确要求简短答案或不生成报告时服从。只读查询,不修改数据。"""
|
|
agent = call(c, "POST", "/api/agent", json_body={"name":"质量中心只读试验 DataAgent","description":"基于发布指标副本的本地只读分析试跑","prompt":prompt,"category":"质量分析","tags":"汽车交通,质量指标,试验"}, safe_request={"name":"质量中心只读试验 DataAgent","description":"based on isolated read-only KPI duplicate","prompt":"business semantics and evidence policy only"})
|
|
aid = agent.get("id")
|
|
if not aid: raise RuntimeError("agent creation did not return an id")
|
|
state["agent_id"] = aid
|
|
call(c, "POST", f"/api/agent/{aid}/datasources/{dsid}", safe_request={"agent_id":aid,"datasource_id":dsid})
|
|
tables = call(c, "GET", f"/api/datasource/{dsid}/tables")
|
|
names = tables if isinstance(tables, list) else (tables or {}).get("data", [])
|
|
chosen = [x for x in names if x in {"official_scope_month_metrics", "official_product_month_rows"}]
|
|
if len(chosen) != 2:
|
|
raise RuntimeError("DataAgent saw unexpected number of quality tables")
|
|
state["tables"] = chosen
|
|
call(c, "POST", f"/api/agent/{aid}/datasources/tables", json_body={"datasourceId":dsid,"tables":chosen}, safe_request={"agent_id":aid,"datasource_id":dsid,"tables":chosen})
|
|
state["schema_init"] = call(c, "POST", f"/api/agent/{aid}/datasources/init", safe_request={"agent_id":aid,"datasource_id":dsid,"tables":chosen}, timeout=300)
|
|
(TRIAL/"trial-state.json").write_text(json.dumps(state,ensure_ascii=False,indent=2))
|
|
(TRIAL/"trial-state.json").chmod(0o600)
|
|
print(json.dumps({"setup":"complete","agent_id":state["agent_id"],"datasource_id":state["datasource_id"],"tables":state["tables"],"schema_init":state.get("schema_init")},ensure_ascii=False))
|
|
|
|
|
|
def finalize_nodes(label: str, session_id: str, nodes: list[dict], export_html: bool = False):
|
|
"""Persist the upstream Nuxt-style timeline and export its Markdown as HTML."""
|
|
node_blocks=[]
|
|
current_key=None
|
|
report_markdown=""
|
|
report_block=None
|
|
for n in nodes:
|
|
node_name=n.get("nodeName")
|
|
step_key=n.get("stepId") or f"{node_name}:{n.get('attempt',1)}"
|
|
if step_key != current_key:
|
|
node_blocks.append([])
|
|
current_key=step_key
|
|
block=node_blocks[-1]
|
|
if node_name=="ReportGeneratorNode":
|
|
if report_block is None:
|
|
report_block=dict(n)
|
|
report_block["text"]=""
|
|
report_block["textType"]="MARK_DOWN"
|
|
block.append(report_block)
|
|
if n.get("textType")=="MARK_DOWN":
|
|
report_markdown += str(n.get("text", ""))
|
|
report_block["text"] = report_markdown
|
|
continue
|
|
block.append(n)
|
|
timeline_content=json.dumps(node_blocks,ensure_ascii=False)
|
|
html_file=None
|
|
html_status=None
|
|
with httpx.Client(timeout=60) as c:
|
|
timeline_resp=call(c,"POST",f"/api/sessions/{session_id}/messages",json_body={"sessionId":session_id,"role":"assistant","content":timeline_content,"messageType":"timeline"},safe_request={"session_id":session_id,"role":"assistant","messageType":"timeline","node_blocks":len(node_blocks),"content_chars":len(timeline_content)},timeout=60,log_response=False)
|
|
if report_markdown and export_html:
|
|
html_resp=c.post(API+f"/api/sessions/{session_id}/reports/html",content=report_markdown.encode("utf-8"),headers={"Content-Type":"text/plain;charset=utf-8"},timeout=60)
|
|
html_status=html_resp.status_code
|
|
html_path=OUT/f"{label}.html"
|
|
if html_resp.status_code==200:
|
|
html_path.write_bytes(html_resp.content)
|
|
html_path.chmod(0o600)
|
|
html_file=str(html_path)
|
|
record("POST",f"/api/sessions/{session_id}/reports/html",html_resp.status_code,{"content_type":html_resp.headers.get("content-type"),"bytes":len(html_resp.content),"saved_file":html_file}, {"session_id":session_id,"markdown_chars":len(report_markdown)})
|
|
if html_resp.status_code>=400:
|
|
raise RuntimeError(f"HTML report export returned HTTP {html_resp.status_code}")
|
|
return {"report_markdown_chars":len(report_markdown),"report_html_status":html_status,"report_html_file":html_file,"timeline_saved":bool(timeline_resp),"node_blocks":len(node_blocks)}
|
|
|
|
|
|
def stream_question(label: str, query: str, session_id: str | None = None, timeout: int = 600, export_html: bool = False):
|
|
"""Issue one original DataAgent Graph request and store raw SSE evidence (credentials redacted)."""
|
|
from scripts.dataagent_trial import load_private
|
|
p = load_private()
|
|
state = json.loads((TRIAL/"trial-state.json").read_text())
|
|
aid = state["agent_id"]
|
|
started = time.time()
|
|
with httpx.Client(timeout=None) as c:
|
|
if session_id is None:
|
|
s = call(c, "POST", f"/api/agent/{aid}/sessions", json_body={"title":label}, safe_request={"agent_id":aid,"title":label})
|
|
session_id = s.get("id")
|
|
if not session_id: raise RuntimeError("chat session creation did not return id")
|
|
msg = {"sessionId":session_id,"role":"user","content":query,"messageType":"text"}
|
|
call(c, "POST", f"/api/sessions/{session_id}/messages", json_body=msg, safe_request={"session_id":session_id,"role":"user","messageType":"text","content":query})
|
|
params={"agentId":str(aid),"conversationId":session_id,"query":query,"humanFeedback":"false","rejectedPlan":"false","nl2sqlOnly":"false"}
|
|
req = c.build_request("GET", API+"/api/stream/search", params=params, headers={"Accept":"text/event-stream"}, timeout=timeout)
|
|
raw = bytearray()
|
|
try:
|
|
response = c.send(req, stream=True)
|
|
try:
|
|
response.raise_for_status()
|
|
for chunk in response.iter_raw():
|
|
raw.extend(chunk)
|
|
finally:
|
|
response.close()
|
|
except Exception as exc:
|
|
path=OUT/f"{label}.sse.redacted.txt"
|
|
safe=bytes(raw).decode("utf-8",errors="replace")
|
|
for k in ("upstream_key","embedding_key","upstream_base","embedding_base","db_password"):
|
|
if p.get(k): safe=safe.replace(p[k],"[REDACTED]")
|
|
path.write_text(safe,encoding="utf-8"); path.chmod(0o600)
|
|
record("GET","/api/stream/search",0,{"error_type":type(exc).__name__,"duration_ms":round((time.time()-started)*1000)}, {"agent_id":aid,"session_id":session_id,"query":query})
|
|
raise
|
|
text=bytes(raw).decode("utf-8",errors="replace")
|
|
for k in ("upstream_key","embedding_key","upstream_base","embedding_base","db_password"):
|
|
if p.get(k): text=text.replace(p[k],"[REDACTED]")
|
|
path=OUT/f"{label}.sse.redacted.txt"
|
|
path.write_text(text,encoding="utf-8"); path.chmod(0o600)
|
|
nodes=[]
|
|
for block in re.split(r"\n\n",text):
|
|
if not block.startswith("data:"): continue
|
|
payload=block[len("data:"):].strip()
|
|
try: nodes.append(json.loads(payload))
|
|
except Exception: pass
|
|
final_text="\n".join(str(n.get("text","")) for n in nodes if n.get("eventType")=="FINAL_ANSWER")
|
|
finalization=finalize_nodes(label,session_id,nodes,export_html=export_html)
|
|
summaries=[]
|
|
for n in nodes:
|
|
item={k:n.get(k) for k in ("nodeName","eventType","textType","error","stepId","attempt") if k in n}
|
|
if n.get("text"):
|
|
item["text_preview"]=str(n["text"])[:800]
|
|
summaries.append(item)
|
|
timeline={"label":label,"query":query,"session_id":session_id,"duration_seconds":round(time.time()-started,2),"sse_http":200,"sse_event_count":len(nodes),"final_answer":final_text,**finalization,"nodes":summaries}
|
|
(OUT/f"{label}.summary.json").write_text(json.dumps(timeline,ensure_ascii=False,indent=2)); (OUT/f"{label}.summary.json").chmod(0o600)
|
|
record("GET","/api/stream/search",200,{"sse_event_count":len(nodes),"duration_seconds":timeline["duration_seconds"],"session_id":session_id,"output_file":str(path.relative_to(OUT))},{"agent_id":aid,"session_id":session_id,"query":query})
|
|
print(json.dumps({"label":label,"session_id":session_id,"duration_seconds":timeline["duration_seconds"],"sse_event_count":len(nodes),"sse_file":str(path),"summary_file":str(OUT/f"{label}.summary.json")},ensure_ascii=False))
|
|
return timeline
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
ap=argparse.ArgumentParser()
|
|
ap.add_argument("action",choices=["setup","short-report","finalize-short-report","followup","deep"])
|
|
args=ap.parse_args()
|
|
if args.action=="setup": setup()
|
|
elif args.action=="short-report": stream_question("task3-short-report","请查询2026年8月和9月汽车交通合计的正式综合合格率、黑片合格率和亮片合格率,用查询结果生成一页精简中文HTML报告,注明环比变化和数据来源。报告不超过600字。",timeout=600,export_html=True)
|
|
elif args.action=="finalize-short-report":
|
|
label="task3-short-report"; summary_path=OUT/f"{label}.summary.json"; summary=json.loads(summary_path.read_text()); raw=(OUT/f"{label}.sse.redacted.txt").read_text(); nodes=[]
|
|
for block in re.split(r"\n\n",raw):
|
|
if block.startswith("data:"):
|
|
try: nodes.append(json.loads(block[5:].strip()))
|
|
except Exception: pass
|
|
summary.update(finalize_nodes(label,summary["session_id"],nodes,export_html=True)); summary_path.write_text(json.dumps(summary,ensure_ascii=False,indent=2)); summary_path.chmod(0o600)
|
|
print(json.dumps({k:summary.get(k) for k in ("session_id","report_markdown_chars","report_html_status","report_html_file","timeline_saved","node_blocks")},ensure_ascii=False))
|
|
elif args.action=="followup":
|
|
short=json.loads((OUT/"task3-short-report.summary.json").read_text())
|
|
stream_question("task2-followup","那国内9月合格绿呢,比上月变化多大?直接简短回答,不生成报告。",session_id=short["session_id"],timeout=300)
|
|
elif args.action=="deep": stream_question("task1-deep","9月份汽车交通合格率为什么降低?请自行查询数据,研究主要因素。发现线索后继续追查,并说明哪些判断有证据、哪些还需验证。先给我分析结论,下一轮再生成报告。",timeout=600)
|
|
|
|
if __name__=="__main__":
|
|
main()
|