初始化:质量数据分析平台源码
This commit is contained in:
@@ -0,0 +1,143 @@
|
||||
"""Run real nao quality conversations, with evidence and an active cancellation budget."""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request
|
||||
|
||||
from nao_trial import ROOT, model_settings
|
||||
from nao_trial_chat import BASE, opener
|
||||
|
||||
OUT = ROOT / "output/nao-quality"
|
||||
|
||||
|
||||
def run(label, question, chat_id=None, timeout=240):
|
||||
if not re.fullmatch(r"[a-zA-Z0-9_-]{1,80}", label):
|
||||
raise ValueError("Use a short alphanumeric evidence label")
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
client = opener()
|
||||
model = model_settings()[2]
|
||||
body = {"message": {"text": question},
|
||||
"model": {"provider": "openaiCompatible", "modelId": model},
|
||||
"timezone": "Asia/Shanghai"}
|
||||
if chat_id:
|
||||
body["chatId"] = chat_id
|
||||
(OUT / (label + "-request.json")).write_text(json.dumps(body, ensure_ascii=False, indent=2))
|
||||
state = {"chatId": chat_id, "budgetExpired": False, "stopRequest": None}
|
||||
events, text_blocks, tools, tool_outputs = [], {}, [], {}
|
||||
finished = threading.Event()
|
||||
response_ref = []
|
||||
|
||||
def cancel_at_budget():
|
||||
if finished.wait(timeout):
|
||||
return
|
||||
state["budgetExpired"] = True
|
||||
if state.get("chatId"):
|
||||
request = Request(BASE + "/api/trpc/chat.stop", method="POST",
|
||||
data=json.dumps({"json": {"chatId": state["chatId"]}}).encode(),
|
||||
headers={"Content-Type": "application/json", "Origin": BASE})
|
||||
try:
|
||||
with client.open(request, timeout=15) as response:
|
||||
state["stopRequest"] = {"httpStatus": response.status}
|
||||
except HTTPError as error:
|
||||
state["stopRequest"] = {"httpStatus": error.code}
|
||||
except OSError as error:
|
||||
state["stopRequest"] = {"errorType": type(error).__name__}
|
||||
if response_ref:
|
||||
response_ref[0].close()
|
||||
|
||||
timer = threading.Thread(target=cancel_at_budget, daemon=True)
|
||||
started = time.monotonic()
|
||||
timer.start()
|
||||
error_type = None
|
||||
model_base, model_key, _ = model_settings()
|
||||
request = Request(BASE + "/api/agent/", data=json.dumps(body, ensure_ascii=False).encode(),
|
||||
headers={"Content-Type": "application/json", "Origin": BASE,
|
||||
"Accept": "text/event-stream"})
|
||||
try:
|
||||
with client.open(request, timeout=60) as response, (OUT / (label + ".sse")).open("w") as stream:
|
||||
response_ref.append(response)
|
||||
for raw in response:
|
||||
line = raw.decode("utf-8", errors="replace").replace(model_key, "[REDACTED_KEY]")
|
||||
line = line.replace(model_base, "[REDACTED_MODEL_ENDPOINT]")
|
||||
stream.write(line)
|
||||
stream.flush()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
raw_data = line[5:].strip()
|
||||
if raw_data == "[DONE]":
|
||||
break
|
||||
try:
|
||||
event = json.loads(raw_data)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
events.append(event)
|
||||
kind = event.get("type")
|
||||
if kind == "data-newChat":
|
||||
state["chatId"] = event.get("data", {}).get("id") or state["chatId"]
|
||||
elif kind == "text-delta":
|
||||
block_id = event.get("id", "text")
|
||||
text_blocks[block_id] = text_blocks.get(block_id, "") + event.get("delta", "")
|
||||
elif kind == "tool-input-available":
|
||||
tools.append({"toolCallId": event.get("toolCallId"), "tool": event.get("toolName"),
|
||||
"input": event.get("input")})
|
||||
print("tool", event.get("toolName"), flush=True)
|
||||
elif kind == "tool-output-available":
|
||||
tool_outputs[event.get("toolCallId")] = event.get("output")
|
||||
except (OSError, ValueError, AttributeError) as error:
|
||||
error_type = type(error).__name__
|
||||
finally:
|
||||
finished.set()
|
||||
timer.join(timeout=1)
|
||||
for tool in tools:
|
||||
tool["output"] = tool_outputs.get(tool["toolCallId"])
|
||||
texts = [text.strip() for text in text_blocks.values() if text.strip()]
|
||||
final_text = texts[-1] if texts else ""
|
||||
finish_events = [event for event in events if event.get("type") == "finish"]
|
||||
truncated = any(event.get("finishReason") == "length" for event in finish_events)
|
||||
error_events = [event for event in events if event.get("type") in ("error", "tool-output-error")]
|
||||
summary = {
|
||||
"label": label, "question": question, "model": model, "chatId": state["chatId"],
|
||||
"durationSeconds": round(time.monotonic() - started, 2),
|
||||
"status": "budget_cancelled" if state["budgetExpired"] else
|
||||
"truncated" if truncated else
|
||||
"finished" if finish_events and final_text and not any(e.get("type") == "error" for e in error_events)
|
||||
else "incomplete",
|
||||
"finishReceived": bool(finish_events), "finishReasons": [event.get("finishReason") for event in finish_events], "errors": error_events,
|
||||
"transportErrorType": error_type, "stopRequest": state["stopRequest"],
|
||||
"toolCalls": tools, "eventCount": len(events), "answer": final_text,
|
||||
}
|
||||
(OUT / (label + "-events.json")).write_text(json.dumps(events, ensure_ascii=False, indent=2))
|
||||
(OUT / (label + "-run.json")).write_text(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
(OUT / (label + "-answer.md")).write_text(final_text)
|
||||
(OUT / (label + "-all-text.md")).write_text("\n\n---\n\n".join(texts))
|
||||
for tool in tools:
|
||||
result = tool.get("output")
|
||||
if tool["tool"] != "story" or not isinstance(result, dict) or not result.get("success"):
|
||||
continue
|
||||
slug = result.get("id")
|
||||
(OUT / (label + "-story.md")).write_text(result.get("code", ""))
|
||||
data = {"json": {"chatId": state["chatId"], "storySlug": slug, "format": "html"}}
|
||||
request = Request(BASE + "/api/trpc/story.download?input=" + quote(json.dumps(data)),
|
||||
headers={"Origin": BASE})
|
||||
with client.open(request, timeout=30) as response:
|
||||
export = json.load(response)["result"]["data"]["json"]
|
||||
(OUT / (label + "-report.html")).write_bytes(base64.b64decode(export["data"]))
|
||||
print(json.dumps({key: summary[key] for key in
|
||||
("label", "chatId", "durationSeconds", "status", "eventCount")}, ensure_ascii=False))
|
||||
return summary
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("label")
|
||||
parser.add_argument("question")
|
||||
parser.add_argument("--chat-id")
|
||||
parser.add_argument("--timeout", type=int, default=240)
|
||||
args = parser.parse_args()
|
||||
run(args.label, args.question, args.chat_id, args.timeout)
|
||||
Reference in New Issue
Block a user