144 lines
6.9 KiB
Python
144 lines
6.9 KiB
Python
"""Run real nao quality conversations, with evidence and an active cancellation budget."""
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import re
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError
|
|
from urllib.parse import quote
|
|
from urllib.request import Request
|
|
|
|
from nao_trial import ROOT, model_settings
|
|
from nao_trial_chat import BASE, opener
|
|
|
|
OUT = ROOT / "output/nao-quality"
|
|
|
|
|
|
def run(label, question, chat_id=None, timeout=240):
|
|
if not re.fullmatch(r"[a-zA-Z0-9_-]{1,80}", label):
|
|
raise ValueError("Use a short alphanumeric evidence label")
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
client = opener()
|
|
model = model_settings()[2]
|
|
body = {"message": {"text": question},
|
|
"model": {"provider": "openaiCompatible", "modelId": model},
|
|
"timezone": "Asia/Shanghai"}
|
|
if chat_id:
|
|
body["chatId"] = chat_id
|
|
(OUT / (label + "-request.json")).write_text(json.dumps(body, ensure_ascii=False, indent=2))
|
|
state = {"chatId": chat_id, "budgetExpired": False, "stopRequest": None}
|
|
events, text_blocks, tools, tool_outputs = [], {}, [], {}
|
|
finished = threading.Event()
|
|
response_ref = []
|
|
|
|
def cancel_at_budget():
|
|
if finished.wait(timeout):
|
|
return
|
|
state["budgetExpired"] = True
|
|
if state.get("chatId"):
|
|
request = Request(BASE + "/api/trpc/chat.stop", method="POST",
|
|
data=json.dumps({"json": {"chatId": state["chatId"]}}).encode(),
|
|
headers={"Content-Type": "application/json", "Origin": BASE})
|
|
try:
|
|
with client.open(request, timeout=15) as response:
|
|
state["stopRequest"] = {"httpStatus": response.status}
|
|
except HTTPError as error:
|
|
state["stopRequest"] = {"httpStatus": error.code}
|
|
except OSError as error:
|
|
state["stopRequest"] = {"errorType": type(error).__name__}
|
|
if response_ref:
|
|
response_ref[0].close()
|
|
|
|
timer = threading.Thread(target=cancel_at_budget, daemon=True)
|
|
started = time.monotonic()
|
|
timer.start()
|
|
error_type = None
|
|
model_base, model_key, _ = model_settings()
|
|
request = Request(BASE + "/api/agent/", data=json.dumps(body, ensure_ascii=False).encode(),
|
|
headers={"Content-Type": "application/json", "Origin": BASE,
|
|
"Accept": "text/event-stream"})
|
|
try:
|
|
with client.open(request, timeout=60) as response, (OUT / (label + ".sse")).open("w") as stream:
|
|
response_ref.append(response)
|
|
for raw in response:
|
|
line = raw.decode("utf-8", errors="replace").replace(model_key, "[REDACTED_KEY]")
|
|
line = line.replace(model_base, "[REDACTED_MODEL_ENDPOINT]")
|
|
stream.write(line)
|
|
stream.flush()
|
|
if not line.startswith("data:"):
|
|
continue
|
|
raw_data = line[5:].strip()
|
|
if raw_data == "[DONE]":
|
|
break
|
|
try:
|
|
event = json.loads(raw_data)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
events.append(event)
|
|
kind = event.get("type")
|
|
if kind == "data-newChat":
|
|
state["chatId"] = event.get("data", {}).get("id") or state["chatId"]
|
|
elif kind == "text-delta":
|
|
block_id = event.get("id", "text")
|
|
text_blocks[block_id] = text_blocks.get(block_id, "") + event.get("delta", "")
|
|
elif kind == "tool-input-available":
|
|
tools.append({"toolCallId": event.get("toolCallId"), "tool": event.get("toolName"),
|
|
"input": event.get("input")})
|
|
print("tool", event.get("toolName"), flush=True)
|
|
elif kind == "tool-output-available":
|
|
tool_outputs[event.get("toolCallId")] = event.get("output")
|
|
except (OSError, ValueError, AttributeError) as error:
|
|
error_type = type(error).__name__
|
|
finally:
|
|
finished.set()
|
|
timer.join(timeout=1)
|
|
for tool in tools:
|
|
tool["output"] = tool_outputs.get(tool["toolCallId"])
|
|
texts = [text.strip() for text in text_blocks.values() if text.strip()]
|
|
final_text = texts[-1] if texts else ""
|
|
finish_events = [event for event in events if event.get("type") == "finish"]
|
|
truncated = any(event.get("finishReason") == "length" for event in finish_events)
|
|
error_events = [event for event in events if event.get("type") in ("error", "tool-output-error")]
|
|
summary = {
|
|
"label": label, "question": question, "model": model, "chatId": state["chatId"],
|
|
"durationSeconds": round(time.monotonic() - started, 2),
|
|
"status": "budget_cancelled" if state["budgetExpired"] else
|
|
"truncated" if truncated else
|
|
"finished" if finish_events and final_text and not any(e.get("type") == "error" for e in error_events)
|
|
else "incomplete",
|
|
"finishReceived": bool(finish_events), "finishReasons": [event.get("finishReason") for event in finish_events], "errors": error_events,
|
|
"transportErrorType": error_type, "stopRequest": state["stopRequest"],
|
|
"toolCalls": tools, "eventCount": len(events), "answer": final_text,
|
|
}
|
|
(OUT / (label + "-events.json")).write_text(json.dumps(events, ensure_ascii=False, indent=2))
|
|
(OUT / (label + "-run.json")).write_text(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
(OUT / (label + "-answer.md")).write_text(final_text)
|
|
(OUT / (label + "-all-text.md")).write_text("\n\n---\n\n".join(texts))
|
|
for tool in tools:
|
|
result = tool.get("output")
|
|
if tool["tool"] != "story" or not isinstance(result, dict) or not result.get("success"):
|
|
continue
|
|
slug = result.get("id")
|
|
(OUT / (label + "-story.md")).write_text(result.get("code", ""))
|
|
data = {"json": {"chatId": state["chatId"], "storySlug": slug, "format": "html"}}
|
|
request = Request(BASE + "/api/trpc/story.download?input=" + quote(json.dumps(data)),
|
|
headers={"Origin": BASE})
|
|
with client.open(request, timeout=30) as response:
|
|
export = json.load(response)["result"]["data"]["json"]
|
|
(OUT / (label + "-report.html")).write_bytes(base64.b64decode(export["data"]))
|
|
print(json.dumps({key: summary[key] for key in
|
|
("label", "chatId", "durationSeconds", "status", "eventCount")}, ensure_ascii=False))
|
|
return summary
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("label")
|
|
parser.add_argument("question")
|
|
parser.add_argument("--chat-id")
|
|
parser.add_argument("--timeout", type=int, default=240)
|
|
args = parser.parse_args()
|
|
run(args.label, args.question, args.chat_id, args.timeout)
|