"""Run real nao quality conversations, with evidence and an active cancellation budget.""" import argparse import base64 import json import re import threading import time from pathlib import Path from urllib.error import HTTPError from urllib.parse import quote from urllib.request import Request from nao_trial import ROOT, model_settings from nao_trial_chat import BASE, opener OUT = ROOT / "output/nao-quality" def run(label, question, chat_id=None, timeout=240): if not re.fullmatch(r"[a-zA-Z0-9_-]{1,80}", label): raise ValueError("Use a short alphanumeric evidence label") OUT.mkdir(parents=True, exist_ok=True) client = opener() model = model_settings()[2] body = {"message": {"text": question}, "model": {"provider": "openaiCompatible", "modelId": model}, "timezone": "Asia/Shanghai"} if chat_id: body["chatId"] = chat_id (OUT / (label + "-request.json")).write_text(json.dumps(body, ensure_ascii=False, indent=2)) state = {"chatId": chat_id, "budgetExpired": False, "stopRequest": None} events, text_blocks, tools, tool_outputs = [], {}, [], {} finished = threading.Event() response_ref = [] def cancel_at_budget(): if finished.wait(timeout): return state["budgetExpired"] = True if state.get("chatId"): request = Request(BASE + "/api/trpc/chat.stop", method="POST", data=json.dumps({"json": {"chatId": state["chatId"]}}).encode(), headers={"Content-Type": "application/json", "Origin": BASE}) try: with client.open(request, timeout=15) as response: state["stopRequest"] = {"httpStatus": response.status} except HTTPError as error: state["stopRequest"] = {"httpStatus": error.code} except OSError as error: state["stopRequest"] = {"errorType": type(error).__name__} if response_ref: response_ref[0].close() timer = threading.Thread(target=cancel_at_budget, daemon=True) started = time.monotonic() timer.start() error_type = None model_base, model_key, _ = model_settings() request = Request(BASE + "/api/agent/", data=json.dumps(body, ensure_ascii=False).encode(), headers={"Content-Type": "application/json", "Origin": BASE, "Accept": "text/event-stream"}) try: with client.open(request, timeout=60) as response, (OUT / (label + ".sse")).open("w") as stream: response_ref.append(response) for raw in response: line = raw.decode("utf-8", errors="replace").replace(model_key, "[REDACTED_KEY]") line = line.replace(model_base, "[REDACTED_MODEL_ENDPOINT]") stream.write(line) stream.flush() if not line.startswith("data:"): continue raw_data = line[5:].strip() if raw_data == "[DONE]": break try: event = json.loads(raw_data) except json.JSONDecodeError: continue events.append(event) kind = event.get("type") if kind == "data-newChat": state["chatId"] = event.get("data", {}).get("id") or state["chatId"] elif kind == "text-delta": block_id = event.get("id", "text") text_blocks[block_id] = text_blocks.get(block_id, "") + event.get("delta", "") elif kind == "tool-input-available": tools.append({"toolCallId": event.get("toolCallId"), "tool": event.get("toolName"), "input": event.get("input")}) print("tool", event.get("toolName"), flush=True) elif kind == "tool-output-available": tool_outputs[event.get("toolCallId")] = event.get("output") except (OSError, ValueError, AttributeError) as error: error_type = type(error).__name__ finally: finished.set() timer.join(timeout=1) for tool in tools: tool["output"] = tool_outputs.get(tool["toolCallId"]) texts = [text.strip() for text in text_blocks.values() if text.strip()] final_text = texts[-1] if texts else "" finish_events = [event for event in events if event.get("type") == "finish"] truncated = any(event.get("finishReason") == "length" for event in finish_events) error_events = [event for event in events if event.get("type") in ("error", "tool-output-error")] summary = { "label": label, "question": question, "model": model, "chatId": state["chatId"], "durationSeconds": round(time.monotonic() - started, 2), "status": "budget_cancelled" if state["budgetExpired"] else "truncated" if truncated else "finished" if finish_events and final_text and not any(e.get("type") == "error" for e in error_events) else "incomplete", "finishReceived": bool(finish_events), "finishReasons": [event.get("finishReason") for event in finish_events], "errors": error_events, "transportErrorType": error_type, "stopRequest": state["stopRequest"], "toolCalls": tools, "eventCount": len(events), "answer": final_text, } (OUT / (label + "-events.json")).write_text(json.dumps(events, ensure_ascii=False, indent=2)) (OUT / (label + "-run.json")).write_text(json.dumps(summary, ensure_ascii=False, indent=2)) (OUT / (label + "-answer.md")).write_text(final_text) (OUT / (label + "-all-text.md")).write_text("\n\n---\n\n".join(texts)) for tool in tools: result = tool.get("output") if tool["tool"] != "story" or not isinstance(result, dict) or not result.get("success"): continue slug = result.get("id") (OUT / (label + "-story.md")).write_text(result.get("code", "")) data = {"json": {"chatId": state["chatId"], "storySlug": slug, "format": "html"}} request = Request(BASE + "/api/trpc/story.download?input=" + quote(json.dumps(data)), headers={"Origin": BASE}) with client.open(request, timeout=30) as response: export = json.load(response)["result"]["data"]["json"] (OUT / (label + "-report.html")).write_bytes(base64.b64decode(export["data"])) print(json.dumps({key: summary[key] for key in ("label", "chatId", "durationSeconds", "status", "eventCount")}, ensure_ascii=False)) return summary if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("label") parser.add_argument("question") parser.add_argument("--chat-id") parser.add_argument("--timeout", type=int, default=240) args = parser.parse_args() run(args.label, args.question, args.chat_id, args.timeout)