Files
quality_analyse/scripts/nao_quality_chat.py

144 lines
6.9 KiB
Python

"""Run real nao quality conversations, with evidence and an active cancellation budget."""
import argparse
import base64
import json
import re
import threading
import time
from pathlib import Path
from urllib.error import HTTPError
from urllib.parse import quote
from urllib.request import Request
from nao_trial import ROOT, model_settings
from nao_trial_chat import BASE, opener
OUT = ROOT / "output/nao-quality"
def run(label, question, chat_id=None, timeout=240):
if not re.fullmatch(r"[a-zA-Z0-9_-]{1,80}", label):
raise ValueError("Use a short alphanumeric evidence label")
OUT.mkdir(parents=True, exist_ok=True)
client = opener()
model = model_settings()[2]
body = {"message": {"text": question},
"model": {"provider": "openaiCompatible", "modelId": model},
"timezone": "Asia/Shanghai"}
if chat_id:
body["chatId"] = chat_id
(OUT / (label + "-request.json")).write_text(json.dumps(body, ensure_ascii=False, indent=2))
state = {"chatId": chat_id, "budgetExpired": False, "stopRequest": None}
events, text_blocks, tools, tool_outputs = [], {}, [], {}
finished = threading.Event()
response_ref = []
def cancel_at_budget():
if finished.wait(timeout):
return
state["budgetExpired"] = True
if state.get("chatId"):
request = Request(BASE + "/api/trpc/chat.stop", method="POST",
data=json.dumps({"json": {"chatId": state["chatId"]}}).encode(),
headers={"Content-Type": "application/json", "Origin": BASE})
try:
with client.open(request, timeout=15) as response:
state["stopRequest"] = {"httpStatus": response.status}
except HTTPError as error:
state["stopRequest"] = {"httpStatus": error.code}
except OSError as error:
state["stopRequest"] = {"errorType": type(error).__name__}
if response_ref:
response_ref[0].close()
timer = threading.Thread(target=cancel_at_budget, daemon=True)
started = time.monotonic()
timer.start()
error_type = None
model_base, model_key, _ = model_settings()
request = Request(BASE + "/api/agent/", data=json.dumps(body, ensure_ascii=False).encode(),
headers={"Content-Type": "application/json", "Origin": BASE,
"Accept": "text/event-stream"})
try:
with client.open(request, timeout=60) as response, (OUT / (label + ".sse")).open("w") as stream:
response_ref.append(response)
for raw in response:
line = raw.decode("utf-8", errors="replace").replace(model_key, "[REDACTED_KEY]")
line = line.replace(model_base, "[REDACTED_MODEL_ENDPOINT]")
stream.write(line)
stream.flush()
if not line.startswith("data:"):
continue
raw_data = line[5:].strip()
if raw_data == "[DONE]":
break
try:
event = json.loads(raw_data)
except json.JSONDecodeError:
continue
events.append(event)
kind = event.get("type")
if kind == "data-newChat":
state["chatId"] = event.get("data", {}).get("id") or state["chatId"]
elif kind == "text-delta":
block_id = event.get("id", "text")
text_blocks[block_id] = text_blocks.get(block_id, "") + event.get("delta", "")
elif kind == "tool-input-available":
tools.append({"toolCallId": event.get("toolCallId"), "tool": event.get("toolName"),
"input": event.get("input")})
print("tool", event.get("toolName"), flush=True)
elif kind == "tool-output-available":
tool_outputs[event.get("toolCallId")] = event.get("output")
except (OSError, ValueError, AttributeError) as error:
error_type = type(error).__name__
finally:
finished.set()
timer.join(timeout=1)
for tool in tools:
tool["output"] = tool_outputs.get(tool["toolCallId"])
texts = [text.strip() for text in text_blocks.values() if text.strip()]
final_text = texts[-1] if texts else ""
finish_events = [event for event in events if event.get("type") == "finish"]
truncated = any(event.get("finishReason") == "length" for event in finish_events)
error_events = [event for event in events if event.get("type") in ("error", "tool-output-error")]
summary = {
"label": label, "question": question, "model": model, "chatId": state["chatId"],
"durationSeconds": round(time.monotonic() - started, 2),
"status": "budget_cancelled" if state["budgetExpired"] else
"truncated" if truncated else
"finished" if finish_events and final_text and not any(e.get("type") == "error" for e in error_events)
else "incomplete",
"finishReceived": bool(finish_events), "finishReasons": [event.get("finishReason") for event in finish_events], "errors": error_events,
"transportErrorType": error_type, "stopRequest": state["stopRequest"],
"toolCalls": tools, "eventCount": len(events), "answer": final_text,
}
(OUT / (label + "-events.json")).write_text(json.dumps(events, ensure_ascii=False, indent=2))
(OUT / (label + "-run.json")).write_text(json.dumps(summary, ensure_ascii=False, indent=2))
(OUT / (label + "-answer.md")).write_text(final_text)
(OUT / (label + "-all-text.md")).write_text("\n\n---\n\n".join(texts))
for tool in tools:
result = tool.get("output")
if tool["tool"] != "story" or not isinstance(result, dict) or not result.get("success"):
continue
slug = result.get("id")
(OUT / (label + "-story.md")).write_text(result.get("code", ""))
data = {"json": {"chatId": state["chatId"], "storySlug": slug, "format": "html"}}
request = Request(BASE + "/api/trpc/story.download?input=" + quote(json.dumps(data)),
headers={"Origin": BASE})
with client.open(request, timeout=30) as response:
export = json.load(response)["result"]["data"]["json"]
(OUT / (label + "-report.html")).write_bytes(base64.b64decode(export["data"]))
print(json.dumps({key: summary[key] for key in
("label", "chatId", "durationSeconds", "status", "eventCount")}, ensure_ascii=False))
return summary
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("label")
parser.add_argument("question")
parser.add_argument("--chat-id")
parser.add_argument("--timeout", type=int, default=240)
args = parser.parse_args()
run(args.label, args.question, args.chat_id, args.timeout)