"""Security and stream-state tests for the delegated NAO runtime (no model calls).""" from __future__ import annotations import io import base64 import json import tempfile import threading import time import unittest import uuid from pathlib import Path from urllib.parse import parse_qs, urlsplit from unittest.mock import patch import sys sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) import nao_quality_runtime as runtime class FakeResponse(io.BytesIO): status = 200 def __enter__(self): return self def __exit__(self, *_): self.close() class FakeOpener: def __init__(self, body=b"{}"): self.body = body self.calls = [] def open(self, request, timeout=None): self.calls.append((request, timeout)) return FakeResponse(self.body) class RuntimeAccessTests(unittest.TestCase): def test_complete_batch_ranking_progress_does_not_claim_arbitrary_truncation(self): text = runtime.public_tool_result({'kind':'quality_changes','completeRanking':True, 'rows':[{'key':'甲'}], 'rowsTruncated':True}) self.assertIn('完整授权范围排序',text) self.assertNotIn('不代表全量排名',text) def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix="test-nao-users-") self.users = Path(self.temp.name).resolve() self.actor = str(uuid.uuid4()) self.release = str(uuid.uuid4()) self.actor_root = self.users / self.actor self.actor_root.mkdir(mode=0o700) self.access = self.actor_root / "access.json" runtime._atomic_json(self.access, { "token": "delegated-secret-for-test", "releaseId": self.release, "queryUrl": runtime.RUNTIME_QUERY_URL, }) def tearDown(self): self.temp.cleanup() def test_delegated_login_only_validates_file_and_never_calls_admin_login(self): class NoNetwork: calls = 0 def open(self, *_args, **_kwargs): self.calls += 1 raise AssertionError("login must not call NAO or admin auth") opener = NoNetwork() api = runtime.DelegatedApi(self.access, runtime_root=self.users, expected_actor_id=self.actor, opener=opener) self.assertIs(api.login(), api) self.assertEqual(opener.calls, 0) def test_every_call_uses_fixed_loopback_delegate_and_pinned_release(self): opener = FakeOpener(b'{"items":[]}') api = runtime.DelegatedApi(self.access, runtime_root=self.users, expected_actor_id=self.actor, opener=opener).login() self.assertEqual(api.call("/api/releases", method="GET"), {"items": []}) request, _ = opener.calls[0] body = json.loads(request.data.decode()) self.assertEqual(request.full_url, runtime.RUNTIME_QUERY_URL) self.assertEqual(set(body), {"token", "method", "path", "payload"}) self.assertEqual(body["token"], "delegated-secret-for-test") with self.assertRaisesRegex(ValueError, "releaseId"): api.call("/api/metrics?releaseId=" + str(uuid.uuid4()), method="GET") with self.assertRaisesRegex(ValueError, "允许范围"): api.call("/api/admin/config", method="GET") self.assertEqual(len(opener.calls), 1) echoed = runtime.DelegatedApi( self.access, runtime_root=self.users, expected_actor_id=self.actor, opener=FakeOpener(b'{"token":"delegated-secret-for-test","note":"echo delegated-secret-for-test"}'), ).call("/api/releases", method="GET") self.assertEqual(echoed, {"note": "echo [REDACTED]"}) def test_scoped_query_envelope_binds_node_and_scope_without_changing_url(self): opener = FakeOpener(b'{"ok":true}') api = runtime.DelegatedApi(self.access, runtime_root=self.users, expected_actor_id=self.actor, opener=opener).login() self.assertEqual(api.call('/api/metrics?releaseId=' + self.release, method='GET', node_id='trend_a', scope_id='execution:trend_a:1'), {'ok': True}) request, _ = opener.calls[0] wrapper = json.loads(request.data.decode()) self.assertEqual(set(wrapper), {'token', 'method', 'path', 'payload', 'nodeId', 'scopeId'}) self.assertEqual(wrapper['nodeId'], 'trend_a') self.assertEqual(wrapper['scopeId'], 'execution:trend_a:1') self.assertEqual(wrapper['path'], '/api/metrics?releaseId=' + self.release) with self.assertRaisesRegex(ValueError, '同时提供'): api.call('/api/metrics?releaseId=' + self.release, method='GET', node_id='trend_a') self.assertEqual(len(opener.calls), 1) def test_batch_changes_reaches_delegate_with_exact_protocol_scope(self): opener=FakeOpener(b'{"ok":true}') api=runtime.DelegatedApi(self.access,runtime_root=self.users, expected_actor_id=self.actor,opener=opener).login() payload={'releaseId':self.release,'period':{'year':2026,'month':9}, 'comparisonPeriod':{'year':2026,'month':8},'filters':{'group':'消费电子-3C'}, 'dimension':'product','sort':'current_rate','stage':None,'limit':3, 'fields':['rate','totalWeight','scrapWeight']} self.assertEqual(api.call('/api/analysis/changes',payload=payload,method='POST', node_id='products',scope_id='scope:products'),{'ok':True}) request,_=opener.calls[0];wrapper=json.loads(request.data) self.assertEqual(wrapper['payload'],payload) self.assertEqual(wrapper['nodeId'],'products') self.assertEqual(wrapper['scopeId'],'scope:products') self.assertEqual(request.full_url,runtime.RUNTIME_QUERY_URL) def test_malformed_access_and_non_loopback_endpoint_are_rejected(self): runtime._atomic_json(self.access, {"token": "x", "releaseId": self.release, "queryUrl": "http://example.com/api/analysis/runtime/query"}) with self.assertRaisesRegex(ValueError, "本机接口"): runtime.load_access_file(self.access, runtime_root=self.users, expected_actor_id=self.actor) runtime._atomic_json(self.access, {"token": "", "releaseId": self.release, "queryUrl": runtime.RUNTIME_QUERY_URL}) with self.assertRaisesRegex(ValueError, "不完整"): runtime.load_access_file(self.access, runtime_root=self.users, expected_actor_id=self.actor) def test_access_file_cannot_escape_actor_directory(self): outside = self.users.parent / self.actor / "access.json" with self.assertRaisesRegex(ValueError, "越界"): runtime.load_access_file(outside, runtime_root=self.users) def test_access_symlinks_are_rejected(self): link = self.actor_root / "linked-access.json" link.symlink_to(self.access) with self.assertRaisesRegex(ValueError, "路径无效"): runtime.load_access_file(link, runtime_root=self.users) root_link = self.users.parent / (self.users.name + "-link") root_link.symlink_to(self.users, target_is_directory=True) try: with self.assertRaisesRegex(ValueError, "运行目录无效"): runtime.load_access_file(self.access, runtime_root=root_link) finally: root_link.unlink() def test_session_to_native_chat_mapping_is_actor_scoped(self): session_id = str(uuid.uuid4()) first = runtime.SessionState(self.actor_root, self.actor) other_actor = str(uuid.uuid4()) other_root = self.users / other_actor other_root.mkdir(mode=0o700) second = runtime.SessionState(other_root, other_actor) first.set_chat_id(session_id, "native-chat-one") second.set_chat_id(session_id, "native-chat-two") self.assertEqual(first.get_chat_id(session_id), "native-chat-one") self.assertEqual(second.get_chat_id(session_id), "native-chat-two") self.assertNotEqual(first.path, second.path) class RuntimeStreamTests(unittest.TestCase): def test_accumulator_keeps_provider_reasoning_separate_from_answer(self): updates = [] stream = runtime.ConversationAccumulator(updates.append) stream.feed({"type": "reasoning-delta", "delta": "private thought"}) stream.feed({"type": "text-start", "id": "first"}) stream.feed({"type": "text-delta", "id": "first", "delta": "early block"}) stream.feed({"type": "text-start", "id": "final"}) stream.feed({"type": "text-delta", "id": "final", "delta": "final answer"}) self.assertEqual(stream.answer, "final answer") self.assertNotIn("private thought", updates) self.assertEqual(stream.model_reasoning, "private thought") def test_provider_reasoning_is_bounded_and_never_inferred_from_answer(self): stream = runtime.ConversationAccumulator() stream.feed({"type": "text-delta", "delta": "Only a final answer"}) self.assertEqual(stream.model_reasoning, "") stream.feed({"type": "reasoning-delta", "id": "r1", "delta": "x" * 199_999}) stream.feed({"type": "reasoning-delta", "id": "r1", "delta": "yz"}) stream.feed({"type": "reasoning-delta", "id": "r2", "delta": "more"}) self.assertEqual(len(stream.model_reasoning.strip()), 200_000) self.assertTrue(stream.reasoning_truncated) self.assertEqual(stream.answer, "Only a final answer") def test_length_and_stringified_mcp_error_are_partial(self): stream = runtime.ConversationAccumulator() stream.feed({"type": "tool-input-available", "toolName": "quality_data", "toolCallId": "q1"}) stream.feed({"type": "tool-output-available", "toolCallId": "q1", "output": '{"isError":true,"content":[{"text":"forbidden"}]}'}) stream.feed({"type": "text-delta", "id": "answer", "delta": "查询受限,说明原因"}) stream.feed({"type": "finish", "finishReason": "stop"}) self.assertEqual(stream.tool_calls[0]["status"], "failed") self.assertEqual(stream.status()[0], "PARTIAL") output_error = runtime.ConversationAccumulator() output_error.feed({"type": "tool-input-available", "toolName": "details", "toolCallId": "q2"}) output_error.feed({"type": "tool-output-error", "toolCallId": "q2"}) self.assertEqual(output_error.tool_calls[0]["status"], "failed") truncated = runtime.ConversationAccumulator() truncated.feed({"type": "text-delta", "id": "a", "delta": "正文未完"}) truncated.feed({"type": "finish", "finishReason": "length"}) self.assertEqual(truncated.status(), ("PARTIAL", "length")) def test_tool_errors_are_read_from_envelopes_not_business_data_cells(self): for flag in ('mcpValidationError','mcpAuthRequired'): stream=runtime.ConversationAccumulator() stream.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'q'}) stream.feed({'type':'tool-output-available','toolCallId':'q','output':{flag:True,'server':'quality'}}) stream.feed({'type':'text-delta','id':'final','delta':'尚未查到数据。'}) stream.feed({'type':'finish','finishReason':'stop'}) self.assertEqual(stream.tool_calls[0]['status'],'failed') self.assertEqual(stream.status(),('PARTIAL','mcp_error')) payload={'kind':'quality_records','rows':[{'error':'缺角','isError':True}], 'note':'{"error":"原始单元格内容"}'} stream=runtime.ConversationAccumulator() stream.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'q'}) stream.feed({'type':'tool-output-available','toolCallId':'q','output':{'content':[{'type':'text','text':json.dumps(payload)}]}}) stream.feed({'type':'text-delta','id':'final','delta':'已查询并保留原始字段。'}) stream.feed({'type':'finish','finishReason':'stop'}) self.assertEqual(stream.tool_calls[0]['status'],'completed') self.assertEqual(stream.status(),('COMPLETED','stop')) def test_transport_done_is_not_a_normal_model_finish(self): stream = runtime.ConversationAccumulator() stream.feed({'type': 'text-delta', 'id': 'progress', 'delta': '先查一下。'}) stream.feed({'type': 'tool-input-available', 'toolName': 'mcp_call', 'toolCallId': 'q'}) stream.feed({'type': 'tool-output-available', 'toolCallId': 'q', 'output': {}}) stream.feed({'type': 'finish', 'finishReason': 'tool-calls'}) stream.feed({'type': '[DONE]'}) self.assertNotEqual(stream.status()[0], 'COMPLETED') self.assertEqual(stream.answer, '先查一下。') def test_progress_before_a_tool_or_unsettled_tool_cannot_count_as_final_answer(self): for settled in (False, True): stream = runtime.ConversationAccumulator() stream.feed({'type': 'text-delta', 'id': 'progress', 'delta': '正在查询。'}) stream.feed({'type': 'tool-input-available', 'toolName': 'mcp_call', 'toolCallId': 'q'}) if settled:stream.feed({'type': 'tool-output-available', 'toolCallId': 'q', 'output': {}}) stream.feed({'type': 'finish', 'finishReason': 'stop'}) self.assertNotEqual(stream.status()[0], 'COMPLETED') stream.feed({'type': 'text-delta', 'id': 'final', 'delta': '已核实的最终回答。'}) self.assertEqual(stream.status()[0], 'COMPLETED' if settled else 'PARTIAL') def test_only_the_terminal_finish_and_successful_stop_control_can_complete(self): for reasons, stopped in ((['stop', 'tool-calls'], None), (['stop'], False)): stream = runtime.ConversationAccumulator() stream.feed({'type': 'text-delta', 'id': 'answer', 'delta': '保留原文'}) for reason in reasons:stream.feed({'type': 'finish', 'finishReason': reason}) stream.feed({'type': '[DONE]'}) self.assertNotEqual(stream.status(stop_succeeded=stopped)[0], 'COMPLETED') def test_native_stop_controller_stops_before_closing_stream_on_cancel(self): calls = [] response = FakeResponse() controller = runtime.NativeStopController(lambda chat_id: calls.append(chat_id) or True, timeout=5) controller.set_chat_id("native-id") controller.set_response(response) controller.start() controller.request_cancel("user") controller.close() self.assertEqual(calls, ["native-id"]) self.assertTrue(response.closed) stream = runtime.ConversationAccumulator() self.assertEqual(stream.status(cancel_cause="user", stop_succeeded=True), ("CANCELLED", "user_cancel")) def test_watchdog_timeout_invokes_native_stop(self): calls = [] controller = runtime.NativeStopController(lambda chat_id: calls.append(chat_id) or True, timeout=0.03) controller.set_chat_id("timed-native-id") controller.start() deadline = time.monotonic() + 1 while controller.thread.is_alive() and time.monotonic() < deadline: time.sleep(0.01) controller.close() self.assertEqual(calls, ["timed-native-id"]) self.assertEqual(controller.cause, "timeout") def test_complete_evidence_can_stop_native_before_turn_timeout(self): calls = [] controller = runtime.NativeStopController( lambda chat_id: calls.append(chat_id) or True, timeout=3, completion_ready=lambda: True) controller.set_chat_id("covered-native-id") controller.start() deadline = time.monotonic() + 2 while controller.thread.is_alive() and time.monotonic() < deadline: time.sleep(0.02) controller.close() self.assertEqual(calls, ["covered-native-id"]) self.assertEqual(controller.cause, "coverage_ready") stream = runtime.ConversationAccumulator() self.assertEqual(stream.status(cancel_cause="coverage_ready", stop_succeeded=True), ("FAILED", "coverage_ready")) def test_transport_failure_stops_native_and_falls_back_to_owned_instance(self): class BrokenClient: def open(self, *_args, **_kwargs): raise OSError("private transport detail must not escape") with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp: users = Path(temp).resolve() actor_id = str(uuid.uuid4()) actor_root = users / actor_id actor_root.mkdir(mode=0o700) session_id = str(uuid.uuid4()) state = runtime.SessionState(actor_root, actor_id) with patch.object(runtime, "_stop_native", return_value=False) as native_stop, \ patch.object(runtime, "_stop_instance", return_value=True) as instance_stop: result = runtime._sse_request( client=BrokenClient(), base_url="http://127.0.0.1:10000", model="test", question="q", release_id=str(uuid.uuid4()), context=None, chat_id="existing-native-id", session_state=state, session_id=session_id, actor_root=actor_root, emit=lambda _event: None, timeout=5, ) self.assertEqual(result[1], "FAILED") native_stop.assert_called_once_with(unittest.mock.ANY, "http://127.0.0.1:10000", "existing-native-id") instance_stop.assert_called_once_with(actor_root, grace_seconds=1.0) def test_early_eof_without_finish_or_done_stops_native_task(self): class EofClient: def open(self, *_args, **_kwargs): return FakeResponse(b'data: {"type":"text-delta","id":"a","delta":"partial"}\n') with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp: users = Path(temp).resolve() actor_id = str(uuid.uuid4()) actor_root = users / actor_id actor_root.mkdir(mode=0o700) session_id = str(uuid.uuid4()) with patch.object(runtime, "_stop_native", return_value=True) as native_stop: result = runtime._sse_request( client=EofClient(), base_url="http://127.0.0.1:10000", model="test", question="q", release_id=str(uuid.uuid4()), context=None, chat_id="existing-native-id", session_state=runtime.SessionState(actor_root, actor_id), session_id=session_id, actor_root=actor_root, emit=lambda _event: None, timeout=5, ) self.assertEqual(result[1], "PARTIAL") self.assertEqual(result[2], "transport_or_stream_error") native_stop.assert_called_once_with(unittest.mock.ANY, "http://127.0.0.1:10000", "existing-native-id") def test_an_already_finished_native_reply_cannot_be_cancelled_by_stale_no_progress_state(self): controls=[];ready=[];controller=runtime.NativeStopController def capture(*args): value=controller(*args);controls.append(value);return value class Response: def __enter__(self):return self def __exit__(self,*_):pass def close(self):pass def __iter__(self): yield b'data: {"type":"text-delta","id":"final","delta":"final reply"}\n' yield b'data: {"type":"finish","finishReason":"stop"}\n' ready.append(controls[0].completion_ready()) yield b'data: [DONE]\n' with tempfile.TemporaryDirectory() as temp: root=Path(temp)/str(uuid.uuid4());root.mkdir();actor=root.name;sid=str(uuid.uuid4()) runtime._atomic_json(root/'private/native-exploration-state.json',{'executionId':'e','halted':True}) client=unittest.mock.Mock();client.open.return_value=Response() with patch.object(runtime,'NativeStopController',side_effect=capture),patch('nao_agent_task.task_from_context',return_value={'executionMode':'native_agent','freshQuery':True}): result=runtime._sse_request(client=client,base_url='http://native.invalid',model='fixture',question='q',release_id=str(uuid.uuid4()), context={'_nativeExecutionId':'e'},chat_id='chat',session_state=runtime.SessionState(root,actor),session_id=sid,actor_root=root,emit=lambda _:None,timeout=5) self.assertEqual(ready,[False]);self.assertEqual(result[1],'COMPLETED') def test_story_export_uses_last_successful_call_and_matching_markdown(self): class StoryClient: def __init__(self): self.slugs = [] def open(self, request, timeout=None): encoded_input = parse_qs(urlsplit(request.full_url).query)["input"][0] query = json.loads(encoded_input)["json"] slug = query["storySlug"] self.slugs.append(slug) html = f"{slug}".encode() payload = {"result": {"data": {"json": { "data": base64.b64encode(html).decode("ascii") }}}} return FakeResponse(json.dumps(payload).encode()) client = StoryClient() calls = [ {"tool": "story", "toolCallId": "create"}, {"tool": "story", "toolCallId": "replace"}, {"tool": "story", "toolCallId": "failed-replacement"}, ] outputs = { "create": {"success": True, "id": "old-report", "code": "# Initial revision"}, "replace": {"success": True, "id": "final-report", "code": "# Final revision"}, "failed-replacement": {"success": False, "id": "broken-report", "code": "# Failed"}, } markdown, html = runtime._story_files(client, "http://127.0.0.1:10000", "chat", calls, outputs) self.assertEqual(markdown, "# Final revision") self.assertEqual(html, "final-report") self.assertEqual(client.slugs, ["final-report"]) def test_sse_uses_idle_timeout_and_warm_session_configuration_is_reused(self): class CaptureClient: def __init__(self): self.timeout = None def open(self, request, timeout=None): self.timeout = timeout return FakeResponse(b'data: {"type":"reasoning-delta","id":"r","delta":"provider reasoning"}\n' b'data: {"type":"reasoning-end","id":"r"}\n' b'data: {"type":"text-delta","id":"a","delta":"answer"}\n' b'data: {"type":"finish","finishReason":"stop"}\n' b'data: [DONE]\n') with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp: users = Path(temp).resolve() actor_id = str(uuid.uuid4()) session_id = str(uuid.uuid4()) actor_root = users / actor_id actor_root.mkdir(mode=0o700) project = actor_root / "project" project.mkdir() current_session = actor_root / "private" / "current-session.json" runtime._atomic_json(current_session, {"actorId": actor_id, "sessionId": session_id}) runtime._write_project(actor_root, session_id, "test-model", refresh_templates=True, refresh_mcp=True, backup_schema=False) mcp_path = project / "agent" / "mcps" / "mcp.json" original_config = mcp_path.read_bytes() runtime._write_project(actor_root, session_id, "test-model", refresh_templates=False, refresh_mcp=False, backup_schema=False) self.assertEqual(mcp_path.read_bytes(), original_config) client = CaptureClient() events = [] result = runtime._sse_request( client=client, base_url="http://127.0.0.1:10000", model="test-model", question="q", release_id=str(uuid.uuid4()), context=None, chat_id=None, session_state=runtime.SessionState(actor_root, actor_id), session_id=session_id, actor_root=actor_root, emit=events.append, timeout=5, ) self.assertEqual(client.timeout, runtime.SSE_IDLE_TIMEOUT_SECONDS) self.assertEqual(result[1], "COMPLETED") self.assertEqual(result[0].answer, "answer") self.assertFalse(any("modelReasoning" in event for event in events)) self.assertTrue(any(event.get("timeline") for event in events)) next_session = str(uuid.uuid4()) runtime._write_project(actor_root, next_session, "test-model", refresh_templates=False, refresh_mcp=True, backup_schema=False) updated = json.loads(mcp_path.read_text(encoding="utf-8")) server_env = updated["mcpServers"]["quality"]["env"] self.assertEqual(server_env["QUALITY_AGENT_SESSION_ID"], next_session) self.assertEqual(json.loads(current_session.read_text())["sessionId"], next_session) class RuntimeManagerTests(unittest.TestCase): def test_ensure_instance_executes_project_setup_and_registry_path(self): with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp: users = Path(temp).resolve() manager = runtime.RuntimeManager(users_root=users) actor_id = str(uuid.uuid4()) session_id = str(uuid.uuid4()) actor_root, actor_lock = manager.acquire_actor(actor_id) fake_port = 49123 def fake_start(root, model_base, model_key, model_name, fingerprint): self.assertEqual(root, actor_root) runtime._atomic_json(root / "instance.json", { "actorId": actor_id, "serverPort": fake_port, "fingerprint": fingerprint, "processes": [], }) try: with patch.object(runtime, "_model_settings", return_value=("https://model.invalid/v1", "test-key", "test-model")), \ patch.object(runtime, "_config_fingerprint", return_value="test-fingerprint"), \ patch.object(manager, "_start", side_effect=fake_start) as start: config = manager.ensure_instance(actor_root, session_id) self.assertEqual(config, ("https://model.invalid/v1", "test-key", "test-model", f"http://127.0.0.1:{fake_port}")) start.assert_called_once() project = actor_root / "project" self.assertTrue((project / "RULES.md").is_file()) self.assertTrue((project / "nao_config.yaml").is_file()) mcp_config = json.loads((project / "agent" / "mcps" / "mcp.json").read_text()) self.assertEqual(mcp_config["mcpServers"]["quality"]["env"]["QUALITY_AGENT_SESSION_ID"], session_id) self.assertEqual(runtime.RuntimeManager._current_session(actor_root), session_id) finally: manager.release_actor(actor_lock) if __name__ == "__main__": unittest.main()