同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。 补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。 统一午夜黑及其他主题的控件细节、中文和业务数字字体。 排除本地业务数据、凭据、预览、发布与验收产物。 验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
434 lines
22 KiB
Python
434 lines
22 KiB
Python
"""Security and stream-state tests for the delegated NAO runtime (no model calls)."""
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import base64
|
|
import json
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qs, urlsplit
|
|
from unittest.mock import patch
|
|
|
|
import sys
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
|
|
|
import nao_quality_runtime as runtime
|
|
|
|
|
|
class FakeResponse(io.BytesIO):
|
|
status = 200
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
self.close()
|
|
|
|
|
|
class FakeOpener:
|
|
def __init__(self, body=b"{}"):
|
|
self.body = body
|
|
self.calls = []
|
|
|
|
def open(self, request, timeout=None):
|
|
self.calls.append((request, timeout))
|
|
return FakeResponse(self.body)
|
|
|
|
|
|
class RuntimeAccessTests(unittest.TestCase):
|
|
def test_complete_batch_ranking_progress_does_not_claim_arbitrary_truncation(self):
|
|
text = runtime.public_tool_result({'kind':'quality_changes','completeRanking':True,
|
|
'rows':[{'key':'甲'}], 'rowsTruncated':True})
|
|
self.assertIn('完整授权范围排序',text)
|
|
self.assertNotIn('不代表全量排名',text)
|
|
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory(prefix="test-nao-users-")
|
|
self.users = Path(self.temp.name).resolve()
|
|
self.actor = str(uuid.uuid4())
|
|
self.release = str(uuid.uuid4())
|
|
self.actor_root = self.users / self.actor
|
|
self.actor_root.mkdir(mode=0o700)
|
|
self.access = self.actor_root / "access.json"
|
|
runtime._atomic_json(self.access, {
|
|
"token": "delegated-secret-for-test",
|
|
"releaseId": self.release,
|
|
"queryUrl": runtime.RUNTIME_QUERY_URL,
|
|
})
|
|
|
|
def tearDown(self):
|
|
self.temp.cleanup()
|
|
|
|
def test_delegated_login_only_validates_file_and_never_calls_admin_login(self):
|
|
class NoNetwork:
|
|
calls = 0
|
|
|
|
def open(self, *_args, **_kwargs):
|
|
self.calls += 1
|
|
raise AssertionError("login must not call NAO or admin auth")
|
|
|
|
opener = NoNetwork()
|
|
api = runtime.DelegatedApi(self.access, runtime_root=self.users,
|
|
expected_actor_id=self.actor, opener=opener)
|
|
self.assertIs(api.login(), api)
|
|
self.assertEqual(opener.calls, 0)
|
|
|
|
def test_every_call_uses_fixed_loopback_delegate_and_pinned_release(self):
|
|
opener = FakeOpener(b'{"items":[]}')
|
|
api = runtime.DelegatedApi(self.access, runtime_root=self.users,
|
|
expected_actor_id=self.actor, opener=opener).login()
|
|
self.assertEqual(api.call("/api/releases", method="GET"), {"items": []})
|
|
request, _ = opener.calls[0]
|
|
body = json.loads(request.data.decode())
|
|
self.assertEqual(request.full_url, runtime.RUNTIME_QUERY_URL)
|
|
self.assertEqual(set(body), {"token", "method", "path", "payload"})
|
|
self.assertEqual(body["token"], "delegated-secret-for-test")
|
|
with self.assertRaisesRegex(ValueError, "releaseId"):
|
|
api.call("/api/metrics?releaseId=" + str(uuid.uuid4()), method="GET")
|
|
with self.assertRaisesRegex(ValueError, "允许范围"):
|
|
api.call("/api/admin/config", method="GET")
|
|
self.assertEqual(len(opener.calls), 1)
|
|
echoed = runtime.DelegatedApi(
|
|
self.access, runtime_root=self.users, expected_actor_id=self.actor,
|
|
opener=FakeOpener(b'{"token":"delegated-secret-for-test","note":"echo delegated-secret-for-test"}'),
|
|
).call("/api/releases", method="GET")
|
|
self.assertEqual(echoed, {"note": "echo [REDACTED]"})
|
|
|
|
def test_scoped_query_envelope_binds_node_and_scope_without_changing_url(self):
|
|
opener = FakeOpener(b'{"ok":true}')
|
|
api = runtime.DelegatedApi(self.access, runtime_root=self.users,
|
|
expected_actor_id=self.actor, opener=opener).login()
|
|
self.assertEqual(api.call('/api/metrics?releaseId=' + self.release, method='GET',
|
|
node_id='trend_a', scope_id='execution:trend_a:1'), {'ok': True})
|
|
request, _ = opener.calls[0]
|
|
wrapper = json.loads(request.data.decode())
|
|
self.assertEqual(set(wrapper), {'token', 'method', 'path', 'payload', 'nodeId', 'scopeId'})
|
|
self.assertEqual(wrapper['nodeId'], 'trend_a')
|
|
self.assertEqual(wrapper['scopeId'], 'execution:trend_a:1')
|
|
self.assertEqual(wrapper['path'], '/api/metrics?releaseId=' + self.release)
|
|
with self.assertRaisesRegex(ValueError, '同时提供'):
|
|
api.call('/api/metrics?releaseId=' + self.release, method='GET', node_id='trend_a')
|
|
self.assertEqual(len(opener.calls), 1)
|
|
|
|
def test_batch_changes_reaches_delegate_with_exact_protocol_scope(self):
|
|
opener=FakeOpener(b'{"ok":true}')
|
|
api=runtime.DelegatedApi(self.access,runtime_root=self.users,
|
|
expected_actor_id=self.actor,opener=opener).login()
|
|
payload={'releaseId':self.release,'period':{'year':2026,'month':9},
|
|
'comparisonPeriod':{'year':2026,'month':8},'filters':{'group':'消费电子-3C'},
|
|
'dimension':'product','sort':'current_rate','stage':None,'limit':3,
|
|
'fields':['rate','totalWeight','scrapWeight']}
|
|
self.assertEqual(api.call('/api/analysis/changes',payload=payload,method='POST',
|
|
node_id='products',scope_id='scope:products'),{'ok':True})
|
|
request,_=opener.calls[0];wrapper=json.loads(request.data)
|
|
self.assertEqual(wrapper['payload'],payload)
|
|
self.assertEqual(wrapper['nodeId'],'products')
|
|
self.assertEqual(wrapper['scopeId'],'scope:products')
|
|
self.assertEqual(request.full_url,runtime.RUNTIME_QUERY_URL)
|
|
|
|
def test_malformed_access_and_non_loopback_endpoint_are_rejected(self):
|
|
runtime._atomic_json(self.access, {"token": "x", "releaseId": self.release,
|
|
"queryUrl": "http://example.com/api/analysis/runtime/query"})
|
|
with self.assertRaisesRegex(ValueError, "本机接口"):
|
|
runtime.load_access_file(self.access, runtime_root=self.users,
|
|
expected_actor_id=self.actor)
|
|
runtime._atomic_json(self.access, {"token": "", "releaseId": self.release,
|
|
"queryUrl": runtime.RUNTIME_QUERY_URL})
|
|
with self.assertRaisesRegex(ValueError, "不完整"):
|
|
runtime.load_access_file(self.access, runtime_root=self.users,
|
|
expected_actor_id=self.actor)
|
|
|
|
def test_access_file_cannot_escape_actor_directory(self):
|
|
outside = self.users.parent / self.actor / "access.json"
|
|
with self.assertRaisesRegex(ValueError, "越界"):
|
|
runtime.load_access_file(outside, runtime_root=self.users)
|
|
|
|
def test_access_symlinks_are_rejected(self):
|
|
link = self.actor_root / "linked-access.json"
|
|
link.symlink_to(self.access)
|
|
with self.assertRaisesRegex(ValueError, "路径无效"):
|
|
runtime.load_access_file(link, runtime_root=self.users)
|
|
root_link = self.users.parent / (self.users.name + "-link")
|
|
root_link.symlink_to(self.users, target_is_directory=True)
|
|
try:
|
|
with self.assertRaisesRegex(ValueError, "运行目录无效"):
|
|
runtime.load_access_file(self.access, runtime_root=root_link)
|
|
finally:
|
|
root_link.unlink()
|
|
|
|
def test_session_to_native_chat_mapping_is_actor_scoped(self):
|
|
session_id = str(uuid.uuid4())
|
|
first = runtime.SessionState(self.actor_root, self.actor)
|
|
other_actor = str(uuid.uuid4())
|
|
other_root = self.users / other_actor
|
|
other_root.mkdir(mode=0o700)
|
|
second = runtime.SessionState(other_root, other_actor)
|
|
first.set_chat_id(session_id, "native-chat-one")
|
|
second.set_chat_id(session_id, "native-chat-two")
|
|
self.assertEqual(first.get_chat_id(session_id), "native-chat-one")
|
|
self.assertEqual(second.get_chat_id(session_id), "native-chat-two")
|
|
self.assertNotEqual(first.path, second.path)
|
|
|
|
|
|
class RuntimeStreamTests(unittest.TestCase):
|
|
def test_accumulator_keeps_provider_reasoning_separate_from_answer(self):
|
|
updates = []
|
|
stream = runtime.ConversationAccumulator(updates.append)
|
|
stream.feed({"type": "reasoning-delta", "delta": "private thought"})
|
|
stream.feed({"type": "text-start", "id": "first"})
|
|
stream.feed({"type": "text-delta", "id": "first", "delta": "early block"})
|
|
stream.feed({"type": "text-start", "id": "final"})
|
|
stream.feed({"type": "text-delta", "id": "final", "delta": "final answer"})
|
|
self.assertEqual(stream.answer, "final answer")
|
|
self.assertNotIn("private thought", updates)
|
|
self.assertEqual(stream.model_reasoning, "private thought")
|
|
|
|
def test_provider_reasoning_is_bounded_and_never_inferred_from_answer(self):
|
|
stream = runtime.ConversationAccumulator()
|
|
stream.feed({"type": "text-delta", "delta": "Only a final answer"})
|
|
self.assertEqual(stream.model_reasoning, "")
|
|
stream.feed({"type": "reasoning-delta", "id": "r1", "delta": "x" * 199_999})
|
|
stream.feed({"type": "reasoning-delta", "id": "r1", "delta": "yz"})
|
|
stream.feed({"type": "reasoning-delta", "id": "r2", "delta": "more"})
|
|
self.assertEqual(len(stream.model_reasoning.strip()), 200_000)
|
|
self.assertTrue(stream.reasoning_truncated)
|
|
self.assertEqual(stream.answer, "Only a final answer")
|
|
|
|
def test_length_and_stringified_mcp_error_are_partial(self):
|
|
stream = runtime.ConversationAccumulator()
|
|
stream.feed({"type": "tool-input-available", "toolName": "quality_data", "toolCallId": "q1"})
|
|
stream.feed({"type": "tool-output-available", "toolCallId": "q1",
|
|
"output": '{"isError":true,"content":[{"text":"forbidden"}]}'})
|
|
stream.feed({"type": "text-delta", "id": "answer", "delta": "查询受限,说明原因"})
|
|
stream.feed({"type": "finish", "finishReason": "stop"})
|
|
self.assertEqual(stream.tool_calls[0]["status"], "failed")
|
|
self.assertEqual(stream.status()[0], "PARTIAL")
|
|
output_error = runtime.ConversationAccumulator()
|
|
output_error.feed({"type": "tool-input-available", "toolName": "details", "toolCallId": "q2"})
|
|
output_error.feed({"type": "tool-output-error", "toolCallId": "q2"})
|
|
self.assertEqual(output_error.tool_calls[0]["status"], "failed")
|
|
truncated = runtime.ConversationAccumulator()
|
|
truncated.feed({"type": "text-delta", "id": "a", "delta": "正文未完"})
|
|
truncated.feed({"type": "finish", "finishReason": "length"})
|
|
self.assertEqual(truncated.status(), ("PARTIAL", "length"))
|
|
|
|
def test_native_stop_controller_stops_before_closing_stream_on_cancel(self):
|
|
calls = []
|
|
response = FakeResponse()
|
|
controller = runtime.NativeStopController(lambda chat_id: calls.append(chat_id) or True, timeout=5)
|
|
controller.set_chat_id("native-id")
|
|
controller.set_response(response)
|
|
controller.start()
|
|
controller.request_cancel("user")
|
|
controller.close()
|
|
self.assertEqual(calls, ["native-id"])
|
|
self.assertTrue(response.closed)
|
|
stream = runtime.ConversationAccumulator()
|
|
self.assertEqual(stream.status(cancel_cause="user", stop_succeeded=True),
|
|
("CANCELLED", "user_cancel"))
|
|
|
|
def test_watchdog_timeout_invokes_native_stop(self):
|
|
calls = []
|
|
controller = runtime.NativeStopController(lambda chat_id: calls.append(chat_id) or True, timeout=0.03)
|
|
controller.set_chat_id("timed-native-id")
|
|
controller.start()
|
|
deadline = time.monotonic() + 1
|
|
while controller.thread.is_alive() and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
controller.close()
|
|
self.assertEqual(calls, ["timed-native-id"])
|
|
self.assertEqual(controller.cause, "timeout")
|
|
|
|
def test_complete_evidence_can_stop_native_before_turn_timeout(self):
|
|
calls = []
|
|
controller = runtime.NativeStopController(
|
|
lambda chat_id: calls.append(chat_id) or True, timeout=3,
|
|
completion_ready=lambda: True)
|
|
controller.set_chat_id("covered-native-id")
|
|
controller.start()
|
|
deadline = time.monotonic() + 2
|
|
while controller.thread.is_alive() and time.monotonic() < deadline:
|
|
time.sleep(0.02)
|
|
controller.close()
|
|
self.assertEqual(calls, ["covered-native-id"])
|
|
self.assertEqual(controller.cause, "coverage_ready")
|
|
stream = runtime.ConversationAccumulator()
|
|
self.assertEqual(stream.status(cancel_cause="coverage_ready", stop_succeeded=True),
|
|
("FAILED", "coverage_ready"))
|
|
|
|
def test_transport_failure_stops_native_and_falls_back_to_owned_instance(self):
|
|
class BrokenClient:
|
|
def open(self, *_args, **_kwargs):
|
|
raise OSError("private transport detail must not escape")
|
|
|
|
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
|
|
users = Path(temp).resolve()
|
|
actor_id = str(uuid.uuid4())
|
|
actor_root = users / actor_id
|
|
actor_root.mkdir(mode=0o700)
|
|
session_id = str(uuid.uuid4())
|
|
state = runtime.SessionState(actor_root, actor_id)
|
|
with patch.object(runtime, "_stop_native", return_value=False) as native_stop, \
|
|
patch.object(runtime, "_stop_instance", return_value=True) as instance_stop:
|
|
result = runtime._sse_request(
|
|
client=BrokenClient(), base_url="http://127.0.0.1:10000", model="test",
|
|
question="q", release_id=str(uuid.uuid4()), context=None, chat_id="existing-native-id",
|
|
session_state=state, session_id=session_id, actor_root=actor_root,
|
|
emit=lambda _event: None, timeout=5,
|
|
)
|
|
self.assertEqual(result[1], "FAILED")
|
|
native_stop.assert_called_once_with(unittest.mock.ANY, "http://127.0.0.1:10000", "existing-native-id")
|
|
instance_stop.assert_called_once_with(actor_root, grace_seconds=1.0)
|
|
|
|
def test_early_eof_without_finish_or_done_stops_native_task(self):
|
|
class EofClient:
|
|
def open(self, *_args, **_kwargs):
|
|
return FakeResponse(b'data: {"type":"text-delta","id":"a","delta":"partial"}\n')
|
|
|
|
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
|
|
users = Path(temp).resolve()
|
|
actor_id = str(uuid.uuid4())
|
|
actor_root = users / actor_id
|
|
actor_root.mkdir(mode=0o700)
|
|
session_id = str(uuid.uuid4())
|
|
with patch.object(runtime, "_stop_native", return_value=True) as native_stop:
|
|
result = runtime._sse_request(
|
|
client=EofClient(), base_url="http://127.0.0.1:10000", model="test",
|
|
question="q", release_id=str(uuid.uuid4()), context=None, chat_id="existing-native-id",
|
|
session_state=runtime.SessionState(actor_root, actor_id), session_id=session_id,
|
|
actor_root=actor_root, emit=lambda _event: None, timeout=5,
|
|
)
|
|
self.assertEqual(result[1], "PARTIAL")
|
|
self.assertEqual(result[2], "transport_or_stream_error")
|
|
native_stop.assert_called_once_with(unittest.mock.ANY, "http://127.0.0.1:10000", "existing-native-id")
|
|
|
|
def test_story_export_uses_last_successful_call_and_matching_markdown(self):
|
|
class StoryClient:
|
|
def __init__(self):
|
|
self.slugs = []
|
|
|
|
def open(self, request, timeout=None):
|
|
encoded_input = parse_qs(urlsplit(request.full_url).query)["input"][0]
|
|
query = json.loads(encoded_input)["json"]
|
|
slug = query["storySlug"]
|
|
self.slugs.append(slug)
|
|
html = f"<html>{slug}</html>".encode()
|
|
payload = {"result": {"data": {"json": {
|
|
"data": base64.b64encode(html).decode("ascii")
|
|
}}}}
|
|
return FakeResponse(json.dumps(payload).encode())
|
|
|
|
client = StoryClient()
|
|
calls = [
|
|
{"tool": "story", "toolCallId": "create"},
|
|
{"tool": "story", "toolCallId": "replace"},
|
|
{"tool": "story", "toolCallId": "failed-replacement"},
|
|
]
|
|
outputs = {
|
|
"create": {"success": True, "id": "old-report", "code": "# Initial revision"},
|
|
"replace": {"success": True, "id": "final-report", "code": "# Final revision"},
|
|
"failed-replacement": {"success": False, "id": "broken-report", "code": "# Failed"},
|
|
}
|
|
markdown, html = runtime._story_files(client, "http://127.0.0.1:10000", "chat", calls, outputs)
|
|
self.assertEqual(markdown, "# Final revision")
|
|
self.assertEqual(html, "<html>final-report</html>")
|
|
self.assertEqual(client.slugs, ["final-report"])
|
|
|
|
def test_sse_uses_idle_timeout_and_warm_session_configuration_is_reused(self):
|
|
class CaptureClient:
|
|
def __init__(self):
|
|
self.timeout = None
|
|
|
|
def open(self, request, timeout=None):
|
|
self.timeout = timeout
|
|
return FakeResponse(b'data: {"type":"reasoning-delta","id":"r","delta":"provider reasoning"}\n'
|
|
b'data: {"type":"reasoning-end","id":"r"}\n'
|
|
b'data: {"type":"text-delta","id":"a","delta":"answer"}\n'
|
|
b'data: {"type":"finish","finishReason":"stop"}\n'
|
|
b'data: [DONE]\n')
|
|
|
|
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
|
|
users = Path(temp).resolve()
|
|
actor_id = str(uuid.uuid4())
|
|
session_id = str(uuid.uuid4())
|
|
actor_root = users / actor_id
|
|
actor_root.mkdir(mode=0o700)
|
|
project = actor_root / "project"
|
|
project.mkdir()
|
|
current_session = actor_root / "private" / "current-session.json"
|
|
runtime._atomic_json(current_session, {"actorId": actor_id, "sessionId": session_id})
|
|
runtime._write_project(actor_root, session_id, "test-model", refresh_templates=True,
|
|
refresh_mcp=True, backup_schema=False)
|
|
mcp_path = project / "agent" / "mcps" / "mcp.json"
|
|
original_config = mcp_path.read_bytes()
|
|
runtime._write_project(actor_root, session_id, "test-model", refresh_templates=False,
|
|
refresh_mcp=False, backup_schema=False)
|
|
self.assertEqual(mcp_path.read_bytes(), original_config)
|
|
|
|
client = CaptureClient()
|
|
events = []
|
|
result = runtime._sse_request(
|
|
client=client, base_url="http://127.0.0.1:10000", model="test-model",
|
|
question="q", release_id=str(uuid.uuid4()), context=None, chat_id=None,
|
|
session_state=runtime.SessionState(actor_root, actor_id), session_id=session_id,
|
|
actor_root=actor_root, emit=events.append, timeout=5,
|
|
)
|
|
self.assertEqual(client.timeout, runtime.SSE_IDLE_TIMEOUT_SECONDS)
|
|
self.assertEqual(result[1], "COMPLETED")
|
|
self.assertEqual(result[0].answer, "answer")
|
|
self.assertFalse(any("modelReasoning" in event for event in events))
|
|
self.assertTrue(any(event.get("timeline") for event in events))
|
|
|
|
next_session = str(uuid.uuid4())
|
|
runtime._write_project(actor_root, next_session, "test-model", refresh_templates=False,
|
|
refresh_mcp=True, backup_schema=False)
|
|
updated = json.loads(mcp_path.read_text(encoding="utf-8"))
|
|
server_env = updated["mcpServers"]["quality"]["env"]
|
|
self.assertEqual(server_env["QUALITY_AGENT_SESSION_ID"], next_session)
|
|
self.assertEqual(json.loads(current_session.read_text())["sessionId"], next_session)
|
|
|
|
|
|
class RuntimeManagerTests(unittest.TestCase):
|
|
def test_ensure_instance_executes_project_setup_and_registry_path(self):
|
|
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
|
|
users = Path(temp).resolve()
|
|
manager = runtime.RuntimeManager(users_root=users)
|
|
actor_id = str(uuid.uuid4())
|
|
session_id = str(uuid.uuid4())
|
|
actor_root, actor_lock = manager.acquire_actor(actor_id)
|
|
fake_port = 49123
|
|
|
|
def fake_start(root, model_base, model_key, model_name, fingerprint):
|
|
self.assertEqual(root, actor_root)
|
|
runtime._atomic_json(root / "instance.json", {
|
|
"actorId": actor_id, "serverPort": fake_port,
|
|
"fingerprint": fingerprint, "processes": [],
|
|
})
|
|
|
|
try:
|
|
with patch.object(runtime, "_model_settings",
|
|
return_value=("https://model.invalid/v1", "test-key", "test-model")), \
|
|
patch.object(runtime, "_config_fingerprint", return_value="test-fingerprint"), \
|
|
patch.object(manager, "_start", side_effect=fake_start) as start:
|
|
config = manager.ensure_instance(actor_root, session_id)
|
|
self.assertEqual(config, ("https://model.invalid/v1", "test-key", "test-model",
|
|
f"http://127.0.0.1:{fake_port}"))
|
|
start.assert_called_once()
|
|
project = actor_root / "project"
|
|
self.assertTrue((project / "RULES.md").is_file())
|
|
self.assertTrue((project / "nao_config.yaml").is_file())
|
|
mcp_config = json.loads((project / "agent" / "mcps" / "mcp.json").read_text())
|
|
self.assertEqual(mcp_config["mcpServers"]["quality"]["env"]["QUALITY_AGENT_SESSION_ID"],
|
|
session_id)
|
|
self.assertEqual(runtime.RuntimeManager._current_session(actor_root), session_id)
|
|
finally:
|
|
manager.release_actor(actor_lock)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|