Files
quality_analyse/verification/test_nao_model_gate.py
lyun ff7bbe3e46 同步:完善质量分析工作台并更新主题字体
同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。
补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。
统一午夜黑及其他主题的控件细节、中文和业务数字字体。
排除本地业务数据、凭据、预览、发布与验收产物。

验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
2026-10-30 22:40:58 +08:00

248 lines
13 KiB
Python

"""Loopback fake-provider tests: no configured model, credential file or real network service."""
import json
import sys
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_model_gate import ModelGate
class HeldOpenProviderResponse:
"""A provider that sends a terminal SSE event but keeps its transport alive."""
status = 200
headers = {'Content-Type': 'text/event-stream'}
def __init__(self, chunks):
self.chunks = list(chunks)
self.read_count = 0
self.terminal_delivered = threading.Event()
self.read_after_terminal = threading.Event()
self.release = threading.Event()
self.closed = threading.Event()
def __enter__(self): return self
def __exit__(self, *_): self.closed.set()
def read1(self, _size):
self.read_count += 1
if self.chunks:
chunk = self.chunks.pop(0)
if not self.chunks:
self.terminal_delivered.set()
return chunk
self.read_after_terminal.set()
# Heartbeats keep the connection readable without adding completion data.
return b'' if self.release.wait(.02) else b': heartbeat\n\n'
class ModelGateTests(unittest.TestCase):
def setUp(self):
self.allowed = True
self.calls = []
self.provider_status = 200
owner = self
class Provider(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
owner.calls.append(json.loads(self.rfile.read(int(self.headers['Content-Length']))))
data = json.dumps({'choices': [{'message': {'content': 'fake model'}}]} if owner.provider_status == 200
else {'error': 'fake-sensitive-provider-details'}).encode()
self.send_response(owner.provider_status)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(data)))
self.end_headers(); self.wfile.write(data)
self.provider = ThreadingHTTPServer(('127.0.0.1', 0), Provider)
self.provider.daemon_threads = True
self.thread = threading.Thread(target=self.provider.serve_forever, daemon=True); self.thread.start()
self.gate = ModelGate(upstream_base=f'http://127.0.0.1:{self.provider.server_port}/v1', upstream_key='dummy-provider-key',
model='fake-model', port=0, client_key='dummy-native-key', authorize=self.authorize)
# Port zero is only used by this isolated test server.
self.base = f'http://127.0.0.1:{self.gate._server.server_port}/v1'
def tearDown(self):
self.gate.close(); self.provider.shutdown(); self.provider.server_close(); self.thread.join(timeout=1)
def authorize(self):
if not self.allowed: raise PermissionError('revoked or expired delegation')
def request(self, *, key='dummy-native-key', model='fake-model', extra=None, phase=None):
headers={'Content-Type':'application/json','Authorization':'Bearer '+key}
if phase:headers['X-Quality-Analysis-Phase']=phase
request = Request(self.base+'/chat/completions', data=json.dumps({'model': model, 'messages': [{'role': 'user', 'content': 'offline fixture'}],**(extra or {})}).encode(),
headers=headers)
try:
with urlopen(request, timeout=3) as response: return response.status, response.read().decode()
except HTTPError as error: return error.code, error.read().decode()
def test_revocation_blocks_native_and_final_synthesis_requests_before_fake_provider(self):
self.assertEqual(self.request()[0], 200)
self.assertEqual(len(self.calls), 1)
self.allowed = False
self.assertEqual(self.request()[0], 403) # next native provider request
self.assertEqual(self.request()[0], 403) # final synthesis uses this same connection
self.assertEqual(len(self.calls), 1)
def test_initial_denial_or_wrong_native_credential_never_reaches_provider(self):
self.allowed = False
self.assertEqual(self.request()[0], 403)
self.allowed = True
self.assertEqual(self.request(key='wrong-key')[0], 403)
self.assertEqual(self.request(model='different-model')[0], 400)
self.assertEqual(len(self.calls), 0)
def test_quality_api_charts_do_not_advertise_incompatible_sql_renderer(self):
data_tool={'type':'function','function':{'name':'mcp_call','parameters':{'type':'object'}}}
chart_tool={'type':'function','function':{'name':'display_chart','parameters':{'type':'object'}}}
self.assertEqual(self.request(extra={'tools':[data_tool,chart_tool],
'tool_choice':{'type':'function','function':{'name':'display_chart'}}})[0],200)
self.assertEqual(self.calls[-1]['tools'],[data_tool])
self.assertNotIn('tool_choice',self.calls[-1])
self.assertEqual(self.request(extra={'tools':[chart_tool],'tool_choice':'required'})[0],200)
self.assertNotIn('tools',self.calls[-1])
self.assertNotIn('tool_choice',self.calls[-1])
def test_provider_errors_are_bounded_public_failures(self):
self.provider_status = 503
code, body = self.request()
self.assertEqual(code, 503)
self.assertNotIn('fake-sensitive-provider-details', body)
self.assertEqual(len(self.calls), 1)
def test_previous_turn_background_call_cannot_enter_current_thinking_or_provider(self):
marker='quality-turn:current-request'
self.gate._native_turn_marker=marker
self.gate._thinking={'enabled':True,'effort':'medium','protocol':'thinking'}
self.assertEqual(self.request(extra={'messages':[{'role':'user','content':'quality-turn:old-request'}]})[0],403)
self.assertEqual(self.request(extra={'messages':[{'role':'system','content':marker},{'role':'user','content':'old request'}]})[0],403)
self.assertEqual(self.calls,[])
self.assertEqual(self.gate._request_number,0)
self.assertEqual(self.gate.request_audit,[])
self.assertEqual(self.gate.rejected_stale_requests,2)
self.assertEqual(self.request(extra={'messages':[{'role':'user','content':[{'type':'text','text':marker+' current task'}]}]})[0],200)
self.assertEqual(self.request(phase='planning')[0],200)
self.assertEqual(len(self.calls),2)
self.assertEqual(len(self.gate.request_audit),2)
def test_saved_enabled_effort_applies_to_every_request_despite_helper_defaults(self):
self.gate._thinking={'enabled':True,'effort':'high','protocol':'thinking'}
self.assertEqual(self.request(extra={'thinking':{'type':'disabled'},'reasoning_effort':'low','temperature':.2,'max_completion_tokens':4096})[0],200)
body=self.calls[-1]
self.assertEqual(body['thinking'],{'type':'enabled'})
self.assertEqual(body['reasoning_effort'],'high')
self.assertNotIn('temperature',body)
from nao_runtime_policy import MAX_OUTPUT_TOKENS
self.assertEqual(body['max_completion_tokens'],MAX_OUTPUT_TOKENS)
self.assertNotIn('offline fixture',json.dumps(self.gate.request_audit))
def test_qwen38_native_effort_uses_template_low_medium_and_xhigh(self):
self.gate._model='qwen3.8-27b-local'
for selected,native in [('low','low'),('medium','medium'),('high','xhigh')]:
self.gate._thinking={'enabled':True,'effort':selected,'protocol':'chat_template'}
self.assertEqual(self.request(model='qwen3.8-27b-local',extra={'reasoning_effort':'high','chat_template_kwargs':{'reasoning_effort':'low','preserve_thinking':True}})[0],200)
body=self.calls[-1]
self.assertEqual(body['chat_template_kwargs'],{'reasoning_effort':native,'enable_thinking':True,'preserve_thinking':True})
self.assertNotIn('reasoning_effort',body)
self.assertEqual(self.gate.request_audit[-1]['reasoningEffort'],native)
def test_qwen_switch_uses_chat_template_control_and_removes_conflicting_flags(self):
self.gate._thinking={'enabled':False,'effort':'medium','protocol':'chat_template'}
self.assertEqual(self.request(extra={'thinking':{'type':'enabled'},'reasoning_effort':'high','chat_template_kwargs':{'enable_thinking':True}})[0],200)
body=self.calls[-1]
self.assertEqual(body['chat_template_kwargs']['enable_thinking'],False)
self.assertNotIn('thinking',body)
self.assertNotIn('reasoning_effort',body)
def test_explicit_sse_done_finishes_without_eof_for_bridge_and_passthrough(self):
from unittest.mock import Mock
rows = [
{'id': 'fake-completion', 'model': 'fake-model', 'choices': [
{'index': 0, 'delta': {'reasoning_content': '先核对。', 'content': '{"ok":'}}]},
{'choices': [{'index': 0, 'delta': {'content': 'true}'}, 'finish_reason': 'stop'}]},
# The usage-only tail must survive even though the choice already finished.
{'choices': [], 'usage': {'prompt_tokens': 7, 'completion_tokens': 12}},
]
wire = [('data: ' + json.dumps(row, ensure_ascii=False) + '\n\n').encode() for row in rows]
chunks = [*wire, b'data: [DO', b'NE]\n\n']
for stream, observe in ((False, True), (True, True), (True, False)):
with self.subTest(stream=stream, observe=observe):
events = []
self.gate._on_thinking = events.append if observe else None
upstream = HeldOpenProviderResponse(chunks)
self.gate._opener = Mock()
self.gate._opener.open.return_value = upstream
response = []
returned = threading.Event()
def request():
try:
response.append(self.request(extra={'stream': stream}))
except Exception as error:
response.append(error)
finally:
returned.set()
worker = threading.Thread(target=request, daemon=True)
worker.start()
try:
self.assertTrue(upstream.terminal_delivered.wait(1), 'fake provider did not deliver DONE')
completed_before_eof = returned.wait(1)
finally:
upstream.release.set()
worker.join(timeout=2)
self.assertTrue(completed_before_eof, 'ModelGate still waits for provider EOF after explicit DONE')
self.assertFalse(upstream.read_after_terminal.is_set(), 'ModelGate read beyond explicit DONE')
self.assertTrue(upstream.closed.is_set())
self.assertEqual(response[0][0], 200)
if stream:
self.assertEqual(response[0][1].encode(), b''.join(chunks))
else:
completion = json.loads(response[0][1])
self.assertEqual(completion['choices'][0]['message']['content'], '{"ok":true}')
self.assertEqual(completion['choices'][0]['finish_reason'], 'stop')
self.assertEqual(completion['usage'], {'prompt_tokens': 7, 'completion_tokens': 12})
self.assertNotIn('reasoning_content', completion['choices'][0]['message'])
if observe:
self.assertEqual(events[-1]['event'], 'end')
self.assertEqual(events[-1]['finishReason'], 'stop')
self.assertFalse(events[-1]['interrupted'])
def test_explicit_done_without_finish_reason_is_not_a_successful_completion(self):
from unittest.mock import Mock
upstream = HeldOpenProviderResponse([
b'data: {"choices":[{"index":0,"delta":{"content":"incomplete"}}]}\n\n',
b'data: [DONE]\n\n',
])
self.gate._opener = Mock()
self.gate._opener.open.return_value = upstream
events = []
self.gate._on_thinking = events.append
response = []
returned = threading.Event()
def request():
try:
response.append(self.request())
finally:
returned.set()
worker = threading.Thread(target=request, daemon=True)
worker.start()
try:
self.assertTrue(upstream.terminal_delivered.wait(1))
completed_before_eof = returned.wait(1)
finally:
upstream.release.set()
worker.join(timeout=2)
self.assertTrue(completed_before_eof, 'invalid DONE must fail without waiting for EOF')
self.assertEqual(response[0][0], 502)
self.assertTrue(events[-1]['interrupted'])
if __name__ == '__main__': unittest.main()