Files
quality_analyse/verification/test_nao_model_gate.py
lyun 52d92c95d8 修复:智能分析空回答与工具结束条件冲突
统一自主分析工具范围与执行前校验,严格核验正常结束、最终正文和图表恢复状态,并显示空回答及恢复失败原因。

包含两次已上线修复及回归测试,保留已有回答与业务数据。

验证:1101 项 Python、469 项前端、97 项后端测试通过;前后端构建和真实模型、页面验收通过。
2026-11-03 08:56:49 +08:00

313 lines
18 KiB
Python

"""Loopback fake-provider tests: no configured model, credential file or real network service."""
import json
import sys
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_model_gate import ModelGate
class HeldOpenProviderResponse:
"""A provider that sends a terminal SSE event but keeps its transport alive."""
status = 200
headers = {'Content-Type': 'text/event-stream'}
def __init__(self, chunks):
self.chunks = list(chunks)
self.read_count = 0
self.terminal_delivered = threading.Event()
self.read_after_terminal = threading.Event()
self.release = threading.Event()
self.closed = threading.Event()
def __enter__(self): return self
def __exit__(self, *_): self.closed.set()
def read1(self, _size):
self.read_count += 1
if self.chunks:
chunk = self.chunks.pop(0)
if not self.chunks:
self.terminal_delivered.set()
return chunk
self.read_after_terminal.set()
# Heartbeats keep the connection readable without adding completion data.
return b'' if self.release.wait(.02) else b': heartbeat\n\n'
class ModelGateTests(unittest.TestCase):
def setUp(self):
self.allowed = True
self.calls = []
self.provider_status = 200
owner = self
class Provider(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
owner.calls.append(json.loads(self.rfile.read(int(self.headers['Content-Length']))))
data = json.dumps({'choices': [{'message': {'content': 'fake model'}}]} if owner.provider_status == 200
else {'error': 'fake-sensitive-provider-details'}).encode()
self.send_response(owner.provider_status)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(data)))
self.end_headers(); self.wfile.write(data)
self.provider = ThreadingHTTPServer(('127.0.0.1', 0), Provider)
self.provider.daemon_threads = True
self.thread = threading.Thread(target=self.provider.serve_forever, daemon=True); self.thread.start()
self.gate = ModelGate(upstream_base=f'http://127.0.0.1:{self.provider.server_port}/v1', upstream_key='dummy-provider-key',
model='fake-model', port=0, client_key='dummy-native-key', authorize=self.authorize)
# Port zero is only used by this isolated test server.
self.base = f'http://127.0.0.1:{self.gate._server.server_port}/v1'
def tearDown(self):
self.gate.close(); self.provider.shutdown(); self.provider.server_close(); self.thread.join(timeout=1)
def authorize(self):
if not self.allowed: raise PermissionError('revoked or expired delegation')
def request(self, *, key='dummy-native-key', model='fake-model', extra=None, phase=None):
headers={'Content-Type':'application/json','Authorization':'Bearer '+key}
if phase:headers['X-Quality-Analysis-Phase']=phase
request = Request(self.base+'/chat/completions', data=json.dumps({'model': model, 'messages': [{'role': 'user', 'content': 'offline fixture'}],**(extra or {})}).encode(),
headers=headers)
try:
with urlopen(request, timeout=3) as response: return response.status, response.read().decode()
except HTTPError as error: return error.code, error.read().decode()
def test_revocation_blocks_native_and_final_synthesis_requests_before_fake_provider(self):
self.assertEqual(self.request()[0], 200)
self.assertEqual(len(self.calls), 1)
self.allowed = False
self.assertEqual(self.request()[0], 403) # next native provider request
self.assertEqual(self.request()[0], 403) # final synthesis uses this same connection
self.assertEqual(len(self.calls), 1)
def test_initial_denial_or_wrong_native_credential_never_reaches_provider(self):
self.allowed = False
self.assertEqual(self.request()[0], 403)
self.allowed = True
self.assertEqual(self.request(key='wrong-key')[0], 403)
self.assertEqual(self.request(model='different-model')[0], 400)
self.assertEqual(len(self.calls), 0)
def test_quality_api_charts_do_not_advertise_incompatible_sql_renderer(self):
data_tool={'type':'function','function':{'name':'mcp_call','parameters':{'type':'object'}}}
chart_tool={'type':'function','function':{'name':'display_chart','parameters':{'type':'object'}}}
self.assertEqual(self.request(extra={'tools':[data_tool,chart_tool],
'tool_choice':{'type':'function','function':{'name':'display_chart'}}})[0],200)
self.assertEqual(self.calls[-1]['tools'],[data_tool])
self.assertNotIn('tool_choice',self.calls[-1])
self.assertEqual(self.request(extra={'tools':[chart_tool],'tool_choice':'required'})[0],200)
self.assertNotIn('tools',self.calls[-1])
self.assertNotIn('tool_choice',self.calls[-1])
def test_provider_errors_are_bounded_public_failures(self):
self.provider_status = 503
code, body = self.request()
self.assertEqual(code, 503)
self.assertNotIn('fake-sensitive-provider-details', body)
self.assertEqual(len(self.calls), 1)
def test_unadvertised_tool_calls_are_blocked_before_json_or_stream_execution(self):
from unittest.mock import Mock
data_tool={'type':'function','function':{'name':'mcp_call','parameters':{'type':'object'}}}
tool={'id':'call-blocked','type':'function','function':{'name':'clarification','arguments':'{}'}}
reply={'choices':[{'index':0,'message':{'role':'assistant','content':None,'tool_calls':[tool]},'finish_reason':'tool_calls'}]}
upstream=HeldOpenProviderResponse([json.dumps(reply).encode()]);upstream.headers={'Content-Type':'application/json'};upstream.release.set()
self.gate._opener=Mock();self.gate._opener.open.return_value=upstream
code,body=self.request(extra={'tools':[data_tool]})
self.assertEqual(code,502);self.assertNotIn('clarification',body)
for newline in (True,False):
prefix=b'data: {"choices":[{"index":0,"delta":{"content":"Preserved text"}}]}\n\n'
bad=('data: '+json.dumps({'choices':[{'index':0,'delta':{'tool_calls':[tool]},'finish_reason':'tool_calls'}]})+ ('\n\n' if newline else '')).encode()
upstream=HeldOpenProviderResponse([prefix,bad[:19],bad[19:]]);upstream.release.set()
self.gate._opener.open.return_value=upstream
code,body=self.request(extra={'stream':True,'tools':[data_tool]})
self.assertEqual(code,200);self.assertIn('Preserved text',body)
self.assertNotIn('clarification',body);self.assertIn('error',body)
def test_allowed_fragmented_tool_calls_keep_their_exact_wire_data(self):
from unittest.mock import Mock
rows=[{'choices':[{'index':0,'delta':{'tool_calls':[{'index':0,'id':'q','type':'function','function':{'name':'mcp_','arguments':''}}]}}]},
{'choices':[{'index':0,'delta':{'tool_calls':[{'index':0,'function':{'name':'call','arguments':'{}'}}]},'finish_reason':'tool_calls'}]}]
wire=b''.join(('data: '+json.dumps(row)+'\n\n').encode() for row in rows)+b'data: [DONE]\n\n'
upstream=HeldOpenProviderResponse([wire[:23],wire[23:71],wire[71:]]);upstream.release.set()
self.gate._opener=Mock();self.gate._opener.open.return_value=upstream
code,body=self.request(extra={'stream':True,'tools':[{'type':'function','function':{'name':'mcp_call'}}]})
self.assertEqual(code,200);self.assertEqual(body.encode(),wire)
def test_follow_up_tool_cannot_end_analysis_before_the_model_writes_an_answer(self):
data_tool = {'type': 'function', 'function': {'name': 'mcp_call', 'parameters': {'type': 'object'}}}
follow_up = {'type': 'function', 'function': {'name': 'suggest_follow_ups', 'parameters': {'type': 'object'}}}
for choice in ('auto', {'type': 'function', 'function': {'name': 'suggest_follow_ups'}}):
with self.subTest(choice=choice):
self.assertEqual(self.request(extra={'tools': [data_tool, follow_up], 'tool_choice': choice})[0], 200)
self.assertEqual(self.calls[-1]['tools'], [data_tool])
if isinstance(choice, dict):
self.assertNotIn('tool_choice', self.calls[-1])
self.assertEqual(self.request(extra={'tools': [follow_up], 'tool_choice': 'required'})[0], 200)
self.assertNotIn('tools', self.calls[-1])
self.assertNotIn('tool_choice', self.calls[-1])
def test_only_supported_tools_reach_the_native_model_in_each_execution_mode(self):
helpers = ['read', 'list', 'grep', 'search', 'mcp_call', 'mcp_connect']
incompatible = ['clarification', 'request_warehouse_selection', 'request_warehouse_credentials',
'execute_sql', 'execute_semantic_query', 'read_query_result', 'display_chart', 'display_map',
'write', 'str_replace', 'execute_python', 'execute_sandboxed_code', 'task',
'onboarding_command', 'onboarding_progress', 'generate_onboarding_rules',
'suggest_follow_ups', 'future_unknown_tool']
tools = [{'type': 'function', 'function': {'name': n, 'parameters': {'type': 'object'}}}
for n in helpers + ['story'] + incompatible]
for native in (False, True):
self.gate.native_agent = native
self.assertEqual(self.request(extra={'tools': tools})[0], 200)
self.assertEqual([t['function']['name'] for t in self.calls[-1]['tools']],
helpers + ([] if native else ['story']))
def test_forced_unavailable_tool_cannot_survive_even_without_a_tool_definition(self):
for extra in ({'tool_choice': {'type': 'function', 'function': {'name': 'clarification'}}},
{'tool_choice': {'type': 'function', 'function': 'malformed'}},
{'tools': [], 'tool_choice': 'required'},
{'tools': [{'type': 'function', 'function': {'name': 'read'}}],
'tool_choice': {'type': 'function', 'function': {'name': 'execute_sql'}}}):
self.assertEqual(self.request(extra=extra)[0], 200)
self.assertNotIn('tool_choice', self.calls[-1])
def test_previous_turn_background_call_cannot_enter_current_thinking_or_provider(self):
marker='quality-turn:current-request'
self.gate._native_turn_marker=marker
self.gate._thinking={'enabled':True,'effort':'medium','protocol':'thinking'}
self.assertEqual(self.request(extra={'messages':[{'role':'user','content':'quality-turn:old-request'}]})[0],403)
self.assertEqual(self.request(extra={'messages':[{'role':'system','content':marker},{'role':'user','content':'old request'}]})[0],403)
self.assertEqual(self.calls,[])
self.assertEqual(self.gate._request_number,0)
self.assertEqual(self.gate.request_audit,[])
self.assertEqual(self.gate.rejected_stale_requests,2)
self.assertEqual(self.request(extra={'messages':[{'role':'user','content':[{'type':'text','text':marker+' current task'}]}]})[0],200)
self.assertEqual(self.request(phase='planning')[0],200)
self.assertEqual(len(self.calls),2)
self.assertEqual(len(self.gate.request_audit),2)
def test_saved_enabled_effort_applies_to_every_request_despite_helper_defaults(self):
self.gate._thinking={'enabled':True,'effort':'high','protocol':'thinking'}
self.assertEqual(self.request(extra={'thinking':{'type':'disabled'},'reasoning_effort':'low','temperature':.2,'max_completion_tokens':4096})[0],200)
body=self.calls[-1]
self.assertEqual(body['thinking'],{'type':'enabled'})
self.assertEqual(body['reasoning_effort'],'high')
self.assertNotIn('temperature',body)
from nao_runtime_policy import MAX_OUTPUT_TOKENS
self.assertEqual(body['max_completion_tokens'],MAX_OUTPUT_TOKENS)
self.assertNotIn('offline fixture',json.dumps(self.gate.request_audit))
def test_qwen38_native_effort_uses_template_low_medium_and_xhigh(self):
self.gate._model='qwen3.8-27b-local'
for selected,native in [('low','low'),('medium','medium'),('high','xhigh')]:
self.gate._thinking={'enabled':True,'effort':selected,'protocol':'chat_template'}
self.assertEqual(self.request(model='qwen3.8-27b-local',extra={'reasoning_effort':'high','chat_template_kwargs':{'reasoning_effort':'low','preserve_thinking':True}})[0],200)
body=self.calls[-1]
self.assertEqual(body['chat_template_kwargs'],{'reasoning_effort':native,'enable_thinking':True,'preserve_thinking':True})
self.assertNotIn('reasoning_effort',body)
self.assertEqual(self.gate.request_audit[-1]['reasoningEffort'],native)
def test_qwen_switch_uses_chat_template_control_and_removes_conflicting_flags(self):
self.gate._thinking={'enabled':False,'effort':'medium','protocol':'chat_template'}
self.assertEqual(self.request(extra={'thinking':{'type':'enabled'},'reasoning_effort':'high','chat_template_kwargs':{'enable_thinking':True}})[0],200)
body=self.calls[-1]
self.assertEqual(body['chat_template_kwargs']['enable_thinking'],False)
self.assertNotIn('thinking',body)
self.assertNotIn('reasoning_effort',body)
def test_explicit_sse_done_finishes_without_eof_for_bridge_and_passthrough(self):
from unittest.mock import Mock
rows = [
{'id': 'fake-completion', 'model': 'fake-model', 'choices': [
{'index': 0, 'delta': {'reasoning_content': '先核对。', 'content': '{"ok":'}}]},
{'choices': [{'index': 0, 'delta': {'content': 'true}'}, 'finish_reason': 'stop'}]},
# The usage-only tail must survive even though the choice already finished.
{'choices': [], 'usage': {'prompt_tokens': 7, 'completion_tokens': 12}},
]
wire = [('data: ' + json.dumps(row, ensure_ascii=False) + '\n\n').encode() for row in rows]
chunks = [*wire, b'data: [DO', b'NE]\n\n']
for stream, observe in ((False, True), (True, True), (True, False)):
with self.subTest(stream=stream, observe=observe):
events = []
self.gate._on_thinking = events.append if observe else None
upstream = HeldOpenProviderResponse(chunks)
self.gate._opener = Mock()
self.gate._opener.open.return_value = upstream
response = []
returned = threading.Event()
def request():
try:
response.append(self.request(extra={'stream': stream}))
except Exception as error:
response.append(error)
finally:
returned.set()
worker = threading.Thread(target=request, daemon=True)
worker.start()
try:
self.assertTrue(upstream.terminal_delivered.wait(1), 'fake provider did not deliver DONE')
completed_before_eof = returned.wait(1)
finally:
upstream.release.set()
worker.join(timeout=2)
self.assertTrue(completed_before_eof, 'ModelGate still waits for provider EOF after explicit DONE')
self.assertFalse(upstream.read_after_terminal.is_set(), 'ModelGate read beyond explicit DONE')
self.assertTrue(upstream.closed.is_set())
self.assertEqual(response[0][0], 200)
if stream:
self.assertEqual(response[0][1].encode(), b''.join(chunks))
else:
completion = json.loads(response[0][1])
self.assertEqual(completion['choices'][0]['message']['content'], '{"ok":true}')
self.assertEqual(completion['choices'][0]['finish_reason'], 'stop')
self.assertEqual(completion['usage'], {'prompt_tokens': 7, 'completion_tokens': 12})
self.assertNotIn('reasoning_content', completion['choices'][0]['message'])
if observe:
self.assertEqual(events[-1]['event'], 'end')
self.assertEqual(events[-1]['finishReason'], 'stop')
self.assertFalse(events[-1]['interrupted'])
def test_explicit_done_without_finish_reason_is_not_a_successful_completion(self):
from unittest.mock import Mock
upstream = HeldOpenProviderResponse([
b'data: {"choices":[{"index":0,"delta":{"content":"incomplete"}}]}\n\n',
b'data: [DONE]\n\n',
])
self.gate._opener = Mock()
self.gate._opener.open.return_value = upstream
events = []
self.gate._on_thinking = events.append
response = []
returned = threading.Event()
def request():
try:
response.append(self.request())
finally:
returned.set()
worker = threading.Thread(target=request, daemon=True)
worker.start()
try:
self.assertTrue(upstream.terminal_delivered.wait(1))
completed_before_eof = returned.wait(1)
finally:
upstream.release.set()
worker.join(timeout=2)
self.assertTrue(completed_before_eof, 'invalid DONE must fail without waiting for EOF')
self.assertEqual(response[0][0], 502)
self.assertTrue(events[-1]['interrupted'])
if __name__ == '__main__': unittest.main()