修复:智能分析空回答与工具结束条件冲突
统一自主分析工具范围与执行前校验,严格核验正常结束、最终正文和图表恢复状态,并显示空回答及恢复失败原因。 包含两次已上线修复及回归测试,保留已有回答与业务数据。 验证:1101 项 Python、469 项前端、97 项后端测试通过;前后端构建和真实模型、页面验收通过。
This commit is contained in:
@@ -97,7 +97,7 @@ class MinimalQueryTests(unittest.TestCase):
|
||||
context=bind_task({'releaseId':RELEASE},task))
|
||||
self.assertEqual(visual['visualState'],'ready',visual)
|
||||
self.assertEqual(visual['visuals'][0]['metric'],'rate_drop_pp')
|
||||
self.assertEqual(visual['visuals'][0]['series'][0]['values'],[20.0])
|
||||
self.assertEqual(visual['visuals'][0]['series'][0]['values'],[-20.0])
|
||||
result['sort']='current_rate';task['changeQuery']['sort']='current_rate'
|
||||
visual=build_visuals([doc],task['question'],TURN,
|
||||
context=bind_task({'releaseId':RELEASE},task))
|
||||
|
||||
@@ -149,14 +149,24 @@ class EvidenceFinalizerTest(unittest.TestCase):
|
||||
with self.assertRaises(finalizer.FinalizationUnavailable):
|
||||
finalizer.packed_evidence([{'unique': 'a'*finalizer.MAX_INPUT_CHARS}])
|
||||
|
||||
def test_real_second_turn_replay_fits_and_preserves_every_summary_value(self):
|
||||
def test_real_second_turn_replay_preserves_every_value_and_obeys_current_input_budget(self):
|
||||
root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime'
|
||||
directories=list(root.glob('.runtime/nao-users/*/sessions/57676d5a-5138-48f4-9343-bae9731aae5a/evidence'))
|
||||
if not directories: self.skipTest('Private captured acceptance evidence not present')
|
||||
with patch.object(finalizer,'packed_evidence',side_effect=AssertionError('Local archives do not consume model input budget')):
|
||||
docs=finalizer.evidence_summaries(directories[0],'194b0287-9044-4d80-9138-ecd3c481015c')
|
||||
self.assertEqual(len(docs),15)
|
||||
self.assertEqual(unpack(finalizer.packed_evidence(docs)),docs)
|
||||
# Formal display metadata has grown since this private run was captured.
|
||||
# Local archives remain lossless; a bounded model handoff may reject an
|
||||
# oversized summary, but may never silently trim it to fit.
|
||||
limit=finalizer.MAX_INPUT_CHARS
|
||||
with patch.object(finalizer,'MAX_INPUT_CHARS',100_000_000):
|
||||
packed=finalizer.packed_evidence(docs)
|
||||
self.assertEqual(unpack(packed),docs)
|
||||
encoded=json.dumps(packed,ensure_ascii=False,separators=(',',':'))
|
||||
if len(encoded)>limit:
|
||||
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.packed_evidence(docs)
|
||||
else:self.assertEqual(unpack(finalizer.packed_evidence(docs)),docs)
|
||||
# This captured run predates paired black/bright defect reads for each
|
||||
# highlighted product, so it is now correctly labelled incomplete.
|
||||
self.assertFalse(finalizer.coverage_sufficient(docs,'分析质量变化'))
|
||||
|
||||
@@ -114,6 +114,71 @@ class ModelGateTests(unittest.TestCase):
|
||||
self.assertNotIn('fake-sensitive-provider-details', body)
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
|
||||
def test_unadvertised_tool_calls_are_blocked_before_json_or_stream_execution(self):
|
||||
from unittest.mock import Mock
|
||||
data_tool={'type':'function','function':{'name':'mcp_call','parameters':{'type':'object'}}}
|
||||
tool={'id':'call-blocked','type':'function','function':{'name':'clarification','arguments':'{}'}}
|
||||
reply={'choices':[{'index':0,'message':{'role':'assistant','content':None,'tool_calls':[tool]},'finish_reason':'tool_calls'}]}
|
||||
upstream=HeldOpenProviderResponse([json.dumps(reply).encode()]);upstream.headers={'Content-Type':'application/json'};upstream.release.set()
|
||||
self.gate._opener=Mock();self.gate._opener.open.return_value=upstream
|
||||
code,body=self.request(extra={'tools':[data_tool]})
|
||||
self.assertEqual(code,502);self.assertNotIn('clarification',body)
|
||||
for newline in (True,False):
|
||||
prefix=b'data: {"choices":[{"index":0,"delta":{"content":"Preserved text"}}]}\n\n'
|
||||
bad=('data: '+json.dumps({'choices':[{'index':0,'delta':{'tool_calls':[tool]},'finish_reason':'tool_calls'}]})+ ('\n\n' if newline else '')).encode()
|
||||
upstream=HeldOpenProviderResponse([prefix,bad[:19],bad[19:]]);upstream.release.set()
|
||||
self.gate._opener.open.return_value=upstream
|
||||
code,body=self.request(extra={'stream':True,'tools':[data_tool]})
|
||||
self.assertEqual(code,200);self.assertIn('Preserved text',body)
|
||||
self.assertNotIn('clarification',body);self.assertIn('error',body)
|
||||
|
||||
def test_allowed_fragmented_tool_calls_keep_their_exact_wire_data(self):
|
||||
from unittest.mock import Mock
|
||||
rows=[{'choices':[{'index':0,'delta':{'tool_calls':[{'index':0,'id':'q','type':'function','function':{'name':'mcp_','arguments':''}}]}}]},
|
||||
{'choices':[{'index':0,'delta':{'tool_calls':[{'index':0,'function':{'name':'call','arguments':'{}'}}]},'finish_reason':'tool_calls'}]}]
|
||||
wire=b''.join(('data: '+json.dumps(row)+'\n\n').encode() for row in rows)+b'data: [DONE]\n\n'
|
||||
upstream=HeldOpenProviderResponse([wire[:23],wire[23:71],wire[71:]]);upstream.release.set()
|
||||
self.gate._opener=Mock();self.gate._opener.open.return_value=upstream
|
||||
code,body=self.request(extra={'stream':True,'tools':[{'type':'function','function':{'name':'mcp_call'}}]})
|
||||
self.assertEqual(code,200);self.assertEqual(body.encode(),wire)
|
||||
|
||||
def test_follow_up_tool_cannot_end_analysis_before_the_model_writes_an_answer(self):
|
||||
data_tool = {'type': 'function', 'function': {'name': 'mcp_call', 'parameters': {'type': 'object'}}}
|
||||
follow_up = {'type': 'function', 'function': {'name': 'suggest_follow_ups', 'parameters': {'type': 'object'}}}
|
||||
for choice in ('auto', {'type': 'function', 'function': {'name': 'suggest_follow_ups'}}):
|
||||
with self.subTest(choice=choice):
|
||||
self.assertEqual(self.request(extra={'tools': [data_tool, follow_up], 'tool_choice': choice})[0], 200)
|
||||
self.assertEqual(self.calls[-1]['tools'], [data_tool])
|
||||
if isinstance(choice, dict):
|
||||
self.assertNotIn('tool_choice', self.calls[-1])
|
||||
self.assertEqual(self.request(extra={'tools': [follow_up], 'tool_choice': 'required'})[0], 200)
|
||||
self.assertNotIn('tools', self.calls[-1])
|
||||
self.assertNotIn('tool_choice', self.calls[-1])
|
||||
|
||||
def test_only_supported_tools_reach_the_native_model_in_each_execution_mode(self):
|
||||
helpers = ['read', 'list', 'grep', 'search', 'mcp_call', 'mcp_connect']
|
||||
incompatible = ['clarification', 'request_warehouse_selection', 'request_warehouse_credentials',
|
||||
'execute_sql', 'execute_semantic_query', 'read_query_result', 'display_chart', 'display_map',
|
||||
'write', 'str_replace', 'execute_python', 'execute_sandboxed_code', 'task',
|
||||
'onboarding_command', 'onboarding_progress', 'generate_onboarding_rules',
|
||||
'suggest_follow_ups', 'future_unknown_tool']
|
||||
tools = [{'type': 'function', 'function': {'name': n, 'parameters': {'type': 'object'}}}
|
||||
for n in helpers + ['story'] + incompatible]
|
||||
for native in (False, True):
|
||||
self.gate.native_agent = native
|
||||
self.assertEqual(self.request(extra={'tools': tools})[0], 200)
|
||||
self.assertEqual([t['function']['name'] for t in self.calls[-1]['tools']],
|
||||
helpers + ([] if native else ['story']))
|
||||
|
||||
def test_forced_unavailable_tool_cannot_survive_even_without_a_tool_definition(self):
|
||||
for extra in ({'tool_choice': {'type': 'function', 'function': {'name': 'clarification'}}},
|
||||
{'tool_choice': {'type': 'function', 'function': 'malformed'}},
|
||||
{'tools': [], 'tool_choice': 'required'},
|
||||
{'tools': [{'type': 'function', 'function': {'name': 'read'}}],
|
||||
'tool_choice': {'type': 'function', 'function': {'name': 'execute_sql'}}}):
|
||||
self.assertEqual(self.request(extra=extra)[0], 200)
|
||||
self.assertNotIn('tool_choice', self.calls[-1])
|
||||
|
||||
def test_previous_turn_background_call_cannot_enter_current_thinking_or_provider(self):
|
||||
marker='quality-turn:current-request'
|
||||
self.gate._native_turn_marker=marker
|
||||
|
||||
@@ -216,6 +216,53 @@ class RuntimeStreamTests(unittest.TestCase):
|
||||
truncated.feed({"type": "finish", "finishReason": "length"})
|
||||
self.assertEqual(truncated.status(), ("PARTIAL", "length"))
|
||||
|
||||
def test_tool_errors_are_read_from_envelopes_not_business_data_cells(self):
|
||||
for flag in ('mcpValidationError','mcpAuthRequired'):
|
||||
stream=runtime.ConversationAccumulator()
|
||||
stream.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'q'})
|
||||
stream.feed({'type':'tool-output-available','toolCallId':'q','output':{flag:True,'server':'quality'}})
|
||||
stream.feed({'type':'text-delta','id':'final','delta':'尚未查到数据。'})
|
||||
stream.feed({'type':'finish','finishReason':'stop'})
|
||||
self.assertEqual(stream.tool_calls[0]['status'],'failed')
|
||||
self.assertEqual(stream.status(),('PARTIAL','mcp_error'))
|
||||
payload={'kind':'quality_records','rows':[{'error':'缺角','isError':True}], 'note':'{"error":"原始单元格内容"}'}
|
||||
stream=runtime.ConversationAccumulator()
|
||||
stream.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'q'})
|
||||
stream.feed({'type':'tool-output-available','toolCallId':'q','output':{'content':[{'type':'text','text':json.dumps(payload)}]}})
|
||||
stream.feed({'type':'text-delta','id':'final','delta':'已查询并保留原始字段。'})
|
||||
stream.feed({'type':'finish','finishReason':'stop'})
|
||||
self.assertEqual(stream.tool_calls[0]['status'],'completed')
|
||||
self.assertEqual(stream.status(),('COMPLETED','stop'))
|
||||
|
||||
def test_transport_done_is_not_a_normal_model_finish(self):
|
||||
stream = runtime.ConversationAccumulator()
|
||||
stream.feed({'type': 'text-delta', 'id': 'progress', 'delta': '先查一下。'})
|
||||
stream.feed({'type': 'tool-input-available', 'toolName': 'mcp_call', 'toolCallId': 'q'})
|
||||
stream.feed({'type': 'tool-output-available', 'toolCallId': 'q', 'output': {}})
|
||||
stream.feed({'type': 'finish', 'finishReason': 'tool-calls'})
|
||||
stream.feed({'type': '[DONE]'})
|
||||
self.assertNotEqual(stream.status()[0], 'COMPLETED')
|
||||
self.assertEqual(stream.answer, '先查一下。')
|
||||
|
||||
def test_progress_before_a_tool_or_unsettled_tool_cannot_count_as_final_answer(self):
|
||||
for settled in (False, True):
|
||||
stream = runtime.ConversationAccumulator()
|
||||
stream.feed({'type': 'text-delta', 'id': 'progress', 'delta': '正在查询。'})
|
||||
stream.feed({'type': 'tool-input-available', 'toolName': 'mcp_call', 'toolCallId': 'q'})
|
||||
if settled:stream.feed({'type': 'tool-output-available', 'toolCallId': 'q', 'output': {}})
|
||||
stream.feed({'type': 'finish', 'finishReason': 'stop'})
|
||||
self.assertNotEqual(stream.status()[0], 'COMPLETED')
|
||||
stream.feed({'type': 'text-delta', 'id': 'final', 'delta': '已核实的最终回答。'})
|
||||
self.assertEqual(stream.status()[0], 'COMPLETED' if settled else 'PARTIAL')
|
||||
|
||||
def test_only_the_terminal_finish_and_successful_stop_control_can_complete(self):
|
||||
for reasons, stopped in ((['stop', 'tool-calls'], None), (['stop'], False)):
|
||||
stream = runtime.ConversationAccumulator()
|
||||
stream.feed({'type': 'text-delta', 'id': 'answer', 'delta': '保留原文'})
|
||||
for reason in reasons:stream.feed({'type': 'finish', 'finishReason': reason})
|
||||
stream.feed({'type': '[DONE]'})
|
||||
self.assertNotEqual(stream.status(stop_succeeded=stopped)[0], 'COMPLETED')
|
||||
|
||||
def test_native_stop_controller_stops_before_closing_stream_on_cancel(self):
|
||||
calls = []
|
||||
response = FakeResponse()
|
||||
@@ -306,6 +353,28 @@ class RuntimeStreamTests(unittest.TestCase):
|
||||
self.assertEqual(result[2], "transport_or_stream_error")
|
||||
native_stop.assert_called_once_with(unittest.mock.ANY, "http://127.0.0.1:10000", "existing-native-id")
|
||||
|
||||
def test_an_already_finished_native_reply_cannot_be_cancelled_by_stale_no_progress_state(self):
|
||||
controls=[];ready=[];controller=runtime.NativeStopController
|
||||
def capture(*args):
|
||||
value=controller(*args);controls.append(value);return value
|
||||
class Response:
|
||||
def __enter__(self):return self
|
||||
def __exit__(self,*_):pass
|
||||
def close(self):pass
|
||||
def __iter__(self):
|
||||
yield b'data: {"type":"text-delta","id":"final","delta":"final reply"}\n'
|
||||
yield b'data: {"type":"finish","finishReason":"stop"}\n'
|
||||
ready.append(controls[0].completion_ready())
|
||||
yield b'data: [DONE]\n'
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root=Path(temp)/str(uuid.uuid4());root.mkdir();actor=root.name;sid=str(uuid.uuid4())
|
||||
runtime._atomic_json(root/'private/native-exploration-state.json',{'executionId':'e','halted':True})
|
||||
client=unittest.mock.Mock();client.open.return_value=Response()
|
||||
with patch.object(runtime,'NativeStopController',side_effect=capture),patch('nao_agent_task.task_from_context',return_value={'executionMode':'native_agent','freshQuery':True}):
|
||||
result=runtime._sse_request(client=client,base_url='http://native.invalid',model='fixture',question='q',release_id=str(uuid.uuid4()),
|
||||
context={'_nativeExecutionId':'e'},chat_id='chat',session_state=runtime.SessionState(root,actor),session_id=sid,actor_root=root,emit=lambda _:None,timeout=5)
|
||||
self.assertEqual(ready,[False]);self.assertEqual(result[1],'COMPLETED')
|
||||
|
||||
def test_story_export_uses_last_successful_call_and_matching_markdown(self):
|
||||
class StoryClient:
|
||||
def __init__(self):
|
||||
|
||||
@@ -52,6 +52,18 @@ def document(ref, *proofs):
|
||||
|
||||
|
||||
class NativeRunTests(unittest.TestCase):
|
||||
def test_pretool_progress_cannot_be_promoted_by_a_stop_finish(self):
|
||||
from nao_quality_runtime import ConversationAccumulator
|
||||
self.accumulator=ConversationAccumulator()
|
||||
for event in ({'type':'text-delta','id':'progress','delta':'正在查询。'},
|
||||
{'type':'tool-input-available','toolName':'mcp_call','toolCallId':'q'},
|
||||
{'type':'tool-output-available','toolCallId':'q','output':{}},
|
||||
{'type':'finish','finishReason':'stop'}):self.accumulator.feed(event)
|
||||
result=self.invoke()
|
||||
self.assertEqual(result['status'],'PARTIAL')
|
||||
self.assertFalse(result['answerQuality']['narrativeComplete'])
|
||||
self.assertEqual(result['answer'],'正在查询。')
|
||||
|
||||
def test_optional_scope_notice_is_shown_once_without_internal_visual_diagnostics(self):
|
||||
child = {**self.task, 'stages': ['bright']}
|
||||
self.snapshot['scopes'].append({'nodeId': 'bright-detail', 'scopeId': 'bright-detail',
|
||||
|
||||
Reference in New Issue
Block a user