Files
lyun 52d92c95d8 修复:智能分析空回答与工具结束条件冲突
统一自主分析工具范围与执行前校验,严格核验正常结束、最终正文和图表恢复状态,并显示空回答及恢复失败原因。

包含两次已上线修复及回归测试,保留已有回答与业务数据。

验证:1101 项 Python、469 项前端、97 项后端测试通过;前后端构建和真实模型、页面验收通过。
2026-11-03 08:56:49 +08:00

346 lines
21 KiB
Python

"""Exercise the native runner with isolated files and mocked upstream/bridge calls."""
import copy
import importlib.util
import json
import os
import sys
import tempfile
import threading
import unittest
from contextlib import ExitStack
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock, patch
PROJECT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(PROJECT / 'scripts')]
RUNTIME_PATH = Path(os.environ.get('NAO_ORIGINAL_BODY_PATH',
str(PROJECT / 'scripts/nao_native_run.py'))).expanduser().resolve()
RUNTIME_SPEC = importlib.util.spec_from_file_location('nao_native_run_integration', RUNTIME_PATH)
RUNTIME_MODULE = importlib.util.module_from_spec(RUNTIME_SPEC)
sys.modules[RUNTIME_SPEC.name] = RUNTIME_MODULE
RUNTIME_SPEC.loader.exec_module(RUNTIME_MODULE)
run_native_exploration = RUNTIME_MODULE.run_native_exploration
from nao_quality_runtime import _atomic_json
IDENTITY = {'attemptId': 'attempt-current', 'sessionId': 'session-current',
'turnId': 'turn-current', 'revision': 3, 'executionId': 'execution-current'}
RELEASE = 'release-current'
def root_task():
return {'version': 1, 'goal': '解释下降并根据实际结果调查', 'question': '本期下降原因',
'executionMode': 'native_agent', 'kind': 'analysis',
'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2026, 'month': 8},
'filters': {'department': '当前产业部'}, 'fields': ['rate'], 'stages': [],
'dimensions': [], 'dimensionStages': {}, 'visualPreference': 'auto',
'comparisonRequired': True, 'trendRequired': False,
'verifySource': False, 'fullDetails': False}
def proof(query_id, *, node='root', scope='scope-root', execution='execution-current'):
return {**IDENTITY, 'executionId': execution, 'queryId': query_id,
'nodeId': node, 'scopeId': scope, 'responseHash': 'a' * 64}
def document(ref, *proofs):
result = ({'current': {'protocolEvidence': proofs[0]},
'comparison': {'protocolEvidence': proofs[1]}}
if len(proofs) == 2 else {'protocolEvidence': proofs[0]})
return {'evidenceRef': ref, 'result': result}
class NativeRunTests(unittest.TestCase):
def test_pretool_progress_cannot_be_promoted_by_a_stop_finish(self):
from nao_quality_runtime import ConversationAccumulator
self.accumulator=ConversationAccumulator()
for event in ({'type':'text-delta','id':'progress','delta':'正在查询。'},
{'type':'tool-input-available','toolName':'mcp_call','toolCallId':'q'},
{'type':'tool-output-available','toolCallId':'q','output':{}},
{'type':'finish','finishReason':'stop'}):self.accumulator.feed(event)
result=self.invoke()
self.assertEqual(result['status'],'PARTIAL')
self.assertFalse(result['answerQuality']['narrativeComplete'])
self.assertEqual(result['answer'],'正在查询。')
def test_optional_scope_notice_is_shown_once_without_internal_visual_diagnostics(self):
child = {**self.task, 'stages': ['bright']}
self.snapshot['scopes'].append({'nodeId': 'bright-detail', 'scopeId': 'bright-detail',
'purpose': 'defects', 'label': '基础范围', 'task': child})
self.snapshot['scopeCoverage'].append({'nodeId': 'bright-detail', 'scopeId': 'bright-detail',
'complete': False})
self.snapshot['explorationWarnings'] = ['基础范围来源细分缺陷尚未完整覆盖所选阶段和期间']
self.visuals.return_value = {'visualState': 'ready', 'visuals': [{'id': 'main-chart'}],
'missingEvidence': ['节点bright-detail范围基础范围缺少bright来源具体缺陷重量图。']}
result = self.invoke()
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['answerQuality']['notices'],
['亮片细分缺陷尚未完整核实:当前范围;未用于已核实结论。'])
self.assertNotIn('调查局限', result['answer'])
self.assertNotIn('bright', result['answer'])
stored = json.loads((self.actor_root / 'private/native-warning-diagnostics.json').read_text())
self.assertIn('bright', stored['visualWarnings'][0])
def test_original_answer_is_not_replaced_by_a_generated_report(self):
self.accumulator.answer = '原生回答原样保留,局限也照原文保留。'
result = self.invoke()
self.assertEqual(result['answer'], self.accumulator.answer)
self.assertIsNone(result['readingSummary'])
self.synthesis.assert_not_called()
def test_no_second_composition_artifact_is_written(self):
result = self.invoke()
self.assertEqual(result['answer'], self.accumulator.answer)
self.assertFalse((self.actor_root / 'private/native-composition.json').exists())
self.synthesis.assert_not_called()
def setUp(self):
self.folder = tempfile.TemporaryDirectory()
self.addCleanup(self.folder.cleanup)
self.actor_root = Path(self.folder.name)
self.cancel_event = threading.Event()
self.task = root_task()
self.request = {'sessionId': IDENTITY['sessionId'], 'turnId': IDENTITY['turnId'],
'actorId': 'actor-current', 'releaseId': RELEASE, 'question': self.task['question'],
'context': {'product': '已取消的旧规格',
'_serverQuestions': [{'question': '上一轮问题'}],
'_protocol': {'autonomousEnabled': True}}}
self.p_current = proof('query-current')
self.p_compare = proof('query-compare')
self.p_group = proof('query-group', node='group', scope='scope-group')
self.p_failed = proof('query-failed', node='failed', scope='scope-failed')
self.p_old = proof('query-old', execution='execution-old')
self.good = document('Q00000001', self.p_current, self.p_compare)
self.unused = document('Q00000002', self.p_group)
self.documents = [self.good, self.unused]
self.snapshot = {'scopes': [{'nodeId': 'root', 'scopeId': 'scope-root', 'task': self.task},
{'nodeId': 'group', 'scopeId': 'scope-group', 'task': self.task}],
'queries': [{'proof': p} for p in (self.p_current, self.p_compare, self.p_group)],
'scopeCoverage': [{'nodeId': 'root', 'scopeId': 'scope-root', 'complete': True},
{'nodeId': 'group', 'scopeId': 'scope-group', 'complete': True}],
'rootGoalGaps': [], 'gaps': [], 'pending': [], 'explorationWarnings': []}
self.accumulator = SimpleNamespace(answer='原生草稿', finish_reasons=['stop'],
tool_calls=[{'tool': 'quality_data', 'status': 'failed', 'input': {'mode': 'compare'}},
{'tool': 'quality_data', 'status': 'completed', 'input': {'mode': 'compare'}}],
timeline=[{'kind': 'tool_error'}, {'kind': 'tool_completed'}, {'kind': 'stop'}])
self.native_status, self.native_reason, self.stop_succeeded = 'COMPLETED', 'stop', None
self.cancel_during_native = False
self.events = []
self.manager = Mock()
self.manager.ensure_instance.return_value = (None, None, 'fixture-model', 'http://native.invalid')
self.delegated = Mock()
self.delegated.call.side_effect = lambda *args, **kwargs: copy.deepcopy(self.snapshot)
stack = self.enterContext(ExitStack())
self.auth = stack.enter_context(patch('nao_quality_runtime._authenticate', return_value=object()))
self.sse = stack.enter_context(patch('nao_quality_runtime._sse_request', side_effect=self.native))
self.read_evidence = stack.enter_context(patch('nao_evidence_finalizer.evidence_summaries',
side_effect=lambda *args, **kwargs: copy.deepcopy(self.documents)))
self.synthesis = stack.enter_context(patch('nao_evidence_finalizer.synthesize_native',
side_effect=AssertionError('native-agent original body must not be synthesized')))
self.visuals = stack.enter_context(patch('analysis_visual_evidence.protocol_visual_result',
return_value={'visualState': 'not_requested', 'visuals': [], 'missingEvidence': []}))
stack.enter_context(patch('nao_runtime_policy.native_timeout', return_value=30))
def native(self, **kwargs):
# Let the real runner decide when a useful response first became available.
kwargs['emit']({'type': 'progress', 'toolCalls': [
{'status': 'completed', 'input': {'mode': 'catalog'}, 'outputSummary': '读取目录'}]})
kwargs['emit']({'type': 'progress', 'toolCalls': [
{'status': 'completed', 'input': {'mode': 'compare'}, 'outputSummary': '证据:Q00000001'}]})
if self.cancel_during_native:
self.cancel_event.set()
return self.accumulator, self.native_status, self.native_reason, self.stop_succeeded
def invoke(self):
return run_native_exploration(request=copy.deepcopy(self.request), task=copy.deepcopy(self.task),
authority=copy.deepcopy(IDENTITY), actor_root=self.actor_root, delegated=self.delegated,
connection=('https://model.invalid/v1', 'synthetic-fixture-key', 'fixture-model'),
manager=self.manager, state={}, cancel_event=self.cancel_event, emit=self.events.append,
prior_evidence_files={'prior-evidence.json'},
phase_timings={'planningAndReview': 10, 'taskConfirmation': 20})
def test_corrected_tool_error_and_natural_stop_can_complete_with_verified_root(self):
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
self.snapshot['explorationWarnings'] = ['先前一次探索失败,后续已用完整证据核实根目标。']
result = self.invoke()
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['nativeStopReason'], 'mcp_error')
self.assertEqual(result['nativeFinishReason'], 'stop')
self.assertIsNone(result['nativeStopSucceeded'])
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertEqual(result['timeline'], self.accumulator.timeline)
self.assertEqual(result['conclusionProtocolQueryIds'],
['query-current', 'query-compare', 'query-group'])
self.assertEqual(result['answerQuality']['citedEvidenceRefs'], [])
self.assertEqual(result['answerQuality']['narrativeFactBinding'], 'not_assessed')
self.assertIn(self.snapshot['explorationWarnings'][0], result['answerQuality']['notices'])
self.assertEqual(result['answer'], self.accumulator.answer)
self.assertIn('firstUsefulMs', result['phaseTimingsMs'])
self.assertGreaterEqual(result['phaseTimingsMs']['firstUsefulMs'], 30)
self.delegated.call.assert_called_once_with('/api/analysis/protocol/queries', payload={}, method='POST')
context = self.sse.call_args.kwargs['context']
self.assertEqual(context['filters'], self.task['filters'])
self.assertNotIn('product', context)
self.assertIsNone(self.sse.call_args.kwargs['chat_id'])
stored = json.loads((self.actor_root / 'private/native-completion.json').read_text())
self.assertEqual(stored['status'], 'COMPLETED')
self.assertEqual(stored['answer'], self.accumulator.answer)
self.assertEqual(self.events[-1], result)
def test_generated_chart_contract_uses_formal_query_evidence_without_rewriting_answer(self):
self.task['visualPreference']='required'
self.visuals.return_value={'visualState':'ready','missingEvidence':[], 'visuals':[{
'id':'actual-chart','type':'line','metric':'composite_rate','title':'正式趋势',
'scope':{'department':'当前产业部'},'period':{'year':2026,'month':9},
'categories':[{'key':'2026-09'}],'series':[{'values':[.95]}],
'evidenceRefs':['Q00000001']}]}
result=self.invoke()
self.assertEqual(result['status'],'COMPLETED')
self.visuals.assert_called_once()
self.assertEqual(self.visuals.call_args.args[0], self.documents)
self.assertEqual(result['visuals'][0]['id'], 'actual-chart')
self.assertEqual(result['visuals'][0]['evidenceRefs'], ['Q00000001'])
self.assertEqual(result['answer'], self.accumulator.answer)
self.synthesis.assert_not_called()
def test_root_gap_remains_partial_even_with_stop_and_available_evidence(self):
self.snapshot['rootGoalGaps'] = ['亮片必答阶段尚无完整两期依据。']
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertEqual(result['answer'], self.accumulator.answer)
self.assertIn(self.snapshot['rootGoalGaps'][0], result['evidenceCompletion']['missingEvidence'])
self.assertIn(self.snapshot['rootGoalGaps'][0], result['answerQuality']['missingEvidence'])
def test_three_no_progress_attempts_remain_partial_after_natural_stop(self):
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
{'executionId': IDENTITY['executionId'], 'halted': True,
'noProgress': 3, 'queryElapsedMs': 1234})
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
self.assertEqual(result['phaseTimingsMs']['formalQuery'], 1234)
self.assertTrue(any('连续三次' in gap for gap in result['answerQuality']['missingEvidence']))
self.assertEqual(result['answer'], self.accumulator.answer)
def test_controlled_no_progress_stop_does_not_add_generic_tool_failure(self):
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
{'executionId': IDENTITY['executionId'], 'halted': True,
'noProgress': 3, 'queryElapsedMs': 1234})
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
result = self.invoke()
self.assertEqual(result['status'],'PARTIAL')
self.assertEqual(result['nativeStopReason'],'mcp_error')
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
self.assertFalse(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_corrected_error_still_requires_real_stop_and_successful_stop_control(self):
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
for reasons, stopped in (([], None), (['stop'], False)):
with self.subTest(finish_reasons=reasons, stop_succeeded=stopped):
self.accumulator.finish_reasons = reasons
self.stop_succeeded = stopped
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertTrue(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_only_complete_current_scope_documents_reach_visual_generation_and_result(self):
self.snapshot['queries'].extend([{'proof': self.p_failed}, {'proof': self.p_old}])
self.snapshot['scopeCoverage'].append(
{'nodeId': 'failed', 'scopeId': 'scope-failed', 'complete': False})
self.documents.extend([
document('Q00000003', self.p_failed),
document('Q00000004', self.p_old),
document('Q00000005', proof('query-not-in-current-receipts')),
document('Q00000006', self.p_current, self.p_failed),
{'evidenceRef': 'Q00000007', 'result': {'summary': {'unproved': True}}},
])
result = self.invoke()
admitted = self.visuals.call_args.args[0]
self.assertEqual([doc['evidenceRef'] for doc in admitted], ['Q00000001', 'Q00000002'])
self.assertEqual(result['queryEvidenceRefs'], ['Q00000001', 'Q00000002'])
self.assertEqual(result['conclusionProtocolQueryIds'],
['query-current', 'query-compare', 'query-group'])
self.assertNotIn('query-failed', result['conclusionProtocolQueryIds'])
self.assertNotIn('query-old', result['conclusionProtocolQueryIds'])
self.assertNotIn('query-not-in-current-receipts', result['conclusionProtocolQueryIds'])
self.assertEqual(result['answer'], self.accumulator.answer)
self.read_evidence.assert_called_once_with(self.actor_root / 'evidence', RELEASE,
exclude_files={'prior-evidence.json'}, allow_empty=True)
self.synthesis.assert_not_called()
def test_original_citations_map_only_to_actual_current_query_proofs(self):
self.snapshot['queries'].append({'proof': self.p_old})
self.documents.append(document('Q00000003', self.p_old))
self.accumulator.answer = '原文引用当前证据[Q00000001],以及外部证据[Q00000003]。'
result = self.invoke()
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current', 'query-compare'])
self.assertEqual(result['answer'], self.accumulator.answer)
self.assertEqual(result['answerQuality']['unresolvedEvidenceRefs'], ['Q00000003'])
self.assertEqual(result['status'], 'PARTIAL')
self.assertNotIn('query-group', result['conclusionProtocolQueryIds'])
self.assertNotIn('query-old', result['conclusionProtocolQueryIds'])
def test_uppercase_hexadecimal_reference_maps_to_its_actual_receipt(self):
hexadecimal = proof('query-hexadecimal')
self.snapshot['queries'].append({'proof': hexadecimal})
self.documents.append(document('QABCDEF12', hexadecimal))
self.accumulator.answer = '原文显式引用[QABCDEF12]。'
result = self.invoke()
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['answer'], self.accumulator.answer)
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-hexadecimal'])
self.assertEqual(result['answerQuality']['citedEvidenceRefs'], ['QABCDEF12'])
def test_unbound_original_citation_remains_a_metadata_gap(self):
self.accumulator.answer = '原生回答的引用[QABCDEF99]仍按原文保留。'
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertEqual(result['conclusionProtocolQueryIds'], [])
self.assertEqual(result['answer'], self.accumulator.answer)
self.assertEqual(result['answerQuality']['unresolvedEvidenceRefs'], ['QABCDEF99'])
self.assertTrue(any('QABCDEF99' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_cancelled_native_turn_never_reads_query_snapshot_or_synthesizes(self):
for cancellation in ('event', 'native_status'):
with self.subTest(cancellation=cancellation):
self.cancel_event.clear()
self.cancel_during_native = cancellation == 'event'
self.native_status = 'CANCELLED' if cancellation == 'native_status' else 'COMPLETED'
result = self.invoke()
self.assertEqual(result['status'], 'CANCELLED')
self.assertEqual(result['protocolEvidence'], [])
self.assertEqual(result['answer'], '')
self.delegated.call.assert_not_called()
self.read_evidence.assert_not_called()
self.synthesis.assert_not_called()
self.visuals.assert_not_called()
def test_auto_visual_failure_has_one_business_notice_and_required_failure_remains_partial(self):
self.visuals.return_value = {'visualState': 'failed', 'visuals': [],
'missingEvidence': ['图表尚未完整生成,可独立恢复。',
'queryPlan共有15张图表,超过12张呈现上限,已截取前12张;结果为部分完成。']}
for preference, expected in (('auto', 'COMPLETED'), ('required', 'PARTIAL')):
with self.subTest(visualPreference=preference):
self.task['visualPreference'] = preference
result = self.invoke()
self.assertEqual(result['status'], expected)
if preference == 'auto':
self.assertNotIn('图表尚未完整生成,可独立恢复。', result['answer'])
self.assertNotIn('结果为部分完成', result['answer'])
self.assertNotIn('共有15张图表', result['answer'])
self.assertEqual(result['answerQuality']['notices'],
['部分重点图表未能完整展示;已保留核实文字和表格。'])
else:
self.assertNotIn('图表尚未完整生成,可独立恢复。', result['answer'])
self.assertNotIn('结果为部分完成', result['answer'])
self.assertIn('图表尚未完整生成,可独立恢复。', result['answerQuality']['missingEvidence'])
if __name__ == '__main__':
unittest.main()