统一自主分析工具范围与执行前校验,严格核验正常结束、最终正文和图表恢复状态,并显示空回答及恢复失败原因。 包含两次已上线修复及回归测试,保留已有回答与业务数据。 验证:1101 项 Python、469 项前端、97 项后端测试通过;前后端构建和真实模型、页面验收通过。
346 lines
21 KiB
Python
346 lines
21 KiB
Python
"""Exercise the native runner with isolated files and mocked upstream/bridge calls."""
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from contextlib import ExitStack
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock, patch
|
|
|
|
PROJECT = Path(__file__).resolve().parents[1]
|
|
sys.path[:0] = [str(PROJECT / 'scripts')]
|
|
RUNTIME_PATH = Path(os.environ.get('NAO_ORIGINAL_BODY_PATH',
|
|
str(PROJECT / 'scripts/nao_native_run.py'))).expanduser().resolve()
|
|
RUNTIME_SPEC = importlib.util.spec_from_file_location('nao_native_run_integration', RUNTIME_PATH)
|
|
RUNTIME_MODULE = importlib.util.module_from_spec(RUNTIME_SPEC)
|
|
sys.modules[RUNTIME_SPEC.name] = RUNTIME_MODULE
|
|
RUNTIME_SPEC.loader.exec_module(RUNTIME_MODULE)
|
|
run_native_exploration = RUNTIME_MODULE.run_native_exploration
|
|
from nao_quality_runtime import _atomic_json
|
|
|
|
|
|
IDENTITY = {'attemptId': 'attempt-current', 'sessionId': 'session-current',
|
|
'turnId': 'turn-current', 'revision': 3, 'executionId': 'execution-current'}
|
|
RELEASE = 'release-current'
|
|
|
|
|
|
def root_task():
|
|
return {'version': 1, 'goal': '解释下降并根据实际结果调查', 'question': '本期下降原因',
|
|
'executionMode': 'native_agent', 'kind': 'analysis',
|
|
'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2026, 'month': 8},
|
|
'filters': {'department': '当前产业部'}, 'fields': ['rate'], 'stages': [],
|
|
'dimensions': [], 'dimensionStages': {}, 'visualPreference': 'auto',
|
|
'comparisonRequired': True, 'trendRequired': False,
|
|
'verifySource': False, 'fullDetails': False}
|
|
|
|
|
|
def proof(query_id, *, node='root', scope='scope-root', execution='execution-current'):
|
|
return {**IDENTITY, 'executionId': execution, 'queryId': query_id,
|
|
'nodeId': node, 'scopeId': scope, 'responseHash': 'a' * 64}
|
|
|
|
|
|
def document(ref, *proofs):
|
|
result = ({'current': {'protocolEvidence': proofs[0]},
|
|
'comparison': {'protocolEvidence': proofs[1]}}
|
|
if len(proofs) == 2 else {'protocolEvidence': proofs[0]})
|
|
return {'evidenceRef': ref, 'result': result}
|
|
|
|
|
|
class NativeRunTests(unittest.TestCase):
|
|
def test_pretool_progress_cannot_be_promoted_by_a_stop_finish(self):
|
|
from nao_quality_runtime import ConversationAccumulator
|
|
self.accumulator=ConversationAccumulator()
|
|
for event in ({'type':'text-delta','id':'progress','delta':'正在查询。'},
|
|
{'type':'tool-input-available','toolName':'mcp_call','toolCallId':'q'},
|
|
{'type':'tool-output-available','toolCallId':'q','output':{}},
|
|
{'type':'finish','finishReason':'stop'}):self.accumulator.feed(event)
|
|
result=self.invoke()
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
self.assertFalse(result['answerQuality']['narrativeComplete'])
|
|
self.assertEqual(result['answer'],'正在查询。')
|
|
|
|
def test_optional_scope_notice_is_shown_once_without_internal_visual_diagnostics(self):
|
|
child = {**self.task, 'stages': ['bright']}
|
|
self.snapshot['scopes'].append({'nodeId': 'bright-detail', 'scopeId': 'bright-detail',
|
|
'purpose': 'defects', 'label': '基础范围', 'task': child})
|
|
self.snapshot['scopeCoverage'].append({'nodeId': 'bright-detail', 'scopeId': 'bright-detail',
|
|
'complete': False})
|
|
self.snapshot['explorationWarnings'] = ['基础范围来源细分缺陷尚未完整覆盖所选阶段和期间']
|
|
self.visuals.return_value = {'visualState': 'ready', 'visuals': [{'id': 'main-chart'}],
|
|
'missingEvidence': ['节点bright-detail范围基础范围缺少bright来源具体缺陷重量图。']}
|
|
result = self.invoke()
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertEqual(result['answerQuality']['notices'],
|
|
['亮片细分缺陷尚未完整核实:当前范围;未用于已核实结论。'])
|
|
self.assertNotIn('调查局限', result['answer'])
|
|
self.assertNotIn('bright', result['answer'])
|
|
stored = json.loads((self.actor_root / 'private/native-warning-diagnostics.json').read_text())
|
|
self.assertIn('bright', stored['visualWarnings'][0])
|
|
|
|
def test_original_answer_is_not_replaced_by_a_generated_report(self):
|
|
self.accumulator.answer = '原生回答原样保留,局限也照原文保留。'
|
|
result = self.invoke()
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
self.assertIsNone(result['readingSummary'])
|
|
self.synthesis.assert_not_called()
|
|
|
|
def test_no_second_composition_artifact_is_written(self):
|
|
result = self.invoke()
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
self.assertFalse((self.actor_root / 'private/native-composition.json').exists())
|
|
self.synthesis.assert_not_called()
|
|
|
|
def setUp(self):
|
|
self.folder = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.folder.cleanup)
|
|
self.actor_root = Path(self.folder.name)
|
|
self.cancel_event = threading.Event()
|
|
self.task = root_task()
|
|
self.request = {'sessionId': IDENTITY['sessionId'], 'turnId': IDENTITY['turnId'],
|
|
'actorId': 'actor-current', 'releaseId': RELEASE, 'question': self.task['question'],
|
|
'context': {'product': '已取消的旧规格',
|
|
'_serverQuestions': [{'question': '上一轮问题'}],
|
|
'_protocol': {'autonomousEnabled': True}}}
|
|
self.p_current = proof('query-current')
|
|
self.p_compare = proof('query-compare')
|
|
self.p_group = proof('query-group', node='group', scope='scope-group')
|
|
self.p_failed = proof('query-failed', node='failed', scope='scope-failed')
|
|
self.p_old = proof('query-old', execution='execution-old')
|
|
self.good = document('Q00000001', self.p_current, self.p_compare)
|
|
self.unused = document('Q00000002', self.p_group)
|
|
self.documents = [self.good, self.unused]
|
|
self.snapshot = {'scopes': [{'nodeId': 'root', 'scopeId': 'scope-root', 'task': self.task},
|
|
{'nodeId': 'group', 'scopeId': 'scope-group', 'task': self.task}],
|
|
'queries': [{'proof': p} for p in (self.p_current, self.p_compare, self.p_group)],
|
|
'scopeCoverage': [{'nodeId': 'root', 'scopeId': 'scope-root', 'complete': True},
|
|
{'nodeId': 'group', 'scopeId': 'scope-group', 'complete': True}],
|
|
'rootGoalGaps': [], 'gaps': [], 'pending': [], 'explorationWarnings': []}
|
|
self.accumulator = SimpleNamespace(answer='原生草稿', finish_reasons=['stop'],
|
|
tool_calls=[{'tool': 'quality_data', 'status': 'failed', 'input': {'mode': 'compare'}},
|
|
{'tool': 'quality_data', 'status': 'completed', 'input': {'mode': 'compare'}}],
|
|
timeline=[{'kind': 'tool_error'}, {'kind': 'tool_completed'}, {'kind': 'stop'}])
|
|
self.native_status, self.native_reason, self.stop_succeeded = 'COMPLETED', 'stop', None
|
|
self.cancel_during_native = False
|
|
self.events = []
|
|
self.manager = Mock()
|
|
self.manager.ensure_instance.return_value = (None, None, 'fixture-model', 'http://native.invalid')
|
|
self.delegated = Mock()
|
|
self.delegated.call.side_effect = lambda *args, **kwargs: copy.deepcopy(self.snapshot)
|
|
stack = self.enterContext(ExitStack())
|
|
self.auth = stack.enter_context(patch('nao_quality_runtime._authenticate', return_value=object()))
|
|
self.sse = stack.enter_context(patch('nao_quality_runtime._sse_request', side_effect=self.native))
|
|
self.read_evidence = stack.enter_context(patch('nao_evidence_finalizer.evidence_summaries',
|
|
side_effect=lambda *args, **kwargs: copy.deepcopy(self.documents)))
|
|
self.synthesis = stack.enter_context(patch('nao_evidence_finalizer.synthesize_native',
|
|
side_effect=AssertionError('native-agent original body must not be synthesized')))
|
|
self.visuals = stack.enter_context(patch('analysis_visual_evidence.protocol_visual_result',
|
|
return_value={'visualState': 'not_requested', 'visuals': [], 'missingEvidence': []}))
|
|
stack.enter_context(patch('nao_runtime_policy.native_timeout', return_value=30))
|
|
|
|
def native(self, **kwargs):
|
|
# Let the real runner decide when a useful response first became available.
|
|
kwargs['emit']({'type': 'progress', 'toolCalls': [
|
|
{'status': 'completed', 'input': {'mode': 'catalog'}, 'outputSummary': '读取目录'}]})
|
|
kwargs['emit']({'type': 'progress', 'toolCalls': [
|
|
{'status': 'completed', 'input': {'mode': 'compare'}, 'outputSummary': '证据:Q00000001'}]})
|
|
if self.cancel_during_native:
|
|
self.cancel_event.set()
|
|
return self.accumulator, self.native_status, self.native_reason, self.stop_succeeded
|
|
|
|
def invoke(self):
|
|
return run_native_exploration(request=copy.deepcopy(self.request), task=copy.deepcopy(self.task),
|
|
authority=copy.deepcopy(IDENTITY), actor_root=self.actor_root, delegated=self.delegated,
|
|
connection=('https://model.invalid/v1', 'synthetic-fixture-key', 'fixture-model'),
|
|
manager=self.manager, state={}, cancel_event=self.cancel_event, emit=self.events.append,
|
|
prior_evidence_files={'prior-evidence.json'},
|
|
phase_timings={'planningAndReview': 10, 'taskConfirmation': 20})
|
|
|
|
def test_corrected_tool_error_and_natural_stop_can_complete_with_verified_root(self):
|
|
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
|
|
self.snapshot['explorationWarnings'] = ['先前一次探索失败,后续已用完整证据核实根目标。']
|
|
result = self.invoke()
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertEqual(result['nativeStopReason'], 'mcp_error')
|
|
self.assertEqual(result['nativeFinishReason'], 'stop')
|
|
self.assertIsNone(result['nativeStopSucceeded'])
|
|
|
|
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
|
|
self.assertEqual(result['timeline'], self.accumulator.timeline)
|
|
self.assertEqual(result['conclusionProtocolQueryIds'],
|
|
['query-current', 'query-compare', 'query-group'])
|
|
self.assertEqual(result['answerQuality']['citedEvidenceRefs'], [])
|
|
self.assertEqual(result['answerQuality']['narrativeFactBinding'], 'not_assessed')
|
|
self.assertIn(self.snapshot['explorationWarnings'][0], result['answerQuality']['notices'])
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
self.assertIn('firstUsefulMs', result['phaseTimingsMs'])
|
|
self.assertGreaterEqual(result['phaseTimingsMs']['firstUsefulMs'], 30)
|
|
self.delegated.call.assert_called_once_with('/api/analysis/protocol/queries', payload={}, method='POST')
|
|
context = self.sse.call_args.kwargs['context']
|
|
self.assertEqual(context['filters'], self.task['filters'])
|
|
self.assertNotIn('product', context)
|
|
self.assertIsNone(self.sse.call_args.kwargs['chat_id'])
|
|
stored = json.loads((self.actor_root / 'private/native-completion.json').read_text())
|
|
self.assertEqual(stored['status'], 'COMPLETED')
|
|
self.assertEqual(stored['answer'], self.accumulator.answer)
|
|
self.assertEqual(self.events[-1], result)
|
|
|
|
def test_generated_chart_contract_uses_formal_query_evidence_without_rewriting_answer(self):
|
|
self.task['visualPreference']='required'
|
|
self.visuals.return_value={'visualState':'ready','missingEvidence':[], 'visuals':[{
|
|
'id':'actual-chart','type':'line','metric':'composite_rate','title':'正式趋势',
|
|
'scope':{'department':'当前产业部'},'period':{'year':2026,'month':9},
|
|
'categories':[{'key':'2026-09'}],'series':[{'values':[.95]}],
|
|
'evidenceRefs':['Q00000001']}]}
|
|
result=self.invoke()
|
|
self.assertEqual(result['status'],'COMPLETED')
|
|
self.visuals.assert_called_once()
|
|
self.assertEqual(self.visuals.call_args.args[0], self.documents)
|
|
self.assertEqual(result['visuals'][0]['id'], 'actual-chart')
|
|
self.assertEqual(result['visuals'][0]['evidenceRefs'], ['Q00000001'])
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
self.synthesis.assert_not_called()
|
|
|
|
def test_root_gap_remains_partial_even_with_stop_and_available_evidence(self):
|
|
self.snapshot['rootGoalGaps'] = ['亮片必答阶段尚无完整两期依据。']
|
|
result = self.invoke()
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
self.assertIn(self.snapshot['rootGoalGaps'][0], result['evidenceCompletion']['missingEvidence'])
|
|
self.assertIn(self.snapshot['rootGoalGaps'][0], result['answerQuality']['missingEvidence'])
|
|
|
|
def test_three_no_progress_attempts_remain_partial_after_natural_stop(self):
|
|
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
|
|
{'executionId': IDENTITY['executionId'], 'halted': True,
|
|
'noProgress': 3, 'queryElapsedMs': 1234})
|
|
result = self.invoke()
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
|
self.assertEqual(result['phaseTimingsMs']['formalQuery'], 1234)
|
|
self.assertTrue(any('连续三次' in gap for gap in result['answerQuality']['missingEvidence']))
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
|
|
def test_controlled_no_progress_stop_does_not_add_generic_tool_failure(self):
|
|
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
|
|
{'executionId': IDENTITY['executionId'], 'halted': True,
|
|
'noProgress': 3, 'queryElapsedMs': 1234})
|
|
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
|
|
result = self.invoke()
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
self.assertEqual(result['nativeStopReason'],'mcp_error')
|
|
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
|
self.assertFalse(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
|
|
|
def test_corrected_error_still_requires_real_stop_and_successful_stop_control(self):
|
|
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
|
|
for reasons, stopped in (([], None), (['stop'], False)):
|
|
with self.subTest(finish_reasons=reasons, stop_succeeded=stopped):
|
|
self.accumulator.finish_reasons = reasons
|
|
self.stop_succeeded = stopped
|
|
result = self.invoke()
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertTrue(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
|
|
|
def test_only_complete_current_scope_documents_reach_visual_generation_and_result(self):
|
|
self.snapshot['queries'].extend([{'proof': self.p_failed}, {'proof': self.p_old}])
|
|
self.snapshot['scopeCoverage'].append(
|
|
{'nodeId': 'failed', 'scopeId': 'scope-failed', 'complete': False})
|
|
self.documents.extend([
|
|
document('Q00000003', self.p_failed),
|
|
document('Q00000004', self.p_old),
|
|
document('Q00000005', proof('query-not-in-current-receipts')),
|
|
document('Q00000006', self.p_current, self.p_failed),
|
|
{'evidenceRef': 'Q00000007', 'result': {'summary': {'unproved': True}}},
|
|
])
|
|
result = self.invoke()
|
|
admitted = self.visuals.call_args.args[0]
|
|
self.assertEqual([doc['evidenceRef'] for doc in admitted], ['Q00000001', 'Q00000002'])
|
|
self.assertEqual(result['queryEvidenceRefs'], ['Q00000001', 'Q00000002'])
|
|
self.assertEqual(result['conclusionProtocolQueryIds'],
|
|
['query-current', 'query-compare', 'query-group'])
|
|
self.assertNotIn('query-failed', result['conclusionProtocolQueryIds'])
|
|
self.assertNotIn('query-old', result['conclusionProtocolQueryIds'])
|
|
self.assertNotIn('query-not-in-current-receipts', result['conclusionProtocolQueryIds'])
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
self.read_evidence.assert_called_once_with(self.actor_root / 'evidence', RELEASE,
|
|
exclude_files={'prior-evidence.json'}, allow_empty=True)
|
|
self.synthesis.assert_not_called()
|
|
|
|
def test_original_citations_map_only_to_actual_current_query_proofs(self):
|
|
self.snapshot['queries'].append({'proof': self.p_old})
|
|
self.documents.append(document('Q00000003', self.p_old))
|
|
self.accumulator.answer = '原文引用当前证据[Q00000001],以及外部证据[Q00000003]。'
|
|
result = self.invoke()
|
|
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current', 'query-compare'])
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
self.assertEqual(result['answerQuality']['unresolvedEvidenceRefs'], ['Q00000003'])
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertNotIn('query-group', result['conclusionProtocolQueryIds'])
|
|
self.assertNotIn('query-old', result['conclusionProtocolQueryIds'])
|
|
|
|
def test_uppercase_hexadecimal_reference_maps_to_its_actual_receipt(self):
|
|
hexadecimal = proof('query-hexadecimal')
|
|
self.snapshot['queries'].append({'proof': hexadecimal})
|
|
self.documents.append(document('QABCDEF12', hexadecimal))
|
|
self.accumulator.answer = '原文显式引用[QABCDEF12]。'
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-hexadecimal'])
|
|
self.assertEqual(result['answerQuality']['citedEvidenceRefs'], ['QABCDEF12'])
|
|
|
|
def test_unbound_original_citation_remains_a_metadata_gap(self):
|
|
self.accumulator.answer = '原生回答的引用[QABCDEF99]仍按原文保留。'
|
|
result = self.invoke()
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertEqual(result['conclusionProtocolQueryIds'], [])
|
|
self.assertEqual(result['answer'], self.accumulator.answer)
|
|
self.assertEqual(result['answerQuality']['unresolvedEvidenceRefs'], ['QABCDEF99'])
|
|
self.assertTrue(any('QABCDEF99' in gap for gap in result['evidenceCompletion']['missingEvidence']))
|
|
|
|
def test_cancelled_native_turn_never_reads_query_snapshot_or_synthesizes(self):
|
|
for cancellation in ('event', 'native_status'):
|
|
with self.subTest(cancellation=cancellation):
|
|
self.cancel_event.clear()
|
|
self.cancel_during_native = cancellation == 'event'
|
|
self.native_status = 'CANCELLED' if cancellation == 'native_status' else 'COMPLETED'
|
|
result = self.invoke()
|
|
self.assertEqual(result['status'], 'CANCELLED')
|
|
self.assertEqual(result['protocolEvidence'], [])
|
|
self.assertEqual(result['answer'], '')
|
|
self.delegated.call.assert_not_called()
|
|
self.read_evidence.assert_not_called()
|
|
self.synthesis.assert_not_called()
|
|
self.visuals.assert_not_called()
|
|
|
|
def test_auto_visual_failure_has_one_business_notice_and_required_failure_remains_partial(self):
|
|
self.visuals.return_value = {'visualState': 'failed', 'visuals': [],
|
|
'missingEvidence': ['图表尚未完整生成,可独立恢复。',
|
|
'queryPlan共有15张图表,超过12张呈现上限,已截取前12张;结果为部分完成。']}
|
|
for preference, expected in (('auto', 'COMPLETED'), ('required', 'PARTIAL')):
|
|
with self.subTest(visualPreference=preference):
|
|
self.task['visualPreference'] = preference
|
|
result = self.invoke()
|
|
self.assertEqual(result['status'], expected)
|
|
if preference == 'auto':
|
|
self.assertNotIn('图表尚未完整生成,可独立恢复。', result['answer'])
|
|
self.assertNotIn('结果为部分完成', result['answer'])
|
|
self.assertNotIn('共有15张图表', result['answer'])
|
|
self.assertEqual(result['answerQuality']['notices'],
|
|
['部分重点图表未能完整展示;已保留核实文字和表格。'])
|
|
else:
|
|
self.assertNotIn('图表尚未完整生成,可独立恢复。', result['answer'])
|
|
self.assertNotIn('结果为部分完成', result['answer'])
|
|
self.assertIn('图表尚未完整生成,可独立恢复。', result['answerQuality']['missingEvidence'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|