Files
quality_analyse/verification/test_native_run.py
T

308 lines
19 KiB
Python

"""Exercise the native runner with isolated files and mocked upstream/bridge calls."""
import copy
import json
import sys
import tempfile
import threading
import unittest
from contextlib import ExitStack
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock, patch
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts')]
from nao_native_run import run_native_exploration
from nao_quality_runtime import _atomic_json
IDENTITY = {'attemptId': 'attempt-current', 'sessionId': 'session-current',
'turnId': 'turn-current', 'revision': 3, 'executionId': 'execution-current'}
RELEASE = 'release-current'
def root_task():
return {'version': 1, 'goal': '解释下降并根据实际结果调查', 'question': '本期下降原因',
'executionMode': 'native_agent', 'kind': 'analysis',
'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2026, 'month': 8},
'filters': {'department': '当前产业部'}, 'fields': ['rate'], 'stages': [],
'dimensions': [], 'dimensionStages': {}, 'visualPreference': 'auto',
'comparisonRequired': True, 'trendRequired': False,
'verifySource': False, 'fullDetails': False}
def proof(query_id, *, node='root', scope='scope-root', execution='execution-current'):
return {**IDENTITY, 'executionId': execution, 'queryId': query_id,
'nodeId': node, 'scopeId': scope, 'responseHash': 'a' * 64}
def document(ref, *proofs):
result = ({'current': {'protocolEvidence': proofs[0]},
'comparison': {'protocolEvidence': proofs[1]}}
if len(proofs) == 2 else {'protocolEvidence': proofs[0]})
return {'evidenceRef': ref, 'result': result}
class NativeRunTests(unittest.TestCase):
def test_optional_scope_notice_is_shown_once_without_internal_visual_diagnostics(self):
child = {**self.task, 'stages': ['bright']}
self.snapshot['scopes'].append({'nodeId': 'bright-detail', 'scopeId': 'bright-detail',
'purpose': 'defects', 'label': '基础范围', 'task': child})
self.snapshot['scopeCoverage'].append({'nodeId': 'bright-detail', 'scopeId': 'bright-detail',
'complete': False})
self.snapshot['explorationWarnings'] = ['基础范围来源细分缺陷尚未完整覆盖所选阶段和期间']
self.visuals.return_value = {'visualState': 'ready', 'visuals': [{'id': 'main-chart'}],
'missingEvidence': ['节点bright-detail范围基础范围缺少bright来源具体缺陷重量图。']}
result = self.invoke()
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['readingSummary']['notices'],
['亮片细分缺陷尚未完整核实:当前范围;未用于已核实结论。'])
self.assertNotIn('调查局限', result['answer'])
self.assertNotIn('bright', result['answer'])
stored = json.loads((self.actor_root / 'private/native-warning-diagnostics.json').read_text())
self.assertIn('bright', stored['visualWarnings'][0])
def test_full_facts_remain_private_instead_of_being_appended_to_the_business_answer(self):
self.final['fullEvidenceFacts'] = [{'text': '后台保留的完整统计事实', 'evidenceRef': 'Q00000001'}]
result = self.invoke()
self.assertNotIn('完整统计依据', result['answer'])
self.assertNotIn('后台保留的完整统计事实', result['answer'])
stored = json.loads((self.actor_root / 'private/native-full-evidence-facts.json').read_text())
self.assertEqual(stored, self.final['fullEvidenceFacts'])
def test_issued_binding_catalog_remains_private_for_exact_replay(self):
issued = {'bindingInventory': {'N17': {'factId': 'F2', 'value': '493.9966', 'unit': 'kg'}},
'facts': [{'id': 'F2', 'rawValue': '493.99660000'}]}
self.final['_privateBindingInput'] = issued
result = self.invoke()
stored = json.loads((self.actor_root / 'private/native-composition.json').read_text())
self.assertEqual(stored['bindingInput'], issued)
self.assertNotIn('_privateBindingInput', result)
self.assertNotIn('bindingInput', result)
self.assertNotIn('bindingInventory', json.dumps(self.events, ensure_ascii=False))
def setUp(self):
self.folder = tempfile.TemporaryDirectory()
self.addCleanup(self.folder.cleanup)
self.actor_root = Path(self.folder.name)
self.cancel_event = threading.Event()
self.task = root_task()
self.request = {'sessionId': IDENTITY['sessionId'], 'turnId': IDENTITY['turnId'],
'actorId': 'actor-current', 'releaseId': RELEASE, 'question': self.task['question'],
'context': {'product': '已取消的旧规格',
'_serverQuestions': [{'question': '上一轮问题'}],
'_protocol': {'autonomousEnabled': True}}}
self.p_current = proof('query-current')
self.p_compare = proof('query-compare')
self.p_group = proof('query-group', node='group', scope='scope-group')
self.p_failed = proof('query-failed', node='failed', scope='scope-failed')
self.p_old = proof('query-old', execution='execution-old')
self.good = document('Q00000001', self.p_current, self.p_compare)
self.unused = document('Q00000002', self.p_group)
self.documents = [self.good, self.unused]
self.snapshot = {'scopes': [{'nodeId': 'root', 'scopeId': 'scope-root', 'task': self.task},
{'nodeId': 'group', 'scopeId': 'scope-group', 'task': self.task}],
'queries': [{'proof': p} for p in (self.p_current, self.p_compare, self.p_group)],
'scopeCoverage': [{'nodeId': 'root', 'scopeId': 'scope-root', 'complete': True},
{'nodeId': 'group', 'scopeId': 'scope-group', 'complete': True}],
'rootGoalGaps': [], 'gaps': [], 'pending': [], 'explorationWarnings': []}
self.final = {'answer': '按已核实结果解释[Q00000001]', 'reportTitle': '质量数据分析',
'readingSummary': {'version': 1, 'lead': '已核实解释[Q00000001]',
'evidence': [], 'notices': [], 'evidenceRefs': ['Q00000001']},
'usedEvidenceRefs': ['Q00000001'], 'evidenceSufficient': True,
'missingEvidence': [], 'rawModelResponses': [], 'compositionDiagnostics': {}}
self.accumulator = SimpleNamespace(answer='原生草稿', finish_reasons=['stop'],
tool_calls=[{'tool': 'quality_data', 'status': 'failed', 'input': {'mode': 'compare'}},
{'tool': 'quality_data', 'status': 'completed', 'input': {'mode': 'compare'}}],
timeline=[{'kind': 'tool_error'}, {'kind': 'tool_completed'}, {'kind': 'stop'}])
self.native_status, self.native_reason, self.stop_succeeded = 'COMPLETED', 'stop', None
self.cancel_during_native = False
self.events = []
self.manager = Mock()
self.manager.ensure_instance.return_value = (None, None, 'fixture-model', 'http://native.invalid')
self.delegated = Mock()
self.delegated.call.side_effect = lambda *args, **kwargs: copy.deepcopy(self.snapshot)
stack = self.enterContext(ExitStack())
self.auth = stack.enter_context(patch('nao_quality_runtime._authenticate', return_value=object()))
self.sse = stack.enter_context(patch('nao_quality_runtime._sse_request', side_effect=self.native))
self.read_evidence = stack.enter_context(patch('nao_evidence_finalizer.evidence_summaries',
side_effect=lambda *args, **kwargs: copy.deepcopy(self.documents)))
self.synthesis = stack.enter_context(patch('nao_evidence_finalizer.synthesize_native',
side_effect=lambda **kwargs: copy.deepcopy(self.final)))
self.visuals = stack.enter_context(patch('analysis_visual_evidence.protocol_visual_result',
return_value={'visualState': 'not_requested', 'visuals': [], 'missingEvidence': []}))
stack.enter_context(patch('nao_runtime_policy.native_timeout', return_value=30))
def native(self, **kwargs):
# Let the real runner decide when a useful response first became available.
kwargs['emit']({'type': 'progress', 'toolCalls': [
{'status': 'completed', 'input': {'mode': 'catalog'}, 'outputSummary': '读取目录'}]})
kwargs['emit']({'type': 'progress', 'toolCalls': [
{'status': 'completed', 'input': {'mode': 'compare'}, 'outputSummary': '证据:Q00000001'}]})
if self.cancel_during_native:
self.cancel_event.set()
return self.accumulator, self.native_status, self.native_reason, self.stop_succeeded
def invoke(self):
return run_native_exploration(request=copy.deepcopy(self.request), task=copy.deepcopy(self.task),
authority=copy.deepcopy(IDENTITY), actor_root=self.actor_root, delegated=self.delegated,
connection=('https://model.invalid/v1', 'synthetic-fixture-key', 'fixture-model'),
manager=self.manager, state={}, cancel_event=self.cancel_event, emit=self.events.append,
prior_evidence_files={'prior-evidence.json'},
phase_timings={'planningAndReview': 10, 'taskConfirmation': 20})
def test_corrected_tool_error_and_natural_stop_can_complete_with_verified_root(self):
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
self.snapshot['explorationWarnings'] = ['先前一次探索失败,后续已用完整证据核实根目标。']
result = self.invoke()
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['nativeStopReason'], 'mcp_error')
self.assertEqual(result['nativeFinishReason'], 'stop')
self.assertIsNone(result['nativeStopSucceeded'])
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertEqual(result['timeline'], self.accumulator.timeline)
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current', 'query-compare'])
self.assertIn(self.snapshot['explorationWarnings'][0], result['readingSummary']['notices'])
self.assertIn('firstUsefulMs', result['phaseTimingsMs'])
self.assertGreaterEqual(result['phaseTimingsMs']['firstUsefulMs'], 30)
self.delegated.call.assert_called_once_with('/api/analysis/protocol/queries', payload={}, method='POST')
context = self.sse.call_args.kwargs['context']
self.assertEqual(context['filters'], self.task['filters'])
self.assertNotIn('product', context)
self.assertIsNone(self.sse.call_args.kwargs['chat_id'])
stored = json.loads((self.actor_root / 'private/native-completion.json').read_text())
self.assertEqual(stored['status'], 'COMPLETED')
self.assertEqual(self.events[-1], result)
def test_generated_chart_contract_is_available_before_the_single_synthesis(self):
self.task['visualPreference']='required'
self.visuals.return_value={'visualState':'ready','missingEvidence':[], 'visuals':[{
'id':'actual-chart','type':'line','metric':'composite_rate','title':'正式趋势',
'scope':{'department':'当前产业部'},'period':{'year':2026,'month':9},
'categories':[{'key':'2026-09'}],'series':[{'values':[.95]}],
'evidenceRefs':['Q00000001']}]}
def synthesize(**kwargs):
preview=kwargs['context']['_platformVisuals']
self.assertEqual(preview['visualState'],'ready')
self.assertEqual(preview['charts'][0]['id'],'actual-chart')
self.assertEqual(preview['charts'][0]['evidenceRefs'],['Q00000001'])
self.assertEqual(preview['charts'][0]['pointCount'],1)
self.assertNotIn('series',preview['charts'][0])
return copy.deepcopy(self.final)
self.synthesis.side_effect=synthesize
result=self.invoke()
self.assertEqual(result['status'],'COMPLETED')
self.visuals.assert_called_once();self.synthesis.assert_called_once()
def test_root_gap_remains_partial_even_with_stop_and_available_evidence(self):
self.snapshot['rootGoalGaps'] = ['亮片必答阶段尚无完整两期依据。']
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertIn(self.snapshot['rootGoalGaps'][0], result['answer'])
self.assertIn(self.snapshot['rootGoalGaps'][0], result['readingSummary']['notices'])
def test_three_no_progress_attempts_remain_partial_after_natural_stop(self):
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
{'executionId': IDENTITY['executionId'], 'halted': True,
'noProgress': 3, 'queryElapsedMs': 1234})
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
self.assertEqual(result['phaseTimingsMs']['formalQuery'], 1234)
self.assertTrue(any('连续三次' in notice for notice in result['readingSummary']['notices']))
def test_controlled_no_progress_stop_does_not_add_generic_tool_failure(self):
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
{'executionId': IDENTITY['executionId'], 'halted': True,
'noProgress': 3, 'queryElapsedMs': 1234})
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
result = self.invoke()
self.assertEqual(result['status'],'PARTIAL')
self.assertEqual(result['nativeStopReason'],'mcp_error')
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
self.assertFalse(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_corrected_error_still_requires_real_stop_and_successful_stop_control(self):
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
for reasons, stopped in (([], None), (['stop'], False)):
with self.subTest(finish_reasons=reasons, stop_succeeded=stopped):
self.accumulator.finish_reasons = reasons
self.stop_succeeded = stopped
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertTrue(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_only_complete_current_scope_documents_reach_synthesis(self):
self.snapshot['queries'].extend([{'proof': self.p_failed}, {'proof': self.p_old}])
self.snapshot['scopeCoverage'].append(
{'nodeId': 'failed', 'scopeId': 'scope-failed', 'complete': False})
self.documents.extend([
document('Q00000003', self.p_failed),
document('Q00000004', self.p_old),
document('Q00000005', proof('query-not-in-current-receipts')),
document('Q00000006', self.p_current, self.p_failed),
{'evidenceRef': 'Q00000007', 'result': {'summary': {'unproved': True}}},
])
result = self.invoke()
admitted = self.synthesis.call_args.kwargs['evidence']
self.assertEqual([doc['evidenceRef'] for doc in admitted], ['Q00000001', 'Q00000002'])
self.assertEqual(result['queryEvidenceRefs'], ['Q00000001', 'Q00000002'])
self.read_evidence.assert_called_once_with(self.actor_root / 'evidence', RELEASE,
exclude_files={'prior-evidence.json'}, allow_empty=True)
def test_used_refs_map_only_to_actual_current_proofs_not_unused_or_foreign_documents(self):
self.snapshot['queries'].append({'proof': self.p_old})
self.documents.append(document('Q00000003', self.p_old))
self.final['usedEvidenceRefs'] = ['Q00000001', 'Q00000003', 'QDEADBEEF', 'Q00000001']
result = self.invoke()
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current', 'query-compare'])
self.assertNotIn('query-group', result['conclusionProtocolQueryIds'])
self.assertNotIn('query-old', result['conclusionProtocolQueryIds'])
def test_claimed_sufficient_answer_without_current_cited_proof_remains_partial(self):
self.final['usedEvidenceRefs'] = ['QDEADBEEF']
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertEqual(result['conclusionProtocolQueryIds'], [])
self.assertTrue(any('实际查询证明' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_cancelled_native_turn_never_reads_query_snapshot_or_synthesizes(self):
for cancellation in ('event', 'native_status'):
with self.subTest(cancellation=cancellation):
self.cancel_event.clear()
self.cancel_during_native = cancellation == 'event'
self.native_status = 'CANCELLED' if cancellation == 'native_status' else 'COMPLETED'
result = self.invoke()
self.assertEqual(result['status'], 'CANCELLED')
self.assertEqual(result['protocolEvidence'], [])
self.assertEqual(result['answer'], '')
self.delegated.call.assert_not_called()
self.read_evidence.assert_not_called()
self.synthesis.assert_not_called()
self.visuals.assert_not_called()
def test_auto_visual_failure_has_one_business_notice_and_required_failure_remains_partial(self):
self.visuals.return_value = {'visualState': 'failed', 'visuals': [],
'missingEvidence': ['图表尚未完整生成,可独立恢复。',
'queryPlan共有15张图表,超过12张呈现上限,已截取前12张;结果为部分完成。']}
for preference, expected in (('auto', 'COMPLETED'), ('required', 'PARTIAL')):
with self.subTest(visualPreference=preference):
self.task['visualPreference'] = preference
result = self.invoke()
self.assertEqual(result['status'], expected)
if preference == 'auto':
self.assertNotIn('图表尚未完整生成,可独立恢复。', result['answer'])
self.assertNotIn('结果为部分完成', result['answer'])
self.assertNotIn('共有15张图表', result['answer'])
self.assertEqual(result['readingSummary']['notices'],
['部分重点图表未能完整展示;已保留核实文字和表格。'])
else:
self.assertIn('图表尚未完整生成,可独立恢复。', result['answer'])
self.assertIn('图表尚未完整生成,可独立恢复。', result['readingSummary']['notices'])
self.assertIn('结果为部分完成', result['answer'])
if __name__ == '__main__':
unittest.main()