314 lines
16 KiB
Python
314 lines
16 KiB
Python
"""Regression tests for the isolated native-agent original-body runtime copy."""
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import ModuleType, SimpleNamespace
|
|
from unittest.mock import Mock, patch
|
|
|
|
|
|
PROJECT = Path(__file__).resolve().parents[1]
|
|
RUNTIME_COPY = Path(os.environ.get('NAO_ORIGINAL_BODY_PATH',
|
|
str(PROJECT / 'scripts/nao_native_run.py'))).expanduser().resolve()
|
|
SPEC = importlib.util.spec_from_file_location('nao_native_run_original_body', RUNTIME_COPY)
|
|
NATIVE_RUN = importlib.util.module_from_spec(SPEC)
|
|
sys.modules[SPEC.name] = NATIVE_RUN
|
|
SPEC.loader.exec_module(NATIVE_RUN)
|
|
|
|
|
|
IDENTITY = {'attemptId': 'attempt-current', 'sessionId': 'session-current',
|
|
'turnId': 'turn-current', 'revision': 3, 'executionId': 'execution-current'}
|
|
|
|
|
|
def proof(query_id='query-current'):
|
|
return {**IDENTITY, 'queryId': query_id, 'nodeId': 'root', 'scopeId': 'scope-root',
|
|
'responseHash': 'a' * 64}
|
|
|
|
|
|
def document(ref='Q00000001', evidence_proof=None):
|
|
return {'evidenceRef': ref, 'result': {'protocolEvidence': evidence_proof or proof()}}
|
|
|
|
|
|
def task():
|
|
return {'version': 1, 'goal': '解释下降并根据实际结果调查', 'question': '本期下降原因',
|
|
'executionMode': 'native_agent', 'kind': 'analysis',
|
|
'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2026, 'month': 8},
|
|
'filters': {'department': '当前产业部'}, 'fields': ['rate'], 'stages': [],
|
|
'dimensions': [], 'dimensionStages': {}, 'visualPreference': 'auto',
|
|
'comparisonRequired': True, 'trendRequired': False,
|
|
'verifySource': False, 'fullDetails': False}
|
|
|
|
|
|
class NativeOriginalBodyTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.actor_root = Path(self.temp.name)
|
|
(self.actor_root / 'private').mkdir()
|
|
(self.actor_root / 'evidence').mkdir()
|
|
self.cancel_event = threading.Event()
|
|
self.task = task()
|
|
self.request = {'sessionId': IDENTITY['sessionId'], 'turnId': IDENTITY['turnId'],
|
|
'actorId': 'actor-current', 'releaseId': 'release-current',
|
|
'question': self.task['question'],
|
|
'context': {'_serverHistory': [], '_serverQuestions': [], '_protocol': {}}}
|
|
self.current_proof = proof()
|
|
self.documents = [document(evidence_proof=self.current_proof)]
|
|
self.snapshot = {'scopes': [{'nodeId': 'root', 'scopeId': 'scope-root', 'task': self.task}],
|
|
'queries': [{'proof': self.current_proof}],
|
|
'scopeCoverage': [{'nodeId': 'root', 'scopeId': 'scope-root', 'complete': True}],
|
|
'rootGoalGaps': [], 'gaps': [], 'pending': [], 'explorationWarnings': []}
|
|
self.accumulator = SimpleNamespace(answer='原始回答 [Q00000001]', finish_reasons=['stop'],
|
|
tool_calls=[{'tool': 'quality_data', 'status': 'completed', 'input': {'mode': 'compare'}}],
|
|
timeline=[{'kind': 'assistant', 'text': '原始回答'}])
|
|
self.native_status, self.native_reason, self.stop_succeeded = 'COMPLETED', 'stop', None
|
|
self.events = []
|
|
self.manager = Mock()
|
|
self.manager.ensure_instance.return_value = (None, None, 'fixture-model', 'http://native.invalid')
|
|
self.delegated = Mock()
|
|
self.delegated.call.side_effect = lambda *args, **kwargs: copy.deepcopy(self.snapshot)
|
|
self.synthetic_writer = Mock(side_effect=AssertionError('native original-body path must not synthesize text'))
|
|
|
|
quality_runtime = ModuleType('nao_quality_runtime')
|
|
quality_runtime._atomic_json = self.atomic_json
|
|
quality_runtime._authenticate = Mock(return_value=object())
|
|
quality_runtime._sse_request = self.native
|
|
quality_runtime._read_json = self.read_json
|
|
agent_task = ModuleType('nao_agent_task')
|
|
agent_task.bind_task = lambda seed, current_task: {**copy.deepcopy(current_task), **copy.deepcopy(seed)}
|
|
protocol_execution = ModuleType('nao_protocol_execution')
|
|
protocol_execution._identity = lambda authority: copy.deepcopy(IDENTITY)
|
|
protocol_execution._platform_visual_context = lambda visuals: {'visualState': visuals.get('visualState')}
|
|
protocol_execution.query_evidence_bindings = lambda docs, receipts: [
|
|
{'evidenceRef': item['evidenceRef'], 'proofs': item['result'].get('protocolEvidence')}
|
|
for item in docs]
|
|
evidence_finalizer = ModuleType('nao_evidence_finalizer')
|
|
evidence_finalizer.evidence_summaries = lambda *args, **kwargs: copy.deepcopy(self.documents)
|
|
evidence_finalizer.synthesize_native = self.synthetic_writer
|
|
visual_evidence = ModuleType('analysis_visual_evidence')
|
|
visual_evidence.protocol_visual_result = self.visual_result
|
|
runtime_policy = ModuleType('nao_runtime_policy')
|
|
runtime_policy.native_timeout = lambda: 30
|
|
native_exploration = ModuleType('nao_native_exploration')
|
|
native_exploration._proofs = lambda result: ([result['protocolEvidence']]
|
|
if isinstance(result, dict) and isinstance(result.get('protocolEvidence'), dict) else [])
|
|
self.modules = {'nao_quality_runtime': quality_runtime, 'nao_agent_task': agent_task,
|
|
'nao_protocol_execution': protocol_execution,
|
|
'nao_evidence_finalizer': evidence_finalizer,
|
|
'analysis_visual_evidence': visual_evidence,
|
|
'nao_runtime_policy': runtime_policy,
|
|
'nao_native_exploration': native_exploration}
|
|
self.module_patch = patch.dict(sys.modules, self.modules)
|
|
self.module_patch.start()
|
|
self.addCleanup(self.module_patch.stop)
|
|
|
|
def atomic_json(self, path, payload):
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(payload, ensure_ascii=False), encoding='utf-8')
|
|
|
|
@staticmethod
|
|
def read_json(path):
|
|
return json.loads(Path(path).read_text(encoding='utf-8'))
|
|
|
|
def native(self, **kwargs):
|
|
return self.accumulator, self.native_status, self.native_reason, self.stop_succeeded
|
|
|
|
def visual_result(self, documents, scopes, question, turn_id, **kwargs):
|
|
return {'visualState': 'ready', 'visuals': [{'id': 'formal-query-chart',
|
|
'evidenceRefs': [item['evidenceRef'] for item in documents]}], 'missingEvidence': []}
|
|
|
|
def invoke(self):
|
|
return NATIVE_RUN.run_native_exploration(request=copy.deepcopy(self.request),
|
|
task=copy.deepcopy(self.task), authority=copy.deepcopy(IDENTITY),
|
|
actor_root=self.actor_root, delegated=self.delegated,
|
|
connection=('https://model.invalid/v1', 'synthetic-fixture-key', 'fixture-model'),
|
|
manager=self.manager, state={}, cancel_event=self.cancel_event,
|
|
emit=self.events.append, prior_evidence_files={'prior-evidence.json'},
|
|
phase_timings={'planningAndReview': 10, 'taskConfirmation': 20})
|
|
|
|
def test_persists_the_complete_original_body_and_keeps_formal_charts(self):
|
|
body = '原生回答第一行\n' + ('原生正文' * 1800) + '\n[Q00000001]'
|
|
self.accumulator.answer = body
|
|
self.accumulator.timeline = [{'kind': 'assistant', 'text': 'x' * 8000}]
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertEqual(result['answer'], body)
|
|
self.assertIsNone(result['readingSummary'])
|
|
self.assertEqual(result['visuals'][0]['id'], 'formal-query-chart')
|
|
self.assertEqual(result['queryEvidenceRefs'], ['Q00000001'])
|
|
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current'])
|
|
self.assertEqual(result['answerQuality']['narrativeFactBinding'], 'not_assessed')
|
|
self.assertEqual(result['answerQuality']['backgroundEvidenceQueryIds'], [])
|
|
self.assertIsNone(result['factsComplete'])
|
|
self.assertEqual(self.read_json(self.actor_root / 'private/native-draft.json')['answer'], body)
|
|
self.assertEqual(self.read_json(self.actor_root / 'private/native-completion.json')['answer'], body)
|
|
self.assertEqual(self.read_json(self.actor_root / 'private/native-original-body-diagnostics.json')
|
|
['originalBodyCharacters'], len(body))
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_empty_native_body_stays_empty_and_cannot_be_completed(self):
|
|
self.accumulator.answer = ''
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['answer'], '')
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertFalse(result['answerQuality']['narrativeComplete'])
|
|
self.assertIn('自主分析未生成正文。', result['answerQuality']['missingEvidence'])
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_missing_formal_evidence_is_metadata_and_does_not_rewrite_the_body(self):
|
|
self.documents = []
|
|
body = '原始正文照常保留。'
|
|
self.accumulator.answer = body
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['answer'], body)
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertFalse(result['answerQuality']['dataComplete'])
|
|
self.assertIn('本轮未取得当前执行的可核验查询证据。', result['answerQuality']['missingEvidence'])
|
|
self.assertNotIn('尚未解决', result['answer'])
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_partial_or_interrupted_generation_is_not_rewritten_or_marked_complete(self):
|
|
self.native_status, self.native_reason = 'PARTIAL', 'length'
|
|
self.accumulator.finish_reasons = ['length']
|
|
body = '模型已输出的局限与正文。'
|
|
self.accumulator.answer = body
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['answer'], body)
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertFalse(result['answerQuality']['narrativeComplete'])
|
|
self.assertEqual(result['finishReason'], 'length')
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_completed_status_without_natural_stop_remains_partial(self):
|
|
body = '原生回答保留,结束信号不足。'
|
|
self.accumulator.answer = body
|
|
cases = (
|
|
('missing_stop_finish_reason', [], None),
|
|
('stop_control_failed', ['stop'], False),
|
|
)
|
|
for label, finish_reasons, stop_succeeded in cases:
|
|
with self.subTest(label=label):
|
|
self.accumulator.finish_reasons = finish_reasons
|
|
self.stop_succeeded = stop_succeeded
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['answer'], body)
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertFalse(result['answerQuality']['narrativeComplete'])
|
|
self.assertEqual(result['finishReason'], 'incomplete')
|
|
self.assertIn('自主调查未收到有效的正常结束信号。',
|
|
result['answerQuality']['missingEvidence'])
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_recovered_tool_error_keeps_natural_stop_completion_without_fact_binding(self):
|
|
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
|
|
body = '原生回答保留自己的局限。'
|
|
self.accumulator.answer = body
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertEqual(result['answer'], body)
|
|
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current'])
|
|
self.assertEqual(result['answerQuality']['backgroundEvidenceQueryIds'], ['query-current'])
|
|
self.assertTrue(result['answerQuality']['narrativeComplete'])
|
|
self.assertEqual(result['answerQuality']['narrativeFactBinding'], 'not_assessed')
|
|
self.assertIsNone(result['factsComplete'])
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_no_explicit_refs_cannot_use_docs_without_verified_complete_scope_receipts(self):
|
|
self.snapshot['scopeCoverage'] = None
|
|
body = '正常原生正文不写证据编号。'
|
|
self.accumulator.answer = body
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['answer'], body)
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertEqual(result['conclusionProtocolQueryIds'], [])
|
|
self.assertEqual(result['answerQuality']['backgroundEvidenceQueryIds'], [])
|
|
self.assertFalse(result['answerQuality']['dataComplete'])
|
|
self.assertIn('本轮没有可关联的完整scope正式查询证明。',
|
|
result['answerQuality']['missingEvidence'])
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_unbound_original_citation_is_a_gap_without_rewriting_the_body(self):
|
|
body = '原生回答里保留未绑定的引用。[QABCDEF99]'
|
|
self.accumulator.answer = body
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertEqual(result['answer'], body)
|
|
self.assertFalse(result['answerQuality']['dataComplete'])
|
|
self.assertEqual(result['answerQuality']['unresolvedEvidenceRefs'], ['QABCDEF99'])
|
|
self.assertIn('原文引用未关联本轮完整scope正式查询证据:QABCDEF99。',
|
|
result['answerQuality']['missingEvidence'])
|
|
self.assertNotIn('QABCDEF99', result['conclusionProtocolQueryIds'])
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_explicit_hexadecimal_query_reference_maps_only_its_receipt(self):
|
|
hex_proof = proof('query-hex')
|
|
self.snapshot['queries'].append({'proof': hex_proof})
|
|
self.documents.append(document('QABCDEF12', hex_proof))
|
|
body = '原文引用正式查询[QABCDEF12]。'
|
|
self.accumulator.answer = body
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertEqual(result['answer'], body)
|
|
self.assertEqual(result['answerQuality']['citedEvidenceRefs'], ['QABCDEF12'])
|
|
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-hex'])
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_cancelled_turn_remains_cancelled_and_does_not_read_or_synthesize(self):
|
|
for cancellation in ('event', 'native_status'):
|
|
with self.subTest(cancellation=cancellation):
|
|
self.cancel_event.clear()
|
|
self.native_status, self.native_reason = 'COMPLETED', 'stop'
|
|
if cancellation == 'event':
|
|
self.cancel_event.set()
|
|
else:
|
|
self.native_status = 'CANCELLED'
|
|
self.accumulator.answer = '中断前的草稿'
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['status'], 'CANCELLED')
|
|
self.assertEqual(result['answer'], '')
|
|
self.delegated.call.assert_not_called()
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
def test_server_coverage_gap_keeps_body_and_prevents_completed_status(self):
|
|
self.snapshot['rootGoalGaps'] = ['亮片必答阶段尚无完整两期依据。']
|
|
body = '原生结论及其局限。'
|
|
self.accumulator.answer = body
|
|
|
|
result = self.invoke()
|
|
|
|
self.assertEqual(result['answer'], body)
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
self.assertFalse(result['answerQuality']['dataComplete'])
|
|
self.assertIn('亮片必答阶段尚无完整两期依据。', result['answerQuality']['missingEvidence'])
|
|
self.synthetic_writer.assert_not_called()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|