136 lines
7.0 KiB
Python
136 lines
7.0 KiB
Python
"""Native synthesis through the real ModelGate and a loopback fake provider."""
|
|
import json
|
|
import sys
|
|
import threading
|
|
import unittest
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
|
|
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
|
|
from nao_evidence_finalizer import synthesize_native, FinalizationUnavailable
|
|
from nao_model_gate import ModelGate
|
|
from test_agent_answer import evidence
|
|
|
|
|
|
class NativeAnswerGateTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.allowed = True
|
|
self.cancelled = threading.Event()
|
|
self.calls = []
|
|
self.events = []
|
|
self.reject_before_followup = False
|
|
self.malformed_synthesis = False
|
|
self.cancel_after_synthesis = False
|
|
owner = self
|
|
class Provider(BaseHTTPRequestHandler):
|
|
def log_message(self, *_): pass
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
owner.calls.append(body)
|
|
reviewing = body['messages'][0]['content'].startswith('逐句核验证据')
|
|
payload = json.loads(body['messages'][1]['content'])
|
|
content = ({'reviews': [{'index': 0, 'supported': True}]} if reviewing else
|
|
{'blocks': [{'text': '本期正式率低于基期,需要继续核查阶段差异。',
|
|
'factIds': ['F1'], 'claimType': 'inference'}],
|
|
'summarySelection': {'leadIndex': 0, 'evidenceIndexes': []}})
|
|
if 'draftBlocks' in payload:
|
|
content = {'reviews': [{'sourceId': block['sourceId'], 'role': 'core',
|
|
'factIds': ['F1'], 'supported': True, 'statisticalClaimsSupported': True}
|
|
for block in payload['draftBlocks']], 'missingGoalDescriptions': [], 'visualIds': []}
|
|
if owner.malformed_synthesis and not reviewing:
|
|
content = 'invalid composition'
|
|
data = json.dumps({'model': 'fixture-model', 'choices': [{'finish_reason': 'stop',
|
|
'message': {'content': content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)}}]}).encode()
|
|
self.send_response(200)
|
|
self.send_header('Content-Type', 'application/json')
|
|
self.send_header('Content-Length', str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
self.provider = ThreadingHTTPServer(('127.0.0.1', 0), Provider)
|
|
self.provider.daemon_threads = True
|
|
self.thread = threading.Thread(target=self.provider.serve_forever, daemon=True)
|
|
self.thread.start()
|
|
def authorize():
|
|
if not self.allowed or self.cancelled.is_set():
|
|
raise PermissionError('fixture delegation revoked or cancelled')
|
|
def thinking(event):
|
|
self.events.append(event)
|
|
# Revoke after the gate has fully authorized/read the first
|
|
# completion, exactly before the finalizer's next model request.
|
|
if event.get('event') == 'end' and not event.get('interrupted'):
|
|
if self.reject_before_followup:
|
|
self.allowed = False
|
|
if self.cancel_after_synthesis:
|
|
self.cancelled.set()
|
|
self.gate = ModelGate(upstream_base=f'http://127.0.0.1:{self.provider.server_port}/v1',
|
|
upstream_key='fixture-provider-key', model='fixture-model', port=0,
|
|
client_key='fixture-gate-key', authorize=authorize,
|
|
native_turn_marker='quality-turn:fixture-current',
|
|
thinking_options={'enabled': True, 'effort': 'high', 'protocol': 'thinking'}, on_thinking=thinking)
|
|
self.base = f'http://127.0.0.1:{self.gate._server.server_port}/v1'
|
|
|
|
def tearDown(self):
|
|
self.gate.close()
|
|
self.provider.shutdown()
|
|
self.provider.server_close()
|
|
self.thread.join(timeout=1)
|
|
|
|
def synthesize(self, draft=None):
|
|
# No mocked opener: the finalizer, HTTP gate and upstream all run.
|
|
return synthesize_native(base_url=self.base, api_key='fixture-gate-key', model='fixture-model',
|
|
question='解释质量变化', evidence=evidence(), cancel_event=self.cancelled, native_answer=draft)
|
|
|
|
def test_finished_draft_is_reviewed_once_with_same_thinking_and_authorization(self):
|
|
result = self.synthesize('综合合格率95.00%,低于基期96.00%。')
|
|
self.assertTrue(result['evidenceSufficient'], result['missingEvidence'])
|
|
self.assertEqual(len(self.calls), 1)
|
|
self.assertEqual(result['compositionDiagnostics']['modelPhases'], ['native_draft_review'])
|
|
self.assertEqual(self.calls[0]['thinking'], {'type': 'enabled'})
|
|
self.assertEqual(self.calls[0]['reasoning_effort'], 'high')
|
|
self.assertEqual(self.gate.rejected_stale_requests, 0)
|
|
self.assertEqual({event['phase'] for event in self.events}, {'finalizing'})
|
|
|
|
def test_synthesis_and_review_use_authorized_finalizing_phase_without_native_marker(self):
|
|
result = self.synthesize()
|
|
self.assertTrue(result['evidenceSufficient'], result)
|
|
self.assertEqual(len(self.calls), 2)
|
|
self.assertEqual(self.gate.rejected_stale_requests, 0)
|
|
self.assertEqual(result['compositionDiagnostics']['modelPhases'], ['native_synthesis', 'evidence_review'])
|
|
self.assertTrue(all(call['thinking'] == {'type': 'enabled'} for call in self.calls))
|
|
self.assertTrue(all(call['reasoning_effort'] == 'high' for call in self.calls))
|
|
self.assertFalse(any('quality-turn:fixture-current' in json.dumps(call['messages']) for call in self.calls))
|
|
self.assertTrue(self.events)
|
|
self.assertEqual({event['phase'] for event in self.events}, {'finalizing'})
|
|
|
|
def test_revoked_delegation_blocks_initial_synthesis_before_upstream(self):
|
|
self.allowed = False
|
|
with self.assertRaisesRegex(FinalizationUnavailable, '授权已失效'):
|
|
self.synthesize()
|
|
self.assertEqual(self.calls, [])
|
|
|
|
def test_revocation_is_checked_again_before_review_and_format_repair(self):
|
|
for malformed in (False, True):
|
|
with self.subTest(malformed=malformed):
|
|
self.allowed = True
|
|
self.calls.clear()
|
|
self.reject_before_followup = True
|
|
self.malformed_synthesis = malformed
|
|
with self.assertRaisesRegex(FinalizationUnavailable, '授权已失效'):
|
|
self.synthesize()
|
|
self.assertEqual(len(self.calls), 1)
|
|
|
|
def test_cancellation_blocks_initial_and_subsequent_model_work(self):
|
|
self.cancelled.set()
|
|
with self.assertRaisesRegex(FinalizationUnavailable, '已取消'):
|
|
self.synthesize()
|
|
self.assertEqual(self.calls, [])
|
|
self.cancelled.clear()
|
|
self.cancel_after_synthesis = True
|
|
with self.assertRaises(FinalizationUnavailable):
|
|
self.synthesize()
|
|
self.assertEqual(len(self.calls), 1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|