Files
quality_analyse/verification/test_native_answer_gate.py

136 lines
7.0 KiB
Python

"""Native synthesis through the real ModelGate and a loopback fake provider."""
import json
import sys
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
from nao_evidence_finalizer import synthesize_native, FinalizationUnavailable
from nao_model_gate import ModelGate
from test_agent_answer import evidence
class NativeAnswerGateTests(unittest.TestCase):
def setUp(self):
self.allowed = True
self.cancelled = threading.Event()
self.calls = []
self.events = []
self.reject_before_followup = False
self.malformed_synthesis = False
self.cancel_after_synthesis = False
owner = self
class Provider(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
owner.calls.append(body)
reviewing = body['messages'][0]['content'].startswith('逐句核验证据')
payload = json.loads(body['messages'][1]['content'])
content = ({'reviews': [{'index': 0, 'supported': True}]} if reviewing else
{'blocks': [{'text': '本期正式率低于基期,需要继续核查阶段差异。',
'factIds': ['F1'], 'claimType': 'inference'}],
'summarySelection': {'leadIndex': 0, 'evidenceIndexes': []}})
if 'draftBlocks' in payload:
content = {'reviews': [{'sourceId': block['sourceId'], 'role': 'core',
'factIds': ['F1'], 'supported': True, 'statisticalClaimsSupported': True}
for block in payload['draftBlocks']], 'missingGoalDescriptions': [], 'visualIds': []}
if owner.malformed_synthesis and not reviewing:
content = 'invalid composition'
data = json.dumps({'model': 'fixture-model', 'choices': [{'finish_reason': 'stop',
'message': {'content': content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)}}]}).encode()
self.send_response(200)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(data)))
self.end_headers()
self.wfile.write(data)
self.provider = ThreadingHTTPServer(('127.0.0.1', 0), Provider)
self.provider.daemon_threads = True
self.thread = threading.Thread(target=self.provider.serve_forever, daemon=True)
self.thread.start()
def authorize():
if not self.allowed or self.cancelled.is_set():
raise PermissionError('fixture delegation revoked or cancelled')
def thinking(event):
self.events.append(event)
# Revoke after the gate has fully authorized/read the first
# completion, exactly before the finalizer's next model request.
if event.get('event') == 'end' and not event.get('interrupted'):
if self.reject_before_followup:
self.allowed = False
if self.cancel_after_synthesis:
self.cancelled.set()
self.gate = ModelGate(upstream_base=f'http://127.0.0.1:{self.provider.server_port}/v1',
upstream_key='fixture-provider-key', model='fixture-model', port=0,
client_key='fixture-gate-key', authorize=authorize,
native_turn_marker='quality-turn:fixture-current',
thinking_options={'enabled': True, 'effort': 'high', 'protocol': 'thinking'}, on_thinking=thinking)
self.base = f'http://127.0.0.1:{self.gate._server.server_port}/v1'
def tearDown(self):
self.gate.close()
self.provider.shutdown()
self.provider.server_close()
self.thread.join(timeout=1)
def synthesize(self, draft=None):
# No mocked opener: the finalizer, HTTP gate and upstream all run.
return synthesize_native(base_url=self.base, api_key='fixture-gate-key', model='fixture-model',
question='解释质量变化', evidence=evidence(), cancel_event=self.cancelled, native_answer=draft)
def test_finished_draft_is_reviewed_once_with_same_thinking_and_authorization(self):
result = self.synthesize('综合合格率95.00%,低于基期96.00%。')
self.assertTrue(result['evidenceSufficient'], result['missingEvidence'])
self.assertEqual(len(self.calls), 1)
self.assertEqual(result['compositionDiagnostics']['modelPhases'], ['native_draft_review'])
self.assertEqual(self.calls[0]['thinking'], {'type': 'enabled'})
self.assertEqual(self.calls[0]['reasoning_effort'], 'high')
self.assertEqual(self.gate.rejected_stale_requests, 0)
self.assertEqual({event['phase'] for event in self.events}, {'finalizing'})
def test_synthesis_and_review_use_authorized_finalizing_phase_without_native_marker(self):
result = self.synthesize()
self.assertTrue(result['evidenceSufficient'], result)
self.assertEqual(len(self.calls), 2)
self.assertEqual(self.gate.rejected_stale_requests, 0)
self.assertEqual(result['compositionDiagnostics']['modelPhases'], ['native_synthesis', 'evidence_review'])
self.assertTrue(all(call['thinking'] == {'type': 'enabled'} for call in self.calls))
self.assertTrue(all(call['reasoning_effort'] == 'high' for call in self.calls))
self.assertFalse(any('quality-turn:fixture-current' in json.dumps(call['messages']) for call in self.calls))
self.assertTrue(self.events)
self.assertEqual({event['phase'] for event in self.events}, {'finalizing'})
def test_revoked_delegation_blocks_initial_synthesis_before_upstream(self):
self.allowed = False
with self.assertRaisesRegex(FinalizationUnavailable, '授权已失效'):
self.synthesize()
self.assertEqual(self.calls, [])
def test_revocation_is_checked_again_before_review_and_format_repair(self):
for malformed in (False, True):
with self.subTest(malformed=malformed):
self.allowed = True
self.calls.clear()
self.reject_before_followup = True
self.malformed_synthesis = malformed
with self.assertRaisesRegex(FinalizationUnavailable, '授权已失效'):
self.synthesize()
self.assertEqual(len(self.calls), 1)
def test_cancellation_blocks_initial_and_subsequent_model_work(self):
self.cancelled.set()
with self.assertRaisesRegex(FinalizationUnavailable, '已取消'):
self.synthesize()
self.assertEqual(self.calls, [])
self.cancelled.clear()
self.cancel_after_synthesis = True
with self.assertRaises(FinalizationUnavailable):
self.synthesize()
self.assertEqual(len(self.calls), 1)
if __name__ == '__main__':
unittest.main()