Files
quality_analyse/verification/test_native_draft_finalization.py

214 lines
13 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""A finished native draft is independently reviewed once, never rewritten."""
import io
import json
import sys
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
from nao_evidence_finalizer import synthesize_native
from test_agent_answer import evidence
def response(value):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {
'content': value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)}}]}).encode())
class NativeDraftFinalizationTests(unittest.TestCase):
def invoke(self, draft, provider, **kwargs):
return synthesize_native(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
question='为什么合格率下降', evidence=evidence(), cancel_event=threading.Event(),
context={'_agentTask': {'executionMode': 'native_agent'}}, native_answer=draft, opener=provider, **kwargs)
def verdict(self, payload, **change):
return {'reviews': [{'sourceId': block['sourceId'], 'role': 'core' if i == 0 else 'key_evidence',
'factIds': ['F1'], 'supported': True, 'statisticalClaimsSupported': True, **change}
for i, block in enumerate(payload['draftBlocks'])], 'missingGoalDescriptions': [], 'visualIds': []}
def test_finished_native_draft_has_one_review_and_preserves_markdown(self):
calls = []
draft = '本期综合合格率95.00%,低于基期96.00%。\n\n## 对比\n\n| 期间 | 综合合格率 |\n|---|---|\n| 本期 | 95.00% |\n| 基期 | 96.00% |'
def provider(request, timeout):
body = json.loads(request.data); calls.append(body)
payload = json.loads(body['messages'][1]['content'])
if 'draftBlocks' not in payload:
return response({'blocks': [{'role': 'core', 'text': '本期综合合格率95.00%。', 'factIds': ['F1']}]})
return response(self.verdict(payload))
result = self.invoke(draft, provider)
self.assertEqual(len(calls), 1)
self.assertTrue(result['evidenceSufficient'], result['missingEvidence'])
self.assertIn('| 本期 | 95.00% |', result['answer'])
self.assertIn('## 对比', result['answer'])
self.assertEqual(result['compositionDiagnostics']['modelPhases'], ['native_draft_review'])
def test_reviewer_cannot_supply_replacement_prose(self):
calls = []
def provider(request, timeout):
calls.append(1); payload = json.loads(json.loads(request.data)['messages'][1]['content'])
if 'draftBlocks' not in payload: return response({'blocks': []})
return response(self.verdict(payload, text='设备故障造成下降。'))
result = self.invoke('本期综合合格率95.00%。', provider)
self.assertEqual(len(calls), 2)
self.assertFalse(result['evidenceSufficient'])
self.assertNotIn('设备故障', result['answer'])
def test_exact_unsupported_clause_is_deleted_without_second_model(self):
calls = []
draft = '本期综合合格率95.00%。设备故障造成下降。'
def provider(request, timeout):
calls.append(1); payload = json.loads(json.loads(request.data)['messages'][1]['content'])
if 'draftBlocks' not in payload: return response({'blocks': []})
return response(self.verdict(payload, supported=False, statisticalClaimsSupported=False,
unsupportedClauses=['设备故障造成下降。'], remainingClaimsSupported=True,
remainingStatisticalClaimsSupported=True))
result = self.invoke(draft, provider)
self.assertEqual(len(calls), 1)
self.assertTrue(result['evidenceSufficient'], result['missingEvidence'])
self.assertNotIn('设备故障', result['answer'])
self.assertIn('95.00%', result['answer'])
def test_unbound_number_stays_gap_after_one_bounded_repair(self):
calls = []
def provider(request, timeout):
calls.append(1); payload = json.loads(json.loads(request.data)['messages'][1]['content'])
if 'draftBlocks' not in payload: return response({'blocks': []})
return response(self.verdict(payload))
result = self.invoke('本期综合合格率99.99%。', provider)
self.assertEqual(len(calls), 2)
self.assertFalse(result['evidenceSufficient'])
self.assertNotIn('99.99%', result['answer'])
def test_typed_markdown_cells_keep_units_direction_and_original_rank(self):
fact = {'id': 'F1', 'kind': 'product', 'label': '规格甲', 'rank': 1,
'text': '第1项:规格甲;综合合格率96.00%→95.00%,下降1.00个百分点;投入重量100.00 kg。',
'evidenceRef': 'Q00000001', 'scope': {'dimension': 'product', 'filters': {}}}
report = {'facts': [fact], 'requiredFactIds': [], 'priorityFactIds': [], 'missingEvidence': [],
'rejectedEvidenceRefs': [], 'notices': [], 'sections': []}
draft = ('规格甲合格率95.00%。\n\n## 对比\n\n'
'| 序号 | 产品 | 投入(kg) | 变化(个百分点) |\n|---|---|---|---|\n'
'| 1 | 规格甲 | 100.00 | -1.00 |')
def provider(request, timeout):
payload = json.loads(json.loads(request.data)['messages'][1]['content'])
return response(self.verdict(payload))
with patch('nao_native_answer.build_native_report', return_value=report):
result = self.invoke(draft, provider)
self.assertTrue(result['evidenceSufficient'], result['compositionDiagnostics']['issues'])
self.assertIn('| 1 |', result['answer'])
self.assertIn('100.00 kg', result['answer'])
self.assertIn('-1.00个百分点', result['answer'])
for invalid in (draft.replace('| 1 |', '| 9 |'), draft.replace('投入(kg)', '投入(吨)'),
draft.replace('-1.00 |', '-9.00 |'), draft.replace('-1.00 |', '+1.00 |')):
with self.subTest(invalid=invalid), patch('nao_native_answer.build_native_report', return_value=report):
bad = self.invoke(invalid, provider)
self.assertFalse(bad['evidenceSufficient'])
def test_invalid_support_and_unknown_source_ids_cannot_skip_review(self):
for change in ({'sourceId': 'D999'}, {'supported': 'true'}, {'factIds': ['F999']}):
calls = []
def provider(request, timeout):
calls.append(1); payload = json.loads(json.loads(request.data)['messages'][1]['content'])
return response(self.verdict(payload, **change))
result = self.invoke('综合合格率95.00%。', provider)
self.assertEqual(len(calls), 2)
self.assertFalse(result['evidenceSufficient'])
def test_goal_gap_remains_authoritative(self):
def provider(request, timeout):
payload = json.loads(json.loads(request.data)['messages'][1]['content'])
value = self.verdict(payload)
value['missingGoalDescriptions'] = ['重点规格尚未回答。']
return response(value)
result = self.invoke('综合合格率95.00%。', provider)
self.assertFalse(result['evidenceSufficient'])
self.assertIn('重点规格尚未回答。', result['missingEvidence'])
def test_complete_optional_table_rejection_does_not_create_a_new_goal(self):
calls=[]
draft='综合合格率95.00%。\n\n| 分组 | 变化 |\n|---|---|\n| 未知组甲 | -9.999个百分点 |\n| 未知组乙 | -8.888个百分点 |'
def provider(request,timeout):
calls.append(1);payload=json.loads(json.loads(request.data)['messages'][1]['content']);value=self.verdict(payload)
value['reviews'][1].update(supported=False,statisticalClaimsSupported=False,
factIds=[],unsupportedClauses=['| 未知组甲 | -9.999个百分点 |','| 未知组乙 | -8.888个百分点 |'])
return response(value)
result=self.invoke(draft,provider)
self.assertEqual(len(calls),1);self.assertTrue(result['evidenceSufficient'],result['missingEvidence'])
self.assertFalse(result['compositionDiagnostics']['formatRepairAttempted'])
self.assertNotIn('9.999',result['answer'])
def test_only_core_wholly_rejected_is_business_gap_without_format_retry(self):
calls=[]
def provider(request,timeout):
calls.append(1);payload=json.loads(json.loads(request.data)['messages'][1]['content']);value=self.verdict(payload)
for row,source in zip(value['reviews'],payload['draftBlocks']):
row.update(supported=False,statisticalClaimsSupported=False,unsupportedClauses=[source['text']])
return response(value)
result=self.invoke('综合合格率95.00%。',provider)
self.assertEqual(len(calls),1);self.assertFalse(result['evidenceSufficient'])
self.assertFalse(result['compositionDiagnostics']['formatRepairAttempted'])
self.assertEqual(result['compositionDiagnostics']['issues'],[])
self.assertTrue(any('核心分析' in gap for gap in result['missingEvidence']))
self.assertFalse(any('格式未通过' in gap for gap in result['missingEvidence']))
def test_complete_optional_multisentence_rejection_needs_no_remainder_flags(self):
calls=[]
def provider(request,timeout):
calls.append(1);payload=json.loads(json.loads(request.data)['messages'][1]['content']);value=self.verdict(payload)
value['reviews'][1].update(role='detail',supported=False,statisticalClaimsSupported=False,
unsupportedClauses=['未知指标99.99%。','设备故障导致下降。'])
return response(value)
result=self.invoke('综合合格率95.00%。\n\n未知指标99.99%。设备故障导致下降。',provider)
self.assertEqual(len(calls),1);self.assertTrue(result['evidenceSufficient'],result['missingEvidence'])
self.assertNotIn('设备故障',result['answer'])
def test_uncited_optional_layout_does_not_force_full_review_retry(self):
calls=[]
def provider(request,timeout):
calls.append(1);payload=json.loads(json.loads(request.data)['messages'][1]['content']);value=self.verdict(payload)
value['reviews'][1].update(role='detail',factIds=[])
return response(value)
result=self.invoke('综合合格率95.00%。\n\n### 结论\n\n数据支持的优先排查方向是:',provider)
self.assertEqual(len(calls),1);self.assertTrue(result['evidenceSufficient'],result['missingEvidence'])
self.assertEqual(result['compositionDiagnostics']['omittedDraftLayoutBlocks'],['D2'])
def test_whole_optional_unsupported_block_is_removed_but_core_is_a_gap(self):
calls = []
def provider(request, timeout):
calls.append(1); payload = json.loads(json.loads(request.data)['messages'][1]['content'])
value = self.verdict(payload)
value['reviews'][1].update(role='detail', supported=False, statisticalClaimsSupported=False,
unsupportedClauses=[payload['draftBlocks'][1]['text']])
return response(value)
result = self.invoke('综合合格率95.00%。\n\n未知指标99.99%。', provider)
self.assertEqual(len(calls), 1)
self.assertTrue(result['evidenceSufficient'], result['missingEvidence'])
self.assertNotIn('99.99%', result['answer'])
def unsupported_core(request, timeout):
payload = json.loads(json.loads(request.data)['messages'][1]['content'])
return response(self.verdict(payload, supported=False, statisticalClaimsSupported=False))
result = self.invoke('综合合格率95.00%。', unsupported_core)
self.assertFalse(result['evidenceSufficient'])
def test_cancellation_prevents_any_draft_review(self):
from nao_evidence_finalizer import FinalizationUnavailable
cancelled = threading.Event(); cancelled.set()
with self.assertRaises(FinalizationUnavailable):
synthesize_native(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
question='质量变化', evidence=evidence(), cancel_event=cancelled,
native_answer='综合合格率95.00%。', opener=lambda *args, **kwargs: self.fail('model called'))
def test_citation_format_is_normalized_only_when_bound_to_real_source(self):
def provider(request, timeout):
payload = json.loads(json.loads(request.data)['messages'][1]['content'])
return response(self.verdict(payload))
result = self.invoke('综合合格率95.00%。[Q00000001]', provider)
self.assertTrue(result['evidenceSufficient'])
self.assertEqual(result['answer'].count('[Q00000001]'), 1)
result = self.invoke('综合合格率95.00%。[Q99999999]', provider)
self.assertFalse(result['evidenceSufficient'])
if __name__ == '__main__': unittest.main()