104 lines
7.8 KiB
Python
104 lines
7.8 KiB
Python
"""Only verified service tables reach presentation; the model owns explanation."""
|
|
import copy,io,json,sys,tempfile,threading,unittest
|
|
from unittest.mock import patch
|
|
from pathlib import Path
|
|
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).parent)]
|
|
import nao_native_answer as n
|
|
import test_native_grounded_contract as grounded
|
|
from test_nao_history_report import ranking_history_snapshot,report_task,authority,FakeDelegated,RELEASE,TURN
|
|
from nao_history_report import execute_history_report
|
|
|
|
class RecoveryDataPresentationTests(unittest.TestCase):
|
|
def provider(self,calls,*,bad_prose=False,only_table=False):
|
|
def respond(request,timeout):
|
|
body=json.loads(request.data);p=json.loads(body['messages'][1]['content']);calls.append((body,p))
|
|
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
value={'reviews':[{'supported':True,'statisticalClaimsSupported':True} for _ in p['blocks']],'missingGoalDescriptions':[]}
|
|
else:
|
|
scopes={s['scopeId']:s for s in p['scopes']};periods={}
|
|
for fact in p['facts']:
|
|
periods.setdefault(json.dumps(scopes[fact['scopeId']].get('period'),sort_keys=True),fact['id'])
|
|
ids=list(periods.values())
|
|
blocks=[] if only_table else [{'role':'core','text':'报废重量999999 kg。' if bad_prose else '两个范围采用各自的期间,不能把两组排名直接视为同期变化。','factIds':ids}]
|
|
blocks.append({'role':'key_evidence' if blocks else 'core','text':'| 规格 | 报废重量 |\n|---|---|\n| 伪型号 | 999999 kg |','factIds':ids})
|
|
value={'reportTitle':'多范围规格报废分析','blocks':blocks,'visualIds':[]}
|
|
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(value,ensure_ascii=False)}}]}).encode())
|
|
return respond
|
|
def invoke(self,**options):
|
|
calls=[];snapshot=ranking_history_snapshot();original=copy.deepcopy(snapshot)
|
|
final=n.synthesize_history_snapshot(base_url='https://offline.invalid',api_key='fixture',model='fixture',question='汇总已有结果,不查询新数据',
|
|
snapshot=snapshot,task=report_task(),authority=authority(),release_id=RELEASE,request_turn_id=TURN,
|
|
cancel_event=threading.Event(),opener=self.provider(calls,**options))
|
|
self.assertEqual(snapshot,original);return final,calls
|
|
def test_model_numeric_table_is_not_authority_and_service_rows_are_exact(self):
|
|
final,calls=self.invoke()
|
|
self.assertTrue(final['bodyEvidenceSufficient'],final['compositionDiagnostics']['issues'])
|
|
self.assertEqual(len(calls),2);self.assertNotIn('999999',final['answer']);self.assertNotIn('伪型号',final['answer'])
|
|
self.assertIn('九月规格1',final['answer']);self.assertIn('八月规格1',final['answer'])
|
|
self.assertIn('10.25',final['answer']);self.assertIn('Top 5',final['answer']);self.assertIn('Top 3',final['answer'])
|
|
self.assertNotIn('999999',json.dumps(calls[1][1]['blocks'],ensure_ascii=False))
|
|
self.assertTrue(final['readingSummary']);self.assertTrue(final['titleValid'])
|
|
def test_bad_free_prose_is_still_rejected_and_table_only_is_not_an_explanation(self):
|
|
for options in ({'bad_prose':True},{'only_table':True}):
|
|
final,calls=self.invoke(**options);self.assertFalse(final['bodyEvidenceSufficient'])
|
|
self.assertLessEqual(len(calls),3)
|
|
|
|
def native(self,text,*,semantic=True):
|
|
docs,context=grounded.GroundedNativeContractTests().fixture()
|
|
context['_nativeExecutionId']='fixture-current-execution';context['_agentTask']['kind']='analysis'
|
|
calls=[]
|
|
def provider(request,timeout):
|
|
body=json.loads(request.data);payload=json.loads(body['messages'][1]['content']);calls.append(body)
|
|
self.assertNotIn('draftBlocks',payload)
|
|
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
value={'reviews':[{'supported':semantic,'statisticalClaimsSupported':semantic} for _ in payload['blocks']],
|
|
'missingGoalDescriptions':[]}
|
|
else:
|
|
fid=next(f['id'] for f in payload['facts'] if '黑片外废' in f['text'])
|
|
value={'reportTitle':'阶段报废分析','blocks':[{'role':'core','text':text,'factIds':[fid]}],'visualIds':[]}
|
|
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(value,ensure_ascii=False)}}]}).encode())
|
|
final=n.synthesize_native_answer(base_url='https://offline.invalid',api_key='fixture',model='fixture',question='分析质量变化',
|
|
evidence=docs,context=context,native_answer='内部未核验原稿 999999 kg',cancel_event=threading.Event(),opener=provider)
|
|
self.assertLessEqual(len(calls),2)
|
|
self.assertTrue(all(sum(len(m['content']) for m in c['messages'])<=90000 for c in calls))
|
|
self.assertNotIn('内部未核验原稿',json.dumps(calls,ensure_ascii=False))
|
|
return final,calls
|
|
|
|
def test_complex_native_reuses_atomic_guards_and_exact_service_table(self):
|
|
good,calls=self.native('9月黑片外废报废重量615.249 kg。')
|
|
self.assertTrue(good['bodyEvidenceSufficient'],good['compositionDiagnostics']['issues'])
|
|
self.assertEqual(len(calls),2);self.assertTrue(good['compositionDiagnostics']['nativeAtomicBindings'])
|
|
for text in ('9月亮片外废报废重量615.249 kg。','8月黑片外废报废重量615.249 kg。',
|
|
'9月黑片外废数量615.249件。','9月黑片外废报废重量615.35 kg。',
|
|
'9月黑片外废报废低于615.2 kg。'):
|
|
with self.subTest(text=text):
|
|
final,calls=self.native(text);self.assertFalse(final['bodyEvidenceSufficient'])
|
|
self.assertEqual(len(calls),1)
|
|
|
|
def test_semantic_rejection_of_core_is_a_real_gap(self):
|
|
final,calls=self.native('9月黑片外废报废重量615.249 kg。',semantic=False)
|
|
self.assertFalse(final['bodyEvidenceSufficient']);self.assertEqual(len(calls),2)
|
|
self.assertTrue(final['bodyMissingEvidence'])
|
|
|
|
def test_metrics_path_keeps_existing_native_draft_contract(self):
|
|
docs,context=grounded.GroundedNativeContractTests().fixture()
|
|
context.update(_nativeExecutionId='fixture-current-execution')
|
|
context['_agentTask']['kind']='metrics'
|
|
with patch.object(n,'_synthesize_verified_report',side_effect=lambda **kwargs:kwargs):
|
|
routed=n.synthesize_native_answer(base_url='offline',api_key='fixture',model='fixture',question='查询指标',
|
|
evidence=docs,context=context,native_answer='原有单值稿',cancel_event=threading.Event())
|
|
self.assertEqual(routed['native_answer'],'原有单值稿')
|
|
self.assertNotIn('presentationSections',routed['report'])
|
|
|
|
def test_recovery_uses_only_existing_signed_history_import(self):
|
|
delegated=FakeDelegated(ranking_history_snapshot());calls=[]
|
|
with tempfile.TemporaryDirectory() as folder, patch('nao_evidence_finalizer._open_model',side_effect=self.provider(calls)):
|
|
result=execute_history_report(request={'releaseId':RELEASE,'turnId':TURN,'question':'汇总已有结果,不查询新数据','_autonomousAnswer':True},
|
|
task=report_task(),authority=authority(),actor_root=Path(folder),delegated=delegated,
|
|
connection=('https://offline.invalid','fixture','fixture'),cancel_event=threading.Event(),emit=lambda _:None,timings={})
|
|
self.assertEqual([c[0] for c in delegated.calls],['/api/analysis/protocol/history'])
|
|
self.assertEqual(result['status'],'COMPLETED');self.assertEqual(len(calls),2)
|
|
self.assertIn('九月规格1',result['answer']);self.assertIn('八月规格1',result['answer'])
|
|
|
|
if __name__=='__main__':unittest.main()
|