236 lines
16 KiB
Python
236 lines
16 KiB
Python
"""Content titles share existing finalization calls and stay metadata-only."""
|
|
import io
|
|
import copy
|
|
import json
|
|
import sys
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
|
|
from nao_evidence_finalizer import synthesize_native
|
|
from nao_native_answer import synthesize_history_snapshot, _report_title_required
|
|
from test_agent_answer import evidence
|
|
|
|
|
|
def response(value):
|
|
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {
|
|
'content': json.dumps(value, ensure_ascii=False)}}]}).encode())
|
|
|
|
|
|
class NativeContentTitleTests(unittest.TestCase):
|
|
def invoke(self, title, *, draft=True, report=None):
|
|
calls = []
|
|
def provider(request, timeout):
|
|
body = json.loads(request.data); calls.append(body)
|
|
payload = json.loads(body['messages'][1]['content'])
|
|
if 'draftBlocks' in payload:
|
|
result = {'reviews': [{'sourceId': block['sourceId'], 'role': 'core', 'factIds': ['F1'],
|
|
'supported': True, 'statisticalClaimsSupported': True} for block in payload['draftBlocks']],
|
|
'missingGoalDescriptions': [], 'reportTitle': title}
|
|
elif body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
result = {'reviews': [{'supported': True, 'statisticalClaimsSupported': True}], 'missingGoalDescriptions': []}
|
|
else:
|
|
result = {'blocks': [{'role': 'core', 'text': '综合合格率95.00%。', 'factIds': ['F1']}], 'reportTitle': title}
|
|
return response(result)
|
|
kwargs = dict(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
|
|
question='为什么合格率下降', evidence=evidence(), cancel_event=threading.Event(),
|
|
context={'_agentTask': {'executionMode': 'native_agent'}}, opener=provider,
|
|
native_answer='综合合格率95.00%。' if draft else None)
|
|
if report:
|
|
with patch('nao_native_answer.build_native_report', return_value=report): result = synthesize_native(**kwargs)
|
|
else: result = synthesize_native(**kwargs)
|
|
return result, calls
|
|
|
|
def test_content_title_is_kept_without_another_call_in_both_paths(self):
|
|
for draft, count in ((True, 1), (False, 2)):
|
|
with self.subTest(draft=draft):
|
|
result, calls = self.invoke('2026年8—9月合格率变化', draft=draft)
|
|
self.assertEqual(len(calls), count)
|
|
self.assertEqual(result['reportTitle'], '2026年8—9月合格率变化')
|
|
self.assertTrue(result['titleGeneratedByModel'])
|
|
self.assertTrue(result['titleValid'])
|
|
self.assertEqual(result['titleSource'], 'model_verified_content')
|
|
self.assertTrue(result['evidenceSufficient'])
|
|
|
|
def test_invalid_title_falls_back_without_poisoning_verified_body_or_retry(self):
|
|
for title in (None, '质量数据分析', '为什么合格率下降', '2027年9月质量变化',
|
|
'2026年10月质量变化', 'QABCDEF12质量分析', 'QABCDEFAB质量分析', 'history-fact:F1汇总',
|
|
'scope-1质量分析', '<b>合格率变化</b>', '合格率95.00%质量分析'):
|
|
with self.subTest(title=title):
|
|
result, calls = self.invoke(title)
|
|
self.assertEqual(len(calls), 1)
|
|
self.assertFalse(result.get('titleValid', True))
|
|
self.assertFalse(result['titleGeneratedByModel'])
|
|
self.assertEqual(result['reportTitle'], '质量数据分析')
|
|
self.assertTrue(result['evidenceSufficient'])
|
|
|
|
def test_numeric_business_spec_is_allowed_but_kpi_cannot_supply_a_date(self):
|
|
spec = '16.33X15.5X3.75+T'
|
|
fact = {'id': 'F1', 'kind': 'product', 'label': spec,
|
|
'text': spec + '综合合格率95.00%,投入重量10.00 kg。', 'evidenceRef': 'Q00000001',
|
|
'scope': {'period': {'year': 2026, 'month': 9}, 'filters': {'product': spec}}}
|
|
report = {'facts': [fact], 'requiredFactIds': [], 'priorityFactIds': [], 'missingEvidence': [],
|
|
'rejectedEvidenceRefs': [], 'notices': [], 'sections': []}
|
|
result, _ = self.invoke('规格' + spec + '质量表现', report=report)
|
|
self.assertTrue(result['titleGeneratedByModel'])
|
|
self.assertEqual(result['reportTitle'], '规格' + spec + '质量表现')
|
|
result, _ = self.invoke('2026年10月质量表现', report=report)
|
|
self.assertFalse(result.get('titleValid', True))
|
|
|
|
def test_title_top_count_requires_actual_contiguous_ranking(self):
|
|
facts = [{'id': 'F' + str(i), 'kind': 'product', 'label': '规格' + chr(64+i), 'rank': i,
|
|
'text': '第' + str(i) + '项规格' + chr(64+i) + '合格率95.00%。', 'evidenceRef': 'Q00000001',
|
|
'scope': {'period': {'year': 2026, 'month': 9}, 'dimension': 'product', 'filters': {}}}
|
|
for i in range(1, 6)]
|
|
report = {'facts': facts, 'requiredFactIds': [], 'priorityFactIds': [], 'missingEvidence': [],
|
|
'rejectedEvidenceRefs': [], 'notices': [], 'sections': []}
|
|
result, calls = self.invoke('2026年9月产品Top5观察', report=report)
|
|
self.assertTrue(result['titleValid'])
|
|
self.assertEqual(len(calls), 1)
|
|
result, _ = self.invoke('2026年9月产品Top6观察', report=report)
|
|
self.assertFalse(result['titleValid'])
|
|
|
|
def test_aggregate_year_or_quarter_does_not_invent_a_narrow_month_title(self):
|
|
fact = {'id': 'F1', 'kind': 'overall', 'label': '综合合格率',
|
|
'text': '2026年9月综合合格率95.00%。', 'evidenceRef': 'Q00000001',
|
|
'scope': {'period': {'year': 2026}, 'filters': {}}}
|
|
report = {'facts': [fact], 'requiredFactIds': [], 'priorityFactIds': [], 'missingEvidence': [],
|
|
'rejectedEvidenceRefs': [], 'notices': [], 'sections': []}
|
|
for aggregate in ({'year': 2026}, {'year': 2026, 'quarter': 3}):
|
|
fact['scope']['period'] = aggregate
|
|
result, _ = self.invoke('2026年9月合格率观察', report=report)
|
|
self.assertFalse(result['titleValid'])
|
|
result, _ = self.invoke('2026年合格率观察', report=report)
|
|
self.assertTrue(result['titleValid'])
|
|
|
|
def test_native_history_keeps_same_response_title_instead_of_overwriting(self):
|
|
from test_nao_history_report import snapshot, report_task, authority, RELEASE, TURN
|
|
calls = []
|
|
def provider(request, timeout):
|
|
body = json.loads(request.data); calls.append(body)
|
|
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
return response({'reviews': [{'supported': True, 'statisticalClaimsSupported': True}], 'missingGoalDescriptions': []})
|
|
return response({'reportTitle': '2025至2026年多范围报废表现', 'blocks': [{'role': 'core',
|
|
'text': '历史来源期间和筛选分别适用,应分别解读报废重量。', 'factIds': ['F1', 'F2']}]})
|
|
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
|
|
question='汇总历史', snapshot=snapshot(), task=report_task(), authority=authority(),
|
|
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
|
|
self.assertEqual(len(calls), 2)
|
|
self.assertEqual(result['reportTitle'], '2025至2026年多范围报废表现')
|
|
self.assertTrue(result['titleValid'])
|
|
self.assertEqual(result['titleSource'], 'model_verified_content')
|
|
self.assertTrue(result['titleRequired'])
|
|
|
|
def test_title_required_uses_confirmed_task_or_trusted_history_path(self):
|
|
history=[{'validity':{'verified':True,'basis':'protocol_history_snapshot'}}]
|
|
native=[{'validity':{'verified':True,'basis':'quality_comparison'}}]
|
|
self.assertTrue(_report_title_required({'_reportTitleRequired':True},history))
|
|
self.assertFalse(_report_title_required({'_reportTitleRequired':True},native))
|
|
self.assertTrue(_report_title_required({'_agentTask':{'kind':'report'}},native))
|
|
self.assertFalse(_report_title_required({'question':'生成报告','_agentTask':{'kind':'analysis'}},native))
|
|
self.assertFalse(_report_title_required({'_agentTask':{'kind':'metrics'}},native))
|
|
|
|
def test_actual_history_caller_requires_title_when_presentation_loses_kind(self):
|
|
import nao_native_answer as composer
|
|
from test_nao_history_report import snapshot,report_task,authority,RELEASE,TURN
|
|
original=composer._synthesize_verified_report;seen=[]
|
|
def forward(**kwargs):
|
|
seen.append(kwargs['context'].get('_reportTitleRequired'))
|
|
kwargs['context']=copy.deepcopy(kwargs['context']);kwargs['context']['_agentTask'].pop('kind',None)
|
|
return original(**kwargs)
|
|
def provider(request,timeout):
|
|
body=json.loads(request.data)
|
|
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
return response({'reviews':[{'supported':True,'statisticalClaimsSupported':True}],'missingGoalDescriptions':[]})
|
|
self.assertTrue(json.loads(body['messages'][1]['content'])['reportTitleGuide']['required'])
|
|
return response({'reportTitle':'2025至2026年多范围报废表现','blocks':[{'role':'core',
|
|
'text':'历史来源期间和筛选分别适用,应分别解读报废重量。','factIds':['F1','F2']}]})
|
|
with patch('nao_native_answer._synthesize_verified_report',side_effect=forward):
|
|
result=synthesize_history_snapshot(base_url='https://offline.invalid',api_key='fixture',model='fixture',
|
|
question='汇总历史',snapshot=snapshot(),task=report_task(),authority=authority(),release_id=RELEASE,
|
|
request_turn_id=TURN,cancel_event=threading.Event(),opener=provider)
|
|
self.assertEqual(seen,[True]);self.assertTrue(result['titleRequired']);self.assertTrue(result['titleValid'])
|
|
|
|
def required_report(self, provider, *, draft=False, required=True):
|
|
return synthesize_native(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
|
|
question='汇总合格率变化',evidence=evidence(),cancel_event=threading.Event(),
|
|
context={'_agentTask':{'kind':'report' if required else 'analysis','executionMode':'native_agent'}},
|
|
native_answer='综合合格率95.00%。' if draft else None,opener=provider)
|
|
|
|
def test_report_schema_and_combined_repair_both_require_report_title(self):
|
|
requests=[];generation=[]
|
|
def provider(request,timeout):
|
|
body=json.loads(request.data);requests.append(body)
|
|
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
return response({'reviews':[{'supported':True,'statisticalClaimsSupported':True}],'missingGoalDescriptions':[]})
|
|
generation.append(body)
|
|
return response({'blocks':[{'role':'core','text':'综合合格率99.99%。' if len(generation)==1 else '综合合格率95.00%。','factIds':['F1']}],
|
|
**({'reportTitle':'2026年8—9月合格率变化'} if len(generation)>1 else {})})
|
|
result=self.required_report(provider)
|
|
self.assertEqual(len(requests),3);self.assertEqual(len(generation),2)
|
|
self.assertIn('输出结构:{"reportTitle":',requests[0]['messages'][0]['content'])
|
|
self.assertIn('reportTitle',generation[1]['messages'][-1]['content'])
|
|
self.assertNotIn('仍只返回blocks JSON',generation[1]['messages'][-1]['content'])
|
|
self.assertTrue(result['titleValid']);self.assertTrue(result['bodyEvidenceSufficient']);self.assertEqual(result['titleMissingEvidence'],[])
|
|
|
|
def test_title_only_bounded_repair_reuses_body_without_rewriting(self):
|
|
for draft in (False,True):
|
|
requests=[];generation=[]
|
|
def provider(request,timeout):
|
|
body=json.loads(request.data);requests.append(body);payload=json.loads(body['messages'][1]['content'])
|
|
if 'draftBlocks' not in payload and body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
return response({'reviews':[{'supported':True,'statisticalClaimsSupported':True}],'missingGoalDescriptions':[]})
|
|
generation.append(body)
|
|
if len(generation)>1:return response({'reportTitle':'2026年8—9月合格率变化'})
|
|
if draft:return response({'reviews':[{'sourceId':b['sourceId'],'role':'core','factIds':['F1'],
|
|
'supported':True,'statisticalClaimsSupported':True} for b in payload['draftBlocks']],'missingGoalDescriptions':[]})
|
|
return response({'blocks':[{'role':'core','text':'综合合格率95.00%。','factIds':['F1']}]})
|
|
with self.subTest(draft=draft):
|
|
result=self.required_report(provider,draft=draft)
|
|
self.assertEqual(len(requests),2 if draft else 3)
|
|
self.assertEqual(len(generation),2)
|
|
self.assertIn('titleMetadataOnlyRepair',generation[1]['messages'][-1]['content'])
|
|
self.assertTrue(result['bodyEvidenceSufficient']);self.assertTrue(result['titleValid'])
|
|
self.assertIn('95.00%',result['answer']);self.assertTrue(result['readingSummary'])
|
|
|
|
def test_required_title_failure_preserves_verified_body_and_separate_gap(self):
|
|
requests=[]
|
|
def provider(request,timeout):
|
|
body=json.loads(request.data);requests.append(body)
|
|
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
return response({'reviews':[{'supported':True,'statisticalClaimsSupported':True}],'missingGoalDescriptions':[]})
|
|
return response({'blocks':[{'role':'core','text':'综合合格率95.00%。','factIds':['F1']}]})
|
|
result=self.required_report(provider)
|
|
self.assertEqual(len(requests),3);self.assertTrue(result['titleRequired']);self.assertFalse(result['titleValid'])
|
|
self.assertTrue(result['evidenceSufficient']);self.assertTrue(result['bodyEvidenceSufficient'])
|
|
self.assertEqual(result['missingEvidence'],[]);self.assertEqual(result['bodyMissingEvidence'],[])
|
|
self.assertTrue(result['titleMissingEvidence']);self.assertIn('95.00%',result['answer'])
|
|
self.assertNotIn('格式未通过',result['answer'])
|
|
|
|
def test_optional_chat_title_never_retries_or_poisons_body(self):
|
|
requests=[]
|
|
def provider(request,timeout):
|
|
body=json.loads(request.data);requests.append(body)
|
|
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
return response({'reviews':[{'supported':True,'statisticalClaimsSupported':True}],'missingGoalDescriptions':[]})
|
|
return response({'blocks':[{'role':'core','text':'综合合格率95.00%。','factIds':['F1']}]})
|
|
result=self.required_report(provider,required=False)
|
|
self.assertEqual(len(requests),2);self.assertFalse(result['titleRequired'])
|
|
self.assertTrue(result['bodyEvidenceSufficient']);self.assertEqual(result['titleMissingEvidence'],[])
|
|
|
|
def test_title_failure_never_marks_rejected_body_sufficient(self):
|
|
def provider(request,timeout):
|
|
body=json.loads(request.data)
|
|
if body['messages'][0]['content'].startswith('逐句核验证据'):
|
|
return response({'reviews':[{'supported':False,'statisticalClaimsSupported':False}],
|
|
'missingGoalDescriptions':['尚未覆盖所需比较关系。']})
|
|
return response({'blocks':[{'role':'core','text':'综合合格率95.00%。','factIds':['F1']}]})
|
|
result=self.required_report(provider)
|
|
self.assertFalse(result['bodyEvidenceSufficient']);self.assertFalse(result['evidenceSufficient'])
|
|
self.assertTrue(result['bodyMissingEvidence']);self.assertTrue(result['titleMissingEvidence'])
|
|
|
|
|
|
if __name__ == '__main__': unittest.main()
|