"""Evidence-bound composed answers; uses only mocked OpenAI-compatible responses.""" import io import json import sys import threading import tempfile from urllib.error import HTTPError import unittest from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) import nao_evidence_finalizer as finalizer def evidence(ref='Q00000001', rate=.95): return [{'evidenceRef': ref, 'mode': 'compare', 'result': { 'kind': 'quality_comparison', 'filterBinding': {}, 'current': {'period': {'year': 2026, 'month': 9}, 'compositeRate': rate, 'coverage': {'sourceMonths': [{'year': 2026, 'month': 9, 'blackRows': 2, 'brightRows': 2}]}}, 'comparison': {'period': {'year': 2026, 'month': 8}, 'compositeRate': .96, 'coverage': {'sourceMonths': [{'year': 2026, 'month': 8, 'blackRows': 2, 'brightRows': 2}]}}, 'delta': {'available': True, 'percentagePoints': (rate - .96) * 100}, }}] def response(parsed): payload = {'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(parsed, ensure_ascii=False)}}]} return io.BytesIO(json.dumps(payload).encode()) class AgentAnswerTest(unittest.TestCase): def test_product_drilldown_fallback_keeps_each_queried_specification(self): from unittest.mock import patch facts = [{'id':'F1','kind':'overall','text':'总体合格率上升。','evidenceRef':'Q00000001'}] docs = evidence() for index, product in enumerate(('规格甲','规格乙','规格丙'),2): ref = f'Q{index:08X}' facts.append({'id':f'F{index}','kind':'cross','text':product+'来源缺角变化。','evidenceRef':ref}) docs.append({'evidenceRef':ref,'result':{'filters':{'product':product}}}) report = {'facts':facts,'sections':['不应输出冗长全表'],'headline':'总体','policy':'关联不作为生产根因。','missingEvidence':[]} invalid = {'answerBlocks':[{'type':'bullet','text':'{{F1}} 虚构改善50%'}], 'evidenceSufficient':True,'missingEvidence':[]} with patch('nao_agent_report.build_agent_report',return_value=report): result = finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture', question='fixture',evidence=docs,cancel_event=threading.Event(), context={'_agentTask':{'version':1,'kind':'analysis','defectScope':'products','defectProductCount':3}}, opener=lambda *args,**kwargs:response(invalid)) self.assertTrue(result['selectionFallback']) self.assertTrue(result['compositionRepairAttempted']) for product in ('规格甲','规格乙','规格丙'): self.assertIn(product,result['answer']) self.assertIn('建议核查',result['answer']) self.assertNotIn('50%',result['answer']) self.assertNotIn('不应输出冗长全表',result['answer']) def test_invalid_composition_gets_one_model_repair_with_same_global_budget(self): calls = [] def provider(request, timeout): body = json.loads(request.data) calls.append(body) if len(calls) == 1: return response({'reportTitle':'合格率变化分析','answerBlocks':[{'type':'bullet','text':'本期改善50% {{F1}}'}], 'evidenceSufficient':True,'missingEvidence':[]}) return response({'answerBlocks':[{'type':'bullet','text':'{{F1}}'}], 'evidenceSufficient':True,'missingEvidence':[]}) result = finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture', question='比较质量',evidence=evidence(),cancel_event=threading.Event(),opener=provider) self.assertEqual(len(calls),2) self.assertNotIn('50%',result['answer']) self.assertFalse(result.get('selectionFallback',False)) self.assertEqual(result['reportTitle'],'合格率变化分析') self.assertTrue(result['titleGeneratedByModel']) from nao_runtime_policy import MAX_OUTPUT_TOKENS self.assertTrue(all(body['max_completion_tokens']==MAX_OUTPUT_TOKENS for body in calls)) self.assertEqual(len(calls[1]['messages']),4) def test_report_uses_model_selected_priorities_without_dumping_all_tables(self): result = self.synthesize({'reportFactIds': ['F1'], 'reportTitle': '合格率变化分析', 'evidenceSufficient': True, 'missingEvidence': []}, context={'_agentTask': {'version': 1, 'kind': 'report', 'historyMode': 'all', 'question': '综合以上生成报告'}}) self.assertIn('## 核心结论', result['answer']) self.assertIn('下降1.00个百分点', result['answer']) self.assertIn('## 下一步核查', result['answer']) self.assertNotIn('| ---', result['answer']) self.assertLess(len(result['answer']), 1000) self.assertFalse(result.get('selectionFallback', False)) def test_report_rejects_unknown_focus_id_and_still_has_bounded_fallback(self): result = self.synthesize({'reportFactIds': ['F999'], 'evidenceSufficient': True, 'missingEvidence': []}, context={'_agentTask': {'version': 1, 'kind': 'report', 'historyMode': 'all', 'question': '综合以上生成报告'}}) self.assertTrue(result['selectionFallback']) self.assertNotIn('F999', result['answer']) self.assertNotIn('| ---', result['answer']) def synthesize(self, parsed, *, docs=None, model='vendor/compatible-chat', context=None, capture=None): def opener(request, timeout): if capture is not None: capture['body'] = json.loads(request.data) capture['timeout'] = timeout return response(parsed) return finalizer.synthesize( base_url='https://offline.invalid/v1', api_key='synthetic-only', model=model, question='请概括质量变化', evidence=evidence() if docs is None else docs, cancel_event=threading.Event(), context=context, opener=opener) def test_model_composes_prose_around_exact_server_fact_and_reference(self): captured = {} result = self.synthesize({ 'answerBlocks': [ {'type': 'heading', 'text': '变化概览'}, {'type': 'paragraph', 'text': '建议结合现场记录开展复核;当前结论只限于本次查询指标:{{F1}}'}, ], 'evidenceSufficient': True, 'missingEvidence': [], }, context={'_agentTask': {'version': 1, 'kind': 'analysis', 'responseStyle': 'brief', 'dimensions': ['group'], 'question': '简要回答'}}, capture=captured) self.assertIn('### 变化概览', result['answer']) self.assertIn('建议结合现场记录开展复核', result['answer']) self.assertIn('下降1.00个百分点', result['answer']) self.assertIn('[Q00000001]', result['answer']) self.assertEqual(result['selectedFactIds'], ['F1']) self.assertFalse(result.get('selectionFallback', False)) self.assertEqual(captured['body']['model'], 'vendor/compatible-chat') self.assertEqual(captured['body']['thinking'], {'type':'disabled'}) from nao_runtime_policy import MAX_OUTPUT_TOKENS self.assertEqual(captured['body']['max_completion_tokens'], MAX_OUTPUT_TOKENS) selector = json.loads(captured['body']['messages'][1]['content']) self.assertEqual(selector['agentTask']['responseStyle'], 'brief') self.assertTrue(selector['facts']) def test_unsupported_digits_force_verified_report_fallback(self): result = self.synthesize({ 'answerBlocks': [{'type': 'paragraph', 'text': '本期合格率达到99%,依据{{F1}}'}], 'evidenceSufficient': True, 'missingEvidence': [], }) self.assertTrue(result['selectionFallback']) self.assertNotIn('99%', result['answer']) self.assertIn('下降1.00个百分点', result['answer']) def test_unknown_fact_reference_forces_fallback(self): result = self.synthesize({ 'answerBlocks': [{'type': 'paragraph', 'text': '结论{{F99}}'}], 'evidenceSufficient': True, 'missingEvidence': [], }) self.assertTrue(result['selectionFallback']) self.assertNotIn('{{F99}}', result['answer']) self.assertIn('下降1.00个百分点', result['answer']) def test_unsupported_direction_and_cause_prose_forces_fallback(self): result=self.synthesize({ 'answerBlocks':[{'type':'paragraph','text':'质量已持续改善,且确认由设备磨损导致。{{F1}}'}], 'evidenceSufficient':True,'missingEvidence':[]}) self.assertTrue(result['selectionFallback']) self.assertNotIn('设备磨损',result['answer']) def test_cause_hypothesis_passes_only_when_explicitly_unconfirmed(self): result=self.synthesize({ 'answerBlocks':[{'type':'paragraph','text':'待验证:设备磨损是否可能导致报废增加,建议检查。{{F1}}'}], 'evidenceSufficient':True,'missingEvidence':[]}) self.assertFalse(result.get('selectionFallback',False)) self.assertIn('待验证:设备磨损是否可能导致报废增加',result['answer']) absolute=self.synthesize({ 'answerBlocks':[{'type':'paragraph','text':'所有缺陷都已完全排除。{{F1}}'}], 'evidenceSufficient':True,'missingEvidence':[]}) self.assertTrue(absolute['selectionFallback']) def test_thinking_capability_is_retried_without_model_name_branch(self): bodies=[] parsed={'answerBlocks':[{'type':'paragraph','text':'总体变化:{{F1}}'}], 'evidenceSufficient':True,'missingEvidence':[]} def opener(request,timeout): bodies.append(json.loads(request.data)) if len(bodies)==1: raise HTTPError(request.full_url,400,'thinking unsupported',None,io.BytesIO(b'')) return response(parsed) result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic-only', model='some/openai-compatible-model',question='请概括质量变化',evidence=evidence(), cancel_event=threading.Event(),opener=opener) self.assertTrue(result['evidenceSufficient']) self.assertEqual(len(bodies),2) self.assertEqual(bodies[0]['thinking'],{'type':'disabled'}) self.assertNotIn('thinking',bodies[1]) def test_non_mimo_model_is_eligible_and_incomplete_server_evidence_stays_partial(self): calls = [{'status': 'completed'}] self.assertTrue(finalizer.eligible('provider/model-x', 'PARTIAL', 'timeout', calls, False, True)) self.assertFalse(finalizer.eligible(' ', 'PARTIAL', 'timeout', calls, False, True)) from test_nao_evidence_report import coverage_comparison doc=coverage_comparison() doc['result']['current']['coverage']['sourceMonths'][0].update(blackRows=0,brightRows=0) result = self.synthesize({ 'answerBlocks': [{'type': 'paragraph', 'text': '当前正式变化:{{F1}}'}], 'evidenceSufficient': True, 'missingEvidence': [], }, docs=[doc]) self.assertFalse(result['evidenceSufficient']) self.assertTrue(result['missingEvidence']) self.assertTrue(any('没有可引用的正式综合合格率' in item for item in result['missingEvidence'])) self.assertIn('未覆盖内容', result['answer']) def test_legacy_selector_response_still_uses_verified_report(self): result = self.synthesize({ 'otherFactIds': ['F1'], 'groupFactIds': [], 'productFactIds': [], 'defectFactIds': [], 'evidenceSufficient': True, 'missingEvidence': [], }) self.assertTrue(result['evidenceSufficient']) self.assertIn('下降1.00个百分点', result['answer']) self.assertIn('### 重点关注', result['answer']) self.assertFalse(result.get('selectionFallback', False)) def test_source_verification_task_requires_complete_source_proof(self): question = '请核对来源' period={'year':2026,'month':9} context = {'releaseId':'r1','_agentTask': {'version': 1, 'question': question, 'kind': 'source_verify', 'verifySource': True, 'fullDetails': False,'period':period, 'comparisonPeriod':None,'filters':{'product':'P'},'stages':['bright']}} query = {'mode': 'compare', 'result': {'kind': 'quality_query'}} proof = {'complete': True, 'expectedRows':2, 'verifiedRows': 2, 'missingRows':0, 'mismatchRows':0,'allSelectedValuesMatchStored': True} side_context={'releaseId':'r1','year':2026,'month':9,'stage':'bright','category':None, 'product':'P','includeUnassigned':False,'extraFilters':{}} source = {'mode': 'investigate', 'result': {'kind':'quality_investigation','releaseId':'r1', 'period':period,'filters':{'product':'P'},'stage':'bright','action':'source','current': { 'context':side_context,'totalRows': 2,'aggregateComplete':True,'sourceVerification': proof}}} self.assertFalse(finalizer.coverage_sufficient([query], question, context)) self.assertTrue(finalizer.coverage_sufficient([source], question, context)) source['result']['current']['sourceVerification'] = {**proof, 'complete': False} self.assertFalse(finalizer.coverage_sufficient([source], question, context)) def test_source_proofs_must_cover_every_requested_stage_side_and_scope(self): question='核验9月与8月原表' current={'year':2026,'month':9};previous={'year':2026,'month':8} context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'source_verify', 'verifySource':True,'fullDetails':False,'period':current,'comparisonPeriod':previous, 'filters':{'product':'P'},'stages':['black','bright']}} query={'mode':'compare','result':{'kind':'quality_query'}} def source(stage, product='P'): proof={'complete':True,'expectedRows':1,'verifiedRows':1,'missingRows':0,'mismatchRows':0, 'allSelectedValuesMatchStored':True} def side(month): return {'context':{'releaseId':'r1','year':2026,'month':month,'stage':stage,'category':None, 'product':product,'includeUnassigned':False,'extraFilters':{}}, 'totalRows':1,'aggregateComplete':True,'sourceVerification':proof} return {'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1', 'period':current,'comparisonPeriod':previous,'filters':{'product':product},'stage':stage, 'category':None,'action':'source','current':side(9),'comparison':side(8)}} wrong_scope=source('black',product='OTHER') self.assertFalse(finalizer.coverage_sufficient([query,wrong_scope],question,context)) black=source('black');bright=source('bright') self.assertFalse(finalizer.coverage_sufficient([black],question,context)) self.assertTrue(finalizer.coverage_sufficient([black,bright],question,context)) def test_full_details_task_requires_contiguous_complete_pages(self): question = '把所有记录列出来' period={'year':2026,'month':9} context = {'releaseId':'r1','_agentTask': {'version': 1, 'question': question, 'kind': 'details', 'verifySource': False, 'fullDetails': True,'period':period, 'comparisonPeriod':None,'filters':{},'stages':['black']}} side={'context':{'releaseId':'r1','year':2026,'month':9,'stage':'black','category':None, 'includeUnassigned':False,'extraFilters':{}},'recordFilter':'all', 'matchingRows':2,'offset':0,'returnedRows':2,'hasMore':False,'differenceRecords':[{},{}]} rows={'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1','period':period, 'filters':{},'stage':'black','category':None,'action':'rows','current':side}} self.assertTrue(finalizer.coverage_sufficient([rows], question, context)) rows['result']['current']['hasMore'] = True self.assertFalse(finalizer.coverage_sufficient([rows], question, context)) rows['result']['current']['hasMore'] = False rows['result']['current']['recordFilter'] = 'differences' self.assertFalse(finalizer.coverage_sufficient([rows], question, context)) def test_full_details_require_every_requested_stage_and_both_periods(self): question='列出黑片和亮片两期的全部明细' current={'year':2026,'month':9};previous={'year':2026,'month':8} context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'details', 'verifySource':False,'fullDetails':True,'period':current,'comparisonPeriod':previous, 'filters':{'group':'G'},'stages':['black','bright']}} def rows(stage): def side(month): return {'context':{'releaseId':'r1','year':2026,'month':month,'stage':stage,'category':None, 'group':'G','includeUnassigned':False,'extraFilters':{}}, 'recordFilter':'all','matchingRows':1,'offset':0,'returnedRows':1, 'hasMore':False,'differenceRecords':[{}]} return {'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1', 'period':current,'comparisonPeriod':previous,'filters':{'group':'G'},'stage':stage, 'category':None,'action':'rows','current':side(9),'comparison':side(8)}} black=rows('black');bright=rows('bright') self.assertFalse(finalizer.coverage_sufficient([black],question,context)) self.assertTrue(finalizer.coverage_sufficient([black,bright],question,context)) bright['result']['comparison']['recordFilter']='differences' self.assertFalse(finalizer.coverage_sufficient([black,bright],question,context)) def test_unbound_task_period_requires_a_matching_query_root(self): question='列出当前范围的全部明细' context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'details', 'verifySource':False,'fullDetails':True,'period':None,'comparisonPeriod':None, 'filters':{'group':'G'},'stages':['black']}} query={'mode':'compare','result':{'kind':'quality_comparison','releaseId':'r1', 'filterBinding':{'group':'G'},'current':{'period':{'year':2026,'month':9}}, 'comparison':{'period':{'year':2026,'month':8}}}} page={'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1', 'period':{'year':2026,'month':9},'filters':{'group':'G'},'stage':'black','category':None, 'action':'rows','current':{'context':{'releaseId':'r1','year':2026,'month':9,'stage':'black', 'category':None,'group':'G','includeUnassigned':False,'extraFilters':{}},'recordFilter':'all', 'matchingRows':1,'offset':0,'returnedRows':1,'hasMore':False,'differenceRecords':[{}]}}} self.assertTrue(finalizer.coverage_sufficient([query,page],question,context)) self.assertFalse(finalizer.coverage_sufficient([page],question,context)) query['result']['filterBinding']={'group':'OTHER'} self.assertFalse(finalizer.coverage_sufficient([query,page],question,context)) def test_empty_evidence_is_allowed_only_for_planning_callers(self): with tempfile.TemporaryDirectory() as folder: with self.assertRaises(finalizer.FinalizationUnavailable): finalizer.evidence_summaries(Path(folder),'release-a') self.assertEqual(finalizer.evidence_summaries(Path(folder),'release-a',allow_empty=True),[]) def test_semantic_metrics_and_single_period_dimensions_gate_completion(self): from test_nao_metric_requests import metric_evidence, QUESTION task={'version':1,'question':QUESTION,'kind':'metrics','period':{'year':2026,'month':9}, 'comparisonPeriod':{'year':2026,'month':8},'filters':{'product':'SYNTH-16X15+T'}, 'stages':['black','bright'],'fields':['rate','totalWeight','scrapWeight']} context={'releaseId':'release-a','_agentTask':task} docs=metric_evidence() self.assertTrue(finalizer.coverage_sufficient(docs,QUESTION,context)) docs[0]['result']['current']['bright'].pop('totalWeight') self.assertFalse(finalizer.coverage_sufficient(docs,QUESTION,context)) question='本期产品情况' single={'version':1,'question':question,'kind':'analysis','period':{'year':2026,'month':9}, 'comparisonPeriod':None,'filters':{},'dimensions':['product'],'stages':[]} frame={'releaseId':'r1','_agentTask':single} query={'mode':'query','result':{'kind':'quality_metrics','releaseId':'r1', 'period':{'year':2026,'month':9},'filters':{},'view':'overall'}} self.assertFalse(finalizer.coverage_sufficient([query],question,frame)) query['result']['view']='product' self.assertTrue(finalizer.coverage_sufficient([query],question,frame)) if __name__ == '__main__': unittest.main()