"""Evidence-size, completeness and bounded synthesis regressions; no model calls.""" import io import copy import json import sys import tempfile import threading import ssl import unittest import uuid from unittest.mock import MagicMock, patch from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) import nao_evidence_finalizer as finalizer import nao_quality_runtime as runtime FORMAL_PRODUCT_QUESTION = '只看SYNTH-16X15+T,请重新查询2026年9月与8月黑片、亮片的合格率、报废重量和两期投入,附正式数据依据。' FORMAL_PRODUCT_GAPS = ['缺少仅针对SYNTH-16X15+T规格的两期分规格投入重量正式数据', '缺少仅针对该规格的黑片、亮片报废重量分规格正式数据'] def formal_product_evidence(): result = {'kind': 'quality_comparison', 'releaseId': 'release-a', 'filterBinding': {'product': 'SYNTH-16X15+T', 'includeUnassigned': False}, 'delta': {'available': True, 'percentagePoints': -2}, 'stageRateChangesPP': {'black': -1, 'bright': -1}} for side, month, inputs, scraps in [('current', 9, (1200.5, 2400.5), (36.1, 72.1)), ('comparison', 8, (1000.5, 2000.5), (20.1, 40.1))]: result[side] = {'period': {'year': 2026, 'month': month}, 'compositeRate': .95, 'coverage': {'sourceMonths': [{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}]}} for index, stage in enumerate(('black', 'bright')): result[side][stage] = {'rate': .97, 'totalWeight': inputs[index], 'scrapWeight': scraps[index]} return [{'evidenceRef': 'Q00000001', 'mode': 'compare', 'result': result}] def unpack(packed): def decode(value): if isinstance(value, list): return [decode(item) for item in value] if not isinstance(value, dict): return value if set(value) == {'$shared'}: return decode(packed['shared'][value['$shared']]) if set(value) == {'$columns', '$values'}: return [dict(zip(decode(value['$columns']), decode(row))) for row in decode(value['$values'])] return {key: decode(item) for key,item in value.items()} return decode(packed['evidence']) class EvidenceFinalizerTest(unittest.TestCase): def test_real_connection_keeps_system_certificate_and_hostname_verification(self): with patch.object(finalizer,'urlopen',return_value=io.BytesIO(b'{}')) as request: finalizer._open_model('request',35) context=request.call_args.kwargs['context'] self.assertEqual(context.verify_mode,ssl.CERT_REQUIRED) self.assertTrue(context.check_hostname) def test_worker_finalizes_only_this_turn_and_preserves_partial_and_cancelled_states(self): for outcome in ('complete','retry','retry_no_queries','fresh_no_queries','restatement','native_complete','native_query','insufficient','contradictory','server_incomplete','finalizer_error','cancel','cancel_after_stop','stop_failed'): with self.subTest(outcome=outcome),tempfile.TemporaryDirectory() as folder: root=Path(folder);(root/'evidence').mkdir() old=root/'evidence'/('nao-quality-'+'b'*32+'.json') old.write_text('{}');old.chmod(0o600) payload={key:str(uuid.uuid4()) for key in ('turnId','sessionId','actorId','releaseId')} payload.update(question='全部明细' if outcome=='server_incomplete' else '比较质量',context={},accessToken='test-delegation') if outcome == 'fresh_no_queries': payload['question'] = '请重新查询刚才的数据' if outcome == 'restatement': payload['question'] = '请复述上一轮结论' if outcome in ('retry','retry_no_queries'):payload['context']={'_restartConversation':True} manager=MagicMock();manager.acquire_actor.return_value=(root,None) manager.ensure_instance.return_value=('http://127.0.0.1:49191/v1','test-gate-key','mimo-v2.6-flash','http://127.0.0.1:5000') gate=MagicMock();gate.base_url='http://127.0.0.1:49191/v1' stream=runtime.ConversationAccumulator();stream.blocks['first']='查询已返回';stream.block_order.append('first') stream.record_event('assistant',text='查询已返回') stream.tool_calls=[{'tool':'mcp_call','status':'completed'}] cancellation=[] def sse(**kwargs): if outcome in ('retry','retry_no_queries'): self.assertIsNone(kwargs['chat_id']) self.assertEqual(kwargs['context'],{}) if outcome in ('retry_no_queries', 'fresh_no_queries', 'restatement'): stream.tool_calls=[] return stream,'COMPLETED','stop',None cancellation.append(kwargs['cancel_event']) document={'tool':'quality_data','releaseId':payload['releaseId'],'evidenceRef':'Q00000001', 'mode':'compare','result':{'kind':'quality_query' if outcome=='native_query' else 'quality_comparison','releaseId':payload['releaseId'], 'current':{'period':{'year':2026,'month':9},'compositeRate':.95}, 'comparison':{'period':{'year':2026,'month':8},'compositeRate':.96}, 'delta':{'available':True,'percentagePoints':-1}}} path=root/'evidence'/('nao-quality-'+'a'*32+'.json');path.write_text(json.dumps(document));path.chmod(0o600) if outcome == 'cancel_after_stop': kwargs['cancel_event'].set() return stream, 'PARTIAL', 'coverage_ready', True if outcome in ('native_complete','native_query','finalizer_error'):return stream,'COMPLETED','stop',None return stream,'PARTIAL','timeout',outcome!='stop_failed' def synthesize(**kwargs): self.assertEqual([d['evidenceRef'] for d in kwargs['evidence']],['Q00000001']) self.assertEqual(kwargs['base_url'],'http://127.0.0.1:49191/v1') self.assertEqual(kwargs['api_key'],'test-gate-key') self.assertEqual(kwargs['context'],{}) if outcome=='finalizer_error':raise finalizer.FinalizationUnavailable('整理失败') if outcome=='cancel':cancellation[0].set() return {'answer':'最终统计 [Q00000001]','model':'mimo-v2.6-flash','usage':{},'finishReason':'stop', 'evidenceSufficient':outcome!='insufficient','missingEvidence':['缺少用户要求的对照期'] if outcome=='insufficient' else [], 'modelDeclaredEvidenceSufficient':outcome not in ('insufficient','contradictory'), 'completionAdjusted':outcome=='contradictory'} with patch.object(runtime,'RuntimeManager',return_value=manager),patch.object(runtime,'write_access_file',return_value=root/'access.json'), \ patch.object(runtime,'_authenticate'),patch.object(runtime,'_input_cancel_listener'),patch.object(runtime,'SessionState'), \ patch.object(runtime,'_sse_request',side_effect=sse),patch.object(runtime,'_story_files',return_value=(None,None)), \ patch.object(runtime,'_model_settings',return_value=('https://model.test/v1','test-key','mimo-v2.6-flash',{'enabled':False,'effort':'auto','protocol':'thinking'})) as selected, \ patch.object(runtime,'plan_agent_task',return_value=None,create=True), \ patch.object(runtime,'_model_gate_settings',return_value=(49191,'test-gate-key')),patch.object(runtime,'DelegatedApi'), \ patch('nao_model_gate.ModelGate',return_value=gate) as gate_factory, \ patch.object(finalizer,'synthesize',side_effect=synthesize) as request, \ patch.object(finalizer,'coverage_sufficient',return_value=outcome not in ('finalizer_error','server_incomplete')): result=runtime.run_turn(payload,stdout=io.StringIO(),stdin=io.StringIO()) selected.assert_called_once() self.assertEqual(gate_factory.call_args.kwargs['upstream_key'],'test-key') gate.close.assert_called_once() expected={'complete':'COMPLETED','retry':'COMPLETED','retry_no_queries':'PARTIAL','fresh_no_queries':'PARTIAL','restatement':'COMPLETED','native_complete':'COMPLETED','native_query':'COMPLETED','insufficient':'PARTIAL','contradictory':'COMPLETED','server_incomplete':'PARTIAL','finalizer_error':'PARTIAL','cancel':'CANCELLED','cancel_after_stop':'CANCELLED','stop_failed':'PARTIAL'}[outcome] self.assertEqual(result['status'],expected) self.assertEqual(request.call_count,0 if outcome in ('stop_failed','native_query','retry_no_queries','fresh_no_queries','restatement','cancel_after_stop') else 1) if outcome in ('retry_no_queries', 'fresh_no_queries'): self.assertEqual(result['stopReason'],'evidence_incomplete') self.assertIn('未取得新的查询证据',result['answer']) if outcome=='native_query': self.assertEqual(result['stopReason'],'stop') self.assertEqual(result['answer'],'查询已返回') self.assertFalse(any(call['tool']=='model.finalize' for call in result['toolCalls'])) self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],'COMPLETED') if outcome in ('complete','native_complete'):self.assertEqual(result['stopReason'],'evidence_finalized') if outcome in ('complete','native_complete'): self.assertEqual(len([entry for entry in result['timeline'] if entry['kind']=='assistant']),1) self.assertEqual(result['timeline'][-1]['text'],'最终统计 [Q00000001]') self.assertEqual(json.loads((root/'private/last-visible-answer.json').read_text())['answer'],'最终统计 [Q00000001]') if outcome in ('complete','native_complete','insufficient','contradictory','server_incomplete','finalizer_error'): self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],expected) if outcome=='contradictory': self.assertFalse(result['evidenceCompletion']['modelDeclaredEvidenceSufficient']) self.assertTrue(result['evidenceCompletion']['completionAdjusted']) self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient']) if outcome=='server_incomplete': self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient']) def test_columnar_and_shared_encoding_roundtrips_without_rounding_or_null_loss(self): evidence=[{'rows':[{'field':'电镀磕边重量','weight':None,'count':4120,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}}, {'field':'镀废合计','weight':855.801072,'count':431,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}}]}]*4 self.assertEqual(unpack(finalizer.packed_evidence(evidence)),evidence) def test_oversize_evidence_fails_closed_without_discarding_statistics(self): with self.assertRaises(finalizer.FinalizationUnavailable): finalizer.packed_evidence([{'unique': 'a'*finalizer.MAX_INPUT_CHARS}]) def test_real_second_turn_replay_preserves_every_value_and_obeys_current_input_budget(self): root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime' directories=list(root.glob('.runtime/nao-users/*/sessions/57676d5a-5138-48f4-9343-bae9731aae5a/evidence')) if not directories: self.skipTest('Private captured acceptance evidence not present') with patch.object(finalizer,'packed_evidence',side_effect=AssertionError('Local archives do not consume model input budget')): docs=finalizer.evidence_summaries(directories[0],'194b0287-9044-4d80-9138-ecd3c481015c') self.assertEqual(len(docs),15) # Formal display metadata has grown since this private run was captured. # Local archives remain lossless; a bounded model handoff may reject an # oversized summary, but may never silently trim it to fit. limit=finalizer.MAX_INPUT_CHARS with patch.object(finalizer,'MAX_INPUT_CHARS',100_000_000): packed=finalizer.packed_evidence(docs) self.assertEqual(unpack(packed),docs) encoded=json.dumps(packed,ensure_ascii=False,separators=(',',':')) if len(encoded)>limit: with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.packed_evidence(docs) else:self.assertEqual(unpack(finalizer.packed_evidence(docs)),docs) # This captured run predates paired black/bright defect reads for each # highlighted product, so it is now correctly labelled incomplete. self.assertFalse(finalizer.coverage_sufficient(docs,'分析质量变化')) # Ordinary analysis must not be blocked by minor source reconciliation issues. missing=[d for d in docs if not(d['result'].get('action')=='source' and d['result'].get('stage')=='bright')] self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化')) # Full raw difference pagination is optional for an ordinary statistics question. missing=[d for d in docs if not(d['result'].get('action')=='rows' and d['result'].get('stage')=='bright')] self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化')) def test_previous_turn_evidence_is_excluded(self): with tempfile.TemporaryDirectory() as directory: root=Path(directory) for index in range(2): path=root/('nao-quality-'+str(index)*32+'.json') path.write_text(json.dumps({'tool':'quality_data','releaseId':'release','evidenceRef':'Q0000000'+str(index), 'mode':'compare','result':{'releaseId':'release'},'createdAt':str(index)})) path.chmod(0o600) docs=finalizer.evidence_summaries(root,'release',exclude_files={'nao-quality-'+'0'*32+'.json'}) self.assertEqual([d['evidenceRef'] for d in docs],['Q00000001']) def test_identity_permissions_and_duplicate_refs_are_rejected(self): with tempfile.TemporaryDirectory() as directory: root=Path(directory);path=root/('nao-quality-'+'a'*32+'.json') doc={'tool':'quality_data','releaseId':'r','evidenceRef':'Q00000001','mode':'query','result':{'releaseId':'r'}} path.write_text(json.dumps(doc));path.chmod(0o644) with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r') path.chmod(0o600) with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'other') twin=root/('nao-quality-'+'b'*32+'.json');twin.write_text(json.dumps(doc));twin.chmod(0o600) with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r') def test_cancel_failed_stop_and_incomplete_tools_never_start_finalization(self): calls=[{'status':'completed'}] args=['mimo-v2.6-flash','PARTIAL','timeout',calls,False,True] self.assertTrue(finalizer.eligible(*args)) for position,value in ((4,True),(5,False),(5,None),(3,[{'status':'started'}]),(2,'transport_or_stream_error')): changed=list(args);changed[position]=value self.assertFalse(finalizer.eligible(*changed)) self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','timeout',calls,False,True)) self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','coverage_ready',calls,False,True)) def test_full_detail_requires_all_pages_but_accepts_contiguous_completed_pagination(self): evidence=[{'mode':'compare','result':{}}] for offset,count,more in ((0,2,True),(2,1,False)): evidence.append({'mode':'investigate','result':{'action':'rows','current':{ 'context':{'month':9},'matchingRows':3,'offset':offset,'returnedRows':count, 'hasMore':more,'differenceRecords':[{}]*count}}}) self.assertTrue(finalizer.coverage_sufficient(evidence,'全部明细')) self.assertFalse(finalizer.coverage_sufficient(evidence[:-1],'全部明细')) def test_synthesis_is_one_bounded_openai_compatible_request_with_verified_citations(self): calls=[] def opener(request,timeout): calls.append((json.loads(request.data),timeout)) return io.BytesIO(json.dumps({'model':'mimo-v2.6-flash','choices':[{'finish_reason':'stop','message':{ 'content':json.dumps({'otherFactIds':['F1'],'groupFactIds':[],'productFactIds':[],'defectFactIds':[],'answer':'伪造上升方向和QDEADBEEF引用必须忽略','evidenceSufficient':True,'missingEvidence':[]})}}]}).encode()) evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison', 'current':{'period':{'year':2026,'month':9},'compositeRate':0.95, 'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}}, 'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96, 'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}}, 'delta':{'percentagePoints':-1}}}] result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash', question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener) self.assertTrue(result['evidenceSufficient']);self.assertEqual(len(calls),1) self.assertIn('下降1.00个百分点',result['answer']);self.assertNotIn('QDEADBEEF',result['answer']);self.assertNotIn('伪造',result['answer']) self.assertEqual(calls[0][0]['thinking'],{'type':'disabled'}) from nao_runtime_policy import MAX_OUTPUT_TOKENS self.assertEqual(calls[0][0]['max_completion_tokens'],MAX_OUTPUT_TOKENS) self.assertEqual(calls[0][1],finalizer.FINAL_TIMEOUT_SECONDS) cancelled=threading.Event();cancelled.set() with self.assertRaises(finalizer.FinalizationUnavailable): finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash', question='质量如何',evidence=evidence,cancel_event=cancelled,opener=opener) self.assertEqual(len(calls),1) def test_selector_timeout_uses_verified_report_without_claiming_missing_coverage(self): evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{ 'kind':'quality_comparison', 'current':{'period':{'year':2026,'month':9},'compositeRate':0.95, 'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}}, 'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96, 'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}}, 'delta':{'percentagePoints':-1}}}] def timeout(request, timeout): self.assertEqual(timeout, finalizer.FINAL_TIMEOUT_SECONDS) raise TimeoutError('model selector timed out') result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real', model='mimo-v2.6-flash',question='质量如何',evidence=evidence, cancel_event=threading.Event(),opener=timeout) self.assertTrue(result['selectionFallback']) self.assertTrue(result['evidenceSufficient']) self.assertIn('下降1.00个百分点',result['answer']) def test_model_cannot_override_missing_stage_or_ambiguous_frozen_frame(self): from test_nao_evidence_report import coverage_comparison, comparison def opener(request,timeout): return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({ 'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode()) missing=coverage_comparison() missing['result']['current']['coverage']['sourceMonths'][0].update(blackRows=0,brightRows=0) result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash', question='比较8月相对7月',evidence=[missing],cancel_event=threading.Event(),opener=opener) self.assertTrue(result['modelDeclaredEvidenceSufficient']) self.assertFalse(result['evidenceSufficient']) self.assertFalse(finalizer.coverage_sufficient([missing],'比较8月相对7月')) self.assertIn('没有可引用的正式综合合格率',result['answer']) self.assertNotIn('100.00%',result['answer']) docs=[comparison('Q00000001',{},.97),comparison('Q00000002',{'group':'组A'},.80)] unresolved=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash', question='只看组A',evidence=docs,cancel_event=threading.Event(),opener=opener) self.assertFalse(unresolved['evidenceSufficient']) self.assertIn('组A',unresolved['answer']);self.assertIn('80.00%',unresolved['answer']) context={'year':2026,'month':7,'group':'组A'} bound=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash', question='只看组A',evidence=docs,context=context,cancel_event=threading.Event(),opener=opener) self.assertTrue(bound['evidenceSufficient']) self.assertTrue(finalizer.coverage_sufficient(docs,'只看组A',context)) self.assertIn('80.00%',bound['answer']);self.assertNotIn('97.00%',bound['answer']) def test_comparison_required_task_does_not_get_rechecked_as_single_period(self): question = '全面分析质量变化' current = {'year': 2026, 'month': 9} comparison = {'year': 2026, 'month': 8} context = {'releaseId': 'release-a', '_agentTask': { 'version': 1, 'question': question, 'kind': 'analysis', 'freshQuery': True, 'dimensions': ['department', 'group', 'product', 'defect'], 'stages': ['black', 'bright'], 'fields': ['rate'], 'period': current, 'comparisonPeriod': None, 'comparisonRequired': True, 'filters': {}, 'verifySource': False, 'fullDetails': False, }} evidence = [{'evidenceRef': 'Q' + '4' * 8, 'mode': 'compare', 'result': {'kind': 'quality_comparison', 'releaseId': 'release-a', 'filterBinding': {}, 'current': {'period': current}, 'comparison': {'period': comparison}}}] with patch('nao_analysis_coverage.coverage_gaps', return_value=[]), \ patch('nao_analysis_coverage.semantic_single_period_gaps', side_effect=AssertionError('comparison task was treated as single period')), \ patch('nao_analysis_coverage.formal_metric_coverage', return_value=None), \ patch('nao_evidence_report.build_report', return_value={'coverageSufficient': True}): self.assertTrue(finalizer.coverage_sufficient(evidence, question, context)) def test_optional_focus_metadata_cannot_discard_or_invent_verified_statistics(self): evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison', 'current':{'period':{'year':2026,'month':9},'compositeRate':0.95, 'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}}, 'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96, 'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}}, 'delta':{'percentagePoints':-1}}}] def opener(request,timeout): return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({ 'otherFactIds':['invented','F1','F1','F2','F3'], 'groupFactIds':['F1'], 'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode()) result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash', question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener) self.assertTrue(result['selectionAdjusted']) self.assertTrue(result['evidenceSufficient']) self.assertEqual(result['selectedFactIds'],['F1','F2']) self.assertNotIn('invented',result['answer']) self.assertIn('下降1.00个百分点',result['answer']) def test_completion_requires_concrete_missing_items_and_keeps_model_diagnostics(self): evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison', 'current':{'period':{'year':2026,'month':9},'compositeRate':0.95, 'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}}, 'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96, 'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}}, 'delta':{'percentagePoints':-1}}}] for declared,missing,expected in [(False,[],True),(False,[' ',''],True), (False,[' 缺少对照期 ','缺少对照期'],False),(True,['缺少对照期'],False),(True,[],True)]: with self.subTest(declared=declared,missing=missing): def opener(request,timeout): return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({ 'evidenceSufficient':declared,'missingEvidence':missing})}}]}).encode()) result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash', question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener) self.assertEqual(result['evidenceSufficient'],expected) self.assertEqual(result['modelDeclaredEvidenceSufficient'],declared) self.assertEqual(result['modelMissingEvidence'],missing) self.assertEqual(result['completionAdjusted'],not declared and expected) self.assertEqual(result['missingEvidence'],[] if expected else ['缺少对照期']) if not expected:self.assertIn('缺少对照期',result['answer']) def test_captured_3c_with_missing_paired_stage_remains_partial(self): root=Path(__file__).resolve().parents[1] paths=list((root/'.runtime/nao-users').glob('*/sessions/98829d63-0158-4e70-89db-ef0d58fe546a')) if not paths:self.skipTest('Private captured session not present') original=json.loads((paths[0]/'finalization.json').read_text()) self.assertFalse(original['modelDeclaredEvidenceSufficient']) self.assertEqual(original['missingEvidence'],[]) self.assertTrue(original['serverCoverageSufficient']) release=json.loads((root/'output/20261023-incomplete-reason/session.json').read_text())['releaseId'] docs=finalizer.evidence_summaries(paths[0]/'evidence',release) def opener(request,timeout): return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({ 'evidenceSufficient':False,'missingEvidence':[]})}}]}).encode()) result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash', question='3C哪些产品近期表现最差',evidence=docs,cancel_event=threading.Event(),opener=opener) self.assertFalse(result['evidenceSufficient']) self.assertFalse(finalizer.coverage_sufficient(docs,'3C哪些产品近期表现最差')) self.assertFalse(result['completionAdjusted']) self.assertTrue(any('缺少' in item for item in result['missingEvidence'])) self.assertIn('Q039D5FA7',result['answer']) self.assertIn('90.11%',result['answer']) def test_exact_formal_metrics_refute_only_the_model_claims_they_prove_false(self): captured = [] def opener(request, timeout): captured.append(json.loads(request.data)) return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({ 'evidenceSufficient': False, 'missingEvidence': FORMAL_PRODUCT_GAPS})}}]}).encode()) result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash', question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'}, cancel_event=threading.Event(), opener=opener) self.assertTrue(result['evidenceSufficient']) self.assertEqual(result['missingEvidence'], []) self.assertEqual(result['modelMissingEvidence'], FORMAL_PRODUCT_GAPS) self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS) self.assertTrue(result['completionAdjusted']) selector = json.loads(captured[0]['messages'][1]['content']) contract = selector['formalMetricCoverage'] self.assertEqual(contract['filters'], {'product': 'SYNTH-16X15+T'}) self.assertEqual(len(contract['metrics']), 12) self.assertTrue(all(row['covered'] for row in contract['metrics'])) self.assertIn(1200.5, [row['value'] for row in contract['metrics']]) self.assertNotIn('未覆盖内容', result['answer']) def test_formal_metric_contract_rejects_wrong_scope_period_release_and_incomplete_values(self): mutations = { 'product': lambda r: r['filterBinding'].update(product='SYNTH-16X15+T-OTHER'), 'extra_filter': lambda r: r['filterBinding'].update(group='其他分组'), 'period': lambda r: r['comparison']['period'].update(month=7), 'release': lambda r: r.update(releaseId='release-other'), 'input': lambda r: r['current']['black'].pop('totalWeight'), 'scrap': lambda r: r['comparison']['bright'].pop('scrapWeight'), 'rate': lambda r: r['current']['bright'].update(rate=None), 'nonfinite': lambda r: r['current']['black'].update(totalWeight='NaN'), 'source_months': lambda r: r['comparison'].pop('coverage'), 'source_stage': lambda r: r['current']['coverage']['sourceMonths'][0].update(brightRows=0), } for name, mutate in mutations.items(): with self.subTest(name=name): docs = copy.deepcopy(formal_product_evidence()); mutate(docs[0]['result']) def opener(request, timeout): return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({ 'evidenceSufficient': True, 'missingEvidence': []})}}]}).encode()) result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash', question=FORMAL_PRODUCT_QUESTION, evidence=docs, context={'releaseId': 'release-a'}, cancel_event=threading.Event(), opener=opener) self.assertFalse(result['evidenceSufficient']) self.assertTrue(result['missingEvidence']) self.assertFalse(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, {'releaseId': 'release-a'})) def test_unknown_mixed_and_other_scope_model_gaps_still_block_completion(self): for missing in ['缺少该规格投入重量和现场原因证据', '缺少该规格投入重量原始明细', '缺少其他规格的投入重量正式数据', '缺少2026年7月投入重量正式数据', '缺少对照期', '缺少生产原因证据']: with self.subTest(missing=missing): def opener(request, timeout): return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({ 'evidenceSufficient': False, 'missingEvidence': [*FORMAL_PRODUCT_GAPS, missing]})}}]}).encode()) result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash', question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'}, cancel_event=threading.Event(), opener=opener) self.assertFalse(result['evidenceSufficient']) self.assertEqual(result['missingEvidence'], [missing]) self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS) def test_metric_reconciliation_does_not_infer_unspecified_scope_or_hide_later_missing_items(self): from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps for question in [FORMAL_PRODUCT_QUESTION.replace('只看SYNTH-16X15+T,', ''), FORMAL_PRODUCT_QUESTION.replace('2026年', ''), FORMAL_PRODUCT_QUESTION.replace('黑片、亮片', ''), FORMAL_PRODUCT_QUESTION.replace('9月与8月', '13月与8月')]: with self.subTest(question=question): contract = formal_metric_coverage(formal_product_evidence(), question, {'releaseId': 'release-a'}) self.assertIsNone(contract) self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, [])) contract = formal_metric_coverage(formal_product_evidence(), FORMAL_PRODUCT_QUESTION.replace('报废重量和', ''), {'releaseId': 'release-a'}) self.assertEqual(resolve_proven_metric_gaps(contract, [FORMAL_PRODUCT_GAPS[1]]), ([FORMAL_PRODUCT_GAPS[1]], [])) gaps = ['缺少' + '的' * index + '该规格投入重量正式数据' for index in range(12)] + ['缺少生产原因证据'] def opener(request, timeout): return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({ 'evidenceSufficient': False, 'missingEvidence': gaps})}}]}).encode()) result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash', question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'}, cancel_event=threading.Event(), opener=opener) self.assertFalse(result['evidenceSufficient']) self.assertEqual(result['missingEvidence'], ['缺少生产原因证据']) self.assertEqual(result['modelMissingEvidence'], gaps) def test_formal_contract_accepts_the_same_explicit_frozen_filters(self): from nao_analysis_coverage import formal_metric_coverage for filters in [{'group': '合成分组'}, {'department': '合成部门'}, {'group': '合成分组', 'extraFilters': {'region': '合成地区'}, 'includeUnassigned': True}]: with self.subTest(filters=filters): docs = formal_product_evidence() docs[0]['result']['filterBinding'].update(filters) expected = {'product': 'SYNTH-16X15+T', **filters} context = {'releaseId': 'release-a', 'filters': expected} contract = formal_metric_coverage(docs, FORMAL_PRODUCT_QUESTION, context) self.assertEqual(contract['filters'], expected) self.assertEqual(contract['missingEvidence'], []) self.assertTrue(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, context)) def test_unknown_extra_scope_or_analysis_language_does_not_activate_the_contract(self): from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps, coverage_gaps docs = formal_product_evidence() docs[0]['result']['filterBinding']['group'] = '合成分组' for question in [FORMAL_PRODUCT_QUESTION + '仅统计合成分组。', FORMAL_PRODUCT_QUESTION.replace(',请', ',合成部门,分析人员因素,请'), '合成分组中,' + FORMAL_PRODUCT_QUESTION, FORMAL_PRODUCT_QUESTION + '并提供现场生产原因。']: with self.subTest(question=question): contract = formal_metric_coverage(docs, question, {'releaseId': 'release-a'}) self.assertIsNone(contract) self.assertEqual(coverage_gaps(docs, question, {'releaseId': 'release-a'}), []) self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, [])) if __name__=='__main__':unittest.main()