Files
quality_analyse/verification/test_nao_evidence_finalizer.py
lyun ff7bbe3e46 同步:完善质量分析工作台并更新主题字体
同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。
补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。
统一午夜黑及其他主题的控件细节、中文和业务数字字体。
排除本地业务数据、凭据、预览、发布与验收产物。

验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
2026-10-30 22:40:58 +08:00

482 lines
36 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Evidence-size, completeness and bounded synthesis regressions; no model calls."""
import io
import copy
import json
import sys
import tempfile
import threading
import ssl
import unittest
import uuid
from unittest.mock import MagicMock, patch
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_evidence_finalizer as finalizer
import nao_quality_runtime as runtime
FORMAL_PRODUCT_QUESTION = '只看SYNTH-16X15+T,请重新查询2026年9月与8月黑片、亮片的合格率、报废重量和两期投入,附正式数据依据。'
FORMAL_PRODUCT_GAPS = ['缺少仅针对SYNTH-16X15+T规格的两期分规格投入重量正式数据',
'缺少仅针对该规格的黑片、亮片报废重量分规格正式数据']
def formal_product_evidence():
result = {'kind': 'quality_comparison', 'releaseId': 'release-a',
'filterBinding': {'product': 'SYNTH-16X15+T', 'includeUnassigned': False},
'delta': {'available': True, 'percentagePoints': -2},
'stageRateChangesPP': {'black': -1, 'bright': -1}}
for side, month, inputs, scraps in [('current', 9, (1200.5, 2400.5), (36.1, 72.1)),
('comparison', 8, (1000.5, 2000.5), (20.1, 40.1))]:
result[side] = {'period': {'year': 2026, 'month': month}, 'compositeRate': .95,
'coverage': {'sourceMonths': [{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}]}}
for index, stage in enumerate(('black', 'bright')):
result[side][stage] = {'rate': .97, 'totalWeight': inputs[index], 'scrapWeight': scraps[index]}
return [{'evidenceRef': 'Q00000001', 'mode': 'compare', 'result': result}]
def unpack(packed):
def decode(value):
if isinstance(value, list): return [decode(item) for item in value]
if not isinstance(value, dict): return value
if set(value) == {'$shared'}: return decode(packed['shared'][value['$shared']])
if set(value) == {'$columns', '$values'}:
return [dict(zip(decode(value['$columns']), decode(row))) for row in decode(value['$values'])]
return {key: decode(item) for key,item in value.items()}
return decode(packed['evidence'])
class EvidenceFinalizerTest(unittest.TestCase):
def test_real_connection_keeps_system_certificate_and_hostname_verification(self):
with patch.object(finalizer,'urlopen',return_value=io.BytesIO(b'{}')) as request:
finalizer._open_model('request',35)
context=request.call_args.kwargs['context']
self.assertEqual(context.verify_mode,ssl.CERT_REQUIRED)
self.assertTrue(context.check_hostname)
def test_worker_finalizes_only_this_turn_and_preserves_partial_and_cancelled_states(self):
for outcome in ('complete','retry','retry_no_queries','fresh_no_queries','restatement','native_complete','native_query','insufficient','contradictory','server_incomplete','finalizer_error','cancel','cancel_after_stop','stop_failed'):
with self.subTest(outcome=outcome),tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir()
old=root/'evidence'/('nao-quality-'+'b'*32+'.json')
old.write_text('{}');old.chmod(0o600)
payload={key:str(uuid.uuid4()) for key in ('turnId','sessionId','actorId','releaseId')}
payload.update(question='全部明细' if outcome=='server_incomplete' else '比较质量',context={},accessToken='test-delegation')
if outcome == 'fresh_no_queries': payload['question'] = '请重新查询刚才的数据'
if outcome == 'restatement': payload['question'] = '请复述上一轮结论'
if outcome in ('retry','retry_no_queries'):payload['context']={'_restartConversation':True}
manager=MagicMock();manager.acquire_actor.return_value=(root,None)
manager.ensure_instance.return_value=('http://127.0.0.1:49191/v1','test-gate-key','mimo-v2.6-flash','http://127.0.0.1:5000')
gate=MagicMock();gate.base_url='http://127.0.0.1:49191/v1'
stream=runtime.ConversationAccumulator();stream.blocks['first']='查询已返回';stream.block_order.append('first')
stream.record_event('assistant',text='查询已返回')
stream.tool_calls=[{'tool':'mcp_call','status':'completed'}]
cancellation=[]
def sse(**kwargs):
if outcome in ('retry','retry_no_queries'):
self.assertIsNone(kwargs['chat_id'])
self.assertEqual(kwargs['context'],{})
if outcome in ('retry_no_queries', 'fresh_no_queries', 'restatement'):
stream.tool_calls=[]
return stream,'COMPLETED','stop',None
cancellation.append(kwargs['cancel_event'])
document={'tool':'quality_data','releaseId':payload['releaseId'],'evidenceRef':'Q00000001',
'mode':'compare','result':{'kind':'quality_query' if outcome=='native_query' else 'quality_comparison','releaseId':payload['releaseId'],
'current':{'period':{'year':2026,'month':9},'compositeRate':.95},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':.96},
'delta':{'available':True,'percentagePoints':-1}}}
path=root/'evidence'/('nao-quality-'+'a'*32+'.json');path.write_text(json.dumps(document));path.chmod(0o600)
if outcome == 'cancel_after_stop':
kwargs['cancel_event'].set()
return stream, 'PARTIAL', 'coverage_ready', True
if outcome in ('native_complete','native_query','finalizer_error'):return stream,'COMPLETED','stop',None
return stream,'PARTIAL','timeout',outcome!='stop_failed'
def synthesize(**kwargs):
self.assertEqual([d['evidenceRef'] for d in kwargs['evidence']],['Q00000001'])
self.assertEqual(kwargs['base_url'],'http://127.0.0.1:49191/v1')
self.assertEqual(kwargs['api_key'],'test-gate-key')
self.assertEqual(kwargs['context'],{})
if outcome=='finalizer_error':raise finalizer.FinalizationUnavailable('整理失败')
if outcome=='cancel':cancellation[0].set()
return {'answer':'最终统计 [Q00000001]','model':'mimo-v2.6-flash','usage':{},'finishReason':'stop',
'evidenceSufficient':outcome!='insufficient','missingEvidence':['缺少用户要求的对照期'] if outcome=='insufficient' else [],
'modelDeclaredEvidenceSufficient':outcome not in ('insufficient','contradictory'),
'completionAdjusted':outcome=='contradictory'}
with patch.object(runtime,'RuntimeManager',return_value=manager),patch.object(runtime,'write_access_file',return_value=root/'access.json'), \
patch.object(runtime,'_authenticate'),patch.object(runtime,'_input_cancel_listener'),patch.object(runtime,'SessionState'), \
patch.object(runtime,'_sse_request',side_effect=sse),patch.object(runtime,'_story_files',return_value=(None,None)), \
patch.object(runtime,'_model_settings',return_value=('https://model.test/v1','test-key','mimo-v2.6-flash',{'enabled':False,'effort':'auto','protocol':'thinking'})) as selected, \
patch.object(runtime,'plan_agent_task',return_value=None,create=True), \
patch.object(runtime,'_model_gate_settings',return_value=(49191,'test-gate-key')),patch.object(runtime,'DelegatedApi'), \
patch('nao_model_gate.ModelGate',return_value=gate) as gate_factory, \
patch.object(finalizer,'synthesize',side_effect=synthesize) as request, \
patch.object(finalizer,'coverage_sufficient',return_value=outcome not in ('finalizer_error','server_incomplete')):
result=runtime.run_turn(payload,stdout=io.StringIO(),stdin=io.StringIO())
selected.assert_called_once()
self.assertEqual(gate_factory.call_args.kwargs['upstream_key'],'test-key')
gate.close.assert_called_once()
expected={'complete':'COMPLETED','retry':'COMPLETED','retry_no_queries':'PARTIAL','fresh_no_queries':'PARTIAL','restatement':'COMPLETED','native_complete':'COMPLETED','native_query':'COMPLETED','insufficient':'PARTIAL','contradictory':'COMPLETED','server_incomplete':'PARTIAL','finalizer_error':'PARTIAL','cancel':'CANCELLED','cancel_after_stop':'CANCELLED','stop_failed':'PARTIAL'}[outcome]
self.assertEqual(result['status'],expected)
self.assertEqual(request.call_count,0 if outcome in ('stop_failed','native_query','retry_no_queries','fresh_no_queries','restatement','cancel_after_stop') else 1)
if outcome in ('retry_no_queries', 'fresh_no_queries'):
self.assertEqual(result['stopReason'],'evidence_incomplete')
self.assertIn('未取得新的查询证据',result['answer'])
if outcome=='native_query':
self.assertEqual(result['stopReason'],'stop')
self.assertEqual(result['answer'],'查询已返回')
self.assertFalse(any(call['tool']=='model.finalize' for call in result['toolCalls']))
self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],'COMPLETED')
if outcome in ('complete','native_complete'):self.assertEqual(result['stopReason'],'evidence_finalized')
if outcome in ('complete','native_complete'):
self.assertEqual(len([entry for entry in result['timeline'] if entry['kind']=='assistant']),1)
self.assertEqual(result['timeline'][-1]['text'],'最终统计 [Q00000001]')
self.assertEqual(json.loads((root/'private/last-visible-answer.json').read_text())['answer'],'最终统计 [Q00000001]')
if outcome in ('complete','native_complete','insufficient','contradictory','server_incomplete','finalizer_error'):
self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],expected)
if outcome=='contradictory':
self.assertFalse(result['evidenceCompletion']['modelDeclaredEvidenceSufficient'])
self.assertTrue(result['evidenceCompletion']['completionAdjusted'])
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
if outcome=='server_incomplete':
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
def test_columnar_and_shared_encoding_roundtrips_without_rounding_or_null_loss(self):
evidence=[{'rows':[{'field':'电镀磕边重量','weight':None,'count':4120,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}},
{'field':'镀废合计','weight':855.801072,'count':431,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}}]}]*4
self.assertEqual(unpack(finalizer.packed_evidence(evidence)),evidence)
def test_oversize_evidence_fails_closed_without_discarding_statistics(self):
with self.assertRaises(finalizer.FinalizationUnavailable):
finalizer.packed_evidence([{'unique': 'a'*finalizer.MAX_INPUT_CHARS}])
def test_real_second_turn_replay_fits_and_preserves_every_summary_value(self):
root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime'
directories=list(root.glob('.runtime/nao-users/*/sessions/57676d5a-5138-48f4-9343-bae9731aae5a/evidence'))
if not directories: self.skipTest('Private captured acceptance evidence not present')
with patch.object(finalizer,'packed_evidence',side_effect=AssertionError('Local archives do not consume model input budget')):
docs=finalizer.evidence_summaries(directories[0],'194b0287-9044-4d80-9138-ecd3c481015c')
self.assertEqual(len(docs),15)
self.assertEqual(unpack(finalizer.packed_evidence(docs)),docs)
# This captured run predates paired black/bright defect reads for each
# highlighted product, so it is now correctly labelled incomplete.
self.assertFalse(finalizer.coverage_sufficient(docs,'分析质量变化'))
# Ordinary analysis must not be blocked by minor source reconciliation issues.
missing=[d for d in docs if not(d['result'].get('action')=='source' and d['result'].get('stage')=='bright')]
self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化'))
# Full raw difference pagination is optional for an ordinary statistics question.
missing=[d for d in docs if not(d['result'].get('action')=='rows' and d['result'].get('stage')=='bright')]
self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化'))
def test_previous_turn_evidence_is_excluded(self):
with tempfile.TemporaryDirectory() as directory:
root=Path(directory)
for index in range(2):
path=root/('nao-quality-'+str(index)*32+'.json')
path.write_text(json.dumps({'tool':'quality_data','releaseId':'release','evidenceRef':'Q0000000'+str(index),
'mode':'compare','result':{'releaseId':'release'},'createdAt':str(index)}))
path.chmod(0o600)
docs=finalizer.evidence_summaries(root,'release',exclude_files={'nao-quality-'+'0'*32+'.json'})
self.assertEqual([d['evidenceRef'] for d in docs],['Q00000001'])
def test_identity_permissions_and_duplicate_refs_are_rejected(self):
with tempfile.TemporaryDirectory() as directory:
root=Path(directory);path=root/('nao-quality-'+'a'*32+'.json')
doc={'tool':'quality_data','releaseId':'r','evidenceRef':'Q00000001','mode':'query','result':{'releaseId':'r'}}
path.write_text(json.dumps(doc));path.chmod(0o644)
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r')
path.chmod(0o600)
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'other')
twin=root/('nao-quality-'+'b'*32+'.json');twin.write_text(json.dumps(doc));twin.chmod(0o600)
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r')
def test_cancel_failed_stop_and_incomplete_tools_never_start_finalization(self):
calls=[{'status':'completed'}]
args=['mimo-v2.6-flash','PARTIAL','timeout',calls,False,True]
self.assertTrue(finalizer.eligible(*args))
for position,value in ((4,True),(5,False),(5,None),(3,[{'status':'started'}]),(2,'transport_or_stream_error')):
changed=list(args);changed[position]=value
self.assertFalse(finalizer.eligible(*changed))
self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','timeout',calls,False,True))
self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','coverage_ready',calls,False,True))
def test_full_detail_requires_all_pages_but_accepts_contiguous_completed_pagination(self):
evidence=[{'mode':'compare','result':{}}]
for offset,count,more in ((0,2,True),(2,1,False)):
evidence.append({'mode':'investigate','result':{'action':'rows','current':{
'context':{'month':9},'matchingRows':3,'offset':offset,'returnedRows':count,
'hasMore':more,'differenceRecords':[{}]*count}}})
self.assertTrue(finalizer.coverage_sufficient(evidence,'全部明细'))
self.assertFalse(finalizer.coverage_sufficient(evidence[:-1],'全部明细'))
def test_synthesis_is_one_bounded_openai_compatible_request_with_verified_citations(self):
calls=[]
def opener(request,timeout):
calls.append((json.loads(request.data),timeout))
return io.BytesIO(json.dumps({'model':'mimo-v2.6-flash','choices':[{'finish_reason':'stop','message':{
'content':json.dumps({'otherFactIds':['F1'],'groupFactIds':[],'productFactIds':[],'defectFactIds':[],'answer':'伪造上升方向和QDEADBEEF引用必须忽略','evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
'delta':{'percentagePoints':-1}}}]
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
self.assertTrue(result['evidenceSufficient']);self.assertEqual(len(calls),1)
self.assertIn('下降1.00个百分点',result['answer']);self.assertNotIn('QDEADBEEF',result['answer']);self.assertNotIn('伪造',result['answer'])
self.assertEqual(calls[0][0]['thinking'],{'type':'disabled'})
from nao_runtime_policy import MAX_OUTPUT_TOKENS
self.assertEqual(calls[0][0]['max_completion_tokens'],MAX_OUTPUT_TOKENS)
self.assertEqual(calls[0][1],finalizer.FINAL_TIMEOUT_SECONDS)
cancelled=threading.Event();cancelled.set()
with self.assertRaises(finalizer.FinalizationUnavailable):
finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='质量如何',evidence=evidence,cancel_event=cancelled,opener=opener)
self.assertEqual(len(calls),1)
def test_selector_timeout_uses_verified_report_without_claiming_missing_coverage(self):
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{
'kind':'quality_comparison',
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
'delta':{'percentagePoints':-1}}}]
def timeout(request, timeout):
self.assertEqual(timeout, finalizer.FINAL_TIMEOUT_SECONDS)
raise TimeoutError('model selector timed out')
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',
model='mimo-v2.6-flash',question='质量如何',evidence=evidence,
cancel_event=threading.Event(),opener=timeout)
self.assertTrue(result['selectionFallback'])
self.assertTrue(result['evidenceSufficient'])
self.assertIn('下降1.00个百分点',result['answer'])
def test_model_cannot_override_missing_stage_or_ambiguous_frozen_frame(self):
from test_nao_evidence_report import coverage_comparison, comparison
def opener(request,timeout):
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
missing=coverage_comparison()
missing['result']['current']['coverage']['sourceMonths'][0].update(blackRows=0,brightRows=0)
result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
question='比较8月相对7月',evidence=[missing],cancel_event=threading.Event(),opener=opener)
self.assertTrue(result['modelDeclaredEvidenceSufficient'])
self.assertFalse(result['evidenceSufficient'])
self.assertFalse(finalizer.coverage_sufficient([missing],'比较8月相对7月'))
self.assertIn('没有可引用的正式综合合格率',result['answer'])
self.assertNotIn('100.00%',result['answer'])
docs=[comparison('Q00000001',{},.97),comparison('Q00000002',{'group':'组A'},.80)]
unresolved=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
question='只看组A',evidence=docs,cancel_event=threading.Event(),opener=opener)
self.assertFalse(unresolved['evidenceSufficient'])
self.assertIn('组A',unresolved['answer']);self.assertIn('80.00%',unresolved['answer'])
context={'year':2026,'month':7,'group':'组A'}
bound=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
question='只看组A',evidence=docs,context=context,cancel_event=threading.Event(),opener=opener)
self.assertTrue(bound['evidenceSufficient'])
self.assertTrue(finalizer.coverage_sufficient(docs,'只看组A',context))
self.assertIn('80.00%',bound['answer']);self.assertNotIn('97.00%',bound['answer'])
def test_comparison_required_task_does_not_get_rechecked_as_single_period(self):
question = '全面分析质量变化'
current = {'year': 2026, 'month': 9}
comparison = {'year': 2026, 'month': 8}
context = {'releaseId': 'release-a', '_agentTask': {
'version': 1, 'question': question, 'kind': 'analysis', 'freshQuery': True,
'dimensions': ['department', 'group', 'product', 'defect'],
'stages': ['black', 'bright'], 'fields': ['rate'],
'period': current, 'comparisonPeriod': None, 'comparisonRequired': True,
'filters': {}, 'verifySource': False, 'fullDetails': False,
}}
evidence = [{'evidenceRef': 'Q' + '4' * 8, 'mode': 'compare',
'result': {'kind': 'quality_comparison', 'releaseId': 'release-a',
'filterBinding': {},
'current': {'period': current},
'comparison': {'period': comparison}}}]
with patch('nao_analysis_coverage.coverage_gaps', return_value=[]), \
patch('nao_analysis_coverage.semantic_single_period_gaps',
side_effect=AssertionError('comparison task was treated as single period')), \
patch('nao_analysis_coverage.formal_metric_coverage', return_value=None), \
patch('nao_evidence_report.build_report', return_value={'coverageSufficient': True}):
self.assertTrue(finalizer.coverage_sufficient(evidence, question, context))
def test_optional_focus_metadata_cannot_discard_or_invent_verified_statistics(self):
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
'delta':{'percentagePoints':-1}}}]
def opener(request,timeout):
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
'otherFactIds':['invented','F1','F1','F2','F3'], 'groupFactIds':['F1'],
'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
self.assertTrue(result['selectionAdjusted'])
self.assertTrue(result['evidenceSufficient'])
self.assertEqual(result['selectedFactIds'],['F1','F2'])
self.assertNotIn('invented',result['answer'])
self.assertIn('下降1.00个百分点',result['answer'])
def test_completion_requires_concrete_missing_items_and_keeps_model_diagnostics(self):
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
'delta':{'percentagePoints':-1}}}]
for declared,missing,expected in [(False,[],True),(False,[' ',''],True),
(False,[' 缺少对照期 ','缺少对照期'],False),(True,['缺少对照期'],False),(True,[],True)]:
with self.subTest(declared=declared,missing=missing):
def opener(request,timeout):
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
'evidenceSufficient':declared,'missingEvidence':missing})}}]}).encode())
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
self.assertEqual(result['evidenceSufficient'],expected)
self.assertEqual(result['modelDeclaredEvidenceSufficient'],declared)
self.assertEqual(result['modelMissingEvidence'],missing)
self.assertEqual(result['completionAdjusted'],not declared and expected)
self.assertEqual(result['missingEvidence'],[] if expected else ['缺少对照期'])
if not expected:self.assertIn('缺少对照期',result['answer'])
def test_captured_3c_with_missing_paired_stage_remains_partial(self):
root=Path(__file__).resolve().parents[1]
paths=list((root/'.runtime/nao-users').glob('*/sessions/98829d63-0158-4e70-89db-ef0d58fe546a'))
if not paths:self.skipTest('Private captured session not present')
original=json.loads((paths[0]/'finalization.json').read_text())
self.assertFalse(original['modelDeclaredEvidenceSufficient'])
self.assertEqual(original['missingEvidence'],[])
self.assertTrue(original['serverCoverageSufficient'])
release=json.loads((root/'output/20261023-incomplete-reason/session.json').read_text())['releaseId']
docs=finalizer.evidence_summaries(paths[0]/'evidence',release)
def opener(request,timeout):
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
'evidenceSufficient':False,'missingEvidence':[]})}}]}).encode())
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='3C哪些产品近期表现最差',evidence=docs,cancel_event=threading.Event(),opener=opener)
self.assertFalse(result['evidenceSufficient'])
self.assertFalse(finalizer.coverage_sufficient(docs,'3C哪些产品近期表现最差'))
self.assertFalse(result['completionAdjusted'])
self.assertTrue(any('缺少' in item for item in result['missingEvidence']))
self.assertIn('Q039D5FA7',result['answer'])
self.assertIn('90.11%',result['answer'])
def test_exact_formal_metrics_refute_only_the_model_claims_they_prove_false(self):
captured = []
def opener(request, timeout):
captured.append(json.loads(request.data))
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
'evidenceSufficient': False, 'missingEvidence': FORMAL_PRODUCT_GAPS})}}]}).encode())
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=opener)
self.assertTrue(result['evidenceSufficient'])
self.assertEqual(result['missingEvidence'], [])
self.assertEqual(result['modelMissingEvidence'], FORMAL_PRODUCT_GAPS)
self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS)
self.assertTrue(result['completionAdjusted'])
selector = json.loads(captured[0]['messages'][1]['content'])
contract = selector['formalMetricCoverage']
self.assertEqual(contract['filters'], {'product': 'SYNTH-16X15+T'})
self.assertEqual(len(contract['metrics']), 12)
self.assertTrue(all(row['covered'] for row in contract['metrics']))
self.assertIn(1200.5, [row['value'] for row in contract['metrics']])
self.assertNotIn('未覆盖内容', result['answer'])
def test_formal_metric_contract_rejects_wrong_scope_period_release_and_incomplete_values(self):
mutations = {
'product': lambda r: r['filterBinding'].update(product='SYNTH-16X15+T-OTHER'),
'extra_filter': lambda r: r['filterBinding'].update(group='其他分组'),
'period': lambda r: r['comparison']['period'].update(month=7),
'release': lambda r: r.update(releaseId='release-other'),
'input': lambda r: r['current']['black'].pop('totalWeight'),
'scrap': lambda r: r['comparison']['bright'].pop('scrapWeight'),
'rate': lambda r: r['current']['bright'].update(rate=None),
'nonfinite': lambda r: r['current']['black'].update(totalWeight='NaN'),
'source_months': lambda r: r['comparison'].pop('coverage'),
'source_stage': lambda r: r['current']['coverage']['sourceMonths'][0].update(brightRows=0),
}
for name, mutate in mutations.items():
with self.subTest(name=name):
docs = copy.deepcopy(formal_product_evidence()); mutate(docs[0]['result'])
def opener(request, timeout):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
'evidenceSufficient': True, 'missingEvidence': []})}}]}).encode())
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
question=FORMAL_PRODUCT_QUESTION, evidence=docs, context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=opener)
self.assertFalse(result['evidenceSufficient'])
self.assertTrue(result['missingEvidence'])
self.assertFalse(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, {'releaseId': 'release-a'}))
def test_unknown_mixed_and_other_scope_model_gaps_still_block_completion(self):
for missing in ['缺少该规格投入重量和现场原因证据', '缺少该规格投入重量原始明细',
'缺少其他规格的投入重量正式数据', '缺少2026年7月投入重量正式数据',
'缺少对照期', '缺少生产原因证据']:
with self.subTest(missing=missing):
def opener(request, timeout):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
'evidenceSufficient': False, 'missingEvidence': [*FORMAL_PRODUCT_GAPS, missing]})}}]}).encode())
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=opener)
self.assertFalse(result['evidenceSufficient'])
self.assertEqual(result['missingEvidence'], [missing])
self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS)
def test_metric_reconciliation_does_not_infer_unspecified_scope_or_hide_later_missing_items(self):
from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps
for question in [FORMAL_PRODUCT_QUESTION.replace('只看SYNTH-16X15+T,', ''),
FORMAL_PRODUCT_QUESTION.replace('2026年', ''),
FORMAL_PRODUCT_QUESTION.replace('黑片、亮片', ''),
FORMAL_PRODUCT_QUESTION.replace('9月与8月', '13月与8月')]:
with self.subTest(question=question):
contract = formal_metric_coverage(formal_product_evidence(), question, {'releaseId': 'release-a'})
self.assertIsNone(contract)
self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, []))
contract = formal_metric_coverage(formal_product_evidence(), FORMAL_PRODUCT_QUESTION.replace('报废重量和', ''), {'releaseId': 'release-a'})
self.assertEqual(resolve_proven_metric_gaps(contract, [FORMAL_PRODUCT_GAPS[1]]), ([FORMAL_PRODUCT_GAPS[1]], []))
gaps = ['缺少' + '的' * index + '该规格投入重量正式数据' for index in range(12)] + ['缺少生产原因证据']
def opener(request, timeout):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
'evidenceSufficient': False, 'missingEvidence': gaps})}}]}).encode())
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=opener)
self.assertFalse(result['evidenceSufficient'])
self.assertEqual(result['missingEvidence'], ['缺少生产原因证据'])
self.assertEqual(result['modelMissingEvidence'], gaps)
def test_formal_contract_accepts_the_same_explicit_frozen_filters(self):
from nao_analysis_coverage import formal_metric_coverage
for filters in [{'group': '合成分组'}, {'department': '合成部门'},
{'group': '合成分组', 'extraFilters': {'region': '合成地区'}, 'includeUnassigned': True}]:
with self.subTest(filters=filters):
docs = formal_product_evidence()
docs[0]['result']['filterBinding'].update(filters)
expected = {'product': 'SYNTH-16X15+T', **filters}
context = {'releaseId': 'release-a', 'filters': expected}
contract = formal_metric_coverage(docs, FORMAL_PRODUCT_QUESTION, context)
self.assertEqual(contract['filters'], expected)
self.assertEqual(contract['missingEvidence'], [])
self.assertTrue(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, context))
def test_unknown_extra_scope_or_analysis_language_does_not_activate_the_contract(self):
from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps, coverage_gaps
docs = formal_product_evidence()
docs[0]['result']['filterBinding']['group'] = '合成分组'
for question in [FORMAL_PRODUCT_QUESTION + '仅统计合成分组。',
FORMAL_PRODUCT_QUESTION.replace(',请', ',合成部门,分析人员因素,请'),
'合成分组中,' + FORMAL_PRODUCT_QUESTION,
FORMAL_PRODUCT_QUESTION + '并提供现场生产原因。']:
with self.subTest(question=question):
contract = formal_metric_coverage(docs, question, {'releaseId': 'release-a'})
self.assertIsNone(contract)
self.assertEqual(coverage_gaps(docs, question, {'releaseId': 'release-a'}), [])
self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, []))
if __name__=='__main__':unittest.main()