同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。 补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。 统一午夜黑及其他主题的控件细节、中文和业务数字字体。 排除本地业务数据、凭据、预览、发布与验收产物。 验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
482 lines
36 KiB
Python
482 lines
36 KiB
Python
"""Evidence-size, completeness and bounded synthesis regressions; no model calls."""
|
||
import io
|
||
import copy
|
||
import json
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import ssl
|
||
import unittest
|
||
import uuid
|
||
from unittest.mock import MagicMock, patch
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||
import nao_evidence_finalizer as finalizer
|
||
import nao_quality_runtime as runtime
|
||
|
||
|
||
FORMAL_PRODUCT_QUESTION = '只看SYNTH-16X15+T,请重新查询2026年9月与8月黑片、亮片的合格率、报废重量和两期投入,附正式数据依据。'
|
||
FORMAL_PRODUCT_GAPS = ['缺少仅针对SYNTH-16X15+T规格的两期分规格投入重量正式数据',
|
||
'缺少仅针对该规格的黑片、亮片报废重量分规格正式数据']
|
||
|
||
|
||
def formal_product_evidence():
|
||
result = {'kind': 'quality_comparison', 'releaseId': 'release-a',
|
||
'filterBinding': {'product': 'SYNTH-16X15+T', 'includeUnassigned': False},
|
||
'delta': {'available': True, 'percentagePoints': -2},
|
||
'stageRateChangesPP': {'black': -1, 'bright': -1}}
|
||
for side, month, inputs, scraps in [('current', 9, (1200.5, 2400.5), (36.1, 72.1)),
|
||
('comparison', 8, (1000.5, 2000.5), (20.1, 40.1))]:
|
||
result[side] = {'period': {'year': 2026, 'month': month}, 'compositeRate': .95,
|
||
'coverage': {'sourceMonths': [{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}]}}
|
||
for index, stage in enumerate(('black', 'bright')):
|
||
result[side][stage] = {'rate': .97, 'totalWeight': inputs[index], 'scrapWeight': scraps[index]}
|
||
return [{'evidenceRef': 'Q00000001', 'mode': 'compare', 'result': result}]
|
||
|
||
|
||
def unpack(packed):
|
||
def decode(value):
|
||
if isinstance(value, list): return [decode(item) for item in value]
|
||
if not isinstance(value, dict): return value
|
||
if set(value) == {'$shared'}: return decode(packed['shared'][value['$shared']])
|
||
if set(value) == {'$columns', '$values'}:
|
||
return [dict(zip(decode(value['$columns']), decode(row))) for row in decode(value['$values'])]
|
||
return {key: decode(item) for key,item in value.items()}
|
||
return decode(packed['evidence'])
|
||
|
||
|
||
class EvidenceFinalizerTest(unittest.TestCase):
|
||
def test_real_connection_keeps_system_certificate_and_hostname_verification(self):
|
||
with patch.object(finalizer,'urlopen',return_value=io.BytesIO(b'{}')) as request:
|
||
finalizer._open_model('request',35)
|
||
context=request.call_args.kwargs['context']
|
||
self.assertEqual(context.verify_mode,ssl.CERT_REQUIRED)
|
||
self.assertTrue(context.check_hostname)
|
||
|
||
def test_worker_finalizes_only_this_turn_and_preserves_partial_and_cancelled_states(self):
|
||
for outcome in ('complete','retry','retry_no_queries','fresh_no_queries','restatement','native_complete','native_query','insufficient','contradictory','server_incomplete','finalizer_error','cancel','cancel_after_stop','stop_failed'):
|
||
with self.subTest(outcome=outcome),tempfile.TemporaryDirectory() as folder:
|
||
root=Path(folder);(root/'evidence').mkdir()
|
||
old=root/'evidence'/('nao-quality-'+'b'*32+'.json')
|
||
old.write_text('{}');old.chmod(0o600)
|
||
payload={key:str(uuid.uuid4()) for key in ('turnId','sessionId','actorId','releaseId')}
|
||
payload.update(question='全部明细' if outcome=='server_incomplete' else '比较质量',context={},accessToken='test-delegation')
|
||
if outcome == 'fresh_no_queries': payload['question'] = '请重新查询刚才的数据'
|
||
if outcome == 'restatement': payload['question'] = '请复述上一轮结论'
|
||
if outcome in ('retry','retry_no_queries'):payload['context']={'_restartConversation':True}
|
||
manager=MagicMock();manager.acquire_actor.return_value=(root,None)
|
||
manager.ensure_instance.return_value=('http://127.0.0.1:49191/v1','test-gate-key','mimo-v2.6-flash','http://127.0.0.1:5000')
|
||
gate=MagicMock();gate.base_url='http://127.0.0.1:49191/v1'
|
||
stream=runtime.ConversationAccumulator();stream.blocks['first']='查询已返回';stream.block_order.append('first')
|
||
stream.record_event('assistant',text='查询已返回')
|
||
stream.tool_calls=[{'tool':'mcp_call','status':'completed'}]
|
||
cancellation=[]
|
||
def sse(**kwargs):
|
||
if outcome in ('retry','retry_no_queries'):
|
||
self.assertIsNone(kwargs['chat_id'])
|
||
self.assertEqual(kwargs['context'],{})
|
||
if outcome in ('retry_no_queries', 'fresh_no_queries', 'restatement'):
|
||
stream.tool_calls=[]
|
||
return stream,'COMPLETED','stop',None
|
||
cancellation.append(kwargs['cancel_event'])
|
||
document={'tool':'quality_data','releaseId':payload['releaseId'],'evidenceRef':'Q00000001',
|
||
'mode':'compare','result':{'kind':'quality_query' if outcome=='native_query' else 'quality_comparison','releaseId':payload['releaseId'],
|
||
'current':{'period':{'year':2026,'month':9},'compositeRate':.95},
|
||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':.96},
|
||
'delta':{'available':True,'percentagePoints':-1}}}
|
||
path=root/'evidence'/('nao-quality-'+'a'*32+'.json');path.write_text(json.dumps(document));path.chmod(0o600)
|
||
if outcome == 'cancel_after_stop':
|
||
kwargs['cancel_event'].set()
|
||
return stream, 'PARTIAL', 'coverage_ready', True
|
||
if outcome in ('native_complete','native_query','finalizer_error'):return stream,'COMPLETED','stop',None
|
||
return stream,'PARTIAL','timeout',outcome!='stop_failed'
|
||
def synthesize(**kwargs):
|
||
self.assertEqual([d['evidenceRef'] for d in kwargs['evidence']],['Q00000001'])
|
||
self.assertEqual(kwargs['base_url'],'http://127.0.0.1:49191/v1')
|
||
self.assertEqual(kwargs['api_key'],'test-gate-key')
|
||
self.assertEqual(kwargs['context'],{})
|
||
if outcome=='finalizer_error':raise finalizer.FinalizationUnavailable('整理失败')
|
||
if outcome=='cancel':cancellation[0].set()
|
||
return {'answer':'最终统计 [Q00000001]','model':'mimo-v2.6-flash','usage':{},'finishReason':'stop',
|
||
'evidenceSufficient':outcome!='insufficient','missingEvidence':['缺少用户要求的对照期'] if outcome=='insufficient' else [],
|
||
'modelDeclaredEvidenceSufficient':outcome not in ('insufficient','contradictory'),
|
||
'completionAdjusted':outcome=='contradictory'}
|
||
with patch.object(runtime,'RuntimeManager',return_value=manager),patch.object(runtime,'write_access_file',return_value=root/'access.json'), \
|
||
patch.object(runtime,'_authenticate'),patch.object(runtime,'_input_cancel_listener'),patch.object(runtime,'SessionState'), \
|
||
patch.object(runtime,'_sse_request',side_effect=sse),patch.object(runtime,'_story_files',return_value=(None,None)), \
|
||
patch.object(runtime,'_model_settings',return_value=('https://model.test/v1','test-key','mimo-v2.6-flash',{'enabled':False,'effort':'auto','protocol':'thinking'})) as selected, \
|
||
patch.object(runtime,'plan_agent_task',return_value=None,create=True), \
|
||
patch.object(runtime,'_model_gate_settings',return_value=(49191,'test-gate-key')),patch.object(runtime,'DelegatedApi'), \
|
||
patch('nao_model_gate.ModelGate',return_value=gate) as gate_factory, \
|
||
patch.object(finalizer,'synthesize',side_effect=synthesize) as request, \
|
||
patch.object(finalizer,'coverage_sufficient',return_value=outcome not in ('finalizer_error','server_incomplete')):
|
||
result=runtime.run_turn(payload,stdout=io.StringIO(),stdin=io.StringIO())
|
||
selected.assert_called_once()
|
||
self.assertEqual(gate_factory.call_args.kwargs['upstream_key'],'test-key')
|
||
gate.close.assert_called_once()
|
||
expected={'complete':'COMPLETED','retry':'COMPLETED','retry_no_queries':'PARTIAL','fresh_no_queries':'PARTIAL','restatement':'COMPLETED','native_complete':'COMPLETED','native_query':'COMPLETED','insufficient':'PARTIAL','contradictory':'COMPLETED','server_incomplete':'PARTIAL','finalizer_error':'PARTIAL','cancel':'CANCELLED','cancel_after_stop':'CANCELLED','stop_failed':'PARTIAL'}[outcome]
|
||
self.assertEqual(result['status'],expected)
|
||
self.assertEqual(request.call_count,0 if outcome in ('stop_failed','native_query','retry_no_queries','fresh_no_queries','restatement','cancel_after_stop') else 1)
|
||
if outcome in ('retry_no_queries', 'fresh_no_queries'):
|
||
self.assertEqual(result['stopReason'],'evidence_incomplete')
|
||
self.assertIn('未取得新的查询证据',result['answer'])
|
||
if outcome=='native_query':
|
||
self.assertEqual(result['stopReason'],'stop')
|
||
self.assertEqual(result['answer'],'查询已返回')
|
||
self.assertFalse(any(call['tool']=='model.finalize' for call in result['toolCalls']))
|
||
self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],'COMPLETED')
|
||
if outcome in ('complete','native_complete'):self.assertEqual(result['stopReason'],'evidence_finalized')
|
||
if outcome in ('complete','native_complete'):
|
||
self.assertEqual(len([entry for entry in result['timeline'] if entry['kind']=='assistant']),1)
|
||
self.assertEqual(result['timeline'][-1]['text'],'最终统计 [Q00000001]')
|
||
self.assertEqual(json.loads((root/'private/last-visible-answer.json').read_text())['answer'],'最终统计 [Q00000001]')
|
||
if outcome in ('complete','native_complete','insufficient','contradictory','server_incomplete','finalizer_error'):
|
||
self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],expected)
|
||
if outcome=='contradictory':
|
||
self.assertFalse(result['evidenceCompletion']['modelDeclaredEvidenceSufficient'])
|
||
self.assertTrue(result['evidenceCompletion']['completionAdjusted'])
|
||
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
|
||
if outcome=='server_incomplete':
|
||
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
|
||
|
||
def test_columnar_and_shared_encoding_roundtrips_without_rounding_or_null_loss(self):
|
||
evidence=[{'rows':[{'field':'电镀磕边重量','weight':None,'count':4120,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}},
|
||
{'field':'镀废合计','weight':855.801072,'count':431,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}}]}]*4
|
||
self.assertEqual(unpack(finalizer.packed_evidence(evidence)),evidence)
|
||
|
||
def test_oversize_evidence_fails_closed_without_discarding_statistics(self):
|
||
with self.assertRaises(finalizer.FinalizationUnavailable):
|
||
finalizer.packed_evidence([{'unique': 'a'*finalizer.MAX_INPUT_CHARS}])
|
||
|
||
def test_real_second_turn_replay_fits_and_preserves_every_summary_value(self):
|
||
root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime'
|
||
directories=list(root.glob('.runtime/nao-users/*/sessions/57676d5a-5138-48f4-9343-bae9731aae5a/evidence'))
|
||
if not directories: self.skipTest('Private captured acceptance evidence not present')
|
||
with patch.object(finalizer,'packed_evidence',side_effect=AssertionError('Local archives do not consume model input budget')):
|
||
docs=finalizer.evidence_summaries(directories[0],'194b0287-9044-4d80-9138-ecd3c481015c')
|
||
self.assertEqual(len(docs),15)
|
||
self.assertEqual(unpack(finalizer.packed_evidence(docs)),docs)
|
||
# This captured run predates paired black/bright defect reads for each
|
||
# highlighted product, so it is now correctly labelled incomplete.
|
||
self.assertFalse(finalizer.coverage_sufficient(docs,'分析质量变化'))
|
||
# Ordinary analysis must not be blocked by minor source reconciliation issues.
|
||
missing=[d for d in docs if not(d['result'].get('action')=='source' and d['result'].get('stage')=='bright')]
|
||
self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化'))
|
||
# Full raw difference pagination is optional for an ordinary statistics question.
|
||
missing=[d for d in docs if not(d['result'].get('action')=='rows' and d['result'].get('stage')=='bright')]
|
||
self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化'))
|
||
|
||
def test_previous_turn_evidence_is_excluded(self):
|
||
with tempfile.TemporaryDirectory() as directory:
|
||
root=Path(directory)
|
||
for index in range(2):
|
||
path=root/('nao-quality-'+str(index)*32+'.json')
|
||
path.write_text(json.dumps({'tool':'quality_data','releaseId':'release','evidenceRef':'Q0000000'+str(index),
|
||
'mode':'compare','result':{'releaseId':'release'},'createdAt':str(index)}))
|
||
path.chmod(0o600)
|
||
docs=finalizer.evidence_summaries(root,'release',exclude_files={'nao-quality-'+'0'*32+'.json'})
|
||
self.assertEqual([d['evidenceRef'] for d in docs],['Q00000001'])
|
||
|
||
def test_identity_permissions_and_duplicate_refs_are_rejected(self):
|
||
with tempfile.TemporaryDirectory() as directory:
|
||
root=Path(directory);path=root/('nao-quality-'+'a'*32+'.json')
|
||
doc={'tool':'quality_data','releaseId':'r','evidenceRef':'Q00000001','mode':'query','result':{'releaseId':'r'}}
|
||
path.write_text(json.dumps(doc));path.chmod(0o644)
|
||
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r')
|
||
path.chmod(0o600)
|
||
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'other')
|
||
twin=root/('nao-quality-'+'b'*32+'.json');twin.write_text(json.dumps(doc));twin.chmod(0o600)
|
||
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r')
|
||
|
||
def test_cancel_failed_stop_and_incomplete_tools_never_start_finalization(self):
|
||
calls=[{'status':'completed'}]
|
||
args=['mimo-v2.6-flash','PARTIAL','timeout',calls,False,True]
|
||
self.assertTrue(finalizer.eligible(*args))
|
||
for position,value in ((4,True),(5,False),(5,None),(3,[{'status':'started'}]),(2,'transport_or_stream_error')):
|
||
changed=list(args);changed[position]=value
|
||
self.assertFalse(finalizer.eligible(*changed))
|
||
self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','timeout',calls,False,True))
|
||
self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','coverage_ready',calls,False,True))
|
||
|
||
def test_full_detail_requires_all_pages_but_accepts_contiguous_completed_pagination(self):
|
||
evidence=[{'mode':'compare','result':{}}]
|
||
for offset,count,more in ((0,2,True),(2,1,False)):
|
||
evidence.append({'mode':'investigate','result':{'action':'rows','current':{
|
||
'context':{'month':9},'matchingRows':3,'offset':offset,'returnedRows':count,
|
||
'hasMore':more,'differenceRecords':[{}]*count}}})
|
||
self.assertTrue(finalizer.coverage_sufficient(evidence,'全部明细'))
|
||
self.assertFalse(finalizer.coverage_sufficient(evidence[:-1],'全部明细'))
|
||
|
||
def test_synthesis_is_one_bounded_openai_compatible_request_with_verified_citations(self):
|
||
calls=[]
|
||
def opener(request,timeout):
|
||
calls.append((json.loads(request.data),timeout))
|
||
return io.BytesIO(json.dumps({'model':'mimo-v2.6-flash','choices':[{'finish_reason':'stop','message':{
|
||
'content':json.dumps({'otherFactIds':['F1'],'groupFactIds':[],'productFactIds':[],'defectFactIds':[],'answer':'伪造上升方向和QDEADBEEF引用必须忽略','evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
|
||
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
|
||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
|
||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
|
||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
|
||
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
|
||
'delta':{'percentagePoints':-1}}}]
|
||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
|
||
self.assertTrue(result['evidenceSufficient']);self.assertEqual(len(calls),1)
|
||
self.assertIn('下降1.00个百分点',result['answer']);self.assertNotIn('QDEADBEEF',result['answer']);self.assertNotIn('伪造',result['answer'])
|
||
self.assertEqual(calls[0][0]['thinking'],{'type':'disabled'})
|
||
from nao_runtime_policy import MAX_OUTPUT_TOKENS
|
||
self.assertEqual(calls[0][0]['max_completion_tokens'],MAX_OUTPUT_TOKENS)
|
||
self.assertEqual(calls[0][1],finalizer.FINAL_TIMEOUT_SECONDS)
|
||
cancelled=threading.Event();cancelled.set()
|
||
with self.assertRaises(finalizer.FinalizationUnavailable):
|
||
finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||
question='质量如何',evidence=evidence,cancel_event=cancelled,opener=opener)
|
||
self.assertEqual(len(calls),1)
|
||
|
||
def test_selector_timeout_uses_verified_report_without_claiming_missing_coverage(self):
|
||
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{
|
||
'kind':'quality_comparison',
|
||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
|
||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
|
||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
|
||
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
|
||
'delta':{'percentagePoints':-1}}}]
|
||
def timeout(request, timeout):
|
||
self.assertEqual(timeout, finalizer.FINAL_TIMEOUT_SECONDS)
|
||
raise TimeoutError('model selector timed out')
|
||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',
|
||
model='mimo-v2.6-flash',question='质量如何',evidence=evidence,
|
||
cancel_event=threading.Event(),opener=timeout)
|
||
self.assertTrue(result['selectionFallback'])
|
||
self.assertTrue(result['evidenceSufficient'])
|
||
self.assertIn('下降1.00个百分点',result['answer'])
|
||
|
||
def test_model_cannot_override_missing_stage_or_ambiguous_frozen_frame(self):
|
||
from test_nao_evidence_report import coverage_comparison, comparison
|
||
def opener(request,timeout):
|
||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
|
||
'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
|
||
missing=coverage_comparison()
|
||
missing['result']['current']['coverage']['sourceMonths'][0].update(blackRows=0,brightRows=0)
|
||
result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
|
||
question='比较8月相对7月',evidence=[missing],cancel_event=threading.Event(),opener=opener)
|
||
self.assertTrue(result['modelDeclaredEvidenceSufficient'])
|
||
self.assertFalse(result['evidenceSufficient'])
|
||
self.assertFalse(finalizer.coverage_sufficient([missing],'比较8月相对7月'))
|
||
self.assertIn('没有可引用的正式综合合格率',result['answer'])
|
||
self.assertNotIn('100.00%',result['answer'])
|
||
docs=[comparison('Q00000001',{},.97),comparison('Q00000002',{'group':'组A'},.80)]
|
||
unresolved=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
|
||
question='只看组A',evidence=docs,cancel_event=threading.Event(),opener=opener)
|
||
self.assertFalse(unresolved['evidenceSufficient'])
|
||
self.assertIn('组A',unresolved['answer']);self.assertIn('80.00%',unresolved['answer'])
|
||
context={'year':2026,'month':7,'group':'组A'}
|
||
bound=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
|
||
question='只看组A',evidence=docs,context=context,cancel_event=threading.Event(),opener=opener)
|
||
self.assertTrue(bound['evidenceSufficient'])
|
||
self.assertTrue(finalizer.coverage_sufficient(docs,'只看组A',context))
|
||
self.assertIn('80.00%',bound['answer']);self.assertNotIn('97.00%',bound['answer'])
|
||
|
||
def test_comparison_required_task_does_not_get_rechecked_as_single_period(self):
|
||
question = '全面分析质量变化'
|
||
current = {'year': 2026, 'month': 9}
|
||
comparison = {'year': 2026, 'month': 8}
|
||
context = {'releaseId': 'release-a', '_agentTask': {
|
||
'version': 1, 'question': question, 'kind': 'analysis', 'freshQuery': True,
|
||
'dimensions': ['department', 'group', 'product', 'defect'],
|
||
'stages': ['black', 'bright'], 'fields': ['rate'],
|
||
'period': current, 'comparisonPeriod': None, 'comparisonRequired': True,
|
||
'filters': {}, 'verifySource': False, 'fullDetails': False,
|
||
}}
|
||
evidence = [{'evidenceRef': 'Q' + '4' * 8, 'mode': 'compare',
|
||
'result': {'kind': 'quality_comparison', 'releaseId': 'release-a',
|
||
'filterBinding': {},
|
||
'current': {'period': current},
|
||
'comparison': {'period': comparison}}}]
|
||
with patch('nao_analysis_coverage.coverage_gaps', return_value=[]), \
|
||
patch('nao_analysis_coverage.semantic_single_period_gaps',
|
||
side_effect=AssertionError('comparison task was treated as single period')), \
|
||
patch('nao_analysis_coverage.formal_metric_coverage', return_value=None), \
|
||
patch('nao_evidence_report.build_report', return_value={'coverageSufficient': True}):
|
||
self.assertTrue(finalizer.coverage_sufficient(evidence, question, context))
|
||
|
||
def test_optional_focus_metadata_cannot_discard_or_invent_verified_statistics(self):
|
||
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
|
||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
|
||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
|
||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
|
||
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
|
||
'delta':{'percentagePoints':-1}}}]
|
||
def opener(request,timeout):
|
||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
|
||
'otherFactIds':['invented','F1','F1','F2','F3'], 'groupFactIds':['F1'],
|
||
'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
|
||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
|
||
self.assertTrue(result['selectionAdjusted'])
|
||
self.assertTrue(result['evidenceSufficient'])
|
||
self.assertEqual(result['selectedFactIds'],['F1','F2'])
|
||
self.assertNotIn('invented',result['answer'])
|
||
self.assertIn('下降1.00个百分点',result['answer'])
|
||
|
||
def test_completion_requires_concrete_missing_items_and_keeps_model_diagnostics(self):
|
||
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
|
||
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
|
||
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
|
||
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
|
||
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
|
||
'delta':{'percentagePoints':-1}}}]
|
||
for declared,missing,expected in [(False,[],True),(False,[' ',''],True),
|
||
(False,[' 缺少对照期 ','缺少对照期'],False),(True,['缺少对照期'],False),(True,[],True)]:
|
||
with self.subTest(declared=declared,missing=missing):
|
||
def opener(request,timeout):
|
||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
|
||
'evidenceSufficient':declared,'missingEvidence':missing})}}]}).encode())
|
||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
|
||
self.assertEqual(result['evidenceSufficient'],expected)
|
||
self.assertEqual(result['modelDeclaredEvidenceSufficient'],declared)
|
||
self.assertEqual(result['modelMissingEvidence'],missing)
|
||
self.assertEqual(result['completionAdjusted'],not declared and expected)
|
||
self.assertEqual(result['missingEvidence'],[] if expected else ['缺少对照期'])
|
||
if not expected:self.assertIn('缺少对照期',result['answer'])
|
||
|
||
def test_captured_3c_with_missing_paired_stage_remains_partial(self):
|
||
root=Path(__file__).resolve().parents[1]
|
||
paths=list((root/'.runtime/nao-users').glob('*/sessions/98829d63-0158-4e70-89db-ef0d58fe546a'))
|
||
if not paths:self.skipTest('Private captured session not present')
|
||
original=json.loads((paths[0]/'finalization.json').read_text())
|
||
self.assertFalse(original['modelDeclaredEvidenceSufficient'])
|
||
self.assertEqual(original['missingEvidence'],[])
|
||
self.assertTrue(original['serverCoverageSufficient'])
|
||
release=json.loads((root/'output/20261023-incomplete-reason/session.json').read_text())['releaseId']
|
||
docs=finalizer.evidence_summaries(paths[0]/'evidence',release)
|
||
def opener(request,timeout):
|
||
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
|
||
'evidenceSufficient':False,'missingEvidence':[]})}}]}).encode())
|
||
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
|
||
question='3C哪些产品近期表现最差',evidence=docs,cancel_event=threading.Event(),opener=opener)
|
||
self.assertFalse(result['evidenceSufficient'])
|
||
self.assertFalse(finalizer.coverage_sufficient(docs,'3C哪些产品近期表现最差'))
|
||
self.assertFalse(result['completionAdjusted'])
|
||
self.assertTrue(any('缺少' in item for item in result['missingEvidence']))
|
||
self.assertIn('Q039D5FA7',result['answer'])
|
||
self.assertIn('90.11%',result['answer'])
|
||
|
||
def test_exact_formal_metrics_refute_only_the_model_claims_they_prove_false(self):
|
||
captured = []
|
||
def opener(request, timeout):
|
||
captured.append(json.loads(request.data))
|
||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
|
||
'evidenceSufficient': False, 'missingEvidence': FORMAL_PRODUCT_GAPS})}}]}).encode())
|
||
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
|
||
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
|
||
cancel_event=threading.Event(), opener=opener)
|
||
self.assertTrue(result['evidenceSufficient'])
|
||
self.assertEqual(result['missingEvidence'], [])
|
||
self.assertEqual(result['modelMissingEvidence'], FORMAL_PRODUCT_GAPS)
|
||
self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS)
|
||
self.assertTrue(result['completionAdjusted'])
|
||
selector = json.loads(captured[0]['messages'][1]['content'])
|
||
contract = selector['formalMetricCoverage']
|
||
self.assertEqual(contract['filters'], {'product': 'SYNTH-16X15+T'})
|
||
self.assertEqual(len(contract['metrics']), 12)
|
||
self.assertTrue(all(row['covered'] for row in contract['metrics']))
|
||
self.assertIn(1200.5, [row['value'] for row in contract['metrics']])
|
||
self.assertNotIn('未覆盖内容', result['answer'])
|
||
|
||
def test_formal_metric_contract_rejects_wrong_scope_period_release_and_incomplete_values(self):
|
||
mutations = {
|
||
'product': lambda r: r['filterBinding'].update(product='SYNTH-16X15+T-OTHER'),
|
||
'extra_filter': lambda r: r['filterBinding'].update(group='其他分组'),
|
||
'period': lambda r: r['comparison']['period'].update(month=7),
|
||
'release': lambda r: r.update(releaseId='release-other'),
|
||
'input': lambda r: r['current']['black'].pop('totalWeight'),
|
||
'scrap': lambda r: r['comparison']['bright'].pop('scrapWeight'),
|
||
'rate': lambda r: r['current']['bright'].update(rate=None),
|
||
'nonfinite': lambda r: r['current']['black'].update(totalWeight='NaN'),
|
||
'source_months': lambda r: r['comparison'].pop('coverage'),
|
||
'source_stage': lambda r: r['current']['coverage']['sourceMonths'][0].update(brightRows=0),
|
||
}
|
||
for name, mutate in mutations.items():
|
||
with self.subTest(name=name):
|
||
docs = copy.deepcopy(formal_product_evidence()); mutate(docs[0]['result'])
|
||
def opener(request, timeout):
|
||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
|
||
'evidenceSufficient': True, 'missingEvidence': []})}}]}).encode())
|
||
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
|
||
question=FORMAL_PRODUCT_QUESTION, evidence=docs, context={'releaseId': 'release-a'},
|
||
cancel_event=threading.Event(), opener=opener)
|
||
self.assertFalse(result['evidenceSufficient'])
|
||
self.assertTrue(result['missingEvidence'])
|
||
self.assertFalse(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, {'releaseId': 'release-a'}))
|
||
|
||
def test_unknown_mixed_and_other_scope_model_gaps_still_block_completion(self):
|
||
for missing in ['缺少该规格投入重量和现场原因证据', '缺少该规格投入重量原始明细',
|
||
'缺少其他规格的投入重量正式数据', '缺少2026年7月投入重量正式数据',
|
||
'缺少对照期', '缺少生产原因证据']:
|
||
with self.subTest(missing=missing):
|
||
def opener(request, timeout):
|
||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
|
||
'evidenceSufficient': False, 'missingEvidence': [*FORMAL_PRODUCT_GAPS, missing]})}}]}).encode())
|
||
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
|
||
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
|
||
cancel_event=threading.Event(), opener=opener)
|
||
self.assertFalse(result['evidenceSufficient'])
|
||
self.assertEqual(result['missingEvidence'], [missing])
|
||
self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS)
|
||
|
||
def test_metric_reconciliation_does_not_infer_unspecified_scope_or_hide_later_missing_items(self):
|
||
from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps
|
||
for question in [FORMAL_PRODUCT_QUESTION.replace('只看SYNTH-16X15+T,', ''),
|
||
FORMAL_PRODUCT_QUESTION.replace('2026年', ''),
|
||
FORMAL_PRODUCT_QUESTION.replace('黑片、亮片', ''),
|
||
FORMAL_PRODUCT_QUESTION.replace('9月与8月', '13月与8月')]:
|
||
with self.subTest(question=question):
|
||
contract = formal_metric_coverage(formal_product_evidence(), question, {'releaseId': 'release-a'})
|
||
self.assertIsNone(contract)
|
||
self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, []))
|
||
contract = formal_metric_coverage(formal_product_evidence(), FORMAL_PRODUCT_QUESTION.replace('报废重量和', ''), {'releaseId': 'release-a'})
|
||
self.assertEqual(resolve_proven_metric_gaps(contract, [FORMAL_PRODUCT_GAPS[1]]), ([FORMAL_PRODUCT_GAPS[1]], []))
|
||
gaps = ['缺少' + '的' * index + '该规格投入重量正式数据' for index in range(12)] + ['缺少生产原因证据']
|
||
def opener(request, timeout):
|
||
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
|
||
'evidenceSufficient': False, 'missingEvidence': gaps})}}]}).encode())
|
||
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
|
||
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
|
||
cancel_event=threading.Event(), opener=opener)
|
||
self.assertFalse(result['evidenceSufficient'])
|
||
self.assertEqual(result['missingEvidence'], ['缺少生产原因证据'])
|
||
self.assertEqual(result['modelMissingEvidence'], gaps)
|
||
|
||
def test_formal_contract_accepts_the_same_explicit_frozen_filters(self):
|
||
from nao_analysis_coverage import formal_metric_coverage
|
||
for filters in [{'group': '合成分组'}, {'department': '合成部门'},
|
||
{'group': '合成分组', 'extraFilters': {'region': '合成地区'}, 'includeUnassigned': True}]:
|
||
with self.subTest(filters=filters):
|
||
docs = formal_product_evidence()
|
||
docs[0]['result']['filterBinding'].update(filters)
|
||
expected = {'product': 'SYNTH-16X15+T', **filters}
|
||
context = {'releaseId': 'release-a', 'filters': expected}
|
||
contract = formal_metric_coverage(docs, FORMAL_PRODUCT_QUESTION, context)
|
||
self.assertEqual(contract['filters'], expected)
|
||
self.assertEqual(contract['missingEvidence'], [])
|
||
self.assertTrue(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, context))
|
||
|
||
def test_unknown_extra_scope_or_analysis_language_does_not_activate_the_contract(self):
|
||
from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps, coverage_gaps
|
||
docs = formal_product_evidence()
|
||
docs[0]['result']['filterBinding']['group'] = '合成分组'
|
||
for question in [FORMAL_PRODUCT_QUESTION + '仅统计合成分组。',
|
||
FORMAL_PRODUCT_QUESTION.replace(',请', ',合成部门,分析人员因素,请'),
|
||
'合成分组中,' + FORMAL_PRODUCT_QUESTION,
|
||
FORMAL_PRODUCT_QUESTION + '并提供现场生产原因。']:
|
||
with self.subTest(question=question):
|
||
contract = formal_metric_coverage(docs, question, {'releaseId': 'release-a'})
|
||
self.assertIsNone(contract)
|
||
self.assertEqual(coverage_gaps(docs, question, {'releaseId': 'release-a'}), [])
|
||
self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, []))
|
||
|
||
if __name__=='__main__':unittest.main()
|