同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。 补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。 统一午夜黑及其他主题的控件细节、中文和业务数字字体。 排除本地业务数据、凭据、预览、发布与验收产物。 验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
351 lines
21 KiB
Python
351 lines
21 KiB
Python
"""Evidence-bound composed answers; uses only mocked OpenAI-compatible responses."""
|
|
import io
|
|
import json
|
|
import sys
|
|
import threading
|
|
import tempfile
|
|
from urllib.error import HTTPError
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
|
import nao_evidence_finalizer as finalizer
|
|
|
|
|
|
def evidence(ref='Q00000001', rate=.95):
|
|
return [{'evidenceRef': ref, 'mode': 'compare', 'result': {
|
|
'kind': 'quality_comparison', 'filterBinding': {},
|
|
'current': {'period': {'year': 2026, 'month': 9}, 'compositeRate': rate,
|
|
'coverage': {'sourceMonths': [{'year': 2026, 'month': 9,
|
|
'blackRows': 2, 'brightRows': 2}]}},
|
|
'comparison': {'period': {'year': 2026, 'month': 8}, 'compositeRate': .96,
|
|
'coverage': {'sourceMonths': [{'year': 2026, 'month': 8,
|
|
'blackRows': 2, 'brightRows': 2}]}},
|
|
'delta': {'available': True, 'percentagePoints': (rate - .96) * 100},
|
|
}}]
|
|
|
|
|
|
def response(parsed):
|
|
payload = {'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(parsed, ensure_ascii=False)}}]}
|
|
return io.BytesIO(json.dumps(payload).encode())
|
|
|
|
|
|
class AgentAnswerTest(unittest.TestCase):
|
|
def test_product_drilldown_fallback_keeps_each_queried_specification(self):
|
|
from unittest.mock import patch
|
|
facts = [{'id':'F1','kind':'overall','text':'总体合格率上升。','evidenceRef':'Q00000001'}]
|
|
docs = evidence()
|
|
for index, product in enumerate(('规格甲','规格乙','规格丙'),2):
|
|
ref = f'Q{index:08X}'
|
|
facts.append({'id':f'F{index}','kind':'cross','text':product+'来源缺角变化。','evidenceRef':ref})
|
|
docs.append({'evidenceRef':ref,'result':{'filters':{'product':product}}})
|
|
report = {'facts':facts,'sections':['不应输出冗长全表'],'headline':'总体','policy':'关联不作为生产根因。','missingEvidence':[]}
|
|
invalid = {'answerBlocks':[{'type':'bullet','text':'{{F1}} 虚构改善50%'}],
|
|
'evidenceSufficient':True,'missingEvidence':[]}
|
|
with patch('nao_agent_report.build_agent_report',return_value=report):
|
|
result = finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
|
|
question='fixture',evidence=docs,cancel_event=threading.Event(),
|
|
context={'_agentTask':{'version':1,'kind':'analysis','defectScope':'products','defectProductCount':3}},
|
|
opener=lambda *args,**kwargs:response(invalid))
|
|
self.assertTrue(result['selectionFallback'])
|
|
self.assertTrue(result['compositionRepairAttempted'])
|
|
for product in ('规格甲','规格乙','规格丙'):
|
|
self.assertIn(product,result['answer'])
|
|
self.assertIn('建议核查',result['answer'])
|
|
self.assertNotIn('50%',result['answer'])
|
|
self.assertNotIn('不应输出冗长全表',result['answer'])
|
|
|
|
def test_invalid_composition_gets_one_model_repair_with_same_global_budget(self):
|
|
calls = []
|
|
def provider(request, timeout):
|
|
body = json.loads(request.data)
|
|
calls.append(body)
|
|
if len(calls) == 1:
|
|
return response({'reportTitle':'合格率变化分析','answerBlocks':[{'type':'bullet','text':'本期改善50% {{F1}}'}],
|
|
'evidenceSufficient':True,'missingEvidence':[]})
|
|
return response({'answerBlocks':[{'type':'bullet','text':'{{F1}}'}],
|
|
'evidenceSufficient':True,'missingEvidence':[]})
|
|
result = finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
|
|
question='比较质量',evidence=evidence(),cancel_event=threading.Event(),opener=provider)
|
|
self.assertEqual(len(calls),2)
|
|
self.assertNotIn('50%',result['answer'])
|
|
self.assertFalse(result.get('selectionFallback',False))
|
|
self.assertEqual(result['reportTitle'],'合格率变化分析')
|
|
self.assertTrue(result['titleGeneratedByModel'])
|
|
from nao_runtime_policy import MAX_OUTPUT_TOKENS
|
|
self.assertTrue(all(body['max_completion_tokens']==MAX_OUTPUT_TOKENS for body in calls))
|
|
self.assertEqual(len(calls[1]['messages']),4)
|
|
|
|
def test_report_uses_model_selected_priorities_without_dumping_all_tables(self):
|
|
result = self.synthesize({'reportFactIds': ['F1'], 'reportTitle': '合格率变化分析',
|
|
'evidenceSufficient': True, 'missingEvidence': []},
|
|
context={'_agentTask': {'version': 1, 'kind': 'report', 'historyMode': 'all',
|
|
'question': '综合以上生成报告'}})
|
|
self.assertIn('## 核心结论', result['answer'])
|
|
self.assertIn('下降1.00个百分点', result['answer'])
|
|
self.assertIn('## 下一步核查', result['answer'])
|
|
self.assertNotIn('| ---', result['answer'])
|
|
self.assertLess(len(result['answer']), 1000)
|
|
self.assertFalse(result.get('selectionFallback', False))
|
|
|
|
def test_report_rejects_unknown_focus_id_and_still_has_bounded_fallback(self):
|
|
result = self.synthesize({'reportFactIds': ['F999'],
|
|
'evidenceSufficient': True, 'missingEvidence': []},
|
|
context={'_agentTask': {'version': 1, 'kind': 'report', 'historyMode': 'all',
|
|
'question': '综合以上生成报告'}})
|
|
self.assertTrue(result['selectionFallback'])
|
|
self.assertNotIn('F999', result['answer'])
|
|
self.assertNotIn('| ---', result['answer'])
|
|
|
|
def synthesize(self, parsed, *, docs=None, model='vendor/compatible-chat', context=None, capture=None):
|
|
def opener(request, timeout):
|
|
if capture is not None:
|
|
capture['body'] = json.loads(request.data)
|
|
capture['timeout'] = timeout
|
|
return response(parsed)
|
|
|
|
return finalizer.synthesize(
|
|
base_url='https://offline.invalid/v1', api_key='synthetic-only', model=model,
|
|
question='请概括质量变化', evidence=evidence() if docs is None else docs,
|
|
cancel_event=threading.Event(), context=context, opener=opener)
|
|
|
|
def test_model_composes_prose_around_exact_server_fact_and_reference(self):
|
|
captured = {}
|
|
result = self.synthesize({
|
|
'answerBlocks': [
|
|
{'type': 'heading', 'text': '变化概览'},
|
|
{'type': 'paragraph', 'text': '建议结合现场记录开展复核;当前结论只限于本次查询指标:{{F1}}'},
|
|
],
|
|
'evidenceSufficient': True, 'missingEvidence': [],
|
|
}, context={'_agentTask': {'version': 1, 'kind': 'analysis', 'responseStyle': 'brief',
|
|
'dimensions': ['group'], 'question': '简要回答'}}, capture=captured)
|
|
|
|
self.assertIn('### 变化概览', result['answer'])
|
|
self.assertIn('建议结合现场记录开展复核', result['answer'])
|
|
self.assertIn('下降1.00个百分点', result['answer'])
|
|
self.assertIn('[Q00000001]', result['answer'])
|
|
self.assertEqual(result['selectedFactIds'], ['F1'])
|
|
self.assertFalse(result.get('selectionFallback', False))
|
|
self.assertEqual(captured['body']['model'], 'vendor/compatible-chat')
|
|
self.assertEqual(captured['body']['thinking'], {'type':'disabled'})
|
|
from nao_runtime_policy import MAX_OUTPUT_TOKENS
|
|
self.assertEqual(captured['body']['max_completion_tokens'], MAX_OUTPUT_TOKENS)
|
|
selector = json.loads(captured['body']['messages'][1]['content'])
|
|
self.assertEqual(selector['agentTask']['responseStyle'], 'brief')
|
|
self.assertTrue(selector['facts'])
|
|
|
|
def test_unsupported_digits_force_verified_report_fallback(self):
|
|
result = self.synthesize({
|
|
'answerBlocks': [{'type': 'paragraph', 'text': '本期合格率达到99%,依据{{F1}}'}],
|
|
'evidenceSufficient': True, 'missingEvidence': [],
|
|
})
|
|
|
|
self.assertTrue(result['selectionFallback'])
|
|
self.assertNotIn('99%', result['answer'])
|
|
self.assertIn('下降1.00个百分点', result['answer'])
|
|
|
|
def test_unknown_fact_reference_forces_fallback(self):
|
|
result = self.synthesize({
|
|
'answerBlocks': [{'type': 'paragraph', 'text': '结论{{F99}}'}],
|
|
'evidenceSufficient': True, 'missingEvidence': [],
|
|
})
|
|
|
|
self.assertTrue(result['selectionFallback'])
|
|
self.assertNotIn('{{F99}}', result['answer'])
|
|
self.assertIn('下降1.00个百分点', result['answer'])
|
|
|
|
def test_unsupported_direction_and_cause_prose_forces_fallback(self):
|
|
result=self.synthesize({
|
|
'answerBlocks':[{'type':'paragraph','text':'质量已持续改善,且确认由设备磨损导致。{{F1}}'}],
|
|
'evidenceSufficient':True,'missingEvidence':[]})
|
|
self.assertTrue(result['selectionFallback'])
|
|
self.assertNotIn('设备磨损',result['answer'])
|
|
|
|
def test_cause_hypothesis_passes_only_when_explicitly_unconfirmed(self):
|
|
result=self.synthesize({
|
|
'answerBlocks':[{'type':'paragraph','text':'待验证:设备磨损是否可能导致报废增加,建议检查。{{F1}}'}],
|
|
'evidenceSufficient':True,'missingEvidence':[]})
|
|
self.assertFalse(result.get('selectionFallback',False))
|
|
self.assertIn('待验证:设备磨损是否可能导致报废增加',result['answer'])
|
|
|
|
absolute=self.synthesize({
|
|
'answerBlocks':[{'type':'paragraph','text':'所有缺陷都已完全排除。{{F1}}'}],
|
|
'evidenceSufficient':True,'missingEvidence':[]})
|
|
self.assertTrue(absolute['selectionFallback'])
|
|
|
|
def test_thinking_capability_is_retried_without_model_name_branch(self):
|
|
bodies=[]
|
|
parsed={'answerBlocks':[{'type':'paragraph','text':'总体变化:{{F1}}'}],
|
|
'evidenceSufficient':True,'missingEvidence':[]}
|
|
def opener(request,timeout):
|
|
bodies.append(json.loads(request.data))
|
|
if len(bodies)==1:
|
|
raise HTTPError(request.full_url,400,'thinking unsupported',None,io.BytesIO(b''))
|
|
return response(parsed)
|
|
result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic-only',
|
|
model='some/openai-compatible-model',question='请概括质量变化',evidence=evidence(),
|
|
cancel_event=threading.Event(),opener=opener)
|
|
self.assertTrue(result['evidenceSufficient'])
|
|
self.assertEqual(len(bodies),2)
|
|
self.assertEqual(bodies[0]['thinking'],{'type':'disabled'})
|
|
self.assertNotIn('thinking',bodies[1])
|
|
|
|
def test_non_mimo_model_is_eligible_and_incomplete_server_evidence_stays_partial(self):
|
|
calls = [{'status': 'completed'}]
|
|
self.assertTrue(finalizer.eligible('provider/model-x', 'PARTIAL', 'timeout', calls, False, True))
|
|
self.assertFalse(finalizer.eligible(' ', 'PARTIAL', 'timeout', calls, False, True))
|
|
|
|
from test_nao_evidence_report import coverage_comparison
|
|
doc=coverage_comparison()
|
|
doc['result']['current']['coverage']['sourceMonths'][0].update(blackRows=0,brightRows=0)
|
|
result = self.synthesize({
|
|
'answerBlocks': [{'type': 'paragraph', 'text': '当前正式变化:{{F1}}'}],
|
|
'evidenceSufficient': True, 'missingEvidence': [],
|
|
}, docs=[doc])
|
|
|
|
self.assertFalse(result['evidenceSufficient'])
|
|
self.assertTrue(result['missingEvidence'])
|
|
self.assertTrue(any('没有可引用的正式综合合格率' in item for item in result['missingEvidence']))
|
|
self.assertIn('未覆盖内容', result['answer'])
|
|
|
|
def test_legacy_selector_response_still_uses_verified_report(self):
|
|
result = self.synthesize({
|
|
'otherFactIds': ['F1'], 'groupFactIds': [], 'productFactIds': [], 'defectFactIds': [],
|
|
'evidenceSufficient': True, 'missingEvidence': [],
|
|
})
|
|
|
|
self.assertTrue(result['evidenceSufficient'])
|
|
self.assertIn('下降1.00个百分点', result['answer'])
|
|
self.assertIn('### 重点关注', result['answer'])
|
|
self.assertFalse(result.get('selectionFallback', False))
|
|
|
|
def test_source_verification_task_requires_complete_source_proof(self):
|
|
question = '请核对来源'
|
|
period={'year':2026,'month':9}
|
|
context = {'releaseId':'r1','_agentTask': {'version': 1, 'question': question, 'kind': 'source_verify',
|
|
'verifySource': True, 'fullDetails': False,'period':period,
|
|
'comparisonPeriod':None,'filters':{'product':'P'},'stages':['bright']}}
|
|
query = {'mode': 'compare', 'result': {'kind': 'quality_query'}}
|
|
proof = {'complete': True, 'expectedRows':2, 'verifiedRows': 2, 'missingRows':0,
|
|
'mismatchRows':0,'allSelectedValuesMatchStored': True}
|
|
side_context={'releaseId':'r1','year':2026,'month':9,'stage':'bright','category':None,
|
|
'product':'P','includeUnassigned':False,'extraFilters':{}}
|
|
source = {'mode': 'investigate', 'result': {'kind':'quality_investigation','releaseId':'r1',
|
|
'period':period,'filters':{'product':'P'},'stage':'bright','action':'source','current': {
|
|
'context':side_context,'totalRows': 2,'aggregateComplete':True,'sourceVerification': proof}}}
|
|
self.assertFalse(finalizer.coverage_sufficient([query], question, context))
|
|
self.assertTrue(finalizer.coverage_sufficient([source], question, context))
|
|
source['result']['current']['sourceVerification'] = {**proof, 'complete': False}
|
|
self.assertFalse(finalizer.coverage_sufficient([source], question, context))
|
|
|
|
def test_source_proofs_must_cover_every_requested_stage_side_and_scope(self):
|
|
question='核验9月与8月原表'
|
|
current={'year':2026,'month':9};previous={'year':2026,'month':8}
|
|
context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'source_verify',
|
|
'verifySource':True,'fullDetails':False,'period':current,'comparisonPeriod':previous,
|
|
'filters':{'product':'P'},'stages':['black','bright']}}
|
|
query={'mode':'compare','result':{'kind':'quality_query'}}
|
|
def source(stage, product='P'):
|
|
proof={'complete':True,'expectedRows':1,'verifiedRows':1,'missingRows':0,'mismatchRows':0,
|
|
'allSelectedValuesMatchStored':True}
|
|
def side(month):
|
|
return {'context':{'releaseId':'r1','year':2026,'month':month,'stage':stage,'category':None,
|
|
'product':product,'includeUnassigned':False,'extraFilters':{}},
|
|
'totalRows':1,'aggregateComplete':True,'sourceVerification':proof}
|
|
return {'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1',
|
|
'period':current,'comparisonPeriod':previous,'filters':{'product':product},'stage':stage,
|
|
'category':None,'action':'source','current':side(9),'comparison':side(8)}}
|
|
wrong_scope=source('black',product='OTHER')
|
|
self.assertFalse(finalizer.coverage_sufficient([query,wrong_scope],question,context))
|
|
black=source('black');bright=source('bright')
|
|
self.assertFalse(finalizer.coverage_sufficient([black],question,context))
|
|
self.assertTrue(finalizer.coverage_sufficient([black,bright],question,context))
|
|
|
|
def test_full_details_task_requires_contiguous_complete_pages(self):
|
|
question = '把所有记录列出来'
|
|
period={'year':2026,'month':9}
|
|
context = {'releaseId':'r1','_agentTask': {'version': 1, 'question': question, 'kind': 'details',
|
|
'verifySource': False, 'fullDetails': True,'period':period,
|
|
'comparisonPeriod':None,'filters':{},'stages':['black']}}
|
|
side={'context':{'releaseId':'r1','year':2026,'month':9,'stage':'black','category':None,
|
|
'includeUnassigned':False,'extraFilters':{}},'recordFilter':'all',
|
|
'matchingRows':2,'offset':0,'returnedRows':2,'hasMore':False,'differenceRecords':[{},{}]}
|
|
rows={'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1','period':period,
|
|
'filters':{},'stage':'black','category':None,'action':'rows','current':side}}
|
|
self.assertTrue(finalizer.coverage_sufficient([rows], question, context))
|
|
rows['result']['current']['hasMore'] = True
|
|
self.assertFalse(finalizer.coverage_sufficient([rows], question, context))
|
|
rows['result']['current']['hasMore'] = False
|
|
rows['result']['current']['recordFilter'] = 'differences'
|
|
self.assertFalse(finalizer.coverage_sufficient([rows], question, context))
|
|
|
|
def test_full_details_require_every_requested_stage_and_both_periods(self):
|
|
question='列出黑片和亮片两期的全部明细'
|
|
current={'year':2026,'month':9};previous={'year':2026,'month':8}
|
|
context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'details',
|
|
'verifySource':False,'fullDetails':True,'period':current,'comparisonPeriod':previous,
|
|
'filters':{'group':'G'},'stages':['black','bright']}}
|
|
def rows(stage):
|
|
def side(month):
|
|
return {'context':{'releaseId':'r1','year':2026,'month':month,'stage':stage,'category':None,
|
|
'group':'G','includeUnassigned':False,'extraFilters':{}},
|
|
'recordFilter':'all','matchingRows':1,'offset':0,'returnedRows':1,
|
|
'hasMore':False,'differenceRecords':[{}]}
|
|
return {'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1',
|
|
'period':current,'comparisonPeriod':previous,'filters':{'group':'G'},'stage':stage,
|
|
'category':None,'action':'rows','current':side(9),'comparison':side(8)}}
|
|
black=rows('black');bright=rows('bright')
|
|
self.assertFalse(finalizer.coverage_sufficient([black],question,context))
|
|
self.assertTrue(finalizer.coverage_sufficient([black,bright],question,context))
|
|
bright['result']['comparison']['recordFilter']='differences'
|
|
self.assertFalse(finalizer.coverage_sufficient([black,bright],question,context))
|
|
|
|
def test_unbound_task_period_requires_a_matching_query_root(self):
|
|
question='列出当前范围的全部明细'
|
|
context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'details',
|
|
'verifySource':False,'fullDetails':True,'period':None,'comparisonPeriod':None,
|
|
'filters':{'group':'G'},'stages':['black']}}
|
|
query={'mode':'compare','result':{'kind':'quality_comparison','releaseId':'r1',
|
|
'filterBinding':{'group':'G'},'current':{'period':{'year':2026,'month':9}},
|
|
'comparison':{'period':{'year':2026,'month':8}}}}
|
|
page={'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1',
|
|
'period':{'year':2026,'month':9},'filters':{'group':'G'},'stage':'black','category':None,
|
|
'action':'rows','current':{'context':{'releaseId':'r1','year':2026,'month':9,'stage':'black',
|
|
'category':None,'group':'G','includeUnassigned':False,'extraFilters':{}},'recordFilter':'all',
|
|
'matchingRows':1,'offset':0,'returnedRows':1,'hasMore':False,'differenceRecords':[{}]}}}
|
|
self.assertTrue(finalizer.coverage_sufficient([query,page],question,context))
|
|
self.assertFalse(finalizer.coverage_sufficient([page],question,context))
|
|
query['result']['filterBinding']={'group':'OTHER'}
|
|
self.assertFalse(finalizer.coverage_sufficient([query,page],question,context))
|
|
|
|
def test_empty_evidence_is_allowed_only_for_planning_callers(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
with self.assertRaises(finalizer.FinalizationUnavailable):
|
|
finalizer.evidence_summaries(Path(folder),'release-a')
|
|
self.assertEqual(finalizer.evidence_summaries(Path(folder),'release-a',allow_empty=True),[])
|
|
|
|
def test_semantic_metrics_and_single_period_dimensions_gate_completion(self):
|
|
from test_nao_metric_requests import metric_evidence, QUESTION
|
|
task={'version':1,'question':QUESTION,'kind':'metrics','period':{'year':2026,'month':9},
|
|
'comparisonPeriod':{'year':2026,'month':8},'filters':{'product':'SYNTH-16X15+T'},
|
|
'stages':['black','bright'],'fields':['rate','totalWeight','scrapWeight']}
|
|
context={'releaseId':'release-a','_agentTask':task}
|
|
docs=metric_evidence()
|
|
self.assertTrue(finalizer.coverage_sufficient(docs,QUESTION,context))
|
|
docs[0]['result']['current']['bright'].pop('totalWeight')
|
|
self.assertFalse(finalizer.coverage_sufficient(docs,QUESTION,context))
|
|
|
|
question='本期产品情况'
|
|
single={'version':1,'question':question,'kind':'analysis','period':{'year':2026,'month':9},
|
|
'comparisonPeriod':None,'filters':{},'dimensions':['product'],'stages':[]}
|
|
frame={'releaseId':'r1','_agentTask':single}
|
|
query={'mode':'query','result':{'kind':'quality_metrics','releaseId':'r1',
|
|
'period':{'year':2026,'month':9},'filters':{},'view':'overall'}}
|
|
self.assertFalse(finalizer.coverage_sufficient([query],question,frame))
|
|
query['result']['view']='product'
|
|
self.assertTrue(finalizer.coverage_sufficient([query],question,frame))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|