统一自主分析工具范围与执行前校验,严格核验正常结束、最终正文和图表恢复状态,并显示空回答及恢复失败原因。 包含两次已上线修复及回归测试,保留已有回答与业务数据。 验证:1101 项 Python、469 项前端、97 项后端测试通过;前后端构建和真实模型、页面验收通过。
272 lines
18 KiB
Python
272 lines
18 KiB
Python
"""Regression: a grouped comparison must not expand into per-object reads."""
|
|
import copy
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
|
|
str(Path(__file__).resolve().parent)]
|
|
from nao_agent_task import normalize_task
|
|
from nao_protocol_execution import _scoped_arguments, _validate_protocol_scope, execute
|
|
from test_nao_agent_task import task_value
|
|
from test_nao_protocol_execution import BoundApi, RELEASE, TURN, identity
|
|
|
|
|
|
def changes_node(**updates):
|
|
return {'nodeId': 'groups', 'purpose': 'changes', 'dimension': 'group',
|
|
'fields': ['rate'], 'stages': [], 'sort': 'rate_drop', 'limit': 1,
|
|
'selector': {'kind': 'base'}, 'comparisonRequired': True,
|
|
'visualPreference': 'none', 'visualType': None, **updates}
|
|
|
|
|
|
def normalized(nodes):
|
|
dimensions = list(dict.fromkeys(n.get('dimension') for n in nodes if n.get('dimension')))
|
|
stages = list(dict.fromkeys(s for n in nodes for s in n['stages']))
|
|
fields = list(dict.fromkeys(f for n in nodes for f in n['fields']))
|
|
return normalize_task(task_value(kind='analysis', queryPlan=nodes,
|
|
dimensions=dimensions, dimensionStages={}, fields=fields, stages=stages,
|
|
trendRequired=False, comparisonRequired=True,
|
|
period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2026, 'month': 8},
|
|
visualPreference='none'), '9月份哪个业务分组综合合格率下滑最多',
|
|
{'_protocol': {'version': 1}})
|
|
|
|
|
|
class MinimalQueryTests(unittest.TestCase):
|
|
def test_auto_batch_style_hint_does_not_report_missing_chart(self):
|
|
from analysis_visual_evidence import protocol_visual_result
|
|
task=normalized([changes_node(visualPreference='auto',visualType='bar')])
|
|
task.pop('queryPlan');task['visualPreference']='auto';task['visualType']='bar'
|
|
scope={'nodeId':'groups','scopeId':'batch','purpose':'changes','task':task}
|
|
with patch('analysis_visuals.build_visuals',return_value={'visuals':[]}):
|
|
result=protocol_visual_result([], [scope],task['question'],TURN,release_id=RELEASE)
|
|
self.assertEqual(result['visualState'],'not_requested',result)
|
|
self.assertEqual(result['missingEvidence'],[])
|
|
task['visualPreference']='required'
|
|
result=protocol_visual_result([], [scope],task['question'],TURN,release_id=RELEASE)
|
|
self.assertEqual(result['visualState'],'unavailable',result)
|
|
self.assertTrue(result['missingEvidence'])
|
|
task['visualPreference']='auto'
|
|
with patch('analysis_visuals.build_visuals',return_value={'visuals':[{'metric':'rate_drop_pp'}]}):
|
|
result=protocol_visual_result([], [scope],task['question'],TURN,release_id=RELEASE)
|
|
self.assertEqual(result['visualState'],'ready',result)
|
|
|
|
def test_requested_weights_cannot_be_completed_when_hidden(self):
|
|
from nao_changes import validate_changes
|
|
task=normalized([changes_node(fields=['rate','totalWeight','scrapWeight'])])
|
|
task['changeQuery']={'dimension':'group','sort':'rate_drop','limit':1}
|
|
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
|
|
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
|
'dimension':'group','stage':None,'sort':'rate_drop','completeRanking':True,'totalRows':1,
|
|
'rows':[{'rank':1,'key':'甲','current':{'rate':0.7},'comparison':{'rate':0.9}}]}
|
|
with self.assertRaisesRegex(ValueError,'正式重量'):
|
|
validate_changes(result,task,RELEASE)
|
|
|
|
def test_group_rate_answer_does_not_force_weight_details(self):
|
|
from nao_changes import changes_report
|
|
task=normalized([changes_node()]);task['changeQuery']={'dimension':'group','sort':'rate_drop','limit':1}
|
|
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
|
|
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
|
'dimension':'group','stage':None,'sort':'rate_drop','completeRanking':True,'totalRows':10,
|
|
'rows':[{'rank':1,'key':'甲','rateDrop':0.2,
|
|
'current':{'rate':0.7},'comparison':{'rate':0.9}}]}
|
|
report=changes_report([{'evidenceRef':'Q12345678','result':result}],task,RELEASE)
|
|
self.assertEqual(len(report['requiredFactIds']),1)
|
|
self.assertNotIn('投入',report['facts'][0]['text'])
|
|
self.assertIn('下降20',report['facts'][0]['text'])
|
|
self.assertNotIn('报废 kg','\n'.join(report['sections']))
|
|
self.assertNotIn('投入 kg','\n'.join(report['sections']))
|
|
|
|
def test_batch_comparison_chart_keeps_both_periods_without_extra_metric_read(self):
|
|
from analysis_visuals import build_visuals
|
|
from nao_agent_task import bind_task
|
|
task=normalized([changes_node(visualPreference='required')])
|
|
task.pop('queryPlan');task['visualPreference']='required'
|
|
task['changeQuery']={'dimension':'group','sort':'rate_drop','limit':1}
|
|
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
|
|
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
|
'dimension':'group','stage':None,'sort':'rate_drop','completeRanking':True,
|
|
'totalRows':10,'returnedRows':1,'truncated':True,
|
|
'rows':[{'rank':1,'key':'甲','label':'甲','rateDrop':0.2,
|
|
'current':{'rate':0.7},'comparison':{'rate':0.9}}]}
|
|
doc={'evidenceRef':'Q12345678','releaseId':RELEASE,'mode':'changes','result':result}
|
|
visual=build_visuals([doc],task['question'],TURN,
|
|
context=bind_task({'releaseId':RELEASE},task))
|
|
self.assertEqual(visual['visualState'],'ready',visual)
|
|
self.assertEqual(visual['visuals'][0]['metric'],'rate_drop_pp')
|
|
self.assertEqual(visual['visuals'][0]['series'][0]['values'],[-20.0])
|
|
result['sort']='current_rate';task['changeQuery']['sort']='current_rate'
|
|
visual=build_visuals([doc],task['question'],TURN,
|
|
context=bind_task({'releaseId':RELEASE},task))
|
|
self.assertEqual(visual['visualState'],'ready',visual)
|
|
self.assertEqual(visual['visuals'][0]['metric'],'composite_rate')
|
|
self.assertEqual(visual['visuals'][0]['series'][0]['values'],[0.7])
|
|
|
|
def test_whole_group_comparison_is_one_batch_node(self):
|
|
task = normalized([changes_node()])
|
|
child = copy.deepcopy(task); child.pop('queryPlan')
|
|
child['changeQuery'] = {'dimension': 'group', 'sort': 'rate_drop', 'limit': 1}
|
|
scope = {'nodeId': 'groups', 'scopeId': 'batch', 'purpose': 'changes',
|
|
'label': '业务分组批量对比', 'task': child, 'limit': 1}
|
|
_validate_protocol_scope(scope, task['queryPlan'][0], task, 'release')
|
|
args = _scoped_arguments({'releaseId': 'release'}, scope, task['queryPlan'][0])
|
|
self.assertEqual(len(args), 1)
|
|
self.assertEqual(args[0]['mode'], 'changes')
|
|
self.assertEqual(args[0]['limit'], 1)
|
|
self.assertNotIn('group', args[0]['filters'])
|
|
|
|
def test_optional_drilldown_depends_on_actual_leading_result(self):
|
|
task = normalized([changes_node(limit=3), changes_node(nodeId='products', dimension='product',
|
|
optional=True, selector={'kind':'changes_top','sourceNodeId':'groups',
|
|
'count':1,'stage':'dominant'})])
|
|
self.assertEqual(task['queryPlan'][1]['selector']['sourceNodeId'], 'groups')
|
|
self.assertTrue(task['queryPlan'][1]['optional'])
|
|
|
|
def test_cannot_select_unqueried_or_future_result(self):
|
|
with self.assertRaises(ValueError):
|
|
normalized([changes_node(selector={'kind':'changes_top','sourceNodeId':'future',
|
|
'count':1,'stage':'overall'})])
|
|
|
|
def test_overall_and_stage_rate_nodes_cannot_reread_the_same_baseline(self):
|
|
rate={'nodeId':'overall','purpose':'metrics','fields':['rate'],'stages':[],
|
|
'selector':{'kind':'base'},'comparisonRequired':True,'visualPreference':'auto','visualType':None}
|
|
with self.assertRaisesRegex(ValueError,'无需重复读取'):
|
|
normalized([rate,{**rate,'nodeId':'stage_rate','stages':['black','bright']}])
|
|
|
|
def test_cause_does_not_rank_tiny_products_by_rate_alone(self):
|
|
task=normalized([changes_node(dimension='product')])
|
|
from nao_query_plan import normalize_query_plan
|
|
task['question']='9月份合格率下降原因'
|
|
with self.assertRaisesRegex(ValueError,'小批量'):
|
|
normalize_query_plan(task['queryPlan'],root_task=task)
|
|
|
|
def test_performance_core_keeps_rate_ranking_and_scale_is_only_supporting(self):
|
|
from nao_changes import changes_report
|
|
task=normalized([changes_node(dimension='product',sort='current_rate',fields=['rate','totalWeight','scrapWeight'])])
|
|
task['question']='3C哪些产品近期表现最差';task['changeQuery']={'dimension':'product','sort':'current_rate','limit':1}
|
|
side=lambda rate,weight:{'rate':rate,'scrapWeight':weight,
|
|
'black':{'rate':rate,'totalWeight':1000,'scrapWeight':weight},
|
|
'bright':{'rate':1,'totalWeight':1000,'scrapWeight':0},
|
|
'blackInputWeight':1000,'brightInputWeight':1000}
|
|
tiny={'rank':1,'key':'小样','label':'小样','rateDrop':1,'current':side(0,1),'comparison':side(1,0)}
|
|
# A large scrap volume can coexist with an improving rate. It cannot
|
|
# supplant the actual lowest-rate product in the requested ranking.
|
|
large={'key':'批量产品','label':'批量产品','scrapDelta':200,'current':side(.9,300),'comparison':side(.8,100)}
|
|
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
|
|
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
|
'dimension':'product','stage':None,'sort':'current_rate','completeRanking':True,'totalRows':10,
|
|
'rows':[tiny],'prioritySignals':{'currentScrap':[large],'scrapIncrease':[large]}}
|
|
report=changes_report([{'evidenceRef':'Q12345678','result':result}],task,RELEASE)
|
|
core=[f['text'] for f in report['facts'] if f['id'] in report['requiredFactIds']]
|
|
self.assertTrue(core)
|
|
self.assertIn('小样',core[0]);self.assertIn('0.00%',core[0])
|
|
self.assertFalse(any('批量产品' in text for text in core))
|
|
signals=[f for f in report['facts'] if '批量产品' in f['text']]
|
|
self.assertTrue(signals)
|
|
self.assertTrue(all(f['kind']=='supporting' for f in signals))
|
|
self.assertIn('小样','\n'.join(report['sections']))
|
|
|
|
def test_real_executor_reads_only_batch_and_honors_conditional_stop(self):
|
|
task = normalized([changes_node(), changes_node(nodeId='products', dimension='product',
|
|
optional=True, selector={'kind':'changes_top','sourceNodeId':'groups',
|
|
'count':1,'stage':'overall'})])
|
|
child = copy.deepcopy(task);child.pop('queryPlan')
|
|
child.update(dimensions=['group'],dimensionStages={'group':[]},
|
|
changeQuery={'dimension':'group','sort':'rate_drop','limit':1})
|
|
scope = {'nodeId':'groups','scopeId':'batch','purpose':'changes','label':'业务分组',
|
|
'task':child,'limit':1}
|
|
|
|
class Api(BoundApi):
|
|
def __init__(self):
|
|
super().__init__();self.reads=[]
|
|
def call(self,path,payload=None,method=None,*,node_id=None,scope_id=None):
|
|
if path == '/api/analysis/protocol/queries':
|
|
return {'version':1,'scopes':[scope], 'pending':[], 'gaps':[],
|
|
'skipped':[{'nodeId':'products','reason':'no_adverse_change'}]}
|
|
if path == '/api/analysis/changes':
|
|
self.reads.append((path,copy.deepcopy(payload)))
|
|
return {'kind':'quality_changes','releaseId':RELEASE,
|
|
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
|
|
'filters':task['filters'],'dimension':'group','stage':None,'sort':'rate_drop',
|
|
'totalRows':10,'returnedRows':1,'truncated':True,'completeRanking':True,
|
|
'coverage':{'rateComparable':True},
|
|
'rows':[{'rank':1,'key':'组甲','label':'组甲','rateDrop':-0.01,'scrapDelta':-10,
|
|
'current':{'rate':0.97,'scrapWeight':20},
|
|
'comparison':{'rate':0.96,'scrapWeight':30}}],
|
|
'protocolEvidence':{**identity(),'queryId':'batch-query','responseHash':'a'*64,
|
|
'nodeId':node_id,'scopeId':scope_id}}
|
|
if path.startswith('/api/metrics') or path.endswith('/defects'):
|
|
raise AssertionError('排名问题不能附加逐对象指标或缺陷读取')
|
|
return super().call(path,payload,method)
|
|
|
|
api=Api();api.task=task
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
|
|
with patch('nao_evidence_finalizer._open_model', side_effect=OSError('offline synthesis')):
|
|
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':task['question']},
|
|
task=task,authority={**identity(),'task':task},actor_root=root,delegated=api,
|
|
connection=('http://offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
|
|
self.assertEqual(result['status'],'COMPLETED',result.get('evidenceCompletion'))
|
|
self.assertEqual(len(api.reads),1)
|
|
self.assertEqual(len(result['protocolEvidence']),1)
|
|
self.assertNotIn('没有取得', result['answer'])
|
|
self.assertNotIn('统计数据附录',result['answer'])
|
|
self.assertEqual(result['answer'].count('| 1 | 组甲 | 96.00% | 97.00% |'), 1)
|
|
self.assertIn('| 序号 | 对象 |',result['answer'])
|
|
|
|
|
|
class ConfirmedScalarSelectionTests(unittest.TestCase):
|
|
def scalar(self):
|
|
original='2026年9月18X15X4规格的综合合格率是多少?不要包含18X15X4.1。'
|
|
task=normalize_task(task_value(goal=original,kind='analysis',dimensions=[],dimensionStages={},
|
|
fields=['rate'],stages=[],period={'year':2026,'month':9},comparisonRequired=False,
|
|
trendRequired=False,visualPreference='none',filterUpdates={
|
|
'product':'18X15X4+T','exclusions':{'product':['18X15X4.1+Z']}}),original,{})
|
|
task['rootGoal']=original
|
|
task['productResolution']={'originalQuestion':original,'selectedName':'18X15X4+T',
|
|
'productMatch':{'status':'ambiguous','candidates':[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]}}
|
|
return task
|
|
|
|
def test_only_signed_scalar_choice_dispatches_same_metrics_and_keeps_goal_review(self):
|
|
from nao_protocol_execution import _confirmed_scalar_selection
|
|
task=self.scalar();before=copy.deepcopy(task);api=BoundApi()
|
|
request={'releaseId':RELEASE,'turnId':TURN,'question':'选择18X15X4+T。','_autonomousAnswer':True}
|
|
self.assertTrue(_confirmed_scalar_selection(request,task))
|
|
for sufficient in (True,False):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
|
|
with patch('nao_evidence_finalizer.synthesize',return_value={
|
|
'answer':'已核实本期正式率','evidenceSufficient':sufficient,
|
|
'missingEvidence':[] if sufficient else ['原目标仍有未回答内容']}) as synth:
|
|
result=execute(request=copy.deepcopy(request),task=task,authority=identity(),actor_root=root,
|
|
delegated=api,connection=('offline','unused','model'),cancel_event=threading.Event(),emit=lambda _:None)
|
|
context=synth.call_args.kwargs['context']
|
|
self.assertTrue(context['_autonomousAnswer']);self.assertTrue(context['_confirmedScalarSelection'])
|
|
self.assertEqual(context['_agentTask']['rootGoal'],before['rootGoal'])
|
|
self.assertEqual(result['status'],'COMPLETED' if sufficient else 'PARTIAL')
|
|
self.assertEqual(result['agentTask'],before)
|
|
self.assertEqual(len(result['protocolEvidence']),1)
|
|
self.assertEqual(task,before)
|
|
self.assertTrue(all('exclusions=' in path and 'product=18X15X4%2BT' in path
|
|
for _,path,_ in api.calls if path.startswith('/api/metrics')))
|
|
|
|
def test_old_unselected_investigative_and_non_scalar_tasks_stay_rejected(self):
|
|
from nao_protocol_execution import _confirmed_scalar_selection
|
|
request={'_autonomousAnswer':True};task=self.scalar()
|
|
self.assertFalse(_confirmed_scalar_selection({},task))
|
|
for changes in ({'stages':['black']},{'trendRequired':True},{'comparisonRequired':True},
|
|
{'verifySource':True},{'fullDetails':True},{'executionMode':'native_agent'},
|
|
{'dimensions':['product']},{'fields':['rate','totalWeight']},
|
|
{'period':{'year':2026}},{'productResolution':{}},{'rootGoal':'找出重点规格'}):
|
|
with self.subTest(changes=changes):
|
|
self.assertFalse(_confirmed_scalar_selection(request,{**task,**changes}))
|
|
for original in ('找重点规格的综合合格率是多少?','合格率为什么下降?','最低合格率前五项是多少?'):
|
|
altered=copy.deepcopy(task);altered['rootGoal']=original;altered['productResolution']['originalQuestion']=original
|
|
self.assertFalse(_confirmed_scalar_selection(request,altered))
|
|
|
|
|
|
if __name__ == '__main__': unittest.main()
|