Files
quality_analyse/verification/test_nao_agent_task.py
T

1049 lines
79 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import copy
import io
import json
import sys
import threading
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_agent_task import _explicit_read_safeguards, normalize_task, bind_task, plan_task, plan_repairs, TaskUnavailable
from nao_analysis_coverage import coverage_gaps, plan_missing_queries, formal_metric_coverage, semantic_metric_gaps
from nao_metric_request import metric_request
from nao_evidence_finalizer import coverage_sufficient
from test_nao_analysis_coverage import evidence
from test_nao_metric_requests import metric_evidence
def task_value(**changes):
return {'goal': '完整理解本轮质量问题', 'kind': 'analysis', 'freshQuery': True,
'dimensions': ['department', 'group', 'product', 'defect'], 'stages': ['black', 'bright'],
'fields': ['rate', 'totalWeight', 'scrapWeight'], 'verifySource': False,
'fullDetails': False, 'visualPreference': 'auto', 'visualType': None,
'responseStyle': 'standard', 'needsClarification': False, 'clarification': '',
'filterUpdates': {}, **changes}
def semantic_ok():
return {'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,'requiresAllSourceRows':False,'requiresTrend':False}
class AgentTaskTest(unittest.TestCase):
def context(self, question, **changes):
return bind_task({}, normalize_task(task_value(**changes), question, {}))
def test_negated_lists_do_not_request_raw_rows_but_positive_siblings_still_do(self):
for question in (
'新任务:查询2026年9月客户800001在黑片和亮片中的全部成品规格清单,去重并显示各自来源阶段。这里只要成员清单,不要正式指标、排名或逐行原始记录。',
'只要清单,不需要指标及全部原始记录',
'不要图表或核验原件',
):
with self.subTest(question=question):
self.assertEqual(_explicit_read_safeguards(question),{'requiresSourceVerification':False,'requiresAllSourceRows':False})
for question in ('不要排名,但需要逐行原始记录','不要排名,逐行读取源记录','不要去重且逐行读取全部原始记录','不要图表或排名,请逐行读取原始记录'):
with self.subTest(question=question):
self.assertTrue(_explicit_read_safeguards(question)['requiresAllSourceRows'])
def test_current_read_negations_do_not_force_or_cancel_a_different_purpose(self):
for text in ('不读逐行原始记录', '不读取全部原始记录', '不做逐条原始记录读取', '原始记录不需要逐行读取'):
with self.subTest(text=text):
self.assertFalse(_explicit_read_safeguards(text)['requiresAllSourceRows'])
self.assertEqual(_explicit_read_safeguards('不要核验原件,但请逐行读取原始记录'),
{'requiresSourceVerification': False, 'requiresAllSourceRows': True})
self.assertEqual(_explicit_read_safeguards('不读逐行原始记录,但请核验归档原件'),
{'requiresSourceVerification': True, 'requiresAllSourceRows': False})
self.assertTrue(_explicit_read_safeguards('不要去重且逐行读取全部原始记录')['requiresAllSourceRows'])
def test_defect_only_metrics_plan_is_repaired_instead_of_answering_parent_input(self):
replies=[task_value(kind='metrics',dimensions=['defect'],stages=['black'],fields=['totalWeight'],dimensionStages={'defect':['black']}),
task_value(kind='analysis',dimensions=['defect'],stages=['black'],fields=['scrapWeight'],dimensionStages={'defect':['black']}),semantic_ok()]
payloads=[]
def opener(request,timeout):
payloads.append(json.loads(json.loads(request.data)['messages'][1]['content']))
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(replies.pop(0))}}]}).encode())
planned=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='查看具体缺陷排名',context={},cancel_event=threading.Event(),opener=opener)
self.assertEqual(planned['kind'],'analysis')
self.assertEqual(planned['fields'],['scrapWeight'])
self.assertEqual(payloads[1]['validationDetails']['field'],'kind')
def test_semantic_coverage_is_independent_of_user_wording(self):
for question in ['分析整体质量变化', '帮我全面看看最近的质量表现', '最近生产表现还好吗']:
context = self.context(question)
self.assertEqual(len(plan_missing_queries([evidence('quality_comparison')], question, context)), 5)
self.assertEqual(len(coverage_gaps([evidence('quality_comparison')], question, context)), 5)
question = '哪些产品恶化最明显'
context = self.context(question, dimensions=['product'])
self.assertEqual(len(plan_missing_queries([evidence('quality_comparison')], question, context)), 1)
def test_all_source_defect_fields_are_not_misread_as_all_raw_records(self):
question = ('改为只看黑片,客户800001不变,期间仍是2026年8至9月与7月比较。'
'新增加工单位等于机加工部,并排除规格18X15X4+Z;'
'按缺陷维度列出全部来源细项重量和变化,保留柱状图。')
requirements = _explicit_read_safeguards(question)
self.assertFalse(requirements['requiresAllSourceRows'])
self.assertFalse(requirements['requiresSourceVerification'])
for aggregate_request in ('汇总全部来源细项重量', '列出所有源字段汇总',
'比较全部缺陷类别重量', '逐条列出全部来源缺陷字段汇总'):
with self.subTest(aggregate_request=aggregate_request):
self.assertFalse(_explicit_read_safeguards(aggregate_request)['requiresAllSourceRows'])
def test_explicit_row_level_source_requests_still_require_full_details(self):
for question in ('逐行读取原始记录', '读取全部源记录', '列出所有原始数据行',
'输出全部原始行字段', '每条记录列出所有字段'):
with self.subTest(question=question):
self.assertTrue(_explicit_read_safeguards(question)['requiresAllSourceRows'])
self.assertFalse(_explicit_read_safeguards('不需要逐行读取全部源记录')['requiresAllSourceRows'])
def test_semantic_metrics_accept_aliases_and_date_notation(self):
question = '只看SYNTH-16X15+T,2026-09对2026-08,黑片亮片良率和投入、报废'
value = task_value(kind='metrics', dimensions=[], filterUpdates={'product': 'SYNTH-16X15+T'},
period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2026, 'month': 8})
context = bind_task({}, normalize_task(value, question))
request = metric_request(question, context)
self.assertIsNotNone(request)
self.assertEqual(request['fields'], ['rate', 'totalWeight', 'scrapWeight'])
contract = formal_metric_coverage(metric_evidence(), question, context)
self.assertEqual(contract['missingEvidence'], [])
self.assertEqual(len(contract['metrics']), 12)
def test_semantic_full_records_and_source_verification_cannot_pass_on_query_only(self):
docs = [{'mode': 'query', 'result': {'kind': 'quality_query'}}]
for question, updates in [('把每一条记录都列出来', {'kind': 'details', 'fullDetails': True}),
('对照原始Excel核实数值', {'kind': 'source_verify', 'verifySource': True})]:
self.assertFalse(coverage_sufficient(docs, question, self.context(question, dimensions=[], **updates)))
def test_scope_updates_keep_other_filters_and_release(self):
context = {'releaseId': 'fixed-release', 'period': {'year': 2026, 'month': 9},
'group': '汽车交通-海外', 'filters': {'group': '汽车交通-海外', 'product': 'A'}}
task = normalize_task(task_value(filterUpdates={'group': '汽车交通-国内', 'product': None}), '那国内呢', context)
bound = bind_task(context, task)
self.assertEqual(bound['filters'], {'group': '汽车交通-国内'})
self.assertNotIn('group', bound)
self.assertEqual(bound['releaseId'], 'fixed-release')
self.assertEqual(bound['period'], context['period'])
for bad in ({'filterUpdates': {'releaseId': 'evil'}}, {'freshQuery': 'true'}, {'period': {'year': 2026, 'month': 13}}):
with self.assertRaises(TaskUnavailable): normalize_task(task_value(**bad), 'q', context)
def test_recoverable_filter_shapes_preserve_existing_scope(self):
context={'filters':{'department':'消费电子','product':'OLD'}}
value=task_value(filterUpdates={'product':[' 18X15X4+T '], 'department':' '})
self.assertEqual(normalize_task(value,'q',context)['filters'],{'department':'消费电子','product':'18X15X4+T'})
value['filterUpdates']={'product':[]}
self.assertEqual(normalize_task(value,'q',context)['filters'],context['filters'])
value['filterUpdates']={'product':None}
self.assertNotIn('product',normalize_task(value,'q',context)['filters'])
value['filterUpdates']={'product':['A','B']}
with self.assertRaises(TaskUnavailable) as err: normalize_task(value,'q',context)
self.assertEqual(err.exception.field,'filterUpdates.product')
def test_model_plan_is_validated_and_retried_without_keyword_parser(self):
replies = [{}, task_value(dimensions=['product'], freshQuery=True, dimensionStages={'product': []}),semantic_ok()]
def opener(request, timeout):
self.assertNotIn('apiKey', request.data.decode())
value = replies.pop(0)
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(value)}}]}).encode())
task = plan_task(base_url='https://example.test/v1', api_key='test-only', model='any-compatible-model',
question='按最新数据再跑一遍', context={}, previous=None,
cancel_event=threading.Event(), opener=opener)
self.assertTrue(task['freshQuery'])
self.assertEqual(task['dimensions'], ['product'])
self.assertFalse(replies)
def test_retry_semantics_retain_prior_question_without_using_partial_answer(self):
from unittest.mock import patch
context={'releaseId':'fixed-release',
'_serverQuestions':[{'turnId':'previous','question':'9月汽车交通的合格率为什么下降?','status':'PARTIAL'}]}
wrong=task_value(goal='比较9月报废构成',dimensions=['group'],stages=['black'],fields=['scrapWeight'],
comparisonRequired=False,dimensionStages={'group':['black']},period={'year':2026,'month':9})
correct=task_value(goal='比较国内与海外对9月合格率下降的变化线索',dimensions=['group'],
fields=['rate','totalWeight','scrapWeight'],comparisonRequired=True,dimensionStages={'group':[]},
period={'year':2026,'month':9},comparisonPeriod={'year':2026,'month':8})
rejection={'aligned':False,'reason':'前文问合格率变化,不是单期报废构成'}
replies=[wrong,rejection,rejection,correct,semantic_ok()]
payloads=[]
def response(**kwargs):
payloads.append(kwargs['payload']);return replies.pop(0)
with patch('nao_agent_task._model_json',side_effect=response):
planned=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='主要是国内还是国外的',context=context,cancel_event=threading.Event(),semantic_review=True)
self.assertTrue(planned['comparisonRequired'])
self.assertEqual(planned['comparisonPeriod'],{'year':2026,'month':8})
self.assertEqual(planned['dimensionStages'],{'group':[]})
self.assertEqual(planned['rootGoal'],'9月汽车交通的合格率为什么下降?')
self.assertEqual(payloads[0]['previousUserQuestions'],context['_serverQuestions'])
self.assertNotIn('answer',json.dumps(payloads[0]['previousUserQuestions']))
self.assertFalse(replies)
def test_last_valid_plan_survives_false_rejection_only_with_two_consistent_reviews(self):
from unittest.mock import patch
first=task_value(dimensions='invalid')
second=task_value(fields=['invalid'])
final=task_value(goal='查询今年产品合格率趋势',kind='metrics',dimensions=[],
fields=['rate'],stages=[],dimensionStages={},filterUpdates={'product':['18X15X4+T']},
period={'year':2026})
reject={'aligned':False,'reason':'产品分析缺少group'}
replies=[first,second,final,reject,semantic_ok(),semantic_ok()]
with patch('nao_agent_task._model_json',side_effect=replies) as call:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年18X15X4+T合格率变化趋势',context={},cancel_event=threading.Event())
self.assertEqual(task['filters']['product'],'18X15X4+T')
self.assertEqual(task['period'],{'year':2026})
self.assertEqual(task['fields'],['rate'])
def test_filter_only_followup_reuses_prior_query_plan_and_recomputes_child_scopes(self):
from unittest.mock import patch
plan = [
{'nodeId':'rank','purpose':'ranking','dimension':'product','limit':5,
'fields':['totalWeight'],'stages':['black'],'selector':{'kind':'base'},
'visualType':'pareto','visualPreference':'auto'},
{'nodeId':'trend','purpose':'trend','fields':['scrapRate'],'stages':['black'],
'selector':{'kind':'ranking_top','sourceNodeId':'rank','stage':'black',
'metric':'totalWeight','count':2},
'visualType':'line','visualPreference':'auto'},
]
previous_value = task_value(kind='analysis',queryPlan=plan,dimensions=['product'],
dimensionStages={'product':['black']},stages=['black'],fields=['totalWeight','scrapRate'],
trendRequired=True,comparisonRequired=False,period={'year':2026},periodCoverage='available')
previous = normalize_task(previous_value,'按产品重量排名后查看前两项趋势',{'_protocol':{'version':1}})
context = {'_protocol':{'version':1},'_agentTask':previous,'_serverQuestions':[]}
proposal = {'goal':'保持其他分析不变,只改部门筛选','kind':'analysis','freshQuery':True,
'verifySource':False,'fullDetails':False,'needsClarification':False,
'filterUpdates':{'department':'部门甲'},'periodCoverage':'available'}
reviewed = {'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,
'requiresAllSourceRows':False,'requiresTrend':True,'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[proposal,reviewed]):
planned = plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='其他不变,只把部门改为部门甲',context=context,
cancel_event=threading.Event())
self.assertEqual(planned['filters']['department'],'部门甲')
self.assertEqual(planned['queryPlan'],previous['queryPlan'])
self.assertEqual(planned['fields'],['totalWeight','scrapRate'])
self.assertTrue(planned['trendRequired'])
def test_stage_weight_top_with_same_stage_rate_uses_ranking_top_child(self):
from unittest.mock import patch
cases = [
('black','scrapWeight','黑片','报废重量',3),
('bright','totalWeight','亮片','投入重量',2),
]
for stage, metric, stage_label, metric_label, count in cases:
with self.subTest(stage=stage,metric=metric,count=count):
question = (f'按{stage_label}{metric_label}排名前{count}个规格,'
f'并显示同批规格的{stage_label}合格率')
plan = [
{'nodeId':'rank','purpose':'ranking','dimension':'product','limit':count,
'fields':[metric],'stages':[stage],'selector':{'kind':'base'},
'visualType':'bar','visualPreference':'auto','comparisonRequired':False},
{'nodeId':'top_rate','purpose':'metrics','fields':['rate'],'stages':[stage],
'selector':{'kind':'ranking_top','sourceNodeId':'rank','stage':stage,
'metric':metric,'count':count},
'visualType':None,'visualPreference':'none','comparisonRequired':False},
]
invalid_plan = copy.deepcopy(plan)
invalid_plan[1]['selector'].pop('count')
invalid = task_value(goal=question,kind='analysis',queryPlan=invalid_plan,
dimensions=['product'],fields=[metric,'rate'],stages=[stage],
dimensionStages={'product':[stage]},trendRequired=False,comparisonRequired=False,
period={'year':2026,'month':9},visualPreference='auto')
valid = {**invalid,'queryPlan':plan}
review = {**semantic_ok(),'requiresTrend':False,'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[invalid,valid,review]) as call:
planned = plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context={'_protocol':{'version':1}},
cancel_event=threading.Event())
self.assertEqual(planned['goal'],question)
self.assertEqual(planned['rootGoal'],question)
self.assertEqual(planned['kind'],'analysis')
self.assertEqual(planned['dimensions'],['product'])
self.assertEqual(set(planned['fields']),{metric,'rate'})
self.assertEqual(planned['stages'],[stage])
self.assertEqual(planned['dimensionStages'],{'product':[stage]})
self.assertFalse(planned['trendRequired'])
self.assertFalse(planned['comparisonRequired'])
self.assertEqual(len(planned['queryPlan']),2)
rank, rate = planned['queryPlan']
self.assertEqual((rank['purpose'],rank['fields'],rank['stages'],rank['limit']),
('ranking',[metric],[stage],count))
self.assertEqual(rank['selector'],{'kind':'base'})
self.assertEqual(rank['visualPreference'],'auto')
self.assertEqual(rate['purpose'],'metrics')
self.assertEqual(rate['fields'],['rate'])
self.assertEqual(rate['stages'],[stage])
self.assertEqual(rate['selector'],{'kind':'ranking_top','sourceNodeId':'rank',
'stage':stage,'metric':metric,'count':count})
self.assertIsNone(rate['visualType'])
self.assertEqual(rate['visualPreference'],'none')
self.assertNotIn('defect',planned['dimensions'])
system = call.call_args_list[0].kwargs['system']
self.assertIn('这批对象同阶段rate',system)
self.assertIn('rate节点只查rate',system)
review_system = call.call_args_list[2].kwargs['system']
self.assertIn('source/stage/metric/count与重量ranking一致',review_system)
retry = call.call_args_list[1].kwargs['payload']
self.assertIn('ranking_top for the same objects',
retry['requiredShapes']['queryPlanRankingTopStageRate'])
self.assertIn('保持ranking重量选择与ranking_top rate子查询',
retry['repairInstructions'])
from nao_native_planning import NATIVE_ROOT_PROMPT, STATIC_NODE_CONTRACT
self.assertIn('按一个阶段的totalWeight或scrapWeight选产品Top N',NATIVE_ROOT_PROMPT)
self.assertIn('selector={\"kind\":\"ranking_top\"',STATIC_NODE_CONTRACT)
def test_query_plan_prompt_explains_selector_objects_and_root_stage_union(self):
from unittest.mock import patch
nodes = [
{'nodeId':'rank','purpose':'ranking','dimension':'product','limit':10,
'fields':['totalWeight'],'stages':['black'],'selector':{'kind':'base'},
'visualType':'pareto','visualPreference':'auto','comparisonRequired':False},
{'nodeId':'trend','purpose':'trend','fields':['rate'],'stages':[],
'selector':{'kind':'ranking_top','sourceNodeId':'rank','stage':'black',
'metric':'totalWeight','count':2},
'visualType':'line','visualPreference':'auto','comparisonRequired':False},
]
invalid = task_value(kind='analysis',queryPlan=nodes,dimensions=['product'],
dimensionStages={'product':['black']},stages=['black','bright'],
fields=['rate','totalWeight'],trendRequired=True,comparisonRequired=False,
period={'year':2026,'monthStart':1,'monthEnd':9})
valid = {**invalid,'stages':['black']}
review = {**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[invalid,valid,review]) as call:
planned = plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='按黑片投入重量排名,再看前两项综合合格率走势',
context={'_protocol':{'version':1}},cancel_event=threading.Event())
self.assertEqual(planned['stages'],['black'])
prompt = call.call_args_list[0].kwargs['system']
self.assertIn('selector始终是带kind的JSON对象',prompt)
self.assertIn('{"kind":"base"}',prompt)
self.assertIn('{"kind":"catalog_members","dimension":"group","names":["部门甲-国内","部门甲-海外"]}',prompt)
self.assertIn('{"kind":"ranking_top","sourceNodeId":"rank","stage":"black","metric":"totalWeight","count":2}',prompt)
self.assertIn('{"kind":"previous_ranking","queryId":"trusted-query-id","stage":"black","metric":"totalWeight","ranks":[1,2]}',prompt)
self.assertIn('root stages必须是所有节点stages数组的字面去重并集',prompt)
self.assertIn('"stages":[]',prompt)
self.assertIn('nodeId必须符合[a-z][a-z0-9_-]{0,31}',prompt)
self.assertIn('root dimensionStages记录ranking/changes节点实际排名的维度和阶段',prompt)
self.assertIn('没有ranking节点时必须为{}',prompt)
self.assertIn('root dimensions来自ranking节点的dimension及selector的dimension',prompt)
self.assertIn('父filters中的department不会自动成为分析维度',prompt)
self.assertIn('产业部甲内比较国内和海外组:root dimensions=[group]、stages=[black]、dimensionStages={}',prompt)
self.assertIn('"needsClarification":false',prompt)
self.assertIn('多指定目录对象分别查询指标、比较或趋势时必须用queryPlan和catalog_members',prompt)
self.assertIn('上轮排名对象只能用previous_ranking选择',prompt)
self.assertIn('同比追问在root保留comparisonMode="yoy"',prompt)
self.assertIn('若去年来源缺失,保留同比期间并如实返回部分结果',prompt)
shapes = call.call_args_list[1].kwargs['payload']['requiredShapes']
self.assertEqual(shapes['queryPlanSelectorExamples'][0],{'kind':'base'})
self.assertEqual(shapes['queryPlanSelectorExamples'][2]['kind'],'ranking_top')
self.assertIn('root stages=[black]',shapes['queryPlanRootExample'])
self.assertIn('trend node fields=[rate], stages=[]',shapes['queryPlanRootExample'])
self.assertIn('expands to two product scopes',shapes['queryPlanRootExample'])
self.assertEqual(shapes['queryPlanNodeIdPattern'],
'[a-z][a-z0-9_-]{0,31}; lowercase ASCII initial, 1..32 chars')
self.assertIn('with no ranking nodes use {}',shapes['queryPlanDimensionStages'])
self.assertIn('comparison, trend, or any named/product/mixed defect request requires a bounded queryPlan',
shapes['queryPlanRequiredWhenGroupedMetrics'])
group_example = shapes['queryPlanGroupComparisonExample']
self.assertFalse(group_example['needsClarification'])
self.assertEqual(group_example['dimensions'],['group'])
self.assertEqual(group_example['stages'],['black'])
self.assertEqual(group_example['dimensionStages'],{})
self.assertEqual(group_example['queryPlan'][0]['stages'],[])
self.assertTrue(all(node['comparisonRequired'] for node in group_example['queryPlan']))
self.assertEqual(call.call_count,3)
def test_protocol_v1_grouped_metrics_without_query_plan_enters_planner_repair(self):
from unittest.mock import patch
groups=['部门甲-国内','部门甲-海外']
context={'_protocol':{'version':1},'_businessCatalog':{
'departments':['部门甲'],'groups':groups,
'groupDepartments':{name:'部门甲' for name in groups}}}
question='比较部门甲国内和海外的综合合格率及黑片投入重量同比'
legacy=task_value(goal=question,kind='metrics',dimensions=['group'],
dimensionStages={'group':['black']},fields=['rate','totalWeight'],stages=['black'],
trendRequired=False,comparisonRequired=True,comparisonMode='yoy',
period={'year':2026,'month':9},filterUpdates={'department':'部门甲'},
targetEntities={'group':groups})
with self.assertRaises(TaskUnavailable) as err:
normalize_task(legacy,question,context)
self.assertEqual(err.exception.field,'queryPlan')
self.assertEqual(err.exception.expected,'bounded node plan')
# Non-protocol callers retain the legacy normalization behavior.
self.assertTrue(normalize_task(legacy,question)['comparisonRequired'])
selector={'kind':'catalog_members','dimension':'group','names':groups}
valid={**legacy,'dimensionStages':{},'queryPlan':[
{'nodeId':'group_rate','purpose':'metrics','fields':['rate'],'stages':[],
'selector':selector,'visualType':'column','visualPreference':'auto','comparisonRequired':True},
{'nodeId':'group_weight','purpose':'metrics','fields':['totalWeight'],'stages':['black'],
'selector':selector,'visualType':'column','visualPreference':'auto','comparisonRequired':True},
]}
review={**semantic_ok(),'requiresTrend':False,'requiresComparison':True,
'targetEntities':{'group':groups}}
with patch('nao_agent_task._model_json',side_effect=[legacy,valid,review]) as call:
planned=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context=context,cancel_event=threading.Event())
self.assertEqual(len(planned['queryPlan']),2)
self.assertEqual(planned['comparisonMode'],'yoy')
self.assertEqual(planned['comparisonPeriod'],{'year':2025,'month':9})
self.assertEqual(call.call_args_list[1].kwargs['payload']['requiredShapes']['queryPlanRequiredWhenGroupedMetrics'],
'protocol v1: nonempty dimensions plus rate/scrapRate fields, comparison, trend, or any named/product/mixed defect request requires a bounded queryPlan; keep the full user requirement')
self.assertEqual(call.call_count,3)
def test_last_valid_plan_survives_transient_review_protocol_failure(self):
from unittest.mock import patch
bad_shape=task_value(dimensions='invalid')
bad_fields=task_value(fields=['invalid'])
valid=task_value(goal='查看今年合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
good={**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
responses=[bad_shape,bad_fields,valid,{'aligned':'true'},good]
events=[]
with patch('nao_agent_task._model_json',side_effect=responses) as call:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年合格率走势',context={},cancel_event=threading.Event(),on_validation_error=events.append)
self.assertTrue(task['trendRequired']);self.assertEqual(task['period'],{'year':2026})
self.assertEqual(call.call_count,5)
self.assertEqual(call.call_args_list[3].kwargs['payload']['plannedTask'],call.call_args_list[4].kwargs['payload']['plannedTask'])
self.assertEqual([(e['phase'],e['reviewAttempt'],e['field'],e['receivedType']) for e in events if e.get('phase')=='task_review'],
[('task_review',1,'aligned','str')])
def test_review_protocol_retry_is_bounded_and_keeps_valid_plan(self):
from unittest.mock import patch
plan=task_value(kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[])
malformed={'aligned':True,'requiresEvidence':'true','requiresSourceVerification':False,'requiresAllSourceRows':False}
with patch('nao_agent_task._model_json',side_effect=[plan,malformed,semantic_ok()]) as call:
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='查9月合格率',context={},cancel_event=threading.Event())
self.assertEqual(result['kind'],'metrics');self.assertEqual(call.call_count,3)
with patch('nao_agent_task._model_json',side_effect=[plan,TaskUnavailable('模型超时'),malformed]) as call:
with self.assertRaises(TaskUnavailable) as err:
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='查9月合格率',context={},cancel_event=threading.Event())
self.assertEqual(err.exception.field,'review.protocol');self.assertEqual(call.call_count,3)
def test_review_disagreement_never_bypasses_evidence_or_scope(self):
from unittest.mock import patch
value=task_value(dimensions=['group'],dimensionStages={'group':[]})
rejection={'aligned':False,'reason':'范围错误'}
contradictory={**semantic_ok(),'requiresSourceVerification':True}
with patch('nao_agent_task._model_json',side_effect=[value,rejection,semantic_ok(),contradictory]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='比较各业务分组',context={},cancel_event=threading.Event())
def test_monthly_trend_is_structured_and_semantic_review_rejects_annual_total(self):
from unittest.mock import patch
annual=task_value(goal='查询年度合格率',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026})
trend={**annual,'goal':'查询今年逐月合格率走势','trendRequired':True}
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[annual,review,trend,review]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='看看今年良率怎么变化',context={},cancel_event=threading.Event())
self.assertTrue(task['trendRequired']);self.assertEqual(task['periodCoverage'],'available')
with self.assertRaises(TaskUnavailable):normalize_task({**annual,'trendRequired':'true'},'q')
self.assertFalse(normalize_task(annual,'q')['trendRequired'])
def test_missing_trend_review_gets_bounded_independent_check(self):
from unittest.mock import patch
plan=task_value(goal='查询月度合格率走势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
incomplete={**{k:v for k,v in semantic_ok().items() if k!='requiresTrend'},'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[plan,dict(incomplete),{}, {'requiresTrend':True}]) as call:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年的变化趋势',context={},cancel_event=threading.Event())
self.assertTrue(task['trendRequired']);self.assertEqual(call.call_count,4)
self.assertEqual(call.call_args_list[2].kwargs['payload']['plannedTask']['goal'],plan['goal'])
with patch('nao_agent_task._model_json',side_effect=[plan,dict(incomplete),{},{}]) as call:
with self.assertRaises(TaskUnavailable) as err:
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年的变化趋势',context={},cancel_event=threading.Event())
self.assertEqual(err.exception.field,'requiresTrend.review');self.assertEqual(call.call_count,4)
current={**plan,'goal':'查询9月合格率','period':{'year':2026,'month':9},'trendRequired':False}
with patch('nao_agent_task._model_json',side_effect=[current,dict(incomplete),{'requiresTrend':False}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='9月合格率',context={},cancel_event=threading.Event())
self.assertFalse(task['trendRequired'])
def test_pure_trend_does_not_add_a_comparison_period_but_yoy_keeps_it(self):
from unittest.mock import patch
plan=task_value(goal='今年月度合格率趋势',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],
stages=[],period={'year':2026},trendRequired=True,comparisonRequired=True)
pure={**plan,'comparisonRequired':False}
verdict={**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[plan,verdict,pure,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年合格率每月走势',context={},cancel_event=threading.Event())
self.assertTrue(task['trendRequired']);self.assertFalse(task['comparisonRequired'])
self.assertIsNone(task['comparisonPeriod'])
yoy={**plan,'goal':'今年月度合格率同比趋势','comparisonPeriod':{'year':2025}}
yoy_review={**semantic_ok(),'requiresTrend':True,'requiresComparison':True}
with patch('nao_agent_task._model_json',side_effect=[yoy,yoy_review]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年合格率趋势和去年同比',context={},cancel_event=threading.Event())
self.assertTrue(task['comparisonRequired']);self.assertEqual(task['comparisonPeriod'],{'year':2025})
def test_ambiguous_catalogue_requires_choice_and_keeps_original_trend_for_followup(self):
from unittest.mock import patch
candidates=[{'name':'18X15X4+T','departments':[],'groups':[]},
{'name':'18X15X4+Z','departments':[],'groups':[]}]
product_match={'status':'ambiguous','query':'18x15x4','candidates':candidates,'candidateCount':2,'truncated':False}
context={'_businessCatalog':{'productMatch':product_match},
'_serverQuestions':[{'question':'你好','status':'COMPLETED'}]}
value=task_value(goal='今年18x15x4合格率变化趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True,
filterUpdates={'product':'18X15X4+T'})
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'productScope':'single'}
with patch('nao_agent_task._model_json',side_effect=[value,review]):
pending=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年18x15x4合格率变化趋势',context=context,cancel_event=threading.Event())
self.assertTrue(pending['needsClarification']);self.assertNotIn('product',pending['filters'])
self.assertIn('18X15X4+T',pending['clarification']);self.assertIn('18X15X4+Z',pending['clarification'])
self.assertEqual(pending['productResolution']['originalQuestion'],'今年18x15x4合格率变化趋势')
selected={'status':'exact','candidates':[candidates[0]],'candidateCount':1,'truncated':False,
'selectedFromClarification':True,'previousQuestion':'今年18x15x4合格率变化趋势'}
context['_businessCatalog']['productMatch']=selected
# Simulate a planner forgetting the prior year and trend and the only
# server history being an unrelated greeting.
short=task_value(goal='确认选项',kind='metrics',dimensions=['product'],dimensionStages={'product':[]},
fields=[],stages=[],filterUpdates={})
with patch('nao_agent_task._model_json',side_effect=[short,review]) as model:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选+T',context=context,previous=pending,cancel_event=threading.Event())
self.assertFalse(task['needsClarification']);self.assertEqual(task['filters']['product'],'18X15X4+T')
self.assertEqual(task['period'],{'year':2026});self.assertEqual(task['fields'],['rate'])
self.assertTrue(task['trendRequired']);self.assertEqual(task['dimensions'],[])
self.assertEqual(task['rootGoal'],'今年18x15x4合格率变化趋势')
self.assertEqual(task['productResolution']['selectedName'],'18X15X4+T')
self.assertEqual(model.call_args_list[1].kwargs['payload']['pendingProductQuestion'],task['rootGoal'])
selected['candidates']=[{'name':'18X15X4+OTHER'}]
with patch('nao_agent_task._model_json',side_effect=[short]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选+T',context=context,previous=pending,cancel_event=threading.Event())
def test_selected_product_is_bound_even_with_static_base_query_plan(self):
from unittest.mock import patch
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
previous=normalize_task(task_value(goal='今年18x15x4合格率变化趋势',kind='metrics',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026},
trendRequired=True,needsClarification=True,clarification='请选择规格'), '今年18x15x4合格率变化趋势', {})
previous['filters']={'exclusions':{'product':['18X15X4.1+Z']}}
previous['productResolution']={'originalQuestion':'今年18x15x4合格率变化趋势',
'productMatch':{'status':'ambiguous','candidates':candidates}}
context={'_protocol':{'version':1,'autonomousEnabled':True},'_businessCatalog':{'productMatch':{
'status':'exact','candidates':[candidates[0]],'selectedFromClarification':True,
'previousQuestion':'今年18x15x4合格率变化趋势'}}}
value=task_value(goal='查询已选规格的综合合格率',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9},queryPlan=[{
'nodeId':'value','purpose':'metrics','fields':['rate'],'stages':[],
'selector':{'kind':'base'},'visualPreference':'none','visualType':None}])
repaired={**value,'trendRequired':True,'period':{'year':2026},'queryPlan':[{
**value['queryPlan'][0],'purpose':'trend','visualType':'line'}]}
with patch('nao_agent_task._model_json',side_effect=[value,repaired,{**semantic_ok(),'requiresTrend':True}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选择18X15X4+T。',context=context,previous=previous,cancel_event=threading.Event())
self.assertEqual(task['filters']['product'],'18X15X4+T')
self.assertEqual(task['period'],{'year':2026})
self.assertTrue(task['trendRequired'])
self.assertEqual(task['queryPlan'][0]['selector'],{'kind':'base'})
self.assertEqual(task['queryPlan'][0]['purpose'],'trend')
self.assertEqual(task['filters']['exclusions']['product'],['18X15X4.1+Z'])
def test_new_analysis_without_supported_execution_route_is_repaired_before_confirmation(self):
from unittest.mock import patch
value=task_value(goal='查询9月综合合格率',kind='analysis',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9})
context={'_protocol':{'version':1,'autonomousEnabled':True}}
with patch('nao_agent_task._model_json',side_effect=[value,{**value,'kind':'metrics'}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='查2026年9月综合合格率',context=context,cancel_event=threading.Event())
self.assertEqual(task['kind'],'metrics');self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_explicit_history_only_report_cannot_restart_prior_investigation(self):
from unittest.mock import patch
from nao_native_planning import history_only_requested
question='请用本会话第一轮已核实证据生成分析报告,不重新查询数据。'
self.assertTrue(history_only_requested(question))
self.assertFalse(history_only_requested('不用报告,请重新查询2026年9月数据'))
self.assertFalse(history_only_requested('不重新查询历史数据,但请重新查询最新数据'))
bad=task_value(goal='重新分析9月下降原因',kind='analysis',executionMode='native_agent',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026,'month':9})
good=task_value(goal='复用第一轮证据整理报告',kind='report',historyMode='first',freshQuery=False,
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False,trendRequired=False)
context={'_protocol':{'version':1,'autonomousEnabled':True}}
with patch('nao_agent_task._model_json',side_effect=[bad,good]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context=context,cancel_event=threading.Event())
self.assertEqual(task['kind'],'report');self.assertFalse(task['freshQuery'])
self.assertEqual(task['historyMode'],'first');self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_first_history_mode_requires_new_autonomous_report(self):
value=task_value(kind='report',historyMode='first',freshQuery=False,
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False)
for context in ({}, {'_protocol':{'version':1,'autonomousEnabled':False}}):
with self.subTest(context=context), self.assertRaises(TaskUnavailable):
normalize_task(value,'整理首轮结果',context)
with self.assertRaises(TaskUnavailable):
normalize_task({**value,'kind':'restate'},'整理首轮结果',
{'_protocol':{'version':1,'autonomousEnabled':True}})
def test_history_no_read_phrasings_repair_before_any_fresh_query(self):
from unittest.mock import patch
from nao_native_planning import history_only_requested
questions=('仅用本会话第一轮已核实结果生成报告,不要开展新查询',
'基于第一轮已核实结果,不做新的查询,生成报告',
'只整理第一轮已核实结果,不需要重新读取数据')
bad=task_value(goal='重新查数',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9})
good=task_value(goal='整理第一轮已核实结果',kind='report',historyMode='first',freshQuery=False,
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False)
for question in questions:
with self.subTest(question=question):
self.assertTrue(history_only_requested(question))
with patch('nao_agent_task._model_json',side_effect=[bad,good]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context={'_protocol':{'version':1,'autonomousEnabled':True}},
cancel_event=threading.Event())
self.assertEqual(task['historyMode'],'first')
self.assertFalse(task['freshQuery'])
self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_first_history_request_cannot_silently_select_latest(self):
from unittest.mock import patch
value=task_value(goal='整理首轮已核实证据',kind='report',historyMode='last',freshQuery=False,
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False)
with patch('nao_agent_task._model_json',side_effect=[value,{**value,'historyMode':'first'}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='用本会话第一轮已核实证据整理报告,不重新查询数据。',
context={'_protocol':{'version':1,'autonomousEnabled':True}},cancel_event=threading.Event())
self.assertEqual(task['historyMode'],'first')
self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_history_report_allows_null_period_fields_after_previous_report_retry(self):
from unittest.mock import patch
# A previous report can itself be PARTIAL and have no receipt. Its
# summary task remains useful conversation context, but it must not
# turn an explicit null date in the next report proposal into a fresh
# scope change or become the source set for the report.
previous = task_value(goal='整合会话已确认结果', kind='report', historyMode='all',
freshQuery=False, dimensions=[], dimensionStages={}, fields=[], stages=[],
comparisonRequired=False, trendRequired=False, filterUpdates={},
period=None, comparisonPeriod=None, visualPreference='none',
periodCoverage='complete')
previous['question'] = '基于本会话已完成的结果生成综合报告,按各轮实际期间和范围组织。'
previous.update(historySelectionHash='a' * 64,
historySourceTurnIds=['completed-1', 'completed-2', 'completed-3'])
context = {
'year': 2026, 'month': 9, 'view': 'overall', 'filters': {},
'_agentTask': previous,
'_serverHistory': [
{'turnId': 'completed-1', 'status': 'COMPLETED'},
{'turnId': 'completed-2', 'status': 'COMPLETED'},
{'turnId': 'completed-3', 'status': 'COMPLETED'},
{'turnId': 'partial-report-no-receipt', 'status': 'PARTIAL'},
],
'_serverQuestions': [
{'turnId': 'completed-1', 'question': '9月3C规格Top5', 'status': 'COMPLETED'},
{'turnId': 'completed-2', 'question': '9月Top产品', 'status': 'COMPLETED'},
{'turnId': 'completed-3', 'question': '清除筛选后查看8月全公司Top3', 'status': 'COMPLETED'},
{'turnId': 'partial-report-no-receipt', 'question': '整合会话结果', 'status': 'PARTIAL'},
],
'_protocol': {'version': 1, 'autonomousEnabled': True, 'previousTask': previous},
}
proposal = task_value(goal='基于已完成的会话结果生成报告', kind='report',
historyMode='all', freshQuery=False, dimensions=[], dimensionStages={}, fields=[],
stages=[], comparisonRequired=False, trendRequired=False, filterUpdates={},
period=None, comparisonPeriod=None)
with patch('nao_agent_task._model_json', return_value=proposal):
task = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question='基于本会话已完成的结果生成报告,不查询新数据。', context=context,
previous=previous, cancel_event=threading.Event())
self.assertEqual(task['kind'], 'report')
self.assertEqual(task['historyMode'], 'all')
self.assertFalse(task['freshQuery'])
self.assertIsNone(task['period'])
self.assertIsNone(task['comparisonPeriod'])
self.assertEqual(task['filters'], {})
self.assertNotIn('historySourceTurnIds', task)
self.assertNotIn('historySelectionHash', task)
for scope_change in (
{'period': {'year': 2026, 'month': 9}},
{'comparisonPeriod': {'year': 2026, 'month': 8}},
{'filterUpdates': {'group': '消费电子-3C'}},
):
with self.subTest(scope_change=scope_change), self.assertRaises(TaskUnavailable) as error:
normalize_task({**proposal, **scope_change},
'基于本会话已完成的结果生成报告,不查询新数据。', context)
self.assertEqual(error.exception.field, 'freshQuery')
def test_selection_completes_pending_empty_metric_intent_without_erasing_scope(self):
from unittest.mock import patch
original='2026年9月18X15X4合格率是多少,不含18X15X4.1'
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
previous=normalize_task(task_value(goal=original,kind='analysis',dimensions=[],dimensionStages={},
fields=[],stages=[],period={'year':2026,'month':9},needsClarification=True,
clarification='请选择规格',filterUpdates={'exclusions':{'product':['18X15X4.1+Z']}}),original,{})
previous['productResolution']={'originalQuestion':original,'productMatch':{
'status':'ambiguous','candidates':candidates}}
context={'_protocol':{'version':1,'autonomousEnabled':True},'_businessCatalog':{'productMatch':{
'status':'exact','candidates':[candidates[0]],'selectedFromClarification':True,
'previousQuestion':original}}}
value=task_value(goal='查询选定规格本期综合率',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9})
with patch('nao_agent_task._model_json',side_effect=[{**value,'kind':'analysis'}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选择18X15X4+T。',context=context,previous=previous,cancel_event=threading.Event())
self.assertEqual(task['fields'],['rate']);self.assertEqual(task['kind'],'analysis')
from nao_protocol_execution import _confirmed_scalar_selection
self.assertTrue(_confirmed_scalar_selection({'_autonomousAnswer':True},task))
self.assertEqual(task['filters'],{'product':'18X15X4+T','exclusions':{'product':['18X15X4.1+Z']}})
self.assertEqual(task['rootGoal'],original)
def test_ambiguous_candidates_remain_clarifiable_when_planner_fails(self):
from unittest.mock import patch
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
match={'status':'ambiguous','query':'18x15x4','candidates':candidates,'candidateCount':2,'truncated':False}
context={'_businessCatalog':{'productMatch':match}}
with patch('nao_agent_task._model_json',side_effect=[{}, {}, {}]):
pending=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年18x15x4合格率变化趋势',context=context,cancel_event=threading.Event())
self.assertTrue(pending['needsClarification']);self.assertTrue(pending['productResolution']['plannerDeferred'])
self.assertIn('18X15X4+T',pending['clarification']);self.assertNotIn('product',pending['filters'])
context['_businessCatalog']['productMatch']={'status':'exact','candidates':[candidates[0]],
'selectedFromClarification':True,'previousQuestion':'今年18x15x4合格率变化趋势'}
planned=task_value(goal='今年18X15X4+T合格率变化趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
with patch('nao_agent_task._model_json',side_effect=[planned,{**semantic_ok(),'requiresTrend':True,'requiresComparison':False}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选+T',context=context,previous=pending,cancel_event=threading.Event())
self.assertEqual(task['filters']['product'],'18X15X4+T');self.assertTrue(task['trendRequired'])
self.assertEqual(task['period'],{'year':2026});self.assertEqual(task['fields'],['rate'])
self.assertEqual(task['rootGoal'],'今年18x15x4合格率变化趋势')
def test_customer_code_trend_stays_authorized_and_bound_in_queries_and_followup(self):
from unittest.mock import patch
from nao_query_contract import bind_query
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
context={'_businessCatalog':catalog,'filters':{'department':'消费电子','extraFilters':{'blankProducer':['Y']}}}
value=task_value(goal='查看客户代码 C-2026 的今年月度合格率走势',kind='metrics',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026},
trendRequired=True,filterUpdates={'extraFilters':{'customerCode':'C-2026'}})
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'customerCodes':['C-2026']}
with patch('nao_agent_task._model_json',side_effect=[value,review]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码 C-2026 今年的合格率趋势',context=context,cancel_event=threading.Event())
self.assertEqual(task['filters'],{'department':'消费电子','extraFilters':{'blankProducer':['Y'],'customerCode':['C-2026']}})
self.assertEqual(task['period'],{'year':2026});self.assertTrue(task['trendRequired'])
query=bind_query({'mode':'query'}, {'task':task,'releaseId':'fixed'})
self.assertEqual(query['filters']['extraFilters']['customerCode'],['C-2026'])
with self.assertRaises(ValueError):
bind_query({'mode':'query','filters':{'extraFilters':{'customerCode':['OTHER']}}},
{'task':task,'releaseId':'fixed'})
followup=task_value(goal='继续看客户 C-2026 的今年月度合格率趋势',kind='metrics',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],trendRequired=True,
filterUpdates={})
next_context=bind_task(context,task)
with patch('nao_agent_task._model_json',side_effect=[followup,review]):
next_task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='换成折线图',context=next_context,previous=task,cancel_event=threading.Event())
self.assertEqual(next_task['filters']['extraFilters']['customerCode'],['C-2026'])
self.assertEqual(next_task['period'],{'year':2026});self.assertTrue(next_task['trendRequired'])
denied={'_businessCatalog':{'extraFields':[]}}
with self.assertRaises(TaskUnavailable):normalize_task(value,'q',denied)
def test_customer_code_cannot_be_lost_when_planner_and_review_disagree(self):
from unittest.mock import patch
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
value=task_value(goal='查看客户代码 C-2026 今年月度合格率趋势',kind='metrics',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026},
trendRequired=True,filterUpdates={})
verdict={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'customerCodes':['C-2026']}
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C-2026今年合格率趋势',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertEqual(task['filters']['extraFilters']['customerCode'],['C-2026'])
with patch('nao_agent_task._model_json',side_effect=[value,verdict]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C-2026今年合格率趋势',context={'_businessCatalog':{'extraFields':[]}},cancel_event=threading.Event())
invented={**verdict,'customerCodes':['80099999']}
with patch('nao_agent_task._model_json',side_effect=[value,invented]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C-2026今年合格率趋势',context={'_businessCatalog':catalog},cancel_event=threading.Event())
def test_customer_filter_is_not_invented_or_retained_after_cancellation(self):
from unittest.mock import patch
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
context={'_businessCatalog':catalog,'filters':{'extraFilters':{'customerCode':['C-2026']}}}
keep=task_value(goal='今年整体合格率',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[],
period={'year':2026},filterUpdates={})
clear={**keep,'filterUpdates':{'extraFilters':{'customerCode':None}}}
verdict={**semantic_ok(),'customerCodes':[]}
with patch('nao_agent_task._model_json',side_effect=[keep,verdict,clear,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='不按客户筛选,看今年整体合格率',context=context,cancel_event=threading.Event())
self.assertNotIn('customerCode',task['filters'].get('extraFilters',{}))
invented={**keep,'filterUpdates':{'extraFilters':{'customerCode':['INVENTED']}}}
with patch('nao_agent_task._model_json',side_effect=[invented,verdict]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年整体合格率',context={'_businessCatalog':catalog},cancel_event=threading.Event())
def test_customer_code_switch_preserves_exact_new_case(self):
from unittest.mock import patch
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
context={'_businessCatalog':catalog,'filters':{'extraFilters':{'customerCode':['C1']}}}
value=task_value(goal='客户代码 c1 今年合格率',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[],
period={'year':2026},filterUpdates={'extraFilters':{'customerCode':['c1']}})
verdict={**semantic_ok(),'customerCodes':['c1']}
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码 C1 换成 c1,看今年合格率',context=context,cancel_event=threading.Event())
self.assertEqual(task['filters']['extraFilters']['customerCode'],['c1'])
def test_missing_customer_subject_is_rechecked_without_replanning(self):
from unittest.mock import patch
value=task_value(goal='客户代码C1合格率',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[],filterUpdates={})
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok(),{}, {'customerCodes':['C1']}]) as call:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C1合格率',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertEqual(task['filters']['extraFilters']['customerCode'],['C1']);self.assertEqual(call.call_count,4)
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok(),{},{}]) as call:
with self.assertRaises(TaskUnavailable) as err:
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C1合格率',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertEqual(err.exception.field,'customerCodes.review');self.assertEqual(call.call_count,4)
def test_product_shaped_customer_code_is_not_automatically_a_product(self):
from unittest.mock import patch
from nao_product_identification import _normalized
question='客户代码“18x15x4”今年合格率趋势'
token='18x15x4';at=_normalized(question).index(token)
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
mention={'status':'ambiguous','query':token,'normalizedSpan':[at,at+len(token)],
'candidates':candidates,'candidateCount':2,'truncated':False}
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}],
'productMatch':{**mention,'mentions':[mention]}}
value=task_value(goal='客户代码18x15x4今年按月合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True,
filterUpdates={'extraFilters':{'customerCode':[token]}})
verdict={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'customerCodes':[token],'productScope':'single'}
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertFalse(task['needsClarification']);self.assertNotIn('product',task['filters'])
self.assertEqual(task['filters']['extraFilters']['customerCode'],[token])
double='客户代码18x15x4和产品18x15x4今年合格率趋势'
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=double,context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertTrue(task['needsClarification']);self.assertIn('18X15X4+T',task['clarification'])
def test_single_product_analysis_cannot_lose_product_filter(self):
from unittest.mock import patch
match={'status':'exact','query':'18x15x4','candidates':[{'name':'18X15X4+T'}],
'candidateCount':1,'truncated':False}
catalog={'productMatch':match}
value=task_value(goal='为什么18X15X4+T合格率下降',kind='analysis',dimensions=['product'],
dimensionStages={'product':[]},fields=['rate'],stages=[],filterUpdates={},comparisonRequired=True)
verdict={**semantic_ok(),'productScope':'single'}
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='为什么18X15X4+T合格率下降',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertEqual(task['filters']['product'],'18X15X4+T');self.assertNotIn('product',task['dimensions'])
comparison={**verdict,'productScope':'compare'}
with patch('nao_agent_task._model_json',side_effect=[value,comparison]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='把18X15X4+T与其他产品比较',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertNotIn('product',task['filters']);self.assertIn('product',task['dimensions'])
def test_product_mentioned_as_background_is_not_automatically_applied(self):
from unittest.mock import patch
name='18X15X4+T';match={'status':'exact','query':name,'candidates':[{'name':name}],'candidateCount':1,'truncated':False}
catalog={'productMatch':match,'mentionedProducts':[name]}
value=task_value(goal='先不看18X15X4+T,查看今年总体合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True,filterUpdates={})
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'productScope':'other'}
with patch('nao_agent_task._model_json',side_effect=[value,review]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=value['goal'],context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertNotIn('product',task['filters'])
def test_greeting_does_not_become_the_business_root_question(self):
from unittest.mock import patch
value=task_value(goal='今年客户代码C1合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
context={'_serverQuestions':[{'question':'你好','status':'COMPLETED'}]}
with patch('nao_agent_task._model_json',side_effect=[value,{**semantic_ok(),'requiresTrend':True,'requiresComparison':False}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年客户代码C1合格率趋势',context=context,cancel_event=threading.Event())
self.assertNotEqual(task['rootGoal'],'你好')
def test_retry_does_not_accept_an_unverified_semantic_review(self):
from unittest.mock import patch
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
proposal=task_value(dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[proposal,{'aligned':'yes'},{'aligned':'yes'}]) as call:
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='主要是哪一组',context=context,cancel_event=threading.Event(),semantic_review=True)
self.assertEqual(call.call_count,3)
def test_retry_comparison_replans_a_history_only_answer_as_a_fresh_analysis(self):
from unittest.mock import patch
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
prior=task_value(kind='explain',freshQuery=False,historyMode='last',dimensions=[],fields=[],comparisonRequired=True)
analysis=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[prior,analysis,semantic_ok()]) as call:
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='哪个分组',
context=context,cancel_event=threading.Event(),semantic_review=True,fresh_retry=True)
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery']);self.assertEqual(call.call_count,3)
def test_unverified_business_claim_cannot_use_direct_conversation(self):
from unittest.mock import patch
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
bad={'kind':'explain','goal':'判断合格率下降主要来自哪个分组',
'directResponse':{'purpose':'acknowledgement','requiresMemory':False,
'answer':'主要是国内分组所致;国内合格率降幅明显更大,海外基本持平。'}}
good=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[bad,good,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='主要是国内还是国外',
context=context,cancel_event=threading.Event(),semantic_review=True,fresh_retry=True)
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery'])
def test_direct_response_requiring_evidence_is_replanned(self):
from unittest.mock import patch
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
bad={'kind':'explain','goal':'判断哪个分组需要关注',
'directResponse':{'purpose':'acknowledgement','requiresMemory':False,
'answer':'国内是重点关注对象,海外情况稳定。'}}
good=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[bad,{'aligned':True,'requiresEvidence':True},
good,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='主要是国内还是国外',
context=context,cancel_event=threading.Event(),semantic_review=True,fresh_retry=True)
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery'])
def test_first_question_direct_response_is_reviewed_for_evidence(self):
from unittest.mock import patch
bad={'kind':'explain','goal':'判断哪个分组需要关注',
'directResponse':{'purpose':'acknowledgement','requiresMemory':False,
'answer':'国内是重点关注对象,海外情况稳定。'}}
good=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[bad,{'aligned':True,'requiresEvidence':True},good,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='9月合格率下降主要是国内还是国外',context={},cancel_event=threading.Event())
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery'])
def test_direct_response_review_without_evidence_decision_fails_closed(self):
from unittest.mock import patch
direct={'kind':'explain','goal':'回应问候',
'directResponse':{'purpose':'greeting','requiresMemory':False,'answer':'你好。'}}
with patch('nao_agent_task._model_json',side_effect=[direct,{'aligned':True}]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='你好',context={},cancel_event=threading.Event())
def test_group_targets_are_bound_from_semantic_plan_and_scoped_catalog(self):
from unittest.mock import patch
groups=['汽车交通-国内','汽车交通-海外','其他部门-国内']
context={'_businessCatalog':{'groups':groups,'groupDepartments':{'汽车交通-国内':'新能源汽车磁钢',
'汽车交通-海外':'新能源汽车磁钢','其他部门-国内':'其他部门'}}}
value=task_value(goal='比较国内与海外的合格率变化',dimensions=['group'],dimensionStages={'group':[]},
filterUpdates={'department':'新能源汽车磁钢'})
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='主要是国内还是国外的',context=context,cancel_event=threading.Event())
self.assertEqual(result['targetEntities']['group'],groups[:2])
value.update(goal='比较这两个地区',targetEntities={'group':groups[:2]})
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='请比较这两个地区',context=context,cancel_event=threading.Event())
self.assertEqual(result['targetEntities']['group'],groups[:2])
def test_planner_repairs_with_invalid_plan_and_field_contract_then_continues(self):
replies=[task_value(dimensions='unknown'),task_value(fields=['unsupported']),task_value(dimensions=['product'],dimensionStages={'product': []}),semantic_ok()]
payloads=[]
def opener(request,timeout):
payloads.append(json.loads(json.loads(request.data)['messages'][1]['content']))
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(replies.pop(0))}}]}).encode())
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='绘制产品图',
context={},cancel_event=threading.Event(),opener=opener)
self.assertEqual(task['dimensions'],['product'])
self.assertEqual(len(payloads),4)
self.assertEqual(payloads[1]['invalidPlan']['dimensions'],'unknown')
self.assertEqual(payloads[1]['requiredShapes']['dimensions']['type'],'array')
self.assertIn('validationFeedback',payloads[2])
self.assertFalse(replies)
def test_repair_plan_cannot_widen_pinned_scope_or_period(self):
task = normalize_task(task_value(filterUpdates={'group': '消费电子-美洲'},
period={'year': 2026, 'month': 9}), 'q')
def run(query):
def opener(request, timeout):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({'queries': [query]})}}]}).encode())
return plan_repairs(base_url='https://example.test/v1', api_key='test-only', model='m', task=task,
evidence=[], gaps=['missing'], failures=[], release_id='fixed',
cancel_event=threading.Event(), opener=opener)
self.assertEqual(run({'mode': 'compare', 'filters': {}})[0]['filters']['group'],'消费电子-美洲')
with self.assertRaises(TaskUnavailable): run({'mode':'compare','filters':{'group':'其他分组'}})
with self.assertRaises(TaskUnavailable): run({'mode': 'execute_sql', 'sql': 'SELECT 1'})
with self.assertRaises(TaskUnavailable): run({'mode': 'query', 'filters': task['filters'], 'period': {'year': 2026, 'month': 8}})
self.assertEqual(run({'mode': 'query', 'filters': task['filters'], 'period': task['period'], 'releaseId': 'other'})[0]['releaseId'], 'fixed')
self.assertEqual(run({'mode': 'query', 'filters': task['filters']})[0]['period'], task['period'])
def test_single_period_metrics_bind_scope_stage_fields_and_source_months(self):
import copy
question = '九月黑片良率告诉我一下'
value = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 9}, filterUpdates={'product': 'SYNTH-16X15+T'})
context = bind_task({'releaseId': 'release-a'}, normalize_task(value, question))
side = metric_evidence()[0]['result']['current']
docs = [{'mode': 'query', 'result': {'kind': 'quality_metrics', 'releaseId': 'release-a',
'period': side['period'], 'filters': context['filters'], 'summary': side, 'coverage': side['coverage']}}]
self.assertEqual(semantic_metric_gaps(docs, question, context), [])
for mutation in ('month', 'product', 'rate', 'coverage'):
bad = copy.deepcopy(docs)
result = bad[0]['result']
if mutation == 'month': result['period']['month'] = 8
if mutation == 'product': result['filters']['product'] = 'OTHER'
if mutation == 'rate': result['summary']['black']['rate'] = None
if mutation == 'coverage': result['coverage']['sourceMonths'][0]['blackRows'] = 0
with self.subTest(mutation=mutation):
self.assertTrue(semantic_metric_gaps(bad, question, context))
def test_client_task_cannot_bypass_server_planner(self):
from nao_quality_runtime import _normalize_request
import uuid
payload = {key: str(uuid.uuid4()) for key in ('turnId', 'sessionId', 'actorId', 'releaseId')}
payload.update(question='q', accessToken='test-only', context={'_agentTask': {'freshQuery': False},
'agentContext': {'filters': {}}, '_resolvedMetricRequest': {}})
normalized = _normalize_request(payload)
self.assertEqual(normalized['context'], {})
def test_overall_task_keeps_primary_frame_when_native_drills_into_a_group(self):
from nao_evidence_report import select_comparison_frames
question = '9月份合格率下降原因'
context = self.context(question, dimensions=[], period={'year': 2026, 'month': 9}, comparisonRequired=True)
overall = evidence('quality_comparison')
local = evidence('quality_comparison', filters={'group': '汽车交通-国内'}, ref='Q00000002')
roots, limitations = select_comparison_frames([overall, local], context)
self.assertEqual(roots, [overall])
self.assertEqual(limitations, [])
class WeightRankingDefaultCoverageTests(unittest.TestCase):
def test_weight_ranking_without_explicit_chart_type_defaults_to_published_months(self):
value=task_value(kind='analysis', dimensions=['product'], stages=['bright'], fields=['totalWeight'],
trendRequired=False, comparisonRequired=False, visualType=None)
self.assertEqual(normalize_task(value,'今年亮片投入重量按产品排名')['periodCoverage'],'available')
value['periodCoverage']='complete'
self.assertEqual(normalize_task(value,'要求完整全年齐全')['periodCoverage'],'complete')
if __name__ == '__main__': unittest.main()