import copy import io import json import sys import tempfile import threading import unittest from urllib.parse import parse_qs, urlsplit from contextlib import ExitStack from pathlib import Path from unittest.mock import MagicMock, patch sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts')) import nao_quality_runtime as runtime from nao_protocol_execution import execute from nao_agent_task import normalize_task, bind_task from test_nao_agent_task import task_value from test_nao_quality_tools import FakeApi, RELEASE ACTOR='11111111-1111-4111-8111-111111111111' SESSION='22222222-2222-4222-8222-222222222222' TURN='55555555-5555-4555-8555-555555555555' def task(filters=None): value = normalize_task(task_value(kind='metrics',dimensions=[],stages=[],fields=['rate'],trendRequired=True, period={'year':2026},filterUpdates=filters or {}),'今年合格率月度趋势',{'_protocol':{'version':1}}) if value['filters'].get('columnFilters') == []: value['filters'].pop('columnFilters') return value def ranking_task(**updates): plan=task_value(kind='analysis',dimensions=['product','group'], dimensionStages={'product':['bright','black'],'group':['bright']},stages=['black','bright'], fields=['totalWeight','scrapWeight'],period={'year':2026},trendRequired=False, visualPreference='auto',visualType='pareto') plan.update(updates) value=normalize_task(plan, '按产品和业务分组查看各阶段重量排名',{'_protocol':{'version':1}}) if value['filters'].get('columnFilters') == []:value['filters'].pop('columnFilters') return value def identity(): return {'attemptId':SESSION,'sessionId':ACTOR,'turnId':TURN,'revision':3,'executionId':TURN} class BoundApi(FakeApi): def __init__(self,proof=True,cancel=None): super().__init__();self.proof=proof;self.cancel=cancel def call(self,path,payload=None,method=None): result=super().call(path,payload,method) if path.startswith('/api/metrics'): parsed=parse_qs(urlsplit(path).query) if 'columnFilters' in parsed: result['context']['filters']={'columnFilters':json.loads(parsed['columnFilters'][0])} if self.proof:result['protocolEvidence']={**identity(),'queryId':TURN,'responseHash':'a'*64} if self.cancel:self.cancel.set() return result class BoundRankingApi(BoundApi): def __init__(self,proof=True,cancel=None): super().__init__(proof=proof,cancel=cancel);self.rank_query=0 def call(self,path,payload=None,method=None): result=super().call(path,payload,method) if path.startswith('/api/metrics'): self.rank_query+=1 parsed=parse_qs(urlsplit(path).query) view=parsed['view'][0] rows=[] for stage,total,scrap in (('black',100,10),('bright',150,12)): result['summary'][stage]['totalWeight']=total result['summary'][stage]['scrapWeight']=scrap for month in result['coverage']['sourceMonths']: month['blackRows']=10;month['brightRows']=12 for suffix,share in (('a',.6),('b',.4)): row={'key':view+'-'+suffix,'label':view+' '+suffix,'coverage':{'blackRows':2,'brightRows':2}} for stage in ('black','bright'): summary=result['summary'][stage] row[stage]={key:summary[key]*share for key in ('totalWeight','scrapWeight')} rows.append(row) result['rows']=rows;result['context']['rowCount']=len(rows) if 'monthStart' in parsed: result['context']['monthStart']=int(parsed['monthStart'][0]) result['context']['monthEnd']=int(parsed['monthEnd'][0]) if self.proof: result['protocolEvidence']={**identity(),'queryId':f'{TURN}-{self.rank_query}', 'responseHash':f'{self.rank_query:064x}'} return result class ProtocolExecutionTests(unittest.TestCase): def test_comparison_returns_both_proofs_and_the_confirmed_comparison_context(self): value=task_value(kind='metrics',dimensions=[],stages=['black'],fields=['rate'],trendRequired=False, comparisonRequired=True,period={'year':2026,'month':9},comparisonPeriod={'year':2026,'month':8},visualPreference='none') t=normalize_task(value,'2026年9月对比8月黑片合格率',{'_protocol':{'version':1}}) t['filters'].pop('columnFilters',None) class ComparisonApi(BoundApi): def call(inner,path,payload=None,method=None): response=super().call(path,payload,method) if path.startswith('/api/metrics'): month=parse_qs(urlsplit(path).query).get('month',['0'])[0] response['protocolEvidence']['queryId']='query-'+month return response with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',return_value={'answer':'已核对两个期间','evidenceSufficient':True,'missingEvidence':[]}): root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir() result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,authority=identity(),actor_root=root, delegated=ComparisonApi(),connection=('offline','unused','model'),cancel_event=threading.Event(),emit=lambda _:None) self.assertEqual(result['agentContext']['comparisonPeriod'],{'year':2026,'month':8}) self.assertEqual({p['queryId'] for p in result['protocolEvidence']},{'query-9','query-8'}) self.assertEqual(result['agentContext']['period'],{'year':2026,'month':9}) def exercise(self,filters=None,api=None): t=task(filters);request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']} event=threading.Event();api=api or BoundApi();emitted=[] with tempfile.TemporaryDirectory() as folder: root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir() with patch('nao_evidence_finalizer.synthesize',return_value={'answer':'月度正式走势','evidenceSufficient':True,'missingEvidence':[],'usage':{}}) as synth: result=execute(request=request,task=t,authority={**identity(),'task':t},actor_root=root, delegated=api,connection=('offline','synthetic','model'),cancel_event=event,emit=emitted.append) documents=[json.loads(p.read_text()) for p in (root/'evidence').glob('nao-quality-*.json')] return result,api,documents,synth def test_real_tools_and_visuals_preserve_bound_monthly_series_and_cleared_scope(self): selected,api,docs,_=self.exercise({'product':'18X15X4+T'}) self.assertEqual(selected['status'],'COMPLETED') self.assertEqual(selected['agentContext']['filters'],{'product':'18X15X4+T'}) self.assertTrue(any('product=18X15X4%2BT' in path for _,path,_ in api.calls)) self.assertEqual(len(docs),1) self.assertEqual(len(docs[0]['result']['trend']),12) self.assertEqual(docs[0]['result']['protocolEvidence']['executionId'],TURN) self.assertTrue(selected['visuals']) self.assertEqual(selected['protocolEvidence'][0]['queryId'],TURN) cleared,api,docs,_=self.exercise() self.assertEqual(cleared['agentContext']['filters'],{}) self.assertTrue(all('product=' not in path for _,path,_ in api.calls)) self.assertTrue(all('columnFilters=' not in path for _,path,_ in api.calls)) self.assertEqual(docs[0]['result']['filters'],{'includeUnassigned':False}) def test_query_without_matching_execution_proof_never_reaches_the_model(self): with self.assertRaises(ValueError):self.exercise(api=BoundApi(proof=False)) def test_weight_ranking_reads_dimension_stage_field_cartesian_product_before_shared_finalizer(self): t=ranking_task();api=BoundRankingApi();request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']} with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize', return_value={'answer':'已核验全量重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}) as synth: root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir() result=execute(request=request,task=t,authority={**identity(),'task':t},actor_root=root, delegated=api,connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None) docs=[json.loads(p.read_text()) for p in (root/'evidence').glob('nao-quality-*.json')] docs.sort(key=lambda doc:int(doc['result']['protocolEvidence']['queryId'].rsplit('-',1)[1])) reads=[parse_qs(urlsplit(path).query) for _,path,_ in api.calls if path.startswith('/api/metrics')] self.assertEqual(len(reads),6) self.assertEqual([doc['result']['dimension'] for doc in docs],['product','product','product','product','group','group']) self.assertEqual([doc['result']['stage'] for doc in docs],['bright','bright','black','black','bright','bright']) self.assertEqual([doc['result']['metric'] for doc in docs],['totalWeight','scrapWeight']*3) self.assertEqual(len(result['protocolEvidence']),6) self.assertEqual(len({proof['queryId'] for proof in result['protocolEvidence']}),6) self.assertTrue(all(proof['executionId']==TURN for proof in result['protocolEvidence'])) from nao_weight_ranking import matching_rankings,ranking_gaps bound_context=bind_task({'releaseId':RELEASE},t) self.assertEqual(result['status'],'COMPLETED', (result.get('visualState'), [(v.get('dimension'),v.get('stage'),v.get('metric')) for v in result.get('visuals',[])], result.get('evidenceCompletion'),len(matching_rankings(docs,t,bound_context)), ranking_gaps(docs,t,bound_context),[(d['result'].get('dimension'),d['result'].get('stage'),d['result'].get('metric'),d['result'].get('period'),d['result'].get('periodCoverage'),d['result'].get('rankingCoverage')) for d in docs])) self.assertTrue(result['visuals']) self.assertEqual(result['toolCount'],6) synth.assert_called_once() self.assertFalse(any('rowLimit' in params or 'rowOffset' in params for params in reads)) def test_ranking_and_trend_or_comparison_are_rejected_without_dropping_task_requirements(self): for updates in ({'trendRequired':True}, {'comparisonRequired':True,'comparisonPeriod':{'year':2025}}): with self.subTest(updates=updates): # This test exercises execute's fail-closed guard directly. # Planner normalization now correctly rejects the same legacy # shape earlier, before it could be confirmed as a protocol task. t=ranking_task();t.update(updates);api=BoundRankingApi() with tempfile.TemporaryDirectory() as folder: root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir() with self.assertRaisesRegex(ValueError,'不能与趋势或期间比较合并'): execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t, authority={**identity(),'task':t},actor_root=root,delegated=api, connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None) self.assertEqual(t['trendRequired'],updates.get('trendRequired',False)) self.assertEqual(t.get('comparisonPeriod'),updates.get('comparisonPeriod')) self.assertFalse(any(path.startswith('/api/metrics') for _,path,_ in api.calls)) def test_ranking_with_ready_but_incomplete_visual_set_finishes_partial(self): t=ranking_task();api=BoundRankingApi() with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize', return_value={'answer':'已核验重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}),\ patch('analysis_visual_evidence.visual_result',return_value={ 'visualSchemaVersion':1,'visualState':'ready','visuals':[]}): root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir() result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t, authority={**identity(),'task':t},actor_root=root,delegated=api, connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None) self.assertEqual(result['visualState'],'ready') self.assertEqual(result['status'],'PARTIAL') self.assertTrue(any('重量排名图表未覆盖' in item for item in result['evidenceCompletion']['missingEvidence'])) def test_clarification_executes_no_tool_or_finalizer(self): t=task();t.update(needsClarification=True,clarification='请选完整规格') with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize') as synth: api=MagicMock() result=execute(request={'releaseId':RELEASE,'question':t['question']},task=t,authority=identity(),actor_root=Path(folder), delegated=api,connection=('offline','key','model'),cancel_event=threading.Event(),emit=lambda _:None) self.assertEqual(result['status'],'NEEDS_CLARIFICATION');api.call.assert_not_called();synth.assert_not_called() def test_backend_previous_task_wins_over_local_task_and_chat(self): previous=task({'product':'18X15X4+T'}) planned=task();planned.update(needsClarification=True,clarification='请确认期间') options={'products':['18X15X4+T','18X15X4+Z'],'organizationPaths':[]} captured={} with tempfile.TemporaryDirectory() as folder: root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir() local=root/'private'/'last-agent-task.json' local.write_text(json.dumps({'sessionId':SESSION,'releaseId':RELEASE,'task':task({'product':'WRONG'})}));local.chmod(0o600) manager=MagicMock();manager.acquire_actor.return_value=(root,None) api=MagicMock();api.call.side_effect=lambda path,**kw: options if path.startswith('/api/options') else {'task':planned,**identity()} if path=='/api/analysis/protocol/task' else {} delegated=MagicMock();delegated.login.return_value=api gate=MagicMock();gate.base_url='http://127.0.0.1:1/v1' def plan(**kw):captured.update(copy.deepcopy({k:v for k,v in kw.items() if k!='cancel_event'}));return planned with ExitStack() as stack: for name,value in {'RuntimeManager':MagicMock(return_value=manager),'write_access_file':MagicMock(return_value=root/'access.json'), 'SessionState':MagicMock(),'_input_cancel_listener':MagicMock(),'_model_settings':MagicMock(return_value=('offline','key','model',{})), '_model_gate_settings':MagicMock(return_value=(1,'key')),'DelegatedApi':MagicMock(return_value=delegated),'plan_agent_task':plan}.items(): stack.enter_context(patch.object(runtime,name,value)) stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate)) result=runtime.run_turn({'turnId':TURN,'actorId':ACTOR,'sessionId':SESSION,'releaseId':'33333333-3333-4333-8333-333333333333', 'accessToken':'synthetic','question':'清除规格条件','context':{'_protocol':{'version':1,'attemptId':SESSION,'previousTask':previous}, '_serverHistory':[{'answer':'unverified'}],'lastVisibleAnswer':'unverified'}},stdout=io.StringIO(),stdin=io.StringIO()) self.assertEqual(result['status'],'NEEDS_CLARIFICATION') self.assertEqual(captured['previous'],previous) self.assertNotIn('_serverHistory',captured['context']);self.assertNotIn('lastVisibleAnswer',captured['context']) manager.ensure_instance.assert_not_called() self.assertEqual(json.loads(local.read_text())['task']['filters']['product'],'WRONG') def test_new_task_does_not_inherit_old_filters_or_period(self): context=bind_task({'_protocol':{'version':1}},task({'product':'OLD','department':'OLD'})) t=normalize_task(task_value(newTask=True,kind='metrics',dimensions=[],period={'year':2025}),'另一个新任务',context) self.assertEqual(t['filters'],{'columnFilters':[]});self.assertEqual(t['period'],{'year':2025}) if __name__=='__main__':unittest.main()