Files
quality_analyse/verification/test_nao_protocol_execution.py
T

321 lines
21 KiB
Python

import copy
import io
import json
import sys
import tempfile
import threading
import unittest
from urllib.parse import parse_qs, urlsplit
from contextlib import ExitStack
from pathlib import Path
from unittest.mock import MagicMock, patch
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
import nao_quality_runtime as runtime
from nao_protocol_execution import (execute, _controlled_failure_diagnostic,
_private_failure_diagnostic, _private_module_provenance)
from nao_agent_task import normalize_task, bind_task
from test_nao_agent_task import task_value
from test_nao_quality_tools import FakeApi, RELEASE
ACTOR='11111111-1111-4111-8111-111111111111'
SESSION='22222222-2222-4222-8222-222222222222'
TURN='55555555-5555-4555-8555-555555555555'
def task(filters=None):
value = normalize_task(task_value(kind='metrics',dimensions=[],stages=[],fields=['rate'],trendRequired=True,
period={'year':2026},filterUpdates=filters or {}),'今年合格率月度趋势',{'_protocol':{'version':1}})
if value['filters'].get('columnFilters') == []: value['filters'].pop('columnFilters')
return value
def ranking_task(**updates):
plan=task_value(kind='analysis',dimensions=['product','group'],
dimensionStages={'product':['bright','black'],'group':['bright']},stages=['black','bright'],
fields=['totalWeight','scrapWeight'],period={'year':2026},trendRequired=False,
visualPreference='auto',visualType='pareto')
plan.update(updates)
value=normalize_task(plan,
'按产品和业务分组查看各阶段重量排名',{'_protocol':{'version':1}})
if value['filters'].get('columnFilters') == []:value['filters'].pop('columnFilters')
return value
def identity():
return {'attemptId':SESSION,'sessionId':ACTOR,'turnId':TURN,'revision':3,'executionId':TURN}
class BoundApi(FakeApi):
def __init__(self,proof=True,cancel=None):
super().__init__();self.proof=proof;self.cancel=cancel
def call(self,path,payload=None,method=None):
result=super().call(path,payload,method)
if path.startswith('/api/metrics'):
parsed=parse_qs(urlsplit(path).query)
if 'columnFilters' in parsed:
result['context']['filters']={'columnFilters':json.loads(parsed['columnFilters'][0])}
if self.proof:result['protocolEvidence']={**identity(),'queryId':TURN,'responseHash':'a'*64}
if self.cancel:self.cancel.set()
return result
class BoundRankingApi(BoundApi):
def __init__(self,proof=True,cancel=None):
super().__init__(proof=proof,cancel=cancel);self.rank_query=0
def call(self,path,payload=None,method=None):
result=super().call(path,payload,method)
if path.startswith('/api/metrics'):
self.rank_query+=1
parsed=parse_qs(urlsplit(path).query)
view=parsed['view'][0]
rows=[]
for stage,total,scrap in (('black',100,10),('bright',150,12)):
result['summary'][stage]['totalWeight']=total
result['summary'][stage]['scrapWeight']=scrap
for month in result['coverage']['sourceMonths']:
month['blackRows']=10;month['brightRows']=12
for suffix,share in (('a',.6),('b',.4)):
row={'key':view+'-'+suffix,'label':view+' '+suffix,'coverage':{'blackRows':2,'brightRows':2}}
for stage in ('black','bright'):
summary=result['summary'][stage]
row[stage]={key:summary[key]*share for key in ('totalWeight','scrapWeight')}
rows.append(row)
result['rows']=rows;result['context']['rowCount']=len(rows)
if 'monthStart' in parsed:
result['context']['monthStart']=int(parsed['monthStart'][0])
result['context']['monthEnd']=int(parsed['monthEnd'][0])
if self.proof:
result['protocolEvidence']={**identity(),'queryId':f'{TURN}-{self.rank_query}',
'responseHash':f'{self.rank_query:064x}'}
return result
class ProtocolExecutionTests(unittest.TestCase):
def test_controlled_failure_diagnostics_classify_without_copying_exception_text(self):
from nao_quality_tools import QualityToolError, SourceCoverageUnavailable
source = _controlled_failure_diagnostic(
SourceCoverageUnavailable({'year': 2026, 'month': 9}), node_id='rank', purpose='ranking')
self.assertEqual(source['failureKind'], 'source_coverage')
self.assertEqual(source['nodeId'], 'rank')
self.assertNotIn('message', source)
upstream = runtime.RuntimeErrorSafe('token=secret SELECT * FROM private_table',
retryable=True, http_status=503)
transient = _controlled_failure_diagnostic(upstream, node_id='rank', purpose='ranking')
self.assertEqual(transient['failureKind'], 'transient_http')
self.assertEqual(transient['httpStatus'], 503)
self.assertTrue(transient['retryable'])
self.assertNotIn('secret', str(transient))
self.assertNotIn('private_table', str(transient))
denied = _controlled_failure_diagnostic(
runtime.RuntimeErrorSafe('upstream detail must not escape', retryable=False, http_status=403),
node_id='rank', purpose='ranking')
self.assertEqual(denied['failureKind'], 'http_error')
self.assertEqual(denied['httpStatus'], 403)
self.assertFalse(denied['retryable'])
self.assertNotIn('upstream detail', str(denied))
rejected = _controlled_failure_diagnostic(QualityToolError('scope=private-value'),
node_id='metrics', purpose='metrics')
self.assertEqual(rejected['failureKind'], 'formal_query_rejected')
self.assertNotIn('private-value', str(rejected))
product = '07394-L0.323XR28.96XR28.96X5.398X8.11+R+Z'
import nao_weight_ranking
import nao_combined_ranking
error = QualityToolError('正式指标接口未确认 filters.product;结果已拒绝。' + product)
try:
raise error
except QualityToolError as captured:
captured._quality_private_rank_trace = {
'queryId': 'private-query-id',
'coverageSourceMonths': [{'year': 2026, 'month': 9, 'blackRows': 2, 'brightRows': 15}],
'summaryCoverage': {'blackRows': 2, 'brightRows': 0},
}
error = captured
public = _controlled_failure_diagnostic(error, node_id='rank', purpose='ranking')
private = _private_failure_diagnostic(error, public, {'filters': {'product': product}}, {
'mode': 'rank', 'dimension': 'product', 'metric': 'scrapWeight',
'stage': None, 'limit': 1, 'filters': {'product': product},
})
self.assertNotIn('errorSummary', public)
self.assertNotIn('queryInput', public)
self.assertNotIn('rankResponseTrace', public)
self.assertIn('正式指标接口未确认 filters.product', private['errorSummary'])
self.assertNotIn(product, private['errorSummary'])
self.assertEqual(private['rankResponseTrace']['queryId'], 'private-query-id')
self.assertEqual(private['rankResponseTrace']['summaryCoverage'], {'blackRows': 2, 'brightRows': 0})
self.assertTrue(any(frame['function'] == 'test_controlled_failure_diagnostics_classify_without_copying_exception_text'
and frame['file'] == 'test_nao_protocol_execution.py'
for frame in private['exceptionFrames']))
self.assertNotIn('exceptionFrames', public)
self.assertEqual(private['queryInput'], {
'mode': 'rank', 'dimension': 'product', 'metric': 'scrapWeight',
'stage': None, 'limit': 1,
})
provenance = private['moduleProvenance']
self.assertNotIn('moduleProvenance', public)
for module_name, function_name in (
('nao_weight_ranking', 'rank_response'),
('nao_combined_ranking', 'combined_response')):
with self.subTest(module=module_name):
module = provenance['modules'][module_name]
self.assertTrue(module['loaded'])
self.assertTrue(module['file'].endswith('/scripts/' + module_name + '.py'))
self.assertEqual(len(module['functions'][function_name]), 64)
unsafe = _private_failure_diagnostic(
QualityToolError('SELECT * FROM private_table; token=secret'), public, {})
self.assertEqual(unsafe['errorSummary'], '受控查询校验失败;详细内容已隐藏。')
self.assertNotIn('private_table', unsafe['errorSummary'])
def test_comparison_returns_both_proofs_and_the_confirmed_comparison_context(self):
value=task_value(kind='metrics',dimensions=[],stages=['black'],fields=['rate'],trendRequired=False,
comparisonRequired=True,period={'year':2026,'month':9},comparisonPeriod={'year':2026,'month':8},visualPreference='none')
t=normalize_task(value,'2026年9月对比8月黑片合格率',{'_protocol':{'version':1}})
t['filters'].pop('columnFilters',None)
class ComparisonApi(BoundApi):
def call(inner,path,payload=None,method=None):
response=super().call(path,payload,method)
if path.startswith('/api/metrics'):
month=parse_qs(urlsplit(path).query).get('month',['0'])[0]
response['protocolEvidence']['queryId']='query-'+month
return response
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',return_value={'answer':'已核对两个期间','evidenceSufficient':True,'missingEvidence':[]}):
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,authority=identity(),actor_root=root,
delegated=ComparisonApi(),connection=('offline','unused','model'),cancel_event=threading.Event(),emit=lambda _:None)
self.assertEqual(result['agentContext']['comparisonPeriod'],{'year':2026,'month':8})
self.assertEqual({p['queryId'] for p in result['protocolEvidence']},{'query-9','query-8'})
self.assertEqual(result['agentContext']['period'],{'year':2026,'month':9})
def exercise(self,filters=None,api=None):
t=task(filters);request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']}
event=threading.Event();api=api or BoundApi();emitted=[]
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
with patch('nao_evidence_finalizer.synthesize',return_value={'answer':'月度正式走势','evidenceSufficient':True,'missingEvidence':[],'usage':{}}) as synth:
result=execute(request=request,task=t,authority={**identity(),'task':t},actor_root=root,
delegated=api,connection=('offline','synthetic','model'),cancel_event=event,emit=emitted.append)
documents=[json.loads(p.read_text()) for p in (root/'evidence').glob('nao-quality-*.json')]
return result,api,documents,synth
def test_real_tools_and_visuals_preserve_bound_monthly_series_and_cleared_scope(self):
selected,api,docs,_=self.exercise({'product':'18X15X4+T'})
self.assertEqual(selected['status'],'COMPLETED')
self.assertEqual(selected['agentContext']['filters'],{'product':'18X15X4+T'})
self.assertTrue(any('product=18X15X4%2BT' in path for _,path,_ in api.calls))
self.assertEqual(len(docs),1)
self.assertEqual(len(docs[0]['result']['trend']),12)
self.assertEqual(docs[0]['result']['protocolEvidence']['executionId'],TURN)
self.assertTrue(selected['visuals'])
self.assertEqual(selected['protocolEvidence'][0]['queryId'],TURN)
cleared,api,docs,_=self.exercise()
self.assertEqual(cleared['agentContext']['filters'],{})
self.assertTrue(all('product=' not in path for _,path,_ in api.calls))
self.assertTrue(all('columnFilters=' not in path for _,path,_ in api.calls))
self.assertEqual(docs[0]['result']['filters'],{'includeUnassigned':False})
def test_query_without_matching_execution_proof_never_reaches_the_model(self):
with self.assertRaises(ValueError):self.exercise(api=BoundApi(proof=False))
def test_weight_ranking_reads_dimension_stage_field_cartesian_product_before_shared_finalizer(self):
t=ranking_task();api=BoundRankingApi();request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']}
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',
return_value={'answer':'已核验全量重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}) as synth:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
result=execute(request=request,task=t,authority={**identity(),'task':t},actor_root=root,
delegated=api,connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
docs=[json.loads(p.read_text()) for p in (root/'evidence').glob('nao-quality-*.json')]
docs.sort(key=lambda doc:int(doc['result']['protocolEvidence']['queryId'].rsplit('-',1)[1]))
reads=[parse_qs(urlsplit(path).query) for _,path,_ in api.calls if path.startswith('/api/metrics')]
self.assertEqual(len(reads),6)
self.assertEqual([doc['result']['dimension'] for doc in docs],['product','product','product','product','group','group'])
self.assertEqual([doc['result']['stage'] for doc in docs],['bright','bright','black','black','bright','bright'])
self.assertEqual([doc['result']['metric'] for doc in docs],['totalWeight','scrapWeight']*3)
self.assertEqual(len(result['protocolEvidence']),6)
self.assertEqual(len({proof['queryId'] for proof in result['protocolEvidence']}),6)
self.assertTrue(all(proof['executionId']==TURN for proof in result['protocolEvidence']))
from nao_weight_ranking import matching_rankings,ranking_gaps
bound_context=bind_task({'releaseId':RELEASE},t)
self.assertEqual(result['status'],'COMPLETED', (result.get('visualState'),
[(v.get('dimension'),v.get('stage'),v.get('metric')) for v in result.get('visuals',[])],
result.get('evidenceCompletion'),len(matching_rankings(docs,t,bound_context)),
ranking_gaps(docs,t,bound_context),[(d['result'].get('dimension'),d['result'].get('stage'),d['result'].get('metric'),d['result'].get('period'),d['result'].get('periodCoverage'),d['result'].get('rankingCoverage')) for d in docs]))
self.assertTrue(result['visuals'])
self.assertEqual(result['toolCount'],6)
synth.assert_called_once()
self.assertFalse(any('rowLimit' in params or 'rowOffset' in params for params in reads))
def test_ranking_and_trend_or_comparison_are_rejected_without_dropping_task_requirements(self):
for updates in ({'trendRequired':True},
{'comparisonRequired':True,'comparisonPeriod':{'year':2025}}):
with self.subTest(updates=updates):
# This test exercises execute's fail-closed guard directly.
# Planner normalization now correctly rejects the same legacy
# shape earlier, before it could be confirmed as a protocol task.
t=ranking_task();t.update(updates);api=BoundRankingApi()
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
with self.assertRaisesRegex(ValueError,'不能与趋势或期间比较合并'):
execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,
authority={**identity(),'task':t},actor_root=root,delegated=api,
connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
self.assertEqual(t['trendRequired'],updates.get('trendRequired',False))
self.assertEqual(t.get('comparisonPeriod'),updates.get('comparisonPeriod'))
self.assertFalse(any(path.startswith('/api/metrics') for _,path,_ in api.calls))
def test_ranking_with_ready_but_incomplete_visual_set_finishes_partial(self):
t=ranking_task();api=BoundRankingApi()
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',
return_value={'answer':'已核验重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}),\
patch('analysis_visual_evidence.visual_result',return_value={
'visualSchemaVersion':1,'visualState':'ready','visuals':[]}):
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,
authority={**identity(),'task':t},actor_root=root,delegated=api,
connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
self.assertEqual(result['visualState'],'ready')
self.assertEqual(result['status'],'PARTIAL')
self.assertTrue(any('重量排名图表未覆盖' in item for item in result['evidenceCompletion']['missingEvidence']))
def test_clarification_executes_no_tool_or_finalizer(self):
t=task();t.update(needsClarification=True,clarification='请选完整规格')
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize') as synth:
api=MagicMock()
result=execute(request={'releaseId':RELEASE,'question':t['question']},task=t,authority=identity(),actor_root=Path(folder),
delegated=api,connection=('offline','key','model'),cancel_event=threading.Event(),emit=lambda _:None)
self.assertEqual(result['status'],'NEEDS_CLARIFICATION');api.call.assert_not_called();synth.assert_not_called()
def test_backend_previous_task_wins_over_local_task_and_chat(self):
previous=task({'product':'18X15X4+T'})
planned=task();planned.update(needsClarification=True,clarification='请确认期间')
options={'products':['18X15X4+T','18X15X4+Z'],'organizationPaths':[]}
captured={}
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
local=root/'private'/'last-agent-task.json'
local.write_text(json.dumps({'sessionId':SESSION,'releaseId':RELEASE,'task':task({'product':'WRONG'})}));local.chmod(0o600)
manager=MagicMock();manager.acquire_actor.return_value=(root,None)
api=MagicMock();api.call.side_effect=lambda path,**kw: options if path.startswith('/api/options') else {'task':planned,**identity()} if path=='/api/analysis/protocol/task' else {}
delegated=MagicMock();delegated.login.return_value=api
gate=MagicMock();gate.base_url='http://127.0.0.1:1/v1'
def plan(**kw):captured.update(copy.deepcopy({k:v for k,v in kw.items() if k!='cancel_event'}));return planned
with ExitStack() as stack:
for name,value in {'RuntimeManager':MagicMock(return_value=manager),'write_access_file':MagicMock(return_value=root/'access.json'),
'SessionState':MagicMock(),'_input_cancel_listener':MagicMock(),'_model_settings':MagicMock(return_value=('offline','key','model',{})),
'_model_gate_settings':MagicMock(return_value=(1,'key')),'DelegatedApi':MagicMock(return_value=delegated),'plan_agent_task':plan}.items():
stack.enter_context(patch.object(runtime,name,value))
stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate))
result=runtime.run_turn({'turnId':TURN,'actorId':ACTOR,'sessionId':SESSION,'releaseId':'33333333-3333-4333-8333-333333333333',
'accessToken':'synthetic','question':'清除规格条件','context':{'_protocol':{'version':1,'attemptId':SESSION,'previousTask':previous},
'_serverHistory':[{'answer':'unverified'}],'lastVisibleAnswer':'unverified'}},stdout=io.StringIO(),stdin=io.StringIO())
self.assertEqual(result['status'],'NEEDS_CLARIFICATION')
self.assertEqual(captured['previous'],previous)
self.assertNotIn('_serverHistory',captured['context']);self.assertNotIn('lastVisibleAnswer',captured['context'])
manager.ensure_instance.assert_not_called()
self.assertEqual(json.loads(local.read_text())['task']['filters']['product'],'WRONG')
def test_new_task_does_not_inherit_old_filters_or_period(self):
context=bind_task({'_protocol':{'version':1}},task({'product':'OLD','department':'OLD'}))
t=normalize_task(task_value(newTask=True,kind='metrics',dimensions=[],period={'year':2025}),'另一个新任务',context)
self.assertEqual(t['filters'],{'columnFilters':[]});self.assertEqual(t['period'],{'year':2025})
if __name__=='__main__':unittest.main()