Files
quality_analyse/verification/test_nao_analysis_progress.py
T

100 lines
6.7 KiB
Python

"""Regress the real recovered-compare failure without calling a model or changing old records."""
import json
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_quality_runtime as runtime
class ProgressRecoveryTest(unittest.TestCase):
def test_known_query_error_reason_is_visible_without_private_headers(self):
output={'isError':True,'structuredContent':{'error':{'code':'invalid_or_unavailable','message':'显式 scope 与 filters.group 不能同时指定'}}}
self.assertIn('不能同时指定',runtime.public_tool_failure(output))
self.assertNotIn('private-secret',runtime.public_tool_failure({'error':{'code':'provider_failure','message':'private-secret'}}))
def test_invalid_read_parameter_recovers_only_after_the_same_static_file_succeeds(self):
stream=runtime.ConversationAccumulator()
stream.feed({'type':'tool-input-error','toolName':'read','toolCallId':'bad-read',
'input':{'path':'/agent/mcps/quality/quality_data.json'},'errorText':'An error occurred.'})
stream.feed({'type':'tool-output-error','toolCallId':'bad-read','errorText':'An error occurred.'})
stream.feed({'type':'tool-input-available','toolName':'read','toolCallId':'rules','input':{'file_path':'/RULES.md'}})
stream.feed({'type':'tool-output-available','toolCallId':'rules','output':{'content':'rules'}})
self.assertEqual(self.finished(stream),('PARTIAL','mcp_error'))
stream.feed({'type':'tool-input-available','toolName':'read','toolCallId':'fixed','input':{'file_path':'/agent/mcps/quality/quality_data.json'}})
stream.feed({'type':'tool-output-available','toolCallId':'fixed','output':{'content':'schema'}})
self.assertEqual(self.finished(stream),('COMPLETED','stop'))
self.assertTrue(stream.tool_calls[0]['recovered'])
self.assertEqual(len(stream.tool_calls),3)
def call(self, stream, call_id, args, failed=False):
stream.feed({'type': 'tool-input-available', 'toolName': 'mcp_call', 'toolCallId': call_id,
'input': {'server': 'quality', 'tool': 'quality_data', 'arguments': args}})
stream.feed({'type': 'tool-output-available', 'toolCallId': call_id,
'output': {'isError': failed, 'content': [{'type': 'text', 'text': 'result'}]}})
def finished(self, stream):
stream.feed({'type': 'text-delta', 'id': 'a', 'delta': '结果及数据局限。'})
stream.feed({'type': 'finish', 'finishReason': 'stop'})
return stream.status()
def test_corrected_compare_does_not_poison_completed_turn(self):
stream = runtime.ConversationAccumulator()
args = {'mode': 'compare', 'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2026, 'month': 8}, 'filters': {'group': '工业电机-VCM'}}
self.call(stream, 'bad', {**args, 'view': 'overall'}, failed=True)
self.call(stream, 'fixed', args)
self.assertEqual(self.finished(stream), ('COMPLETED', 'stop'))
self.assertEqual(stream.tool_calls[0]['status'], 'failed')
self.assertTrue(stream.tool_calls[0]['recovered'])
def test_missing_defect_dimension_recovers_only_for_same_stage_and_scope(self):
base = {'mode':'defects','stage':'black','limit':20,'period':{'year':2026,'month':9},
'comparisonPeriod':{'year':2026,'month':8},'filters':{'product':'16.33X15.5X3.75+T'}}
stream = runtime.ConversationAccumulator()
self.call(stream, 'bad', base, failed=True)
self.call(stream, 'unrelated', {**base,'dimension':'defect','stage':'bright'})
self.assertEqual(self.finished(stream), ('PARTIAL','mcp_error'))
self.call(stream, 'fixed', {**base,'dimension':'defect'})
self.assertEqual(self.finished(stream), ('COMPLETED','stop'))
self.assertTrue(stream.tool_calls[0]['recovered'])
def test_corrected_conflicting_overall_scope_requires_same_group_success(self):
args={'mode':'changes','dimension':'product','limit':5,'filters':{'group':'汽车交通-国内'},
'period':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8}}
stream=runtime.ConversationAccumulator();self.call(stream,'bad',{**args,'scope':'overall'},failed=True)
self.call(stream,'other',{**args,'filters':{'group':'汽车交通-海外'}})
self.assertEqual(self.finished(stream),('PARTIAL','mcp_error'))
self.call(stream,'fixed',args)
self.assertEqual(self.finished(stream),('COMPLETED','stop'))
def test_unrelated_success_does_not_hide_an_unresolved_failure(self):
stream = runtime.ConversationAccumulator()
self.call(stream, 'bad', {'mode': 'compare', 'period': {'year': 2026, 'month': 9}}, failed=True)
self.call(stream, 'different', {'mode': 'query', 'period': {'year': 2026, 'month': 9}})
self.assertEqual(self.finished(stream), ('PARTIAL', 'mcp_error'))
def test_corrected_all_defects_recovers_changed_limit_only_when_all_results_returned(self):
args={'mode':'defects','dimension':'defect','stage':'black','limit':100,
'period':{'year':2026,'month':9},'filters':{'department':'工业电机产业部'}}
for complete in (False,True):
stream=runtime.ConversationAccumulator()
self.call(stream,'bad',{**args,'defect':'all'},failed=True)
stream.feed({'type':'tool-input-available','toolName':'mcp_call','toolCallId':'fixed',
'input':{'server':'quality','tool':'quality_data','arguments':{**args,'limit':50}}})
stream.feed({'type':'tool-output-available','toolCallId':'fixed','output':{'content':[{'type':'text','text':json.dumps({
'kind':'quality_fine_defects','rankingComplete':complete,'truncated':not complete,'pairedTruncated':not complete})}]}})
self.assertEqual(self.finished(stream),('COMPLETED','stop') if complete else ('PARTIAL','mcp_error'))
def test_operation_trace_preserves_query_not_secrets_or_file_contents(self):
stream = runtime.ConversationAccumulator()
self.call(stream, 'q', {'mode': 'query', 'releaseId': 'internal-release', 'apiKey': 'private-secret',
'period': {'year': 2026, 'month': 9}, 'filters': {'product': 'N52', 'token': 'private-secret'}})
encoded = json.dumps(stream.tool_calls)
self.assertNotIn('private-secret', encoded)
self.assertNotIn('internal-release', encoded)
self.assertIn('N52', encoded)
self.assertIn('startedAt', encoded)
self.assertIn('finishedAt', encoded)
self.assertNotIn('private prompt', json.dumps(runtime.public_tool_call('read', {'file': 'private prompt'})))
if __name__ == '__main__': unittest.main()