"""Read recovery retries only the failed request, within the original scope and budget.""" import copy import sys import threading import unittest from pathlib import Path from urllib.error import HTTPError from unittest.mock import patch sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).resolve().parent)] import nao_protocol_execution as execution import test_nao_query_plan_execution as fixtures from nao_quality_runtime import RuntimeErrorSafe class Delegate: def __init__(self, outcomes): self.outcomes = list(outcomes) self.requests = [] def call(self, path, payload=None, method=None, **scope): self.requests.append((path, copy.deepcopy(payload), method, scope)) value = self.outcomes.pop(0) if isinstance(value, BaseException): raise value return value def transient(status=503): error = RuntimeErrorSafe('暂时无法读取') error.retryable = True error.http_status = status return error class QueryRecoveryTests(unittest.TestCase): def scoped(self, delegate, **kwargs): return execution.ScopedApi(delegate, 'node', 'scope', cancel_event=threading.Event(), **kwargs) def test_transient_read_retries_identical_envelope_and_counts_each_attempt(self): delegate = Delegate([transient(), {'protocolEvidence': {'queryId': 'success'}}]) events, diagnostics = [], [] api = self.scoped(delegate, emit=events.append, recovery_log=diagnostics, read_limit=2) payload = {'releaseId': 'release', 'offset': 100, 'limit': 100} response = api.call('/api/analysis/investigate', payload, 'POST') self.assertEqual(response['protocolEvidence']['queryId'], 'success') self.assertEqual(delegate.requests[0], delegate.requests[1]) self.assertEqual(api.read_count, 2) self.assertEqual(len(api.investigation_responses), 1) self.assertEqual(diagnostics[-1]['outcome'], 'recovered') self.assertTrue(any('重试' in event['message'] for event in events)) def test_http_retry_statuses_are_bounded_to_three_actual_attempts(self): for status in (429, 502, 503, 504): with self.subTest(status=status): delegate = Delegate([HTTPError('https://offline.invalid', status, '', {}, None) for _ in range(4)]) api = self.scoped(delegate) with self.assertRaises(HTTPError), patch.object(execution, '_QUERY_RETRY_DELAYS', (0, 0)): api.call('/api/metrics?year=2026', method='GET') self.assertEqual(len(delegate.requests), 3) self.assertEqual(api.read_count, 3) def test_permission_parameter_binding_and_unknown_failures_are_not_retried(self): failures = [HTTPError('https://offline.invalid', code, '', {}, None) for code in (400, 401, 403, 409, 422)] failures += [ValueError('正式查询证据未绑定当前执行'), RuntimeErrorSafe('所选期间没有来源数据')] explicit_permanent = transient() explicit_permanent.retryable = False failures.append(explicit_permanent) for failure in failures: with self.subTest(failure=str(failure)): delegate = Delegate([failure, {}]) api = self.scoped(delegate) with self.assertRaises(type(failure)): api.call('/api/metrics?year=2026', method='GET') self.assertEqual(api.read_count, 1) self.assertEqual(len(delegate.requests), 1) def test_mutating_endpoints_are_never_retried(self): for path in ('/api/analysis/protocol/task', '/api/analysis/protocol/finish'): delegate = Delegate([transient(), {}]) api = self.scoped(delegate) with self.assertRaises(RuntimeErrorSafe): api.call(path, {}, 'POST') self.assertEqual(len(delegate.requests), 1) self.assertEqual(api.read_count, 0) def test_budget_exhaustion_does_not_issue_an_extra_request(self): delegate = Delegate([transient(), {}]) api = self.scoped(delegate, read_limit=1) with self.assertRaises(RuntimeErrorSafe): api.call('/api/metrics?year=2026', method='GET') self.assertEqual(api.read_count, 1) self.assertEqual(len(delegate.requests), 1) with self.assertRaisesRegex(ValueError, '查询上限'): api.call('/api/metrics?year=2026', method='GET') self.assertEqual(len(delegate.requests), 1) def test_finalization_reserve_and_cancellation_stop_retries(self): for remaining, cancelled in ((300, False), (3000, True)): with self.subTest(remaining=remaining, cancelled=cancelled): cancel = threading.Event() class Failing(Delegate): def call(self, *args, **kwargs): if cancelled: cancel.set() return super().call(*args, **kwargs) delegate = Failing([transient(), {}]) api = execution.ScopedApi(delegate, 'node', 'scope', cancel_event=cancel) with patch.object(execution, 'remaining_seconds', side_effect=[3000, remaining]): with self.assertRaises((RuntimeErrorSafe, RuntimeError)): api.call('/api/metrics?year=2026', method='GET') self.assertEqual(len(delegate.requests), 1) def test_cancelled_before_query_does_not_call_delegate(self): cancel = threading.Event(); cancel.set() delegate = Delegate([{}]) api = execution.ScopedApi(delegate, 'node', 'scope', cancel_event=cancel) with self.assertRaisesRegex(RuntimeError, 'cancelled'): api.call('/api/metrics?year=2026', method='GET') self.assertEqual(len(delegate.requests), 0) def test_expired_query_time_preserves_finalization_without_starting_a_request(self): delegate = Delegate([{}]) api = self.scoped(delegate) with patch.object(execution, 'remaining_seconds', return_value=300): with self.assertRaisesRegex(ValueError, '时间不足'): api.call('/api/metrics?year=2026', method='GET') self.assertEqual(api.read_count, 0) self.assertEqual(len(delegate.requests), 0) def test_server_receipt_with_missing_data_is_not_retried(self): delegate = Delegate([{'coverage': {'rawRows': 0}}, {}]) api = self.scoped(delegate) self.assertEqual(api.call('/api/metrics?year=2026', method='GET')['coverage']['rawRows'], 0) self.assertEqual(len(delegate.requests), 1) def test_query_socket_deadline_reserve_is_scoped_and_restored(self): class Reserved(Delegate): deadline_reserve_seconds = 60 def call(self, *args, **kwargs): self.observed_reserve = self.deadline_reserve_seconds return super().call(*args, **kwargs) delegate = Reserved([{}]) api = self.scoped(delegate) api.call('/api/metrics?year=2026', method='GET') self.assertEqual(delegate.observed_reserve, execution.FINALIZATION_RESERVE_SECONDS + 5) self.assertEqual(delegate.deadline_reserve_seconds, 60) def test_only_failed_child_request_retries_and_completed_siblings_keep_their_evidence(self): class Intermittent(fixtures.QueryPlanApi): def __init__(self): super().__init__(); self.failed = False; self.failed_requests = [] def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None): if path.startswith('/api/metrics') and scope_id == 'trend-a' and not self.failed: self.failed = True self.failed_requests.append((path, node_id, scope_id)) raise transient(502) return super().call(path, payload, method, node_id=node_id, scope_id=scope_id) api = Intermittent() with patch.object(execution, '_QUERY_RETRY_DELAYS', (0, 0)): result, documents, _ = fixtures.QueryPlanExecutionTests().run_query(api) self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence']) self.assertEqual(len(documents), 3) self.assertEqual(len(result['protocolEvidence']), 3) self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'rank-scope']), 1) self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'trend-b']), 1) self.assertTrue(result['queryRecovery']) def test_exhausted_failed_child_stays_partial_and_other_children_are_not_repeated(self): class Unavailable(fixtures.QueryPlanApi): def __init__(self): super().__init__(); self.failed_attempts = 0 def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None): if path.startswith('/api/metrics') and scope_id == 'trend-a': self.failed_attempts += 1 raise transient(503) return super().call(path, payload, method, node_id=node_id, scope_id=scope_id) api = Unavailable() with patch.object(execution, '_QUERY_RETRY_DELAYS', (0, 0)): result, documents, _ = fixtures.QueryPlanExecutionTests().run_query(api) self.assertEqual(result['status'], 'PARTIAL') self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient']) self.assertEqual(api.failed_attempts, 3) self.assertEqual(len(documents), 2) self.assertEqual(len(result['protocolEvidence']), 2) self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'rank-scope']), 1) self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'trend-b']), 1) self.assertTrue(any(item['outcome'] == 'stopped' for item in result['queryRecovery'])) if __name__ == '__main__': unittest.main()