import copy import json import sys import tempfile import threading import unittest from pathlib import Path from unittest.mock import patch sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).resolve().parent)] from nao_agent_task import normalize_task from nao_protocol_execution import execute from test_nao_agent_task import task_value from test_nao_protocol_execution import BoundApi, RELEASE, TURN, identity class ComparisonPlanApi(BoundApi): def __init__(self, scope_tasks): super().__init__() self.scope_tasks = scope_tasks self.envelopes = [] def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None): self.envelopes.append({'path': path, 'nodeId': node_id, 'scopeId': scope_id}) if path == '/api/analysis/protocol/queries': return {'version': 1, 'scopes': [ {'nodeId': 'groups_compare', 'scopeId': 'scope-' + key, 'label': key, 'purpose': 'metrics', 'task': copy.deepcopy(task)} for key, task in self.scope_tasks.items()], 'pending': [], 'gaps': []} response = super().call(path, payload, method) if path.startswith('/api/metrics'): proof = response.get('protocolEvidence') if isinstance(proof, dict): proof['nodeId'] = node_id proof['scopeId'] = scope_id proof['queryId'] = f'{TURN}-{len(self.envelopes)}' proof['responseHash'] = f'{len(self.envelopes):064x}' return response class QueryPlanComparisonExecutionTests(unittest.TestCase): def test_missing_prior_year_composite_sources_finish_partial_with_root_frame_and_proofs(self): rule = {'nodeId': 'groups_compare', 'purpose': 'metrics', 'fields': ['rate'], 'stages': [], 'selector': {'kind': 'catalog_members', 'dimension': 'group', 'names': ['部门甲-国内', '部门甲-海外']}, 'visualPreference': 'none', 'visualType': None, 'comparisonRequired': True} value = task_value(kind='metrics', queryPlan=[rule], dimensions=['group'], dimensionStages={}, stages=[], fields=['rate'], trendRequired=False, comparisonRequired=True, period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2025, 'month': 9}, visualPreference='none', filterUpdates={'department': '部门甲'}) context = {'_protocol': {'version': 1}, '_businessCatalog': { 'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'], 'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}} task = normalize_task(value, '比较部门甲国内和海外2026年9月与2025年9月的综合合格率', context) child_tasks = {} for name in rule['selector']['names']: child_tasks[name] = {'version': 1, 'kind': 'metrics', 'question': task['question'], 'goal': task['goal'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'], 'filters': {**task['filters'], 'group': name}, 'dimensions': [], 'dimensionStages': {}, 'fields': ['rate'], 'stages': [], 'trendRequired': False, 'comparisonRequired': True, 'periodCoverage': task['periodCoverage'], 'visualPreference': 'none', 'visualType': None, 'freshQuery': True, 'historyMode': 'none'} class MissingPriorYearApi(ComparisonPlanApi): def __init__(self, scope_tasks): super().__init__(scope_tasks) self.actual_metric_responses = [] def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None): response = super().call(path, payload, method, node_id=node_id, scope_id=scope_id) if path.startswith('/api/metrics') and 'year=2025' in path: response['coverage'].update(rawRows=0, includedRows=0, sourceMonths=[], missingStages=['black', 'bright']) response['summary']['compositeRate'] = None if path.startswith('/api/metrics'): self.actual_metric_responses.append(copy.deepcopy(response)) return response api = MissingPriorYearApi(child_tasks) api.task = task request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']} events = [] with tempfile.TemporaryDirectory() as folder: root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir() with patch('nao_evidence_finalizer._open_model', side_effect=AssertionError( '缺少正式来源时不应调用模型补造事实')) as open_model: result = execute(request=request, task=task, authority={**identity(), 'task': task}, actor_root=root, delegated=api, connection=('offline', 'synthetic', 'model'), cancel_event=threading.Event(), emit=events.append) self.assertFalse(list((root / 'evidence').glob('nao-quality-*.json'))) private_completion = json.loads((root / 'private' / 'protocol-completion.json').read_text()) self.assertEqual(result['status'], 'PARTIAL') self.assertEqual(result['agentContext'], {'releaseId': RELEASE, 'filters': task['filters'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod']}) self.assertEqual(len(result['protocolEvidence']), 4, result.get('evidenceCompletion')) self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'groups_compare'}) self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']}, {'scope-部门甲-国内', 'scope-部门甲-海外'}) self.assertEqual(len({proof['queryId'] for proof in result['protocolEvidence']}), 4) self.assertTrue(result['evidenceCompletion']['missingEvidence']) self.assertTrue(all('2025' in gap for gap in result['evidenceCompletion']['missingEvidence']), result['evidenceCompletion']['missingEvidence']) self.assertFalse(result['scopedReport']['facts']) self.assertNotIn('100%', result['answer']) self.assertNotIn('100.00%', result['answer']) self.assertEqual(len([item for item in api.envelopes if item['path'].startswith('/api/metrics') and 'year=2025' in item['path']]), 2) self.assertEqual(len([item for item in api.envelopes if item['path'].startswith('/api/metrics')]), 4) missing_responses = [response for response in api.actual_metric_responses if response.get('context', {}).get('year') == 2025] self.assertEqual(len(missing_responses), 2) self.assertTrue(all(response['coverage'].get('rawRows') == 0 and response['coverage'].get('includedRows') == 0 and response['coverage'].get('sourceMonths') == [] for response in missing_responses)) from types import SimpleNamespace from nao_protocol_execution import _verified_empty_comparison_receipts domestic_responses = [response for response in api.actual_metric_responses if response.get('context', {}).get('group') == '部门甲-国内'] def receipt_observations(responses): return [{'release': copy.deepcopy(response.get('release')), 'context': copy.deepcopy(response.get('context')), 'filters': copy.deepcopy(response.get('filters')), 'coverage': copy.deepcopy(response.get('coverage')), 'protocolEvidence': copy.deepcopy(response.get('protocolEvidence')), 'hasSummary': isinstance(response.get('summary'), dict)} for response in responses] scope = {'nodeId': 'groups_compare', 'scopeId': 'scope-部门甲-国内', 'label': '部门甲-国内'} receipts = _verified_empty_comparison_receipts(SimpleNamespace( metric_responses=receipt_observations(domestic_responses)), scope=scope, task=child_tasks['部门甲-国内'], identity=identity(), release_id=RELEASE) self.assertIsNotNone(receipts) for mutate in ( lambda response: response['context'].pop('year', None), lambda response: response['coverage'].__setitem__('rawRows', -1), lambda response: response['coverage'].__setitem__('includedRows', 2), lambda response: response['coverage'].pop('includedRows', None), lambda response: response['coverage'].pop('rawRows', None), lambda response: response['protocolEvidence'].update(scopeId='forged-scope'), lambda response: response['protocolEvidence'].update(executionId='forged-execution')): invalid = copy.deepcopy(domestic_responses) mutate(invalid[1]) self.assertIsNone(_verified_empty_comparison_receipts(SimpleNamespace( metric_responses=receipt_observations(invalid)), scope=scope, task=child_tasks['部门甲-国内'], identity=identity(), release_id=RELEASE)) self.assertTrue(any('正式查询 4/40' in event.get('message', '') for event in events)) diagnostics = private_completion['queryDiagnostics'] self.assertEqual(len(diagnostics), 4) self.assertEqual(sum(row['rawRows'] == 0 for row in diagnostics), 2) self.assertEqual(sum(row['includedRows'] == 0 for row in diagnostics), 2) self.assertTrue(all(row['nodeId'] == 'groups_compare' and row['queryId'] for row in diagnostics)) open_model.assert_not_called() def test_one_empty_group_keeps_other_group_charts_ready_while_turn_is_partial(self): rule = {'nodeId': 'groups_compare', 'purpose': 'metrics', 'fields': ['rate', 'totalWeight', 'scrapWeight'], 'stages': ['black'], 'selector': {'kind': 'catalog_members', 'dimension': 'group', 'names': ['部门甲-国内', '部门甲-海外']}, 'visualPreference': 'required', 'visualType': 'column', 'comparisonRequired': True} value = task_value(kind='metrics', queryPlan=[rule], dimensions=['group'], dimensionStages={}, stages=['black'], fields=rule['fields'], trendRequired=False, comparisonRequired=True, period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2025, 'month': 9}, visualPreference='required', visualType='column', filterUpdates={'department': '部门甲'}) context = {'_protocol': {'version': 1}, '_businessCatalog': { 'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'], 'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}} task = normalize_task(value, '比较部门甲国内和海外2026年9月与2025年9月黑片合格率和重量', context) child_tasks = {name: {'version': 1, 'kind': 'metrics', 'question': task['question'], 'goal': task['goal'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'], 'filters': {**task['filters'], 'group': name}, 'dimensions': [], 'dimensionStages': {}, 'fields': list(rule['fields']), 'stages': ['black'], 'trendRequired': False, 'comparisonRequired': True, 'periodCoverage': task['periodCoverage'], 'visualPreference': 'required', 'visualType': 'column', 'freshQuery': True, 'historyMode': 'none'} for name in rule['selector']['names']} class MissingOverseasApi(ComparisonPlanApi): def __init__(self, scope_tasks): super().__init__(scope_tasks) self.actual_metric_responses = [] def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None): response = super().call(path, payload, method, node_id=node_id, scope_id=scope_id) if (path.startswith('/api/metrics') and response.get('context', {}).get('group') == '部门甲-海外' and response.get('context', {}).get('year') == 2025): response['coverage'].update(rawRows=0, includedRows=0, sourceMonths=[], missingStages=['black', 'bright']) response['summary']['compositeRate'] = None if path.startswith('/api/metrics'): self.actual_metric_responses.append(copy.deepcopy(response)) return response api = MissingOverseasApi(child_tasks) api.task = task request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']} with tempfile.TemporaryDirectory() as folder: root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir() with patch('nao_evidence_finalizer.synthesize', return_value={ 'answer': '国内对照数据有效,海外缺少去年来源。', 'evidenceSufficient': True, 'missingEvidence': [], 'usage': {}}): result = execute(request=request, task=task, authority={**identity(), 'task': task}, actor_root=root, delegated=api, connection=('offline', 'synthetic', 'model'), cancel_event=threading.Event(), emit=lambda _: None) self.assertEqual(result['status'], 'PARTIAL') self.assertEqual(result['visualState'], 'ready') self.assertTrue(result['visuals']) self.assertEqual({chart.get('scope', {}).get('group') for chart in result['visuals']}, {'部门甲-国内'}) self.assertEqual({chart['metric'] for chart in result['visuals']}, {'black_rate', 'black_total_weight', 'black_scrap_weight'}) self.assertTrue(any('部门甲-海外' in gap and '2025' in gap for gap in result['evidenceCompletion']['missingEvidence'])) self.assertEqual(len(result['protocolEvidence']), 4) self.assertEqual(len(result['queryEvidenceRefs']), 1) def test_catalog_group_compare_uses_current_periods_without_mutating_root_filters(self): rule = {'nodeId': 'groups_compare', 'purpose': 'metrics', 'fields': ['rate', 'totalWeight', 'scrapWeight'], 'stages': ['black'], 'selector': {'kind': 'catalog_members', 'dimension': 'group', 'names': ['部门甲-国内', '部门甲-海外']}, 'visualPreference': 'required', 'visualType': 'column', 'comparisonRequired': True} value = task_value(kind='metrics', queryPlan=[rule], dimensions=['group'], dimensionStages={}, stages=['black'], fields=['rate', 'totalWeight', 'scrapWeight'], trendRequired=False, comparisonRequired=True, period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2026, 'month': 8}, filterUpdates={'department': '部门甲'}) context = {'_protocol': {'version': 1}, '_businessCatalog': { 'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'], 'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}} task = normalize_task(value, '比较部门甲的国内和海外两期黑片合格率', context) child_tasks = {} for name in rule['selector']['names']: child_tasks[name] = {'version': 1, 'kind': 'metrics', 'question': task['question'], 'goal': task['goal'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'], 'filters': {**task['filters'], 'group': name}, 'dimensions': [], 'dimensionStages': {}, 'fields': ['rate', 'totalWeight', 'scrapWeight'], 'stages': ['black'], 'trendRequired': False, 'comparisonRequired': True, 'periodCoverage': task['periodCoverage'], 'visualPreference': 'required', 'visualType': 'column', 'freshQuery': True, 'historyMode': 'none'} api = ComparisonPlanApi(child_tasks) api.task = task request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']} with tempfile.TemporaryDirectory() as folder: root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir() with patch('nao_evidence_finalizer.synthesize', return_value={ 'answer': '已核对国内和海外两期指标', 'evidenceSufficient': True, 'missingEvidence': [], 'usage': {}}) as synth: result = execute(request=request, task=task, authority={**identity(), 'task': task}, actor_root=root, delegated=api, connection=('offline', 'synthetic', 'model'), cancel_event=threading.Event(), emit=lambda _: None) self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence']) self.assertEqual(len(result['protocolEvidence']), 4) self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'groups_compare'}) self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']}, {'scope-部门甲-国内', 'scope-部门甲-海外'}) metric_calls = [call for call in api.envelopes if call['path'].startswith('/api/metrics')] self.assertTrue(metric_calls) self.assertTrue(all(call['nodeId'] == 'groups_compare' and call['scopeId'] for call in metric_calls)) self.assertEqual(result['agentContext']['filters'], task['filters']) self.assertNotIn('group', result['agentContext']['filters']) self.assertEqual(result['agentContext']['comparisonPeriod'], task['comparisonPeriod']) chart_keys = {'id', 'type', 'title', 'subtitle', 'unit', 'categories', 'series', 'evidenceRefs', 'requiredFields', 'scope', 'period', 'comparisonPeriod', 'stage', 'metric', 'coverage', 'note', 'originTurnId', 'sectionKey', 'dimension'} self.assertTrue(result['visuals']) self.assertTrue(all(set(chart) <= chart_keys for chart in result['visuals'])) self.assertTrue(all(not {'protocolNodeId', 'protocolScopeId', 'protocolScopeLabel'} & set(chart) for chart in result['visuals'])) for group in ('部门甲-国内', '部门甲-海外'): scoped_charts = [chart for chart in result['visuals'] if chart.get('scope', {}).get('group') == group] self.assertEqual({chart['metric'] for chart in scoped_charts}, {'black_rate', 'black_total_weight', 'black_scrap_weight'}) self.assertTrue(all(chart['type'] == 'column' for chart in scoped_charts)) synth.assert_called_once() if __name__ == '__main__': unittest.main()