"""Evidence-bound monthly answers must not turn an annual snapshot into a trend.""" import copy import io import json import sys import threading import unittest from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) from nao_agent_report import build_noncomparison_report from nao_evidence_finalizer import coverage_sufficient, synthesize from nao_analysis_coverage import coverage_gaps, plan_missing_queries QUESTION = '查看客户甲今年的综合合格率变化趋势' RELEASE = 'release-trend' def task(**updates): value = {'version': 1, 'kind': 'metrics', 'question': QUESTION, 'goal': QUESTION, 'period': {'year': 2026}, 'comparisonPeriod': None, 'comparisonRequired': False, 'filters': {'extraFilters': {'customerCode': ['客户甲']}}, 'freshQuery': True, 'dimensions': [], 'dimensionStages': {}, 'stages': [], 'fields': ['rate'], 'historyMode': 'none', 'trendRequired': True, 'periodCoverage': 'available', 'visualPreference': 'auto', 'visualType': None, 'responseStyle': 'brief', 'verifySource': False, 'fullDetails': False} value.update(updates) return value def context(**updates): t = task(**updates) return {'releaseId': RELEASE, 'period': t['period'], 'filters': t['filters'], '_agentTask': t} def doc(months=(1, 3, 4), customer='客户甲', release=RELEASE, ref='Q12345678'): rows = [{'period': f'2026-{m:02d}', 'compositeRate': value, 'black': {'rate': value - .01, 'totalWeight': 100}, 'bright': {'rate': value + .01, 'totalWeight': 90}, 'coverage': {'blackRows': 2, 'brightRows': 2}} for m, value in zip(months, (.75, .85, .80, .78))] return {'evidenceRef': ref, 'tool': 'quality_data', 'mode': 'query', 'releaseId': release, 'result': { 'kind': 'quality_metrics', 'releaseId': release, 'view': 'overall', 'period': {'year': 2026}, 'filters': {'extraFilters': {'customerCode': [customer]}}, 'summary': {'compositeRate': .81}, 'trend': rows, 'trendCount': len(rows), 'trendTruncated': False, 'coverage': {'sourceMonths': [ {'year': 2026, 'month': m, 'blackRows': 2, 'brightRows': 2} for m in months]}}} class TrendEvidenceTests(unittest.TestCase): def test_annual_trend_uses_formal_months_and_keeps_missing_month_empty(self): report = build_noncomparison_report([doc()], context(), QUESTION) self.assertTrue(report['coverageSufficient']) self.assertTrue(coverage_sufficient([doc()], QUESTION, context())) text = '\n'.join(f['text'] for f in report['facts']) self.assertIn('1月75.00%', text) self.assertIn('2月未提供', text) self.assertIn('3月85.00%', text) self.assertIn('4月80.00%', text) self.assertIn('首末对比,4月较1月上升', text) self.assertNotIn('81.00%', text) self.assertEqual(report['facts'][0]['evidenceRef'], 'Q12345678') def test_explicit_comparison_still_requires_matching_comparison_evidence(self): comparison = context(comparisonRequired=True, comparisonPeriod={'year': 2025}) self.assertFalse(coverage_sufficient([doc()], QUESTION, comparison)) self.assertTrue(coverage_gaps([doc()], QUESTION, comparison)) def test_client_filter_and_release_must_match_exactly(self): wrong = doc(customer='客户乙') right = doc(ref='Q87654321') stale = doc(release='old', ref='QABCDEF12') for docs in ([wrong], [stale]): report = build_noncomparison_report(docs, context(), QUESTION) self.assertFalse(report['coverageSufficient']) self.assertFalse(coverage_sufficient(docs, QUESTION, context())) self.assertFalse(report['facts']) report = build_noncomparison_report([wrong, stale, right], context(), QUESTION) self.assertTrue(report['coverageSufficient']) self.assertEqual([f['evidenceRef'] for f in report['facts']], ['Q87654321']) def test_history_report_never_relabels_another_customer_as_the_current_scope(self): historical = context(kind='report', historyMode='all', freshQuery=False) wrong = doc(customer='客户乙') report = build_noncomparison_report([wrong], historical, QUESTION) self.assertFalse(report['coverageSufficient']) self.assertFalse(report['facts']) self.assertFalse(coverage_sufficient([wrong], QUESTION, historical)) def test_empty_truncated_wrong_year_and_duplicate_month_are_incomplete(self): cases = [] empty = doc(months=()); cases.append(empty) truncated = doc(); truncated['result']['trendTruncated'] = True; cases.append(truncated) count = doc(); count['result']['trendCount'] += 1; cases.append(count) wrong_year = doc(); wrong_year['result']['trend'][0]['period'] = '2025-01'; cases.append(wrong_year) duplicate = doc(); duplicate['result']['trend'][1]['period'] = '2026-01'; cases.append(duplicate) for item in cases: with self.subTest(item=item): self.assertFalse(build_noncomparison_report([item], context(), QUESTION)['coverageSufficient']) self.assertFalse(coverage_sufficient([item], QUESTION, context())) def test_no_trend_data_returns_an_explanation_without_asking_model_to_invent(self): empty = doc(months=()) def opener(request, timeout): raise AssertionError('No factual content should be sent for narration') result = synthesize(base_url='https://offline.invalid/v1', api_key='synthetic', model='m', question=QUESTION, evidence=[empty], cancel_event=threading.Event(), context=context(), opener=opener) self.assertFalse(result['evidenceSufficient']) self.assertIn('未取得符合本轮范围和期间的可用月度趋势数据', result['answer']) self.assertIn('不足以判断趋势', result['answer']) def test_single_stage_composite_retains_formal_business_rate(self): item = doc() for row in item['result']['trend'][:2]: row['coverage']['brightRows'] = 0 report = build_noncomparison_report([item], context(), QUESTION) self.assertTrue(report['coverageSufficient']) self.assertTrue(coverage_sufficient([item], QUESTION, context())) self.assertIn('75.00%', str(report['facts'])) self.assertIn('85.00%', str(report['facts'])) def test_stage_reference_both_directions_matches_chart_and_customer_scope(self): from analysis_visuals import build_visuals item = doc(months=(1, 2, 3, 4)) rows = item['result']['trend'] rows[0]['coverage']['blackRows'] = 0 rows[1]['coverage']['blackRows'] = 0 rows[1]['coverage']['brightRows'] = 0 rows[2]['coverage']['brightRows'] = 0 report = build_noncomparison_report([item], context(), QUESTION) self.assertTrue(report['coverageSufficient']) text = report['facts'][0]['text'] for value in ('1月75.00%', '2月未提供', '3月80.00%', '4月78.00%'): self.assertIn(value, text) visual = build_visuals([item], QUESTION, 'turn-customer-stage', context=context()) self.assertEqual(visual['visualState'], 'ready') chart = next(c for c in visual['visuals'] if c.get('metric') == 'composite_rate') self.assertEqual(chart['series'][0]['values'], [.75, None, .80, .78]) self.assertEqual(chart['scope']['extraFilters'], {'customerCode': ['客户甲']}) self.assertEqual(chart['evidenceRefs'], ['Q12345678']) self.assertEqual(build_visuals([doc(customer='客户乙')], QUESTION, 'turn-wrong-client', context=context())['visuals'], []) def test_both_stages_missing_does_not_create_observations(self): item = doc() for row in item['result']['trend'][:2]: row['coverage']['blackRows'] = 0 row['coverage']['brightRows'] = 0 report = build_noncomparison_report([item], context(), QUESTION) self.assertFalse(report['coverageSufficient']) self.assertFalse(coverage_sufficient([item], QUESTION, context())) self.assertTrue(any('不足以判断趋势' in value for value in report['missingEvidence'])) self.assertNotIn('75.00%', str(report['facts'])) self.assertNotIn('85.00%', str(report['facts'])) def test_two_valid_months_explain_chart_limit_and_required_chart_is_partial(self): item = doc(months=(1, 2)) report = build_noncomparison_report([item], context(), QUESTION) self.assertTrue(report['coverageSufficient']) self.assertIn('不足3个有效月份', report['facts'][0]['text']) required = context(visualPreference='required') self.assertFalse(build_noncomparison_report([item], required, QUESTION)['coverageSufficient']) self.assertFalse(coverage_sufficient([item], QUESTION, required)) def test_missing_trend_is_requeried_only_in_exact_requested_frame(self): ctx = context() wrong = doc(customer='客户乙') query = {'mode': 'query', 'releaseId': RELEASE, 'period': {'year': 2026}, 'filters': {'extraFilters': {'customerCode': ['客户甲']}}, 'view': 'overall'} self.assertEqual(plan_missing_queries([wrong], QUESTION, ctx), [query]) self.assertTrue(coverage_gaps([wrong], QUESTION, ctx)) self.assertEqual(plan_missing_queries([doc()], QUESTION, ctx), []) self.assertEqual(coverage_gaps([doc()], QUESTION, ctx), []) truncated = doc(); truncated['result']['trendTruncated'] = True self.assertEqual(plan_missing_queries([truncated], QUESTION, ctx), []) self.assertTrue(coverage_gaps([truncated], QUESTION, ctx)) def test_missing_trend_does_not_guess_period_or_drop_scope(self): for ctx in (context(period=None), context(period={'year': 2026, 'month': 9}), context(filters={'extraFilters': {'customerCode': ['客户甲']}, 'exclusions': {'product': ['X']}}), context(trendRequired=False), context(needsClarification=True)): with self.subTest(ctx=ctx): self.assertEqual(plan_missing_queries([], QUESTION, ctx), []) no_release = context(); no_release.pop('releaseId') self.assertEqual(plan_missing_queries([], QUESTION, no_release), []) def test_runtime_fills_missing_trend_and_keeps_failed_reads_incomplete(self): from unittest.mock import MagicMock, patch import nao_quality_runtime as runtime ctx = context() request = {'actorId': 'actor', 'sessionId': 'session', 'releaseId': RELEASE, 'question': QUESTION, 'context': ctx} for fails in (False, True): with self.subTest(fails=fails): docs = [doc(customer='客户乙')] seen = [] def read(arguments): seen.append(arguments) if fails: raise RuntimeError('no data') evidence = doc() docs.append(evidence) return evidence with patch('nao_evidence_finalizer.evidence_summaries', side_effect=lambda *a, **k: list(docs)), \ patch('nao_agent_task.plan_repairs', return_value=[]), \ patch.object(runtime, 'DelegatedApi') as delegated, \ patch('nao_quality_tools.QualityTools') as quality: delegated.return_value.login.return_value = MagicMock() quality.return_value.quality_data.side_effect = read runtime.complete_agent_evidence(task=ctx['_agentTask'], request=request, actor_root=Path('/offline/session'), access_path=Path('/offline/access'), manager=MagicMock(), connection=('offline', 'synthetic', 'm'), accumulator=runtime.ConversationAccumulator(), prior_files=frozenset(), cancel_event=threading.Event(), emit=lambda event: None) self.assertEqual(len(seen), 1) self.assertEqual(seen[0]['filters'], {'extraFilters': {'customerCode': ['客户甲']}}) self.assertEqual(coverage_sufficient(docs, QUESTION, ctx), not fails) def test_finalizer_includes_both_requested_stage_trends_even_when_model_omits_one(self): ctx = context(stages=['black', 'bright']) item = doc() def opener(request, timeout): answer = {'answerBlocks': [{'type': 'paragraph', 'factIds': ['F1']}], 'evidenceSufficient': True, 'missingEvidence': []} return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(answer)}}]}).encode()) result = synthesize(base_url='https://offline.invalid/v1', api_key='synthetic', model='m', question=QUESTION, evidence=[item], cancel_event=threading.Event(), context=ctx, opener=opener) self.assertIn('黑片合格率月度趋势', result['answer']) self.assertIn('亮片合格率月度趋势', result['answer']) self.assertEqual(result['selectedFactIds'], ['F1', 'F2']) if __name__ == '__main__': unittest.main()