"""Synthetic regression cases for bounded metric intent and early completion.""" import copy import json import io import sys import tempfile import threading import unittest import uuid from pathlib import Path from unittest.mock import MagicMock, patch sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) import nao_quality_runtime as runtime import nao_evidence_finalizer as finalizer from nao_analysis_coverage import formal_metric_coverage from nao_metric_request import (resolve_metric_request, prepare_metric_context, frozen_metric_frame, metric_request, requires_fresh_query, is_history_restatement) QUESTION = '只看SYNTH-16X15+T,请重新查询2026年9月与8月黑片、亮片的合格率、报废重量和两期投入,附正式数据依据。' def metric_evidence(): result = {'kind': 'quality_comparison', 'releaseId': 'release-a', 'filterBinding': {'product': 'SYNTH-16X15+T'}, 'delta': {'available': True, 'percentagePoints': -2}, 'stageRateChangesPP': {'black': -1, 'bright': -1}} for side, month in [('current', 9), ('comparison', 8)]: result[side] = {'period': {'year': 2026, 'month': month}, 'compositeRate': .95, 'coverage': {'sourceMonths': [{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}]}, 'black': {'rate': .97, 'totalWeight': 1200.5, 'scrapWeight': 36.1}, 'bright': {'rate': .98, 'totalWeight': 2400.5, 'scrapWeight': 72.1}} return [{'tool': 'quality_data', 'releaseId': 'release-a', 'evidenceRef': 'Q00000001', 'mode': 'compare', 'result': result}] class MetricRequestRegressionTests(unittest.TestCase): def test_common_equivalent_questions_have_the_same_formal_contract(self): questions = [QUESTION, '请对比规格SYNTH-16X15+T在2026年9月和8月的黑片及亮片合格率、投入重量、报废重量。', '帮我查一下SYNTH-16X15+T的2026年9月与8月黑片、亮片合格率、投入量和报废量分别是多少?', 'SYNTH-16X15+T,2026年9月与8月,黑片和亮片的合格率、投入重量及报废重量,给我正式数据。'] for question in questions: with self.subTest(question=question): contract = formal_metric_coverage(metric_evidence(), question, {'releaseId': 'release-a'}) self.assertIsNotNone(contract) self.assertEqual(contract['missingEvidence'], []) self.assertEqual(len(contract['metrics']), 12) def _stream_readiness(self, *, mutate=None, question=QUESTION, model='mimo-v2.6-flash', excluded=False, cancelled=False, pending=False, failed_auxiliary=False): clock = [0.0] gates = [] readiness = [] class Stopper: cause = None stop_succeeded = None def __init__(self, stop, timeout, ready): self.ready = ready; gates.append(self) def set_chat_id(self, value): pass def set_response(self, value): pass def start(self): pass def close(self): pass def request_cancel(self, cause): self.cause = cause class Response: def __enter__(self): return self def __exit__(self, *args): pass def __iter__(self): yield b'data: {"type":"tool-input-available","toolName":"quality_data","toolCallId":"q1"}\n' if not pending: yield b'data: {"type":"tool-output-available","toolCallId":"q1","output":{"success":true}}\n' if failed_auxiliary: yield b'data: {"type":"tool-input-available","toolName":"quality_data","toolCallId":"q2"}\n' yield b'data: {"type":"tool-output-available","toolCallId":"q2","output":{"success":false,"error":"invalid auxiliary query"}}\n' readiness.append(gates[0].ready()) clock[0] = 3.0 readiness.append(gates[0].ready()) yield b'data: {"type":"text-delta","delta":"done"}\n' yield b'data: {"type":"finish","finishReason":"stop"}\n' yield b'data: [DONE]\n' class Client: def open(self, *args, **kwargs): return Response() with tempfile.TemporaryDirectory() as folder: root = Path(folder); (root / 'evidence').mkdir() evidence = root / 'evidence' / ('nao-quality-' + 'a' * 32 + '.json') document = metric_evidence()[0] if mutate: mutate(document['result']) evidence.write_text(json.dumps(document)); evidence.chmod(0o600) cancel_event = threading.Event() if cancelled: cancel_event.set() with patch.object(runtime, 'NativeStopController', Stopper), patch.object(runtime.time, 'monotonic', side_effect=lambda: clock[0]): runtime._sse_request(client=Client(), base_url='http://127.0.0.1:10000', model=model, question=question, release_id='release-a', context={'releaseId': 'release-a'}, chat_id='native', session_state=None, session_id=str(uuid.uuid4()), actor_root=root, emit=lambda event: None, timeout=60, cancel_event=cancel_event, prior_evidence_files={evidence.name} if excluded else frozenset()) return readiness def test_pure_metric_stream_is_ready_within_three_seconds_after_evidence(self): self.assertEqual(self._stream_readiness(), [False, True], 'Covered pure metrics still wait for native model completion') def test_failed_auxiliary_does_not_block_finalizing_already_complete_evidence(self): self.assertEqual(self._stream_readiness(failed_auxiliary=True), [False, True]) self.assertEqual(self._stream_readiness(failed_auxiliary=True, mutate=lambda r:r['current']['black'].pop('totalWeight')), [False,False]) def test_early_completion_keeps_missing_scope_pending_cancel_and_profile_guards(self): cases = [ {'mutate': lambda r: r['current']['black'].pop('totalWeight')}, {'mutate': lambda r: r['current']['coverage']['sourceMonths'][0].update(brightRows=0)}, {'mutate': lambda r: r['filterBinding'].update(group='额外范围')}, {'mutate': lambda r: r['comparison']['period'].update(month=7)}, {'question': QUESTION + '并分析现场原因。'}, {'excluded': True}, {'pending': True}, {'cancelled': True}, ] for case in cases: with self.subTest(case=case): self.assertEqual(self._stream_readiness(**case), [False, False]) self.assertEqual(self._stream_readiness(model='other-model'), [False, True]) def test_followups_inherit_only_verified_dimensions_in_the_same_frozen_frame(self): base = {'releaseId': 'release-a', 'filters': {'group': '合成分组', 'extraFilters': {'region': '合成地区'}}} bound, first = prepare_metric_context(QUESTION, base) record = {'baseMetricFrame': frozen_metric_frame(base), 'metricRequest': first} for question, stage, field in [('那亮片呢?', ['bright'], first['fields']), ('再看黑片投入量。', ['black'], ['totalWeight']), ('该规格报废量是多少?', first['stages'], ['scrapWeight'])]: with self.subTest(question=question): context, request = prepare_metric_context(question, base, record) self.assertEqual(request['stages'], stage) self.assertEqual(request['fields'], field) self.assertEqual(request['product'], first['product']) self.assertEqual(request['periods'], first['periods']) self.assertEqual(request['filters']['group'], '合成分组') self.assertEqual(metric_request(question, context), request) docs = metric_evidence(); docs[0]['result']['filterBinding'].update(base['filters']) self.assertEqual(formal_metric_coverage(docs, question, context)['missingEvidence'], []) for changed in [{**base, 'releaseId': 'release-b'}, {**base, 'filters': {'group': '另一分组'}}, {**base, 'period': {'year': 2025, 'month': 9}}]: with self.subTest(changed=changed): self.assertIsNone(prepare_metric_context('那亮片呢?', changed, record)[1]) self.assertIsNone(prepare_metric_context('那亮片呢?', base)[1]) def test_explicit_new_goal_updates_product_period_stage_and_retains_other_filters(self): base = {'releaseId': 'release-a', 'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2026, 'month': 8}, 'stage': 'black', 'filters': {'product': 'SYNTH-OLD', 'group': '合成分组', 'exclusions': {'product': ['排除品']}}} bound, first = prepare_metric_context(QUESTION, base) self.assertNotIn('stage', bound) self.assertEqual(bound['filters']['product'], 'SYNTH-16X15+T') record = {'baseMetricFrame': frozen_metric_frame(base), 'metricRequest': first} context, changed = prepare_metric_context('换成SYNTH-NEW,2026年10月和9月,亮片投入量是多少?', base, record) self.assertEqual(changed['product'], 'SYNTH-NEW') self.assertEqual(context['period'], {'year': 2026, 'month': 10}) self.assertEqual(context['comparisonPeriod'], {'year': 2026, 'month': 9}) self.assertEqual(context['stage'], 'bright') self.assertEqual(context['filters']['exclusions'], {'product': ['排除品']}) def test_unknown_language_and_ambiguous_dimensions_remain_outside_the_contract(self): for question in [QUESTION + '仅统计合成分组。', QUESTION + '并提供全部原始明细。', QUESTION.replace('2026年', ''), QUESTION.replace('8月', '8月和7月'), QUESTION.replace('8月', '9月'), QUESTION + '并提供生产原因。', QUESTION.replace('SYNTH-16X15+T', 'SYNTH-16X15+T和SYNTH-OTHER'), QUESTION + '去年相比如何?']: with self.subTest(question=question): self.assertIsNone(resolve_metric_request(question)) def test_only_requested_metrics_gate_coverage_and_appear_in_the_answer(self): question = '请对比规格SYNTH-16X15+T在2026年9月和8月的亮片投入量。' docs = metric_evidence() for side in ('current', 'comparison'): docs[0]['result'][side]['black'] = {} docs[0]['result'][side]['coverage']['sourceMonths'][0]['blackRows'] = 0 docs[0]['result'][side]['compositeRate'] = None docs[0]['result'][side]['bright']['rate'] = None self.assertTrue(finalizer.coverage_sufficient(docs, question, {'releaseId': 'release-a'})) reply = {'evidenceSufficient': True, 'missingEvidence': []} result = finalizer.synthesize(base_url='https://synthetic.invalid/v1', api_key='synthetic', model='mimo-v2.6-flash', question=question, evidence=docs, context={'releaseId': 'release-a'}, cancel_event=threading.Event(), opener=lambda *_a, **_kw: io.BytesIO(json.dumps({ 'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(reply)}}]}).encode())) self.assertTrue(result['evidenceSufficient']) self.assertIn('亮片投入重量', result['answer']) self.assertNotIn('综合合格率', result['answer']) self.assertNotIn('黑片', result['answer']) self.assertEqual(result['modelMissingEvidence'], []) docs[0]['result']['comparison']['bright']['totalWeight'] = None self.assertFalse(finalizer.coverage_sufficient(docs, question, {'releaseId': 'release-a'})) def test_fresh_query_detection_and_restatement_are_distinct(self): for question in ['重新查询', '请重新分析刚才的数据', '再查一次', '再次查询亮片投入', '刷新数据']: with self.subTest(question=question): self.assertTrue(requires_fresh_query(question)) self.assertFalse(is_history_restatement(question)) for question in ['把刚才的结果再说一遍', '请复述上一轮结论', '总结刚才的结果']: with self.subTest(question=question): self.assertTrue(is_history_restatement(question)) self.assertFalse(requires_fresh_query(question)) if __name__ == '__main__': unittest.main()