Files
quality_analyse/verification/test_nao_metric_requests.py
T
lyun ff7bbe3e46 同步:完善质量分析工作台并更新主题字体
同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。
补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。
统一午夜黑及其他主题的控件细节、中文和业务数字字体。
排除本地业务数据、凭据、预览、发布与验收产物。

验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
2026-10-30 22:40:58 +08:00

197 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Synthetic regression cases for bounded metric intent and early completion."""
import copy
import json
import io
import sys
import tempfile
import threading
import unittest
import uuid
from pathlib import Path
from unittest.mock import MagicMock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_quality_runtime as runtime
import nao_evidence_finalizer as finalizer
from nao_analysis_coverage import formal_metric_coverage
from nao_metric_request import (resolve_metric_request, prepare_metric_context, frozen_metric_frame,
metric_request, requires_fresh_query, is_history_restatement)
QUESTION = '只看SYNTH-16X15+T,请重新查询2026年9月与8月黑片、亮片的合格率、报废重量和两期投入,附正式数据依据。'
def metric_evidence():
result = {'kind': 'quality_comparison', 'releaseId': 'release-a',
'filterBinding': {'product': 'SYNTH-16X15+T'},
'delta': {'available': True, 'percentagePoints': -2},
'stageRateChangesPP': {'black': -1, 'bright': -1}}
for side, month in [('current', 9), ('comparison', 8)]:
result[side] = {'period': {'year': 2026, 'month': month}, 'compositeRate': .95,
'coverage': {'sourceMonths': [{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}]},
'black': {'rate': .97, 'totalWeight': 1200.5, 'scrapWeight': 36.1},
'bright': {'rate': .98, 'totalWeight': 2400.5, 'scrapWeight': 72.1}}
return [{'tool': 'quality_data', 'releaseId': 'release-a', 'evidenceRef': 'Q00000001',
'mode': 'compare', 'result': result}]
class MetricRequestRegressionTests(unittest.TestCase):
def test_common_equivalent_questions_have_the_same_formal_contract(self):
questions = [QUESTION,
'请对比规格SYNTH-16X15+T在2026年9月和8月的黑片及亮片合格率、投入重量、报废重量。',
'帮我查一下SYNTH-16X15+T的2026年9月与8月黑片、亮片合格率、投入量和报废量分别是多少?',
'SYNTH-16X15+T,2026年9月与8月,黑片和亮片的合格率、投入重量及报废重量,给我正式数据。']
for question in questions:
with self.subTest(question=question):
contract = formal_metric_coverage(metric_evidence(), question, {'releaseId': 'release-a'})
self.assertIsNotNone(contract)
self.assertEqual(contract['missingEvidence'], [])
self.assertEqual(len(contract['metrics']), 12)
def _stream_readiness(self, *, mutate=None, question=QUESTION, model='mimo-v2.6-flash',
excluded=False, cancelled=False, pending=False, failed_auxiliary=False):
clock = [0.0]
gates = []
readiness = []
class Stopper:
cause = None
stop_succeeded = None
def __init__(self, stop, timeout, ready): self.ready = ready; gates.append(self)
def set_chat_id(self, value): pass
def set_response(self, value): pass
def start(self): pass
def close(self): pass
def request_cancel(self, cause): self.cause = cause
class Response:
def __enter__(self): return self
def __exit__(self, *args): pass
def __iter__(self):
yield b'data: {"type":"tool-input-available","toolName":"quality_data","toolCallId":"q1"}\n'
if not pending:
yield b'data: {"type":"tool-output-available","toolCallId":"q1","output":{"success":true}}\n'
if failed_auxiliary:
yield b'data: {"type":"tool-input-available","toolName":"quality_data","toolCallId":"q2"}\n'
yield b'data: {"type":"tool-output-available","toolCallId":"q2","output":{"success":false,"error":"invalid auxiliary query"}}\n'
readiness.append(gates[0].ready())
clock[0] = 3.0
readiness.append(gates[0].ready())
yield b'data: {"type":"text-delta","delta":"done"}\n'
yield b'data: {"type":"finish","finishReason":"stop"}\n'
yield b'data: [DONE]\n'
class Client:
def open(self, *args, **kwargs): return Response()
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir()
evidence = root / 'evidence' / ('nao-quality-' + 'a' * 32 + '.json')
document = metric_evidence()[0]
if mutate: mutate(document['result'])
evidence.write_text(json.dumps(document)); evidence.chmod(0o600)
cancel_event = threading.Event()
if cancelled: cancel_event.set()
with patch.object(runtime, 'NativeStopController', Stopper), patch.object(runtime.time, 'monotonic', side_effect=lambda: clock[0]):
runtime._sse_request(client=Client(), base_url='http://127.0.0.1:10000', model=model,
question=question, release_id='release-a', context={'releaseId': 'release-a'}, chat_id='native',
session_state=None, session_id=str(uuid.uuid4()), actor_root=root, emit=lambda event: None, timeout=60,
cancel_event=cancel_event, prior_evidence_files={evidence.name} if excluded else frozenset())
return readiness
def test_pure_metric_stream_is_ready_within_three_seconds_after_evidence(self):
self.assertEqual(self._stream_readiness(), [False, True], 'Covered pure metrics still wait for native model completion')
def test_failed_auxiliary_does_not_block_finalizing_already_complete_evidence(self):
self.assertEqual(self._stream_readiness(failed_auxiliary=True), [False, True])
self.assertEqual(self._stream_readiness(failed_auxiliary=True, mutate=lambda r:r['current']['black'].pop('totalWeight')), [False,False])
def test_early_completion_keeps_missing_scope_pending_cancel_and_profile_guards(self):
cases = [
{'mutate': lambda r: r['current']['black'].pop('totalWeight')},
{'mutate': lambda r: r['current']['coverage']['sourceMonths'][0].update(brightRows=0)},
{'mutate': lambda r: r['filterBinding'].update(group='额外范围')},
{'mutate': lambda r: r['comparison']['period'].update(month=7)},
{'question': QUESTION + '并分析现场原因。'},
{'excluded': True}, {'pending': True}, {'cancelled': True},
]
for case in cases:
with self.subTest(case=case): self.assertEqual(self._stream_readiness(**case), [False, False])
self.assertEqual(self._stream_readiness(model='other-model'), [False, True])
def test_followups_inherit_only_verified_dimensions_in_the_same_frozen_frame(self):
base = {'releaseId': 'release-a', 'filters': {'group': '合成分组', 'extraFilters': {'region': '合成地区'}}}
bound, first = prepare_metric_context(QUESTION, base)
record = {'baseMetricFrame': frozen_metric_frame(base), 'metricRequest': first}
for question, stage, field in [('那亮片呢?', ['bright'], first['fields']),
('再看黑片投入量。', ['black'], ['totalWeight']),
('该规格报废量是多少?', first['stages'], ['scrapWeight'])]:
with self.subTest(question=question):
context, request = prepare_metric_context(question, base, record)
self.assertEqual(request['stages'], stage)
self.assertEqual(request['fields'], field)
self.assertEqual(request['product'], first['product'])
self.assertEqual(request['periods'], first['periods'])
self.assertEqual(request['filters']['group'], '合成分组')
self.assertEqual(metric_request(question, context), request)
docs = metric_evidence(); docs[0]['result']['filterBinding'].update(base['filters'])
self.assertEqual(formal_metric_coverage(docs, question, context)['missingEvidence'], [])
for changed in [{**base, 'releaseId': 'release-b'}, {**base, 'filters': {'group': '另一分组'}},
{**base, 'period': {'year': 2025, 'month': 9}}]:
with self.subTest(changed=changed):
self.assertIsNone(prepare_metric_context('那亮片呢?', changed, record)[1])
self.assertIsNone(prepare_metric_context('那亮片呢?', base)[1])
def test_explicit_new_goal_updates_product_period_stage_and_retains_other_filters(self):
base = {'releaseId': 'release-a', 'period': {'year': 2026, 'month': 9},
'comparisonPeriod': {'year': 2026, 'month': 8}, 'stage': 'black',
'filters': {'product': 'SYNTH-OLD', 'group': '合成分组', 'exclusions': {'product': ['排除品']}}}
bound, first = prepare_metric_context(QUESTION, base)
self.assertNotIn('stage', bound)
self.assertEqual(bound['filters']['product'], 'SYNTH-16X15+T')
record = {'baseMetricFrame': frozen_metric_frame(base), 'metricRequest': first}
context, changed = prepare_metric_context('换成SYNTH-NEW,2026年10月和9月,亮片投入量是多少?', base, record)
self.assertEqual(changed['product'], 'SYNTH-NEW')
self.assertEqual(context['period'], {'year': 2026, 'month': 10})
self.assertEqual(context['comparisonPeriod'], {'year': 2026, 'month': 9})
self.assertEqual(context['stage'], 'bright')
self.assertEqual(context['filters']['exclusions'], {'product': ['排除品']})
def test_unknown_language_and_ambiguous_dimensions_remain_outside_the_contract(self):
for question in [QUESTION + '仅统计合成分组。', QUESTION + '并提供全部原始明细。',
QUESTION.replace('2026年', ''), QUESTION.replace('8月', '8月和7月'),
QUESTION.replace('8月', '9月'), QUESTION + '并提供生产原因。',
QUESTION.replace('SYNTH-16X15+T', 'SYNTH-16X15+T和SYNTH-OTHER'),
QUESTION + '去年相比如何?']:
with self.subTest(question=question): self.assertIsNone(resolve_metric_request(question))
def test_only_requested_metrics_gate_coverage_and_appear_in_the_answer(self):
question = '请对比规格SYNTH-16X15+T在2026年9月和8月的亮片投入量。'
docs = metric_evidence()
for side in ('current', 'comparison'):
docs[0]['result'][side]['black'] = {}
docs[0]['result'][side]['coverage']['sourceMonths'][0]['blackRows'] = 0
docs[0]['result'][side]['compositeRate'] = None
docs[0]['result'][side]['bright']['rate'] = None
self.assertTrue(finalizer.coverage_sufficient(docs, question, {'releaseId': 'release-a'}))
reply = {'evidenceSufficient': True, 'missingEvidence': []}
result = finalizer.synthesize(base_url='https://synthetic.invalid/v1', api_key='synthetic',
model='mimo-v2.6-flash', question=question, evidence=docs, context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=lambda *_a, **_kw: io.BytesIO(json.dumps({
'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(reply)}}]}).encode()))
self.assertTrue(result['evidenceSufficient'])
self.assertIn('亮片投入重量', result['answer'])
self.assertNotIn('综合合格率', result['answer'])
self.assertNotIn('黑片', result['answer'])
self.assertEqual(result['modelMissingEvidence'], [])
docs[0]['result']['comparison']['bright']['totalWeight'] = None
self.assertFalse(finalizer.coverage_sufficient(docs, question, {'releaseId': 'release-a'}))
def test_fresh_query_detection_and_restatement_are_distinct(self):
for question in ['重新查询', '请重新分析刚才的数据', '再查一次', '再次查询亮片投入', '刷新数据']:
with self.subTest(question=question):
self.assertTrue(requires_fresh_query(question))
self.assertFalse(is_history_restatement(question))
for question in ['把刚才的结果再说一遍', '请复述上一轮结论', '总结刚才的结果']:
with self.subTest(question=question):
self.assertTrue(is_history_restatement(question))
self.assertFalse(requires_fresh_query(question))
if __name__ == '__main__': unittest.main()