同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。 补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。 统一午夜黑及其他主题的控件细节、中文和业务数字字体。 排除本地业务数据、凭据、预览、发布与验收产物。 验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
572 lines
35 KiB
Python
572 lines
35 KiB
Python
"""Runtime/task integration regressions; all model, native and delegated I/O is mocked."""
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from contextlib import ExitStack
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
|
import nao_evidence_finalizer as finalizer
|
|
import nao_quality_runtime as runtime
|
|
from nao_agent_task import TaskUnavailable, normalize_task, plan_repairs
|
|
from nao_quality_tools import QualityTools
|
|
|
|
|
|
ACTOR = '11111111-1111-4111-8111-111111111111'
|
|
SESSION = '22222222-2222-4222-8222-222222222222'
|
|
RELEASE = '33333333-3333-4333-8333-333333333333'
|
|
OTHER_RELEASE = '44444444-4444-4444-8444-444444444444'
|
|
TURN = '55555555-5555-4555-8555-555555555555'
|
|
|
|
|
|
def task(question='查询本月综合合格率', **updates):
|
|
value = {'kind': 'metrics', 'goal': question, 'freshQuery': True,
|
|
'historyMode': 'none', 'dimensions': [], 'stages': [], 'fields': ['rate'],
|
|
'verifySource': False, 'fullDetails': False, 'visualPreference': 'none',
|
|
'visualType': None, 'responseStyle': 'standard', 'needsClarification': False,
|
|
'clarification': '', 'filterUpdates': {}, 'period': {'year': 2026, 'month': 9}}
|
|
value.update(updates)
|
|
return normalize_task(value, question, {'releaseId': RELEASE})
|
|
|
|
|
|
def snapshot(month, *, rate=.95):
|
|
return {'period': {'year': 2026, 'month': month}, 'compositeRate': rate,
|
|
'black': {'rate': .97, 'totalWeight': 100, 'scrapWeight': 3},
|
|
'bright': {'rate': .98, 'totalWeight': 100, 'scrapWeight': 2},
|
|
'coverage': {'sourceMonths': [{'year': 2026, 'month': month,
|
|
'blackRows': 2, 'brightRows': 2}]}}
|
|
|
|
|
|
def metrics(*, month=9, product=None, rate=.95):
|
|
side = snapshot(month, rate=rate)
|
|
return {'kind': 'quality_metrics', 'releaseId': RELEASE,
|
|
'period': side['period'], 'filters': {'product': product} if product else {},
|
|
'view': 'overall', 'summary': {k: v for k, v in side.items() if k != 'period'},
|
|
'coverage': side['coverage'], 'rows': [], 'trend': []}
|
|
|
|
|
|
def comparison(*, month=9, product=None, rate=.95):
|
|
return {'kind': 'quality_comparison', 'releaseId': RELEASE,
|
|
'filterBinding': {'product': product} if product else {},
|
|
'current': snapshot(month, rate=rate), 'comparison': snapshot(month - 1, rate=.96),
|
|
'delta': {'available': True, 'percentagePoints': (rate - .96) * 100},
|
|
'stageRateChangesPP': {'black': 0, 'bright': 0}}
|
|
|
|
|
|
def write_doc(root, result, identifier='a' * 32, *, mode=None):
|
|
mode = mode or ('compare' if result['kind'] == 'quality_comparison' else 'query')
|
|
ref = 'Q' + identifier[:8].upper()
|
|
document = {'tool': 'quality_data', 'mode': mode, 'releaseId': result['releaseId'],
|
|
'evidenceRef': ref, 'createdAt': '2026-10-25T00:00:00Z', 'result': result}
|
|
path = root / 'evidence' / ('nao-quality-' + identifier + '.json')
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(document), encoding='utf-8')
|
|
path.chmod(0o600)
|
|
return document, path
|
|
|
|
|
|
def model_response(request, timeout):
|
|
"""A provider response, not an evidence/coverage mock."""
|
|
selector = json.loads(json.loads(request.data)['messages'][1]['content'])
|
|
facts = selector.get('facts', [])
|
|
payload = {'answerBlocks': [{'type': 'paragraph', 'text': '{{' + f['id'] + '}}'}
|
|
for f in facts[:12]],
|
|
'evidenceSufficient': True, 'missingEvidence': []}
|
|
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop',
|
|
'message': {'content': json.dumps(payload)}}]}).encode())
|
|
|
|
|
|
class AgentRuntimeIntegrationTest(unittest.TestCase):
|
|
def test_repeated_question_does_not_inherit_previous_wrong_answer_or_goal(self):
|
|
context={'_serverQuestions':[{'turnId':'first','question':'9月合格率为什么下降','status':'PARTIAL'},
|
|
{'turnId':'bad1','question':'主要是国内还是国外','status':'COMPLETED'},
|
|
{'turnId':'bad2','question':'主要是国内还是国外','status':'COMPLETED'}],
|
|
'_serverHistory':[{'turnId':'bad1','answer':'单期报废构成','evidenceRefs':['QAAAAAAAA']},
|
|
{'turnId':'bad2','answer':'本期总体','evidenceRefs':['QBBBBBBBB']}],
|
|
'lastVisibleAnswer':'错误的单期总体','filters':{'department':'新能源汽车磁钢'}}
|
|
clean,repeated=runtime._drop_repeated_followup_context('主要是国内还是国外',context)
|
|
self.assertTrue(repeated)
|
|
self.assertEqual([q['turnId'] for q in clean['_serverQuestions']],['first'])
|
|
self.assertEqual(clean['_serverHistory'],[])
|
|
self.assertNotIn('lastVisibleAnswer',clean)
|
|
self.assertEqual(clean['filters'],context['filters'])
|
|
self.assertEqual(len(context['_serverQuestions']),3)
|
|
single,reset=runtime._drop_repeated_followup_context('同一问题',{'_serverQuestions':[{'turnId':'one','question':'同一问题'}],
|
|
'_serverHistory':[{'turnId':'one','answer':'旧结论'}]})
|
|
self.assertTrue(reset);self.assertEqual(single['_serverQuestions'],[]);self.assertEqual(single['_serverHistory'],[])
|
|
|
|
def test_repeat_context_handles_truncation_and_older_same_question_answers(self):
|
|
question='比较某产品质量变化及具体分组差异'*24
|
|
truncated=question[:219]+'…'
|
|
context={'_serverQuestions':[{'turnId':'root','question':'原来的总体分析','status':'PARTIAL'},
|
|
{'turnId':'recent','question':truncated,'status':'COMPLETED','truncated':True}],
|
|
'_serverHistory':[{'turnId':'older','question':truncated,'answer':'过早的错误结论'},
|
|
{'turnId':'recent','question':truncated,'answer':'最近的错误结论'}]}
|
|
clean,repeated=runtime._drop_repeated_followup_context(question,context)
|
|
self.assertTrue(repeated);self.assertEqual(clean['_serverHistory'],[])
|
|
self.assertEqual([x['turnId'] for x in clean['_serverQuestions']],['root'])
|
|
|
|
def run_mocked(self, root, planned, native, *, repair=None, primary=None, history=None):
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
(root / 'evidence').mkdir(exist_ok=True)
|
|
(root / 'private').mkdir(exist_ok=True)
|
|
payload = {'turnId': TURN, 'sessionId': SESSION, 'actorId': ACTOR,
|
|
'releaseId': RELEASE, 'accessToken': 'synthetic-delegation',
|
|
'question': planned['question'], 'context': {'releaseId': RELEASE}}
|
|
if history is not None: payload['context']['_serverHistory'] = history
|
|
manager = MagicMock()
|
|
manager.acquire_actor.return_value = (root, None)
|
|
manager.ensure_instance.return_value = ('http://127.0.0.1:1/v1', 'synthetic-key',
|
|
'synthetic-model', 'http://127.0.0.1:2')
|
|
state = MagicMock()
|
|
state.get_chat_id.return_value = None
|
|
gate = MagicMock()
|
|
gate.base_url = 'http://127.0.0.1:1/v1'
|
|
with ExitStack() as stack:
|
|
replacements = {
|
|
'RuntimeManager': MagicMock(return_value=manager),
|
|
'SessionState': MagicMock(return_value=state),
|
|
'write_access_file': MagicMock(return_value=root / 'access.json'),
|
|
'_authenticate': MagicMock(), '_input_cancel_listener': MagicMock(),
|
|
'_model_settings': MagicMock(return_value=('https://offline.invalid/v1', 'fake-key', 'synthetic-model',{'enabled':False,'effort':'auto','protocol':'thinking'})),
|
|
'_model_gate_settings': MagicMock(return_value=(1, 'synthetic-key')),
|
|
'DelegatedApi': MagicMock(), '_sse_request': native,
|
|
'_story_files': MagicMock(return_value=(None, None)),
|
|
'plan_agent_task': MagicMock(return_value=planned),
|
|
'read_primary_evidence': primary or MagicMock(return_value=None),
|
|
}
|
|
for name, replacement in replacements.items():
|
|
stack.enter_context(patch.object(runtime, name, replacement))
|
|
stack.enter_context(patch('nao_model_gate.ModelGate', return_value=gate))
|
|
stack.enter_context(patch.object(finalizer, '_open_model', side_effect=model_response))
|
|
if repair is not None:
|
|
stack.enter_context(patch.object(QualityTools, 'quality_data', side_effect=repair))
|
|
result = runtime.run_turn(payload, stdout=io.StringIO(), stdin=io.StringIO())
|
|
gate.close.assert_called_once()
|
|
return result
|
|
|
|
@staticmethod
|
|
def accumulator(answer):
|
|
value = runtime.ConversationAccumulator()
|
|
value.blocks['native'] = answer
|
|
value.block_order.append('native')
|
|
value.tool_calls.append({'tool': 'quality_data', 'status': 'completed'})
|
|
return value
|
|
|
|
def test_real_scalar_query_kind_is_fresh_and_completes_with_matching_coverage(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
def native(**kwargs):
|
|
write_doc(root, metrics())
|
|
return self.accumulator('综合合格率95.00% [QAAAAAAAA]'), 'COMPLETED', 'stop', None
|
|
result = self.run_mocked(root, task(), native)
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertNotIn('未取得新的查询证据', result['answer'])
|
|
|
|
def test_scalar_wrong_product_period_and_missing_requested_field_remain_partial(self):
|
|
for defect in ('product', 'period', 'field'):
|
|
with self.subTest(defect=defect), tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
result_doc = metrics()
|
|
if defect == 'product': result_doc['filters'] = {'product': 'other'}
|
|
if defect == 'period': result_doc = metrics(month=8)
|
|
if defect == 'field': result_doc['summary'].pop('compositeRate')
|
|
def native(**kwargs):
|
|
write_doc(root, result_doc)
|
|
return self.accumulator('原生回答'), 'COMPLETED', 'stop', None
|
|
with patch('nao_agent_task.plan_repairs', return_value=[]):
|
|
result = self.run_mocked(root, task(), native)
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
|
|
def test_no_initial_evidence_enters_repair_and_recovers_comparison(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
planned = task(kind='analysis', fields=[], comparisonPeriod={'year': 2026, 'month': 8})
|
|
def native(**kwargs):
|
|
return self.accumulator('尚未取得数据'), 'COMPLETED', 'stop', None
|
|
def repair(arguments):
|
|
document, _ = write_doc(root, comparison())
|
|
return {**document['result'], 'evidenceRef': document['evidenceRef']}
|
|
with patch('nao_agent_task.plan_repairs', return_value=[{'mode': 'compare', 'releaseId': RELEASE,
|
|
'period': planned['period'], 'comparisonPeriod': planned['comparisonPeriod'], 'filters': {}}]) as planner:
|
|
result = self.run_mocked(root, planned, native, repair=repair)
|
|
self.assertTrue(planner.called)
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertEqual(result['completionMethod'], 'bounded_evidence_finalization')
|
|
self.assertNotIn('尚未取得数据', result['answer'])
|
|
self.assertIn('[QAAAAAAAA]', result['answer'])
|
|
|
|
def test_scalar_repair_must_replace_the_native_no_data_answer(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
def native(**kwargs):
|
|
return self.accumulator('尚未取得数据,无法给出合格率。'), 'COMPLETED', 'stop', None
|
|
def repair(arguments):
|
|
document, _ = write_doc(root, metrics())
|
|
return {**document['result'], 'evidenceRef': document['evidenceRef']}
|
|
with patch('nao_agent_task.plan_repairs', return_value=[{'mode': 'query', 'releaseId': RELEASE,
|
|
'period': {'year': 2026, 'month': 9}, 'filters': {}}]):
|
|
result = self.run_mocked(root, task(), native, repair=repair)
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertNotIn('尚未取得数据', result['answer'])
|
|
self.assertIn('95.00%', result['answer'])
|
|
|
|
def test_scalar_native_claim_must_agree_with_queried_value(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
def native(**kwargs):
|
|
write_doc(root, metrics(rate=.95))
|
|
return self.accumulator('综合合格率99.00% [QAAAAAAAA]'), 'COMPLETED', 'stop', None
|
|
result = self.run_mocked(root, task(), native)
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertNotIn('99.00%', result['answer'])
|
|
self.assertIn('95.00%', result['answer'])
|
|
|
|
def test_ai_selected_primary_frame_is_queried_before_local_investigation(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
planned = task(filterUpdates={'product': 'primary-product'})
|
|
def primary(**kwargs):
|
|
self.assertEqual(kwargs['arguments']['mode'], 'query')
|
|
self.assertEqual(kwargs['arguments']['filters'], {'product': 'primary-product'})
|
|
self.assertEqual(kwargs['arguments']['period'], {'year': 2026, 'month': 9})
|
|
result = metrics(product='primary-product', rate=.95)
|
|
document, _ = write_doc(root, result)
|
|
return {**result, 'evidenceRef': document['evidenceRef']}
|
|
def native(**kwargs):
|
|
self.assertEqual(kwargs['context']['primaryEvidence']['summary']['compositeRate'], .95)
|
|
write_doc(root, metrics(product='unrelated-child', rate=.81), identifier='b' * 32)
|
|
return self.accumulator('错误局部81%'), 'COMPLETED', 'stop', None
|
|
result = self.run_mocked(root, planned, native, primary=primary)
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertIn('95.00%', result['answer'])
|
|
self.assertNotIn('81.00%', result['answer'])
|
|
|
|
def test_single_period_analysis_without_breakdowns_still_requires_matching_frame(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
def native(**kwargs):
|
|
write_doc(root, metrics(month=8))
|
|
return self.accumulator('来自八月的旧期间结论'), 'COMPLETED', 'stop', None
|
|
with patch('nao_agent_task.plan_repairs', return_value=[]):
|
|
result = self.run_mocked(root, task(kind='analysis', dimensions=[]), native)
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
|
|
def test_inconsistent_nonfresh_metrics_plan_cannot_complete_without_any_proof(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
def native(**kwargs):
|
|
accumulator = self.accumulator('未经查询的综合合格率99.00%')
|
|
accumulator.tool_calls.clear()
|
|
return accumulator, 'COMPLETED', 'stop', None
|
|
with patch('nao_agent_task.plan_repairs', return_value=[]):
|
|
result = self.run_mocked(root, task(freshQuery=False), native)
|
|
self.assertEqual(result['status'], 'PARTIAL')
|
|
|
|
def test_off_target_native_answer_is_not_completed_when_final_synthesis_fails(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder)
|
|
planned=task(question='主要是国内还是国外的',kind='analysis',dimensions=['group'],
|
|
dimensionStages={'group':[]},comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
|
|
def native(**kwargs):
|
|
write_doc(root,comparison())
|
|
changes={'kind':'quality_changes','releaseId':RELEASE,'dimension':'group','stage':None,'filters':{},
|
|
'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
|
|
'sort':'scrap_delta','totalRows':2,'returnedRows':2,'truncated':False,
|
|
'rows':[{'key':'国内','label':'国内','current':{'rate':.95},'comparison':{'rate':.96},'rateDrop':.01,'scrapDelta':5},
|
|
{'key':'海外','label':'海外','current':{'rate':.96},'comparison':{'rate':.96},'rateDrop':0,'scrapDelta':0}]}
|
|
write_doc(root,changes,identifier='b'*32,mode='changes')
|
|
return self.accumulator('仅有总体指标,没有回答国内或海外。'), 'COMPLETED','stop',None
|
|
with patch.object(finalizer,'synthesize',side_effect=finalizer.FinalizationUnavailable('服务不可用')):
|
|
result=self.run_mocked(root,planned,native)
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
self.assertIn('答复尚未完成核验',result['answer'])
|
|
|
|
def test_wrong_native_metric_is_not_completed_when_final_synthesis_fails(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder)
|
|
def native(**kwargs):
|
|
write_doc(root,metrics(rate=.95))
|
|
return self.accumulator('综合合格率99.00% [QAAAAAAAA]'),'COMPLETED','stop',None
|
|
with patch.object(finalizer,'synthesize',side_effect=finalizer.FinalizationUnavailable('服务不可用')):
|
|
result=self.run_mocked(root,task(),native)
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
self.assertNotIn('99.00%',result['answer'])
|
|
|
|
def test_incomplete_tool_call_cannot_skip_answer_validation(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder)
|
|
def native(**kwargs):
|
|
write_doc(root,metrics(rate=.95))
|
|
value=self.accumulator('综合合格率99.00% [QAAAAAAAA]')
|
|
value.tool_calls[0]['status']='started'
|
|
return value,'COMPLETED','stop',None
|
|
result=self.run_mocked(root,task(),native)
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
self.assertNotIn('99.00%',result['answer'])
|
|
|
|
def test_nonfresh_business_restatement_without_verified_history_is_partial(self):
|
|
for kind in ('explain','restate'):
|
|
with self.subTest(kind=kind),tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder)
|
|
planned=task(question='主要是国内还是国外的',kind=kind,freshQuery=False,
|
|
historyMode='last',comparisonRequired=True,
|
|
comparisonPeriod={'year':2026,'month':8})
|
|
def native(**kwargs):
|
|
result=self.accumulator('主要是国外,沿用上一轮未完成的结论。')
|
|
result.tool_calls.clear()
|
|
return result,'COMPLETED','stop',None
|
|
with patch('nao_agent_task.plan_repairs',return_value=[]):
|
|
result=self.run_mocked(root,planned,native)
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
self.assertIn('未找到同范围、同期间的已核实历史依据',result['answer'])
|
|
|
|
def test_nonfresh_restatement_rejects_answer_that_does_not_match_question(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder);write_doc(root,comparison())
|
|
planned=task(question='主要是国内还是国外的',kind='explain',freshQuery=False,
|
|
historyMode='last',comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
|
|
history=[{'turnId':'prior','question':'为什么下降','answer':'综合率下降','evidenceRefs':['QAAAAAAAA'],
|
|
'context':{'releaseId':RELEASE,'year':2026,'month':9,'comparisonPeriod':{'year':2026,'month':8}}}]
|
|
def native(**kwargs):
|
|
value=self.accumulator('只是总体下降,没有回答分组。');value.tool_calls.clear()
|
|
return value,'COMPLETED','stop',None
|
|
with patch('nao_agent_task.plan_repairs',return_value=[]),patch('nao_agent_task.verify_history_answer',return_value=False,create=True):
|
|
result=self.run_mocked(root,planned,native,history=history)
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
self.assertIn('答复尚未完成核验',result['answer'])
|
|
|
|
def test_nonfresh_restatement_can_use_matching_verified_history(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder);write_doc(root,comparison())
|
|
planned=task(question='综合合格率变化多少',kind='explain',freshQuery=False,
|
|
historyMode='last',comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
|
|
history=[{'turnId':'prior','question':'比较两期','answer':'综合率96%到95%','evidenceRefs':['QAAAAAAAA'],
|
|
'context':{'releaseId':RELEASE,'year':2026,'month':9,'comparisonPeriod':{'year':2026,'month':8}}}]
|
|
def native(**kwargs):
|
|
value=self.accumulator('综合合格率由96%降到95%。');value.tool_calls.clear()
|
|
return value,'COMPLETED','stop',None
|
|
with patch('nao_agent_task.plan_repairs',return_value=[]),patch('nao_agent_task.verify_history_answer',return_value=True):
|
|
result=self.run_mocked(root,planned,native,history=history)
|
|
self.assertEqual(result['status'],'COMPLETED')
|
|
|
|
|
|
class RepairScopeBindingTest(unittest.TestCase):
|
|
def plan(self, query, planned):
|
|
with patch('nao_agent_task._model_json', return_value={'queries': [query]}):
|
|
return plan_repairs(base_url='https://offline.invalid/v1', api_key='synthetic', model='synthetic',
|
|
task=planned, evidence=[], gaps=['missing'], failures=[], release_id=RELEASE,
|
|
cancel_event=threading.Event())
|
|
|
|
def test_changes_inherits_the_only_requested_stage(self):
|
|
planned = task(kind='analysis', dimensions=['product'], stages=['black'],
|
|
comparisonPeriod={'year': 2026, 'month': 8})
|
|
accepted = self.plan({'mode': 'changes', 'dimension': 'product', 'filters': {}}, planned)
|
|
self.assertEqual(accepted[0]['stage'], 'black')
|
|
self.assertEqual(accepted[0]['period'], planned['period'])
|
|
self.assertEqual(accepted[0]['comparisonPeriod'], planned['comparisonPeriod'])
|
|
|
|
def test_changes_cannot_switch_to_unrequested_stage(self):
|
|
planned = task(kind='analysis', dimensions=['product'], stages=['black'],
|
|
comparisonPeriod={'year': 2026, 'month': 8})
|
|
with self.assertRaises(TaskUnavailable):
|
|
self.plan({'mode': 'changes', 'dimension': 'product', 'stage': 'bright', 'filters': {}}, planned)
|
|
|
|
|
|
class TrustedHistoricalReportTest(unittest.TestCase):
|
|
def test_verified_full_report_fallback_is_not_appended_twice(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
document, _ = write_doc(root, comparison())
|
|
with patch.object(finalizer, 'synthesize', return_value={
|
|
'answer': '正式完整统计及明细', 'evidenceSufficient': True,
|
|
'reportTitle': '质量变化分析', 'titleGeneratedByModel': True,
|
|
'selectionFallback': True}):
|
|
result = self.run_report(root, [self.entry(document['evidenceRef'])])
|
|
self.assertTrue(result['reportMarkdown'].startswith('正式完整统计及明细\n\n## 数据明细'))
|
|
self.assertEqual(result['reportMarkdown'].count('正式完整统计及明细'), 1)
|
|
|
|
def run_report(self, root, entries, *, last=False):
|
|
question = '整合前面分析生成报告'
|
|
planned = task(question, kind='report', freshQuery=False, historyMode='last' if last else 'all',
|
|
fields=[], period=None)
|
|
request = {'question': question, 'releaseId': RELEASE, 'turnId': TURN,
|
|
'actorId': ACTOR, 'sessionId': SESSION,
|
|
'context': {'_serverHistory': entries}}
|
|
with patch.object(finalizer, '_open_model', side_effect=model_response):
|
|
return runtime.historical_report(request=request, task=planned, actor_root=root,
|
|
connection=('https://offline.invalid/v1', 'synthetic-only', 'synthetic-model'),
|
|
cancel_event=threading.Event(), emit=lambda value: None)
|
|
|
|
@staticmethod
|
|
def entry(ref, turn=TURN):
|
|
return {'turnId': turn, 'evidenceRefs': [ref]}
|
|
|
|
def test_all_reopens_both_trusted_frames_and_excludes_unreferenced_files(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
a, _ = write_doc(root, comparison(product='产品甲'), 'a' * 32)
|
|
b, _ = write_doc(root, comparison(month=7, product='产品乙'), 'b' * 32)
|
|
write_doc(root, comparison(product='未授权引用产品'), 'c' * 32)
|
|
# An unselected corrupt file must not poison the selected report.
|
|
bad = root / 'evidence' / ('nao-quality-' + 'd' * 32 + '.json')
|
|
bad.write_text('not-json')
|
|
result = self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertIn('产品甲', result['reportMarkdown'])
|
|
self.assertEqual(result['agentContext']['filters'], {}, 'all-history report must clear the previous local-only display scope')
|
|
self.assertIn('产品乙', result['reportMarkdown'])
|
|
self.assertNotIn('未授权引用产品', result['reportMarkdown'])
|
|
|
|
def test_missing_other_session_ref_cannot_be_reopened(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder) / 'current'
|
|
other = Path(folder) / 'other'
|
|
a, _ = write_doc(root, comparison(), 'a' * 32)
|
|
b, _ = write_doc(other, comparison(product='另一会话'), 'b' * 32)
|
|
with self.assertRaises(runtime.RuntimeErrorSafe):
|
|
self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
|
|
|
|
def test_report_rejects_wrong_release_public_permissions_and_symlink(self):
|
|
for defect in ('release', 'permissions', 'symlink'):
|
|
with self.subTest(defect=defect), tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder) / 'current'
|
|
result = comparison()
|
|
if defect == 'release': result['releaseId'] = OTHER_RELEASE
|
|
doc, path = write_doc(root, result)
|
|
if defect == 'permissions': path.chmod(0o644)
|
|
if defect == 'symlink':
|
|
target = Path(folder) / 'external.json'
|
|
path.rename(target)
|
|
path.symlink_to(target)
|
|
with self.assertRaises((runtime.RuntimeErrorSafe, finalizer.FinalizationUnavailable)):
|
|
self.run_report(root, [self.entry(doc['evidenceRef'])])
|
|
|
|
def test_last_does_not_reopen_an_older_unavailable_entry(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
a, _ = write_doc(root, comparison(product='最新范围'))
|
|
result = self.run_report(root, [self.entry('QFFFFFFFF'), self.entry(a['evidenceRef'])], last=True)
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertIn('最新范围', result['reportMarkdown'])
|
|
|
|
def test_all_must_not_claim_a_reference_free_turn_was_verified(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
a, _ = write_doc(root, comparison())
|
|
entries = [self.entry(a['evidenceRef']), {'turnId': 'unverified-turn', 'evidenceRefs': []}]
|
|
result=self.run_report(root, entries)
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
self.assertIn('未纳入本次报告统计',result['answer'])
|
|
self.assertTrue(result['reportMarkdown'])
|
|
self.assertEqual(result['unverifiedHistoryTurnIds'],['unverified-turn'])
|
|
|
|
def test_recorded_non_query_turns_do_not_block_all_or_last_analysis_reports(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder);a,_=write_doc(root,comparison())
|
|
chat_turn='66666666-6666-4666-8666-666666666666'
|
|
runtime._record_non_query_turn(root,{'turnId':chat_turn,'actorId':ACTOR,'sessionId':SESSION,'releaseId':RELEASE})
|
|
entries=[{'turnId':chat_turn,'evidenceRefs':[]},self.entry(a['evidenceRef']),{'turnId':chat_turn,'evidenceRefs':[]}]
|
|
for last in (False,True):
|
|
result=self.run_report(root,entries,last=last)
|
|
self.assertEqual(result['status'],'COMPLETED')
|
|
self.assertEqual(result['unverifiedHistoryTurnIds'],[])
|
|
|
|
def test_non_query_marker_from_another_scope_cannot_hide_unverified_analysis(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder);a,_=write_doc(root,comparison())
|
|
chat_turn='66666666-6666-4666-8666-666666666666'
|
|
runtime._record_non_query_turn(root,{'turnId':chat_turn,'actorId':ACTOR,'sessionId':SESSION,'releaseId':OTHER_RELEASE})
|
|
result=self.run_report(root,[self.entry(a['evidenceRef']),{'turnId':chat_turn,'evidenceRefs':[]}])
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
|
|
def test_all_includes_single_period_turn_in_addition_to_comparison_turn(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
a, _ = write_doc(root, comparison(product='比较范围'), 'a' * 32)
|
|
b, _ = write_doc(root, metrics(month=7, product='单期范围', rate=.81), 'b' * 32)
|
|
result = self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertIn('单期范围', result['reportMarkdown'])
|
|
self.assertIn('81.00%', result['reportMarkdown'])
|
|
|
|
def test_history_can_report_a_verified_source_turn_without_unrelated_metric_query(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
source = {'kind': 'quality_investigation', 'releaseId': RELEASE,
|
|
'period': {'year': 2026, 'month': 9}, 'filters': {}, 'stage': 'black',
|
|
'action': 'source', 'current': {
|
|
'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
|
|
'stage': 'black', 'includeUnassigned': False},
|
|
'aggregateComplete': True, 'totalRows': 2,
|
|
'sourceVerification': {'complete': True, 'allSelectedValuesMatchStored': True,
|
|
'expectedRows': 2, 'verifiedRows': 2,
|
|
'missingRows': 0, 'mismatchRows': 0}}}
|
|
doc, _ = write_doc(root, source, mode='investigate')
|
|
result = self.run_report(root, [self.entry(doc['evidenceRef'])])
|
|
self.assertEqual(result['status'], 'COMPLETED')
|
|
self.assertIn('原表', result['reportMarkdown'])
|
|
self.assertIn('[QAAAAAAAA]', result['answer'])
|
|
|
|
@staticmethod
|
|
def defect(month=9, product=None):
|
|
return {'kind':'quality_fine_defects','releaseId':RELEASE,'dimension':'defect',
|
|
'period':{'year':2026,'month':month},'comparisonPeriod':None,
|
|
'filters':{'product':product} if product else {},'stage':'black',
|
|
'sourceRows':2,'rankingComplete':True,'coverage':{'available':True,'hasValidWeights':True},
|
|
'rows':[{'key':'corner','label':'责废缺角','category':'责废','weight':12.5}]}
|
|
|
|
def test_history_reports_single_period_defects_without_parent_metrics(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder);doc,_=write_doc(root,self.defect(),mode='defects')
|
|
result=self.run_report(root,[self.entry(doc['evidenceRef'])])
|
|
self.assertEqual(result['status'],'COMPLETED')
|
|
self.assertIn('责废缺角',result['reportMarkdown'])
|
|
self.assertIn('12.50',result['reportMarkdown'])
|
|
self.assertNotIn('综合合格率',result['answer'])
|
|
|
|
def test_history_keeps_defect_periods_scopes_and_mixed_metrics(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder)
|
|
a,_=write_doc(root,self.defect(product='范围甲'),'a'*32,mode='defects')
|
|
b,_=write_doc(root,self.defect(month=8,product='范围乙'),'b'*32,mode='defects')
|
|
c,_=write_doc(root,metrics(product='指标范围'),'c'*32)
|
|
result=self.run_report(root,[self.entry(d['evidenceRef']) for d in (a,b,c)])
|
|
self.assertEqual(result['status'],'COMPLETED')
|
|
for text in ('范围甲','范围乙','指标范围','2026年9月','2026年8月','12.50'):
|
|
self.assertIn(text,result['reportMarkdown'])
|
|
|
|
def test_incomplete_historical_defects_are_not_marked_complete(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder);good=self.defect();bad=self.defect(month=8);bad['rankingComplete']=False
|
|
a,_=write_doc(root,good,'a'*32,mode='defects');b,_=write_doc(root,bad,'b'*32,mode='defects')
|
|
result=self.run_report(root,[self.entry(d['evidenceRef']) for d in (a,b)])
|
|
self.assertEqual(result['status'],'PARTIAL')
|
|
self.assertIn('12.50',result['reportMarkdown'])
|
|
|
|
def test_history_reports_weight_ranking_without_parent_metrics(self):
|
|
from nao_weight_ranking import rank_response
|
|
response={'rows':[{'key':'规格甲','black':{'totalWeight':60}},{'key':'规格乙','black':{'totalWeight':40}}],
|
|
'summary':{'black':{'totalWeight':100}},'context':{'rowCount':2},
|
|
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':2}]}}
|
|
for period in ({'year':2026,'month':9},{'year':2026}):
|
|
response['coverage']['sourceMonths']=[{'year':2026,'month':month,'blackRows':2}
|
|
for month in (range(1,10) if 'month' not in period else [9])]
|
|
ranking=rank_response(response,{'id':RELEASE},period,{},'product','black','totalWeight',1)
|
|
with self.subTest(period=period),tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder);doc,_=write_doc(root,ranking,mode='rank')
|
|
result=self.run_report(root,[self.entry(doc['evidenceRef'])])
|
|
self.assertEqual(result['status'],'COMPLETED')
|
|
self.assertIn('规格甲',result['reportMarkdown'])
|
|
self.assertIn('60.00%',result['reportMarkdown'])
|
|
self.assertIn('100.00',result['reportMarkdown'])
|
|
if 'month' not in period:self.assertIn('1—9月已发布数据',result['reportMarkdown'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|