Files
quality_analyse/verification/test_agent_runtime.py
T
lyun ff7bbe3e46 同步:完善质量分析工作台并更新主题字体
同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。
补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。
统一午夜黑及其他主题的控件细节、中文和业务数字字体。
排除本地业务数据、凭据、预览、发布与验收产物。

验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
2026-10-30 22:40:58 +08:00

572 lines
35 KiB
Python

"""Runtime/task integration regressions; all model, native and delegated I/O is mocked."""
import io
import json
import sys
import tempfile
import threading
import unittest
from contextlib import ExitStack
from pathlib import Path
from unittest.mock import MagicMock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_evidence_finalizer as finalizer
import nao_quality_runtime as runtime
from nao_agent_task import TaskUnavailable, normalize_task, plan_repairs
from nao_quality_tools import QualityTools
ACTOR = '11111111-1111-4111-8111-111111111111'
SESSION = '22222222-2222-4222-8222-222222222222'
RELEASE = '33333333-3333-4333-8333-333333333333'
OTHER_RELEASE = '44444444-4444-4444-8444-444444444444'
TURN = '55555555-5555-4555-8555-555555555555'
def task(question='查询本月综合合格率', **updates):
value = {'kind': 'metrics', 'goal': question, 'freshQuery': True,
'historyMode': 'none', 'dimensions': [], 'stages': [], 'fields': ['rate'],
'verifySource': False, 'fullDetails': False, 'visualPreference': 'none',
'visualType': None, 'responseStyle': 'standard', 'needsClarification': False,
'clarification': '', 'filterUpdates': {}, 'period': {'year': 2026, 'month': 9}}
value.update(updates)
return normalize_task(value, question, {'releaseId': RELEASE})
def snapshot(month, *, rate=.95):
return {'period': {'year': 2026, 'month': month}, 'compositeRate': rate,
'black': {'rate': .97, 'totalWeight': 100, 'scrapWeight': 3},
'bright': {'rate': .98, 'totalWeight': 100, 'scrapWeight': 2},
'coverage': {'sourceMonths': [{'year': 2026, 'month': month,
'blackRows': 2, 'brightRows': 2}]}}
def metrics(*, month=9, product=None, rate=.95):
side = snapshot(month, rate=rate)
return {'kind': 'quality_metrics', 'releaseId': RELEASE,
'period': side['period'], 'filters': {'product': product} if product else {},
'view': 'overall', 'summary': {k: v for k, v in side.items() if k != 'period'},
'coverage': side['coverage'], 'rows': [], 'trend': []}
def comparison(*, month=9, product=None, rate=.95):
return {'kind': 'quality_comparison', 'releaseId': RELEASE,
'filterBinding': {'product': product} if product else {},
'current': snapshot(month, rate=rate), 'comparison': snapshot(month - 1, rate=.96),
'delta': {'available': True, 'percentagePoints': (rate - .96) * 100},
'stageRateChangesPP': {'black': 0, 'bright': 0}}
def write_doc(root, result, identifier='a' * 32, *, mode=None):
mode = mode or ('compare' if result['kind'] == 'quality_comparison' else 'query')
ref = 'Q' + identifier[:8].upper()
document = {'tool': 'quality_data', 'mode': mode, 'releaseId': result['releaseId'],
'evidenceRef': ref, 'createdAt': '2026-10-25T00:00:00Z', 'result': result}
path = root / 'evidence' / ('nao-quality-' + identifier + '.json')
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(document), encoding='utf-8')
path.chmod(0o600)
return document, path
def model_response(request, timeout):
"""A provider response, not an evidence/coverage mock."""
selector = json.loads(json.loads(request.data)['messages'][1]['content'])
facts = selector.get('facts', [])
payload = {'answerBlocks': [{'type': 'paragraph', 'text': '{{' + f['id'] + '}}'}
for f in facts[:12]],
'evidenceSufficient': True, 'missingEvidence': []}
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop',
'message': {'content': json.dumps(payload)}}]}).encode())
class AgentRuntimeIntegrationTest(unittest.TestCase):
def test_repeated_question_does_not_inherit_previous_wrong_answer_or_goal(self):
context={'_serverQuestions':[{'turnId':'first','question':'9月合格率为什么下降','status':'PARTIAL'},
{'turnId':'bad1','question':'主要是国内还是国外','status':'COMPLETED'},
{'turnId':'bad2','question':'主要是国内还是国外','status':'COMPLETED'}],
'_serverHistory':[{'turnId':'bad1','answer':'单期报废构成','evidenceRefs':['QAAAAAAAA']},
{'turnId':'bad2','answer':'本期总体','evidenceRefs':['QBBBBBBBB']}],
'lastVisibleAnswer':'错误的单期总体','filters':{'department':'新能源汽车磁钢'}}
clean,repeated=runtime._drop_repeated_followup_context('主要是国内还是国外',context)
self.assertTrue(repeated)
self.assertEqual([q['turnId'] for q in clean['_serverQuestions']],['first'])
self.assertEqual(clean['_serverHistory'],[])
self.assertNotIn('lastVisibleAnswer',clean)
self.assertEqual(clean['filters'],context['filters'])
self.assertEqual(len(context['_serverQuestions']),3)
single,reset=runtime._drop_repeated_followup_context('同一问题',{'_serverQuestions':[{'turnId':'one','question':'同一问题'}],
'_serverHistory':[{'turnId':'one','answer':'旧结论'}]})
self.assertTrue(reset);self.assertEqual(single['_serverQuestions'],[]);self.assertEqual(single['_serverHistory'],[])
def test_repeat_context_handles_truncation_and_older_same_question_answers(self):
question='比较某产品质量变化及具体分组差异'*24
truncated=question[:219]+'…'
context={'_serverQuestions':[{'turnId':'root','question':'原来的总体分析','status':'PARTIAL'},
{'turnId':'recent','question':truncated,'status':'COMPLETED','truncated':True}],
'_serverHistory':[{'turnId':'older','question':truncated,'answer':'过早的错误结论'},
{'turnId':'recent','question':truncated,'answer':'最近的错误结论'}]}
clean,repeated=runtime._drop_repeated_followup_context(question,context)
self.assertTrue(repeated);self.assertEqual(clean['_serverHistory'],[])
self.assertEqual([x['turnId'] for x in clean['_serverQuestions']],['root'])
def run_mocked(self, root, planned, native, *, repair=None, primary=None, history=None):
root.mkdir(parents=True, exist_ok=True)
(root / 'evidence').mkdir(exist_ok=True)
(root / 'private').mkdir(exist_ok=True)
payload = {'turnId': TURN, 'sessionId': SESSION, 'actorId': ACTOR,
'releaseId': RELEASE, 'accessToken': 'synthetic-delegation',
'question': planned['question'], 'context': {'releaseId': RELEASE}}
if history is not None: payload['context']['_serverHistory'] = history
manager = MagicMock()
manager.acquire_actor.return_value = (root, None)
manager.ensure_instance.return_value = ('http://127.0.0.1:1/v1', 'synthetic-key',
'synthetic-model', 'http://127.0.0.1:2')
state = MagicMock()
state.get_chat_id.return_value = None
gate = MagicMock()
gate.base_url = 'http://127.0.0.1:1/v1'
with ExitStack() as stack:
replacements = {
'RuntimeManager': MagicMock(return_value=manager),
'SessionState': MagicMock(return_value=state),
'write_access_file': MagicMock(return_value=root / 'access.json'),
'_authenticate': MagicMock(), '_input_cancel_listener': MagicMock(),
'_model_settings': MagicMock(return_value=('https://offline.invalid/v1', 'fake-key', 'synthetic-model',{'enabled':False,'effort':'auto','protocol':'thinking'})),
'_model_gate_settings': MagicMock(return_value=(1, 'synthetic-key')),
'DelegatedApi': MagicMock(), '_sse_request': native,
'_story_files': MagicMock(return_value=(None, None)),
'plan_agent_task': MagicMock(return_value=planned),
'read_primary_evidence': primary or MagicMock(return_value=None),
}
for name, replacement in replacements.items():
stack.enter_context(patch.object(runtime, name, replacement))
stack.enter_context(patch('nao_model_gate.ModelGate', return_value=gate))
stack.enter_context(patch.object(finalizer, '_open_model', side_effect=model_response))
if repair is not None:
stack.enter_context(patch.object(QualityTools, 'quality_data', side_effect=repair))
result = runtime.run_turn(payload, stdout=io.StringIO(), stdin=io.StringIO())
gate.close.assert_called_once()
return result
@staticmethod
def accumulator(answer):
value = runtime.ConversationAccumulator()
value.blocks['native'] = answer
value.block_order.append('native')
value.tool_calls.append({'tool': 'quality_data', 'status': 'completed'})
return value
def test_real_scalar_query_kind_is_fresh_and_completes_with_matching_coverage(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
write_doc(root, metrics())
return self.accumulator('综合合格率95.00% [QAAAAAAAA]'), 'COMPLETED', 'stop', None
result = self.run_mocked(root, task(), native)
self.assertEqual(result['status'], 'COMPLETED')
self.assertNotIn('未取得新的查询证据', result['answer'])
def test_scalar_wrong_product_period_and_missing_requested_field_remain_partial(self):
for defect in ('product', 'period', 'field'):
with self.subTest(defect=defect), tempfile.TemporaryDirectory() as folder:
root = Path(folder)
result_doc = metrics()
if defect == 'product': result_doc['filters'] = {'product': 'other'}
if defect == 'period': result_doc = metrics(month=8)
if defect == 'field': result_doc['summary'].pop('compositeRate')
def native(**kwargs):
write_doc(root, result_doc)
return self.accumulator('原生回答'), 'COMPLETED', 'stop', None
with patch('nao_agent_task.plan_repairs', return_value=[]):
result = self.run_mocked(root, task(), native)
self.assertEqual(result['status'], 'PARTIAL')
def test_no_initial_evidence_enters_repair_and_recovers_comparison(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
planned = task(kind='analysis', fields=[], comparisonPeriod={'year': 2026, 'month': 8})
def native(**kwargs):
return self.accumulator('尚未取得数据'), 'COMPLETED', 'stop', None
def repair(arguments):
document, _ = write_doc(root, comparison())
return {**document['result'], 'evidenceRef': document['evidenceRef']}
with patch('nao_agent_task.plan_repairs', return_value=[{'mode': 'compare', 'releaseId': RELEASE,
'period': planned['period'], 'comparisonPeriod': planned['comparisonPeriod'], 'filters': {}}]) as planner:
result = self.run_mocked(root, planned, native, repair=repair)
self.assertTrue(planner.called)
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['completionMethod'], 'bounded_evidence_finalization')
self.assertNotIn('尚未取得数据', result['answer'])
self.assertIn('[QAAAAAAAA]', result['answer'])
def test_scalar_repair_must_replace_the_native_no_data_answer(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
return self.accumulator('尚未取得数据,无法给出合格率。'), 'COMPLETED', 'stop', None
def repair(arguments):
document, _ = write_doc(root, metrics())
return {**document['result'], 'evidenceRef': document['evidenceRef']}
with patch('nao_agent_task.plan_repairs', return_value=[{'mode': 'query', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9}, 'filters': {}}]):
result = self.run_mocked(root, task(), native, repair=repair)
self.assertEqual(result['status'], 'COMPLETED')
self.assertNotIn('尚未取得数据', result['answer'])
self.assertIn('95.00%', result['answer'])
def test_scalar_native_claim_must_agree_with_queried_value(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
write_doc(root, metrics(rate=.95))
return self.accumulator('综合合格率99.00% [QAAAAAAAA]'), 'COMPLETED', 'stop', None
result = self.run_mocked(root, task(), native)
self.assertEqual(result['status'], 'COMPLETED')
self.assertNotIn('99.00%', result['answer'])
self.assertIn('95.00%', result['answer'])
def test_ai_selected_primary_frame_is_queried_before_local_investigation(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
planned = task(filterUpdates={'product': 'primary-product'})
def primary(**kwargs):
self.assertEqual(kwargs['arguments']['mode'], 'query')
self.assertEqual(kwargs['arguments']['filters'], {'product': 'primary-product'})
self.assertEqual(kwargs['arguments']['period'], {'year': 2026, 'month': 9})
result = metrics(product='primary-product', rate=.95)
document, _ = write_doc(root, result)
return {**result, 'evidenceRef': document['evidenceRef']}
def native(**kwargs):
self.assertEqual(kwargs['context']['primaryEvidence']['summary']['compositeRate'], .95)
write_doc(root, metrics(product='unrelated-child', rate=.81), identifier='b' * 32)
return self.accumulator('错误局部81%'), 'COMPLETED', 'stop', None
result = self.run_mocked(root, planned, native, primary=primary)
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('95.00%', result['answer'])
self.assertNotIn('81.00%', result['answer'])
def test_single_period_analysis_without_breakdowns_still_requires_matching_frame(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
write_doc(root, metrics(month=8))
return self.accumulator('来自八月的旧期间结论'), 'COMPLETED', 'stop', None
with patch('nao_agent_task.plan_repairs', return_value=[]):
result = self.run_mocked(root, task(kind='analysis', dimensions=[]), native)
self.assertEqual(result['status'], 'PARTIAL')
def test_inconsistent_nonfresh_metrics_plan_cannot_complete_without_any_proof(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
accumulator = self.accumulator('未经查询的综合合格率99.00%')
accumulator.tool_calls.clear()
return accumulator, 'COMPLETED', 'stop', None
with patch('nao_agent_task.plan_repairs', return_value=[]):
result = self.run_mocked(root, task(freshQuery=False), native)
self.assertEqual(result['status'], 'PARTIAL')
def test_off_target_native_answer_is_not_completed_when_final_synthesis_fails(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
planned=task(question='主要是国内还是国外的',kind='analysis',dimensions=['group'],
dimensionStages={'group':[]},comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
def native(**kwargs):
write_doc(root,comparison())
changes={'kind':'quality_changes','releaseId':RELEASE,'dimension':'group','stage':None,'filters':{},
'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
'sort':'scrap_delta','totalRows':2,'returnedRows':2,'truncated':False,
'rows':[{'key':'国内','label':'国内','current':{'rate':.95},'comparison':{'rate':.96},'rateDrop':.01,'scrapDelta':5},
{'key':'海外','label':'海外','current':{'rate':.96},'comparison':{'rate':.96},'rateDrop':0,'scrapDelta':0}]}
write_doc(root,changes,identifier='b'*32,mode='changes')
return self.accumulator('仅有总体指标,没有回答国内或海外。'), 'COMPLETED','stop',None
with patch.object(finalizer,'synthesize',side_effect=finalizer.FinalizationUnavailable('服务不可用')):
result=self.run_mocked(root,planned,native)
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('答复尚未完成核验',result['answer'])
def test_wrong_native_metric_is_not_completed_when_final_synthesis_fails(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
def native(**kwargs):
write_doc(root,metrics(rate=.95))
return self.accumulator('综合合格率99.00% [QAAAAAAAA]'),'COMPLETED','stop',None
with patch.object(finalizer,'synthesize',side_effect=finalizer.FinalizationUnavailable('服务不可用')):
result=self.run_mocked(root,task(),native)
self.assertEqual(result['status'],'PARTIAL')
self.assertNotIn('99.00%',result['answer'])
def test_incomplete_tool_call_cannot_skip_answer_validation(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
def native(**kwargs):
write_doc(root,metrics(rate=.95))
value=self.accumulator('综合合格率99.00% [QAAAAAAAA]')
value.tool_calls[0]['status']='started'
return value,'COMPLETED','stop',None
result=self.run_mocked(root,task(),native)
self.assertEqual(result['status'],'PARTIAL')
self.assertNotIn('99.00%',result['answer'])
def test_nonfresh_business_restatement_without_verified_history_is_partial(self):
for kind in ('explain','restate'):
with self.subTest(kind=kind),tempfile.TemporaryDirectory() as folder:
root=Path(folder)
planned=task(question='主要是国内还是国外的',kind=kind,freshQuery=False,
historyMode='last',comparisonRequired=True,
comparisonPeriod={'year':2026,'month':8})
def native(**kwargs):
result=self.accumulator('主要是国外,沿用上一轮未完成的结论。')
result.tool_calls.clear()
return result,'COMPLETED','stop',None
with patch('nao_agent_task.plan_repairs',return_value=[]):
result=self.run_mocked(root,planned,native)
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('未找到同范围、同期间的已核实历史依据',result['answer'])
def test_nonfresh_restatement_rejects_answer_that_does_not_match_question(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);write_doc(root,comparison())
planned=task(question='主要是国内还是国外的',kind='explain',freshQuery=False,
historyMode='last',comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
history=[{'turnId':'prior','question':'为什么下降','answer':'综合率下降','evidenceRefs':['QAAAAAAAA'],
'context':{'releaseId':RELEASE,'year':2026,'month':9,'comparisonPeriod':{'year':2026,'month':8}}}]
def native(**kwargs):
value=self.accumulator('只是总体下降,没有回答分组。');value.tool_calls.clear()
return value,'COMPLETED','stop',None
with patch('nao_agent_task.plan_repairs',return_value=[]),patch('nao_agent_task.verify_history_answer',return_value=False,create=True):
result=self.run_mocked(root,planned,native,history=history)
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('答复尚未完成核验',result['answer'])
def test_nonfresh_restatement_can_use_matching_verified_history(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);write_doc(root,comparison())
planned=task(question='综合合格率变化多少',kind='explain',freshQuery=False,
historyMode='last',comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
history=[{'turnId':'prior','question':'比较两期','answer':'综合率96%到95%','evidenceRefs':['QAAAAAAAA'],
'context':{'releaseId':RELEASE,'year':2026,'month':9,'comparisonPeriod':{'year':2026,'month':8}}}]
def native(**kwargs):
value=self.accumulator('综合合格率由96%降到95%。');value.tool_calls.clear()
return value,'COMPLETED','stop',None
with patch('nao_agent_task.plan_repairs',return_value=[]),patch('nao_agent_task.verify_history_answer',return_value=True):
result=self.run_mocked(root,planned,native,history=history)
self.assertEqual(result['status'],'COMPLETED')
class RepairScopeBindingTest(unittest.TestCase):
def plan(self, query, planned):
with patch('nao_agent_task._model_json', return_value={'queries': [query]}):
return plan_repairs(base_url='https://offline.invalid/v1', api_key='synthetic', model='synthetic',
task=planned, evidence=[], gaps=['missing'], failures=[], release_id=RELEASE,
cancel_event=threading.Event())
def test_changes_inherits_the_only_requested_stage(self):
planned = task(kind='analysis', dimensions=['product'], stages=['black'],
comparisonPeriod={'year': 2026, 'month': 8})
accepted = self.plan({'mode': 'changes', 'dimension': 'product', 'filters': {}}, planned)
self.assertEqual(accepted[0]['stage'], 'black')
self.assertEqual(accepted[0]['period'], planned['period'])
self.assertEqual(accepted[0]['comparisonPeriod'], planned['comparisonPeriod'])
def test_changes_cannot_switch_to_unrequested_stage(self):
planned = task(kind='analysis', dimensions=['product'], stages=['black'],
comparisonPeriod={'year': 2026, 'month': 8})
with self.assertRaises(TaskUnavailable):
self.plan({'mode': 'changes', 'dimension': 'product', 'stage': 'bright', 'filters': {}}, planned)
class TrustedHistoricalReportTest(unittest.TestCase):
def test_verified_full_report_fallback_is_not_appended_twice(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
document, _ = write_doc(root, comparison())
with patch.object(finalizer, 'synthesize', return_value={
'answer': '正式完整统计及明细', 'evidenceSufficient': True,
'reportTitle': '质量变化分析', 'titleGeneratedByModel': True,
'selectionFallback': True}):
result = self.run_report(root, [self.entry(document['evidenceRef'])])
self.assertTrue(result['reportMarkdown'].startswith('正式完整统计及明细\n\n## 数据明细'))
self.assertEqual(result['reportMarkdown'].count('正式完整统计及明细'), 1)
def run_report(self, root, entries, *, last=False):
question = '整合前面分析生成报告'
planned = task(question, kind='report', freshQuery=False, historyMode='last' if last else 'all',
fields=[], period=None)
request = {'question': question, 'releaseId': RELEASE, 'turnId': TURN,
'actorId': ACTOR, 'sessionId': SESSION,
'context': {'_serverHistory': entries}}
with patch.object(finalizer, '_open_model', side_effect=model_response):
return runtime.historical_report(request=request, task=planned, actor_root=root,
connection=('https://offline.invalid/v1', 'synthetic-only', 'synthetic-model'),
cancel_event=threading.Event(), emit=lambda value: None)
@staticmethod
def entry(ref, turn=TURN):
return {'turnId': turn, 'evidenceRefs': [ref]}
def test_all_reopens_both_trusted_frames_and_excludes_unreferenced_files(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
a, _ = write_doc(root, comparison(product='产品甲'), 'a' * 32)
b, _ = write_doc(root, comparison(month=7, product='产品乙'), 'b' * 32)
write_doc(root, comparison(product='未授权引用产品'), 'c' * 32)
# An unselected corrupt file must not poison the selected report.
bad = root / 'evidence' / ('nao-quality-' + 'd' * 32 + '.json')
bad.write_text('not-json')
result = self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('产品甲', result['reportMarkdown'])
self.assertEqual(result['agentContext']['filters'], {}, 'all-history report must clear the previous local-only display scope')
self.assertIn('产品乙', result['reportMarkdown'])
self.assertNotIn('未授权引用产品', result['reportMarkdown'])
def test_missing_other_session_ref_cannot_be_reopened(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder) / 'current'
other = Path(folder) / 'other'
a, _ = write_doc(root, comparison(), 'a' * 32)
b, _ = write_doc(other, comparison(product='另一会话'), 'b' * 32)
with self.assertRaises(runtime.RuntimeErrorSafe):
self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
def test_report_rejects_wrong_release_public_permissions_and_symlink(self):
for defect in ('release', 'permissions', 'symlink'):
with self.subTest(defect=defect), tempfile.TemporaryDirectory() as folder:
root = Path(folder) / 'current'
result = comparison()
if defect == 'release': result['releaseId'] = OTHER_RELEASE
doc, path = write_doc(root, result)
if defect == 'permissions': path.chmod(0o644)
if defect == 'symlink':
target = Path(folder) / 'external.json'
path.rename(target)
path.symlink_to(target)
with self.assertRaises((runtime.RuntimeErrorSafe, finalizer.FinalizationUnavailable)):
self.run_report(root, [self.entry(doc['evidenceRef'])])
def test_last_does_not_reopen_an_older_unavailable_entry(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
a, _ = write_doc(root, comparison(product='最新范围'))
result = self.run_report(root, [self.entry('QFFFFFFFF'), self.entry(a['evidenceRef'])], last=True)
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('最新范围', result['reportMarkdown'])
def test_all_must_not_claim_a_reference_free_turn_was_verified(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
a, _ = write_doc(root, comparison())
entries = [self.entry(a['evidenceRef']), {'turnId': 'unverified-turn', 'evidenceRefs': []}]
result=self.run_report(root, entries)
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('未纳入本次报告统计',result['answer'])
self.assertTrue(result['reportMarkdown'])
self.assertEqual(result['unverifiedHistoryTurnIds'],['unverified-turn'])
def test_recorded_non_query_turns_do_not_block_all_or_last_analysis_reports(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);a,_=write_doc(root,comparison())
chat_turn='66666666-6666-4666-8666-666666666666'
runtime._record_non_query_turn(root,{'turnId':chat_turn,'actorId':ACTOR,'sessionId':SESSION,'releaseId':RELEASE})
entries=[{'turnId':chat_turn,'evidenceRefs':[]},self.entry(a['evidenceRef']),{'turnId':chat_turn,'evidenceRefs':[]}]
for last in (False,True):
result=self.run_report(root,entries,last=last)
self.assertEqual(result['status'],'COMPLETED')
self.assertEqual(result['unverifiedHistoryTurnIds'],[])
def test_non_query_marker_from_another_scope_cannot_hide_unverified_analysis(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);a,_=write_doc(root,comparison())
chat_turn='66666666-6666-4666-8666-666666666666'
runtime._record_non_query_turn(root,{'turnId':chat_turn,'actorId':ACTOR,'sessionId':SESSION,'releaseId':OTHER_RELEASE})
result=self.run_report(root,[self.entry(a['evidenceRef']),{'turnId':chat_turn,'evidenceRefs':[]}])
self.assertEqual(result['status'],'PARTIAL')
def test_all_includes_single_period_turn_in_addition_to_comparison_turn(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
a, _ = write_doc(root, comparison(product='比较范围'), 'a' * 32)
b, _ = write_doc(root, metrics(month=7, product='单期范围', rate=.81), 'b' * 32)
result = self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('单期范围', result['reportMarkdown'])
self.assertIn('81.00%', result['reportMarkdown'])
def test_history_can_report_a_verified_source_turn_without_unrelated_metric_query(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
source = {'kind': 'quality_investigation', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9}, 'filters': {}, 'stage': 'black',
'action': 'source', 'current': {
'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
'stage': 'black', 'includeUnassigned': False},
'aggregateComplete': True, 'totalRows': 2,
'sourceVerification': {'complete': True, 'allSelectedValuesMatchStored': True,
'expectedRows': 2, 'verifiedRows': 2,
'missingRows': 0, 'mismatchRows': 0}}}
doc, _ = write_doc(root, source, mode='investigate')
result = self.run_report(root, [self.entry(doc['evidenceRef'])])
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('原表', result['reportMarkdown'])
self.assertIn('[QAAAAAAAA]', result['answer'])
@staticmethod
def defect(month=9, product=None):
return {'kind':'quality_fine_defects','releaseId':RELEASE,'dimension':'defect',
'period':{'year':2026,'month':month},'comparisonPeriod':None,
'filters':{'product':product} if product else {},'stage':'black',
'sourceRows':2,'rankingComplete':True,'coverage':{'available':True,'hasValidWeights':True},
'rows':[{'key':'corner','label':'责废缺角','category':'责废','weight':12.5}]}
def test_history_reports_single_period_defects_without_parent_metrics(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);doc,_=write_doc(root,self.defect(),mode='defects')
result=self.run_report(root,[self.entry(doc['evidenceRef'])])
self.assertEqual(result['status'],'COMPLETED')
self.assertIn('责废缺角',result['reportMarkdown'])
self.assertIn('12.50',result['reportMarkdown'])
self.assertNotIn('综合合格率',result['answer'])
def test_history_keeps_defect_periods_scopes_and_mixed_metrics(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
a,_=write_doc(root,self.defect(product='范围甲'),'a'*32,mode='defects')
b,_=write_doc(root,self.defect(month=8,product='范围乙'),'b'*32,mode='defects')
c,_=write_doc(root,metrics(product='指标范围'),'c'*32)
result=self.run_report(root,[self.entry(d['evidenceRef']) for d in (a,b,c)])
self.assertEqual(result['status'],'COMPLETED')
for text in ('范围甲','范围乙','指标范围','2026年9月','2026年8月','12.50'):
self.assertIn(text,result['reportMarkdown'])
def test_incomplete_historical_defects_are_not_marked_complete(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);good=self.defect();bad=self.defect(month=8);bad['rankingComplete']=False
a,_=write_doc(root,good,'a'*32,mode='defects');b,_=write_doc(root,bad,'b'*32,mode='defects')
result=self.run_report(root,[self.entry(d['evidenceRef']) for d in (a,b)])
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('12.50',result['reportMarkdown'])
def test_history_reports_weight_ranking_without_parent_metrics(self):
from nao_weight_ranking import rank_response
response={'rows':[{'key':'规格甲','black':{'totalWeight':60}},{'key':'规格乙','black':{'totalWeight':40}}],
'summary':{'black':{'totalWeight':100}},'context':{'rowCount':2},
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':2}]}}
for period in ({'year':2026,'month':9},{'year':2026}):
response['coverage']['sourceMonths']=[{'year':2026,'month':month,'blackRows':2}
for month in (range(1,10) if 'month' not in period else [9])]
ranking=rank_response(response,{'id':RELEASE},period,{},'product','black','totalWeight',1)
with self.subTest(period=period),tempfile.TemporaryDirectory() as folder:
root=Path(folder);doc,_=write_doc(root,ranking,mode='rank')
result=self.run_report(root,[self.entry(doc['evidenceRef'])])
self.assertEqual(result['status'],'COMPLETED')
self.assertIn('规格甲',result['reportMarkdown'])
self.assertIn('60.00%',result['reportMarkdown'])
self.assertIn('100.00',result['reportMarkdown'])
if 'month' not in period:self.assertIn('1—9月已发布数据',result['reportMarkdown'])
if __name__ == '__main__':
unittest.main()