同步:完善质量分析工作台并更新主题字体

同步已上线的智能分析执行协议、来源查询、图表报告与管理页面配套代码。
补齐数据库迁移、运行依赖和回归测试,保留原有指标计算口径。
统一午夜黑及其他主题的控件细节、中文和业务数字字体。
排除本地业务数据、凭据、预览、发布与验收产物。

验证:生产构建通过,320项前端回归通过;本轮仅同步Git,不再次部署。
This commit is contained in:
2026-10-30 22:40:58 +08:00
parent edef747e3b
commit ff7bbe3e46
393 changed files with 62999 additions and 1783 deletions
@@ -0,0 +1,34 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { activeAnalysisTurn, analysisIsRunning, beginAnalysisSelection, defaultAnalysisSession, rememberAnalysisSession, rememberPendingAnalysisSession } from '../web/src/analysisConversationState.ts';
const memory = () => { const data = new Map(); return { getItem: k => data.get(k) || null, setItem: (k, v) => data.set(k, v), removeItem: k => data.delete(k) }; };
test('last viewed session wins over the most recently updated conversation', () => {
const storage = memory(); rememberAnalysisSession('alice', 'older-viewed', storage);
assert.equal(defaultAnalysisSession('alice', [{ id: 'newer' }, { id: 'older-viewed' }], storage), 'older-viewed');
});
test('another user or deleted session never restores inaccessible history', () => {
const storage = memory(); rememberAnalysisSession('alice', 'a', storage);
assert.equal(defaultAnalysisSession('bob', [{ id: 'b' }], storage), 'b');
assert.equal(defaultAnalysisSession('alice', [{ id: 'replacement' }], storage), 'replacement');
assert.equal(defaultAnalysisSession('alice', [], storage), '');
});
test('disabled storage falls back to accessible server history', () => {
const storage = { getItem() { throw Error(); }, setItem() { throw Error(); }, removeItem() { throw Error(); } };
rememberAnalysisSession('a', 'x', storage);
assert.equal(defaultAnalysisSession('a', [{ id: 'server' }], storage), 'server');
});
test('restores latest active turn without restarting completed analysis', () => {
const rows = [{ id: 'old', status: 'COMPLETED' }, { id: 'queued', status: 'QUEUED' }, { id: 'new', status: 'RUNNING' }];
assert.equal(activeAnalysisTurn(rows).id, 'new');
assert.equal(activeAnalysisTurn([{ id: 'done', status: 'COMPLETED' }]), undefined);
assert.equal(analysisIsRunning('CANCELLED'), false);
});
test('a late background response cannot replace a newer viewed session', () => {
const storage = memory();
const oldSelection = beginAnalysisSelection('alice');
const newSelection = beginAnalysisSelection('alice');
rememberPendingAnalysisSession('alice', 'new', newSelection, storage);
rememberPendingAnalysisSession('alice', 'old', oldSelection, storage);
assert.equal(defaultAnalysisSession('alice', [{ id: 'old' }, { id: 'new' }], storage), 'new');
});
@@ -0,0 +1,25 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { settledAnalysisTurn } from '../web/src/analysisConversationState.ts';
test('a stream closing mid-sentence cannot replace the completed saved answer', () => {
const full = '小结:新能源汽车磁钢下降。完整的后续分析与关注方向。';
const partial = '小结:新能源汽车磁钢下';
const turn = settledAnalysisTurn('t', '问题', { status: 'COMPLETED', answer: full, steps: [1, 2] }, { answer: partial, steps: [1] }, '');
assert.equal(turn.answer, full);
assert.equal(turn.result.answer, full);
assert.deepEqual(turn.steps, [1, 2]);
});
test('a nested final result and an intentionally empty final answer remain authoritative', () => {
for (const answer of ['完整最终回答', '']) {
const turn = settledAnalysisTurn('t', '问题', { status: 'COMPLETED', result: { answer } }, { answer: '过时的中间内容' }, 'COMPLETED');
assert.equal(turn.result.answer, answer);
}
});
test('a terminal event supplies the answer if the final detail read is unavailable', () => {
const turn = settledAnalysisTurn('t', '问题', {}, { answer: '完成事件的完整回答' }, 'COMPLETED');
assert.equal(turn.status, 'COMPLETED');
assert.equal(turn.result.answer, '完成事件的完整回答');
});
+19
View File
@@ -0,0 +1,19 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { consumeEventStream } from '../web/src/eventStream.ts';
const consume=async chunks=>{const events=[];const encoder=new TextEncoder();await consumeEventStream(new ReadableStream({start(c){for(const chunk of chunks)c.enqueue(encoder.encode(chunk));c.close();}}),event=>events.push(event));return events;};
test('a comment before an event cannot hide the recorded result',async()=>{
const events=await consume([': heartbeat\nevent: progress\nid: 12\ndata: {"message":"来源已返回"}\n\n']);
assert.equal(events.length,1);assert.equal(events[0].id,'12');assert.equal(events[0].data.message,'来源已返回');
});
test('an interrupted final frame cannot falsely complete a running turn',async()=>{
const events=await consume(['event: progress\nid: 1\ndata: {"status":"RUNNING"}\n\n','event: completed\nid: 2\ndata: {"status":"COMPL']);
assert.equal(events.length,1);assert.equal(events[0].event,'progress');
});
test('CRLF and chunk boundaries preserve events and multi-line data',async()=>{
const events=await consume(['id: 4\r\nevent: answer\r\ndat','a: {"answer":\r\ndata: "完整回答"}\r\n\r','\n']);
assert.equal(events[0].data.answer,'完整回答');
});
test('a malformed structured terminal event is rejected before it changes task status',async()=>{
await assert.rejects(consume(['event: completed\ndata: {broken}\n\n']),/事件格式异常/);
});
+96
View File
@@ -0,0 +1,96 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { publicErrorMessage, readableMetricFormula } from '../web/src/publicDataDisplay.ts';
import * as conversation from '../web/src/analysisConversationState.ts';
import { sanitizeAnalysisDisplayText } from '../web/src/analysisDisplayText.ts';
test('gateway HTML is a service failure, never rendered as its raw payload', () => {
const error = {message: publicErrorMessage('<html><h1>502 Bad Gateway</h1></html>', 502)};
assert.match(error.message, /服务.*暂不可用/);
assert.doesNotMatch(error.message, /<html>/);
});
test('replay duplicates and old snapshots cannot move the event cursor backwards', () => {
assert.equal(conversation.acceptAnalysisEvent('12', '12'), false);
assert.equal(conversation.acceptAnalysisEvent('11', '12'), false);
assert.equal(conversation.acceptAnalysisEvent('13', '12'), true);
assert.equal(conversation.acceptAnalysisEvent('9007199254740994', '9007199254740993'), true);
});
test('formula wording translates the service policy without replacing its business calculation', () => {
assert.equal(readableMetricFormula('blackRate * brightRate;运算空值=PROPAGATE;零分母=ONE;空结果=NULL'), '黑片合格率 × 亮片合格率;运算中遇到空值:运算结果保持空值;分母为0:取1(率值100%);计算结果为空:保留空值');
assert.equal(readableMetricFormula('blackGood / blackTotal * 0.99'), '黑片合格重量 ÷ 黑片总重量 × 0.99');
assert.equal(readableMetricFormula(undefined), '当前指标服务未提供公式说明');
});
test('an old contradictory report retains its text and gains an explicit evidence-scope warning', () => {
const answer='具体缺陷线索(来源细分,未与正式大类完全对账)。责废缺角增量是黑片报废增长主因。';
assert.equal(conversation.hasUnreconciledEvidence({answer}), true);
assert.equal(conversation.hasUnreconciledEvidence({answer:'未发现对账差异,已完成核对。'}), false);
assert.equal(conversation.hasUnreconciledEvidence({plan:{timeline:[{text:'细分与正式大类未对账:仅作为变化线索'}]}}),true);
});
test('provider whitespace does not duplicate the saved final answer in its event stream', () => {
const answer='**2026年9月正式合格率**\n\n综合76.5%,黑片85%,亮片90%。';
assert.equal(conversation.duplicatesFinalAnalysisAnswer('\n\n'+answer,answer),true);
assert.equal(conversation.duplicatesFinalAnalysisAnswer(answer.slice(0,12),answer),true);
assert.equal(conversation.duplicatesFinalAnalysisAnswer('正在核查合成指标。',answer),false);
assert.equal(conversation.duplicatesFinalAnalysisAnswer(' ',answer),false);
assert.equal(conversation.duplicatesFinalAnalysisAnswer('较早的独立进展',''),false);
});
test('verified VCM coverage supersedes the same scope warning but preserves other unresolved scopes', () => {
const key='0123456789abcdef', other='fedcba9876543210';
const event=(checks)=>({kind:'tool_result',status:'completed',evidenceChecks:checks});
assert.equal(conversation.hasUnreconciledEvidence({answer:'VCM细分未对账',timeline:[event({[key]:'unresolved'}),event({[key]:'expected'})]}),false);
assert.equal(conversation.hasUnreconciledEvidence({timeline:[event({[other]:'unresolved'}),event({[key]:'expected'})]}),true);
assert.equal(conversation.hasUnreconciledEvidence({timeline:[{kind:'assistant',evidenceChecks:{[other]:'checked'},text:'未对账'}]}),true);
});
test('partial pre-tool notes stay in chronology instead of being repeated as final answers', () => {
const note={kind:'assistant',text:'正在核查原表。'}, tool={kind:'tool_result',status:'completed'};
assert.equal(conversation.partialAnswerIsOnlyProgress({status:'PARTIAL',answer:note.text,timeline:[note,tool]}),true);
assert.equal(conversation.partialAnswerIsOnlyProgress({status:'PARTIAL',answer:'已完成部分比较。',timeline:[tool,{kind:'assistant',text:'已完成部分比较。'}]}),false);
assert.equal(conversation.partialAnswerIsOnlyProgress({status:'COMPLETED',answer:note.text,timeline:[note,tool]}),false);
assert.equal(conversation.partialAnswerIsOnlyProgress({status:'PARTIAL',answer:note.text}),false);
});
test('each tool start/result pair renders one folded row without merging different calls', () => {
const input={mode:'compare',period:{month:9}};
const events=[{kind:'tool_start',sequence:1,toolCallId:'a',label:'查询',input},
{kind:'tool_start',sequence:2,toolCallId:'b',label:'查询'},
{kind:'assistant',sequence:3,text:'继续核查'},
{kind:'tool_result',sequence:4,toolCallId:'a',status:'completed',text:'已返回'},
{kind:'tool_result',sequence:5,toolCallId:'b',status:'failed',text:'需重试'}];
const compact=conversation.compactAnalysisTimeline(events);
assert.equal(compact.length,3);
assert.deepEqual(compact.map(row=>row.kind),['tool','tool','assistant']);
assert.equal(compact[0].status,'completed');assert.deepEqual(compact[0].input,input);
assert.equal(compact[1].status,'failed');
assert.equal(events[0].kind,'tool_start');
const replay=conversation.compactAnalysisTimeline([...events,events[0]]);
assert.equal(replay[0].status,'completed');assert.equal(replay.length,3);
});
test('stream updates respect an upward scroll and resume following only at the bottom', () => {
assert.equal(conversation.analysisScrollFollows(40,true,'up'),false);
assert.equal(conversation.analysisScrollFollows(150,false),false);
assert.equal(conversation.analysisScrollFollows(130,false,'down'),false);
assert.equal(conversation.analysisScrollFollows(10,false,'down'),true);
assert.equal(conversation.analysisScrollFollows(120,true),true);
assert.equal(conversation.analysisScrollFollows(10,false),false);
});
test('partial completion explains tool failures separately from model limits', () => {
assert.match(conversation.partialAnalysisNotice('mcp_error'),/数据查询未成功/);
assert.match(conversation.partialAnalysisNotice('length'),/长度上限/);
assert.match(conversation.partialAnalysisNotice('timeout'),/处理时限/);
assert.match(conversation.partialAnalysisNotice('evidence_incomplete'),/被标记为部分完成/);
assert.doesNotMatch(conversation.partialAnalysisNotice('evidence_incomplete'),/数据尚未齐全/);
});
test('provider tool-call markup is hidden while business text stays readable', () => {
assert.equal(sanitizeAnalysisDisplayText('<tool_call><function=read><parameter_path>/RULES.md</parameter></function></tool_call>'), '');
assert.equal(sanitizeAnalysisDisplayText('已核对。<tool_call><function=read>'), '已核对。');
assert.equal(sanitizeAnalysisDisplayText('产品VCM739+Z重量10kg'), '产品VCM739+Z重量10kg');
});
+20
View File
@@ -0,0 +1,20 @@
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {withRequestDeadline,RequestTimeoutError} from '../web/src/requestDeadline.ts';
test('a silent connection expires and aborts its network request',async()=>{
let signal;const start=Date.now();
await assert.rejects(withRequestDeadline(s=>{signal=s;return new Promise(()=>{});},20),RequestTimeoutError);
assert.equal(signal.aborted,true);assert.ok(Date.now()-start<500);
});
test('deadline includes reading a stalled response body',async()=>{
const run=async signal=>{await Promise.resolve();return new Promise(()=>{});};
await assert.rejects(withRequestDeadline(run,20),RequestTimeoutError);
});
test('a completed response cancels its deadline',async()=>{
let signal;assert.equal(await withRequestDeadline(s=>{signal=s;return Promise.resolve('ok');},20),'ok');
await new Promise(r=>setTimeout(r,30));assert.equal(signal.aborted,false);
});
test('HTTP and authentication errors retain their original identity',async()=>{
const original=Object.assign(new Error('unauthorized'),{status:401});
await assert.rejects(withRequestDeadline(async()=>{throw original;},100),e=>e===original);
});
+350
View File
@@ -0,0 +1,350 @@
"""Evidence-bound composed answers; uses only mocked OpenAI-compatible responses."""
import io
import json
import sys
import threading
import tempfile
from urllib.error import HTTPError
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_evidence_finalizer as finalizer
def evidence(ref='Q00000001', rate=.95):
return [{'evidenceRef': ref, 'mode': 'compare', 'result': {
'kind': 'quality_comparison', 'filterBinding': {},
'current': {'period': {'year': 2026, 'month': 9}, 'compositeRate': rate,
'coverage': {'sourceMonths': [{'year': 2026, 'month': 9,
'blackRows': 2, 'brightRows': 2}]}},
'comparison': {'period': {'year': 2026, 'month': 8}, 'compositeRate': .96,
'coverage': {'sourceMonths': [{'year': 2026, 'month': 8,
'blackRows': 2, 'brightRows': 2}]}},
'delta': {'available': True, 'percentagePoints': (rate - .96) * 100},
}}]
def response(parsed):
payload = {'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(parsed, ensure_ascii=False)}}]}
return io.BytesIO(json.dumps(payload).encode())
class AgentAnswerTest(unittest.TestCase):
def test_product_drilldown_fallback_keeps_each_queried_specification(self):
from unittest.mock import patch
facts = [{'id':'F1','kind':'overall','text':'总体合格率上升。','evidenceRef':'Q00000001'}]
docs = evidence()
for index, product in enumerate(('规格甲','规格乙','规格丙'),2):
ref = f'Q{index:08X}'
facts.append({'id':f'F{index}','kind':'cross','text':product+'来源缺角变化。','evidenceRef':ref})
docs.append({'evidenceRef':ref,'result':{'filters':{'product':product}}})
report = {'facts':facts,'sections':['不应输出冗长全表'],'headline':'总体','policy':'关联不作为生产根因。','missingEvidence':[]}
invalid = {'answerBlocks':[{'type':'bullet','text':'{{F1}} 虚构改善50%'}],
'evidenceSufficient':True,'missingEvidence':[]}
with patch('nao_agent_report.build_agent_report',return_value=report):
result = finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
question='fixture',evidence=docs,cancel_event=threading.Event(),
context={'_agentTask':{'version':1,'kind':'analysis','defectScope':'products','defectProductCount':3}},
opener=lambda *args,**kwargs:response(invalid))
self.assertTrue(result['selectionFallback'])
self.assertTrue(result['compositionRepairAttempted'])
for product in ('规格甲','规格乙','规格丙'):
self.assertIn(product,result['answer'])
self.assertIn('建议核查',result['answer'])
self.assertNotIn('50%',result['answer'])
self.assertNotIn('不应输出冗长全表',result['answer'])
def test_invalid_composition_gets_one_model_repair_with_same_global_budget(self):
calls = []
def provider(request, timeout):
body = json.loads(request.data)
calls.append(body)
if len(calls) == 1:
return response({'reportTitle':'合格率变化分析','answerBlocks':[{'type':'bullet','text':'本期改善50% {{F1}}'}],
'evidenceSufficient':True,'missingEvidence':[]})
return response({'answerBlocks':[{'type':'bullet','text':'{{F1}}'}],
'evidenceSufficient':True,'missingEvidence':[]})
result = finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
question='比较质量',evidence=evidence(),cancel_event=threading.Event(),opener=provider)
self.assertEqual(len(calls),2)
self.assertNotIn('50%',result['answer'])
self.assertFalse(result.get('selectionFallback',False))
self.assertEqual(result['reportTitle'],'合格率变化分析')
self.assertTrue(result['titleGeneratedByModel'])
from nao_runtime_policy import MAX_OUTPUT_TOKENS
self.assertTrue(all(body['max_completion_tokens']==MAX_OUTPUT_TOKENS for body in calls))
self.assertEqual(len(calls[1]['messages']),4)
def test_report_uses_model_selected_priorities_without_dumping_all_tables(self):
result = self.synthesize({'reportFactIds': ['F1'], 'reportTitle': '合格率变化分析',
'evidenceSufficient': True, 'missingEvidence': []},
context={'_agentTask': {'version': 1, 'kind': 'report', 'historyMode': 'all',
'question': '综合以上生成报告'}})
self.assertIn('## 核心结论', result['answer'])
self.assertIn('下降1.00个百分点', result['answer'])
self.assertIn('## 下一步核查', result['answer'])
self.assertNotIn('| ---', result['answer'])
self.assertLess(len(result['answer']), 1000)
self.assertFalse(result.get('selectionFallback', False))
def test_report_rejects_unknown_focus_id_and_still_has_bounded_fallback(self):
result = self.synthesize({'reportFactIds': ['F999'],
'evidenceSufficient': True, 'missingEvidence': []},
context={'_agentTask': {'version': 1, 'kind': 'report', 'historyMode': 'all',
'question': '综合以上生成报告'}})
self.assertTrue(result['selectionFallback'])
self.assertNotIn('F999', result['answer'])
self.assertNotIn('| ---', result['answer'])
def synthesize(self, parsed, *, docs=None, model='vendor/compatible-chat', context=None, capture=None):
def opener(request, timeout):
if capture is not None:
capture['body'] = json.loads(request.data)
capture['timeout'] = timeout
return response(parsed)
return finalizer.synthesize(
base_url='https://offline.invalid/v1', api_key='synthetic-only', model=model,
question='请概括质量变化', evidence=evidence() if docs is None else docs,
cancel_event=threading.Event(), context=context, opener=opener)
def test_model_composes_prose_around_exact_server_fact_and_reference(self):
captured = {}
result = self.synthesize({
'answerBlocks': [
{'type': 'heading', 'text': '变化概览'},
{'type': 'paragraph', 'text': '建议结合现场记录开展复核;当前结论只限于本次查询指标:{{F1}}'},
],
'evidenceSufficient': True, 'missingEvidence': [],
}, context={'_agentTask': {'version': 1, 'kind': 'analysis', 'responseStyle': 'brief',
'dimensions': ['group'], 'question': '简要回答'}}, capture=captured)
self.assertIn('### 变化概览', result['answer'])
self.assertIn('建议结合现场记录开展复核', result['answer'])
self.assertIn('下降1.00个百分点', result['answer'])
self.assertIn('[Q00000001]', result['answer'])
self.assertEqual(result['selectedFactIds'], ['F1'])
self.assertFalse(result.get('selectionFallback', False))
self.assertEqual(captured['body']['model'], 'vendor/compatible-chat')
self.assertEqual(captured['body']['thinking'], {'type':'disabled'})
from nao_runtime_policy import MAX_OUTPUT_TOKENS
self.assertEqual(captured['body']['max_completion_tokens'], MAX_OUTPUT_TOKENS)
selector = json.loads(captured['body']['messages'][1]['content'])
self.assertEqual(selector['agentTask']['responseStyle'], 'brief')
self.assertTrue(selector['facts'])
def test_unsupported_digits_force_verified_report_fallback(self):
result = self.synthesize({
'answerBlocks': [{'type': 'paragraph', 'text': '本期合格率达到99%,依据{{F1}}'}],
'evidenceSufficient': True, 'missingEvidence': [],
})
self.assertTrue(result['selectionFallback'])
self.assertNotIn('99%', result['answer'])
self.assertIn('下降1.00个百分点', result['answer'])
def test_unknown_fact_reference_forces_fallback(self):
result = self.synthesize({
'answerBlocks': [{'type': 'paragraph', 'text': '结论{{F99}}'}],
'evidenceSufficient': True, 'missingEvidence': [],
})
self.assertTrue(result['selectionFallback'])
self.assertNotIn('{{F99}}', result['answer'])
self.assertIn('下降1.00个百分点', result['answer'])
def test_unsupported_direction_and_cause_prose_forces_fallback(self):
result=self.synthesize({
'answerBlocks':[{'type':'paragraph','text':'质量已持续改善,且确认由设备磨损导致。{{F1}}'}],
'evidenceSufficient':True,'missingEvidence':[]})
self.assertTrue(result['selectionFallback'])
self.assertNotIn('设备磨损',result['answer'])
def test_cause_hypothesis_passes_only_when_explicitly_unconfirmed(self):
result=self.synthesize({
'answerBlocks':[{'type':'paragraph','text':'待验证:设备磨损是否可能导致报废增加,建议检查。{{F1}}'}],
'evidenceSufficient':True,'missingEvidence':[]})
self.assertFalse(result.get('selectionFallback',False))
self.assertIn('待验证:设备磨损是否可能导致报废增加',result['answer'])
absolute=self.synthesize({
'answerBlocks':[{'type':'paragraph','text':'所有缺陷都已完全排除。{{F1}}'}],
'evidenceSufficient':True,'missingEvidence':[]})
self.assertTrue(absolute['selectionFallback'])
def test_thinking_capability_is_retried_without_model_name_branch(self):
bodies=[]
parsed={'answerBlocks':[{'type':'paragraph','text':'总体变化:{{F1}}'}],
'evidenceSufficient':True,'missingEvidence':[]}
def opener(request,timeout):
bodies.append(json.loads(request.data))
if len(bodies)==1:
raise HTTPError(request.full_url,400,'thinking unsupported',None,io.BytesIO(b''))
return response(parsed)
result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic-only',
model='some/openai-compatible-model',question='请概括质量变化',evidence=evidence(),
cancel_event=threading.Event(),opener=opener)
self.assertTrue(result['evidenceSufficient'])
self.assertEqual(len(bodies),2)
self.assertEqual(bodies[0]['thinking'],{'type':'disabled'})
self.assertNotIn('thinking',bodies[1])
def test_non_mimo_model_is_eligible_and_incomplete_server_evidence_stays_partial(self):
calls = [{'status': 'completed'}]
self.assertTrue(finalizer.eligible('provider/model-x', 'PARTIAL', 'timeout', calls, False, True))
self.assertFalse(finalizer.eligible(' ', 'PARTIAL', 'timeout', calls, False, True))
from test_nao_evidence_report import coverage_comparison
doc=coverage_comparison()
doc['result']['current']['coverage']['sourceMonths'][0].update(blackRows=0,brightRows=0)
result = self.synthesize({
'answerBlocks': [{'type': 'paragraph', 'text': '当前正式变化:{{F1}}'}],
'evidenceSufficient': True, 'missingEvidence': [],
}, docs=[doc])
self.assertFalse(result['evidenceSufficient'])
self.assertTrue(result['missingEvidence'])
self.assertTrue(any('没有可引用的正式综合合格率' in item for item in result['missingEvidence']))
self.assertIn('未覆盖内容', result['answer'])
def test_legacy_selector_response_still_uses_verified_report(self):
result = self.synthesize({
'otherFactIds': ['F1'], 'groupFactIds': [], 'productFactIds': [], 'defectFactIds': [],
'evidenceSufficient': True, 'missingEvidence': [],
})
self.assertTrue(result['evidenceSufficient'])
self.assertIn('下降1.00个百分点', result['answer'])
self.assertIn('### 重点关注', result['answer'])
self.assertFalse(result.get('selectionFallback', False))
def test_source_verification_task_requires_complete_source_proof(self):
question = '请核对来源'
period={'year':2026,'month':9}
context = {'releaseId':'r1','_agentTask': {'version': 1, 'question': question, 'kind': 'source_verify',
'verifySource': True, 'fullDetails': False,'period':period,
'comparisonPeriod':None,'filters':{'product':'P'},'stages':['bright']}}
query = {'mode': 'compare', 'result': {'kind': 'quality_query'}}
proof = {'complete': True, 'expectedRows':2, 'verifiedRows': 2, 'missingRows':0,
'mismatchRows':0,'allSelectedValuesMatchStored': True}
side_context={'releaseId':'r1','year':2026,'month':9,'stage':'bright','category':None,
'product':'P','includeUnassigned':False,'extraFilters':{}}
source = {'mode': 'investigate', 'result': {'kind':'quality_investigation','releaseId':'r1',
'period':period,'filters':{'product':'P'},'stage':'bright','action':'source','current': {
'context':side_context,'totalRows': 2,'aggregateComplete':True,'sourceVerification': proof}}}
self.assertFalse(finalizer.coverage_sufficient([query], question, context))
self.assertTrue(finalizer.coverage_sufficient([source], question, context))
source['result']['current']['sourceVerification'] = {**proof, 'complete': False}
self.assertFalse(finalizer.coverage_sufficient([source], question, context))
def test_source_proofs_must_cover_every_requested_stage_side_and_scope(self):
question='核验9月与8月原表'
current={'year':2026,'month':9};previous={'year':2026,'month':8}
context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'source_verify',
'verifySource':True,'fullDetails':False,'period':current,'comparisonPeriod':previous,
'filters':{'product':'P'},'stages':['black','bright']}}
query={'mode':'compare','result':{'kind':'quality_query'}}
def source(stage, product='P'):
proof={'complete':True,'expectedRows':1,'verifiedRows':1,'missingRows':0,'mismatchRows':0,
'allSelectedValuesMatchStored':True}
def side(month):
return {'context':{'releaseId':'r1','year':2026,'month':month,'stage':stage,'category':None,
'product':product,'includeUnassigned':False,'extraFilters':{}},
'totalRows':1,'aggregateComplete':True,'sourceVerification':proof}
return {'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1',
'period':current,'comparisonPeriod':previous,'filters':{'product':product},'stage':stage,
'category':None,'action':'source','current':side(9),'comparison':side(8)}}
wrong_scope=source('black',product='OTHER')
self.assertFalse(finalizer.coverage_sufficient([query,wrong_scope],question,context))
black=source('black');bright=source('bright')
self.assertFalse(finalizer.coverage_sufficient([black],question,context))
self.assertTrue(finalizer.coverage_sufficient([black,bright],question,context))
def test_full_details_task_requires_contiguous_complete_pages(self):
question = '把所有记录列出来'
period={'year':2026,'month':9}
context = {'releaseId':'r1','_agentTask': {'version': 1, 'question': question, 'kind': 'details',
'verifySource': False, 'fullDetails': True,'period':period,
'comparisonPeriod':None,'filters':{},'stages':['black']}}
side={'context':{'releaseId':'r1','year':2026,'month':9,'stage':'black','category':None,
'includeUnassigned':False,'extraFilters':{}},'recordFilter':'all',
'matchingRows':2,'offset':0,'returnedRows':2,'hasMore':False,'differenceRecords':[{},{}]}
rows={'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1','period':period,
'filters':{},'stage':'black','category':None,'action':'rows','current':side}}
self.assertTrue(finalizer.coverage_sufficient([rows], question, context))
rows['result']['current']['hasMore'] = True
self.assertFalse(finalizer.coverage_sufficient([rows], question, context))
rows['result']['current']['hasMore'] = False
rows['result']['current']['recordFilter'] = 'differences'
self.assertFalse(finalizer.coverage_sufficient([rows], question, context))
def test_full_details_require_every_requested_stage_and_both_periods(self):
question='列出黑片和亮片两期的全部明细'
current={'year':2026,'month':9};previous={'year':2026,'month':8}
context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'details',
'verifySource':False,'fullDetails':True,'period':current,'comparisonPeriod':previous,
'filters':{'group':'G'},'stages':['black','bright']}}
def rows(stage):
def side(month):
return {'context':{'releaseId':'r1','year':2026,'month':month,'stage':stage,'category':None,
'group':'G','includeUnassigned':False,'extraFilters':{}},
'recordFilter':'all','matchingRows':1,'offset':0,'returnedRows':1,
'hasMore':False,'differenceRecords':[{}]}
return {'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1',
'period':current,'comparisonPeriod':previous,'filters':{'group':'G'},'stage':stage,
'category':None,'action':'rows','current':side(9),'comparison':side(8)}}
black=rows('black');bright=rows('bright')
self.assertFalse(finalizer.coverage_sufficient([black],question,context))
self.assertTrue(finalizer.coverage_sufficient([black,bright],question,context))
bright['result']['comparison']['recordFilter']='differences'
self.assertFalse(finalizer.coverage_sufficient([black,bright],question,context))
def test_unbound_task_period_requires_a_matching_query_root(self):
question='列出当前范围的全部明细'
context={'releaseId':'r1','_agentTask':{'version':1,'question':question,'kind':'details',
'verifySource':False,'fullDetails':True,'period':None,'comparisonPeriod':None,
'filters':{'group':'G'},'stages':['black']}}
query={'mode':'compare','result':{'kind':'quality_comparison','releaseId':'r1',
'filterBinding':{'group':'G'},'current':{'period':{'year':2026,'month':9}},
'comparison':{'period':{'year':2026,'month':8}}}}
page={'mode':'investigate','result':{'kind':'quality_investigation','releaseId':'r1',
'period':{'year':2026,'month':9},'filters':{'group':'G'},'stage':'black','category':None,
'action':'rows','current':{'context':{'releaseId':'r1','year':2026,'month':9,'stage':'black',
'category':None,'group':'G','includeUnassigned':False,'extraFilters':{}},'recordFilter':'all',
'matchingRows':1,'offset':0,'returnedRows':1,'hasMore':False,'differenceRecords':[{}]}}}
self.assertTrue(finalizer.coverage_sufficient([query,page],question,context))
self.assertFalse(finalizer.coverage_sufficient([page],question,context))
query['result']['filterBinding']={'group':'OTHER'}
self.assertFalse(finalizer.coverage_sufficient([query,page],question,context))
def test_empty_evidence_is_allowed_only_for_planning_callers(self):
with tempfile.TemporaryDirectory() as folder:
with self.assertRaises(finalizer.FinalizationUnavailable):
finalizer.evidence_summaries(Path(folder),'release-a')
self.assertEqual(finalizer.evidence_summaries(Path(folder),'release-a',allow_empty=True),[])
def test_semantic_metrics_and_single_period_dimensions_gate_completion(self):
from test_nao_metric_requests import metric_evidence, QUESTION
task={'version':1,'question':QUESTION,'kind':'metrics','period':{'year':2026,'month':9},
'comparisonPeriod':{'year':2026,'month':8},'filters':{'product':'SYNTH-16X15+T'},
'stages':['black','bright'],'fields':['rate','totalWeight','scrapWeight']}
context={'releaseId':'release-a','_agentTask':task}
docs=metric_evidence()
self.assertTrue(finalizer.coverage_sufficient(docs,QUESTION,context))
docs[0]['result']['current']['bright'].pop('totalWeight')
self.assertFalse(finalizer.coverage_sufficient(docs,QUESTION,context))
question='本期产品情况'
single={'version':1,'question':question,'kind':'analysis','period':{'year':2026,'month':9},
'comparisonPeriod':None,'filters':{},'dimensions':['product'],'stages':[]}
frame={'releaseId':'r1','_agentTask':single}
query={'mode':'query','result':{'kind':'quality_metrics','releaseId':'r1',
'period':{'year':2026,'month':9},'filters':{},'view':'overall'}}
self.assertFalse(finalizer.coverage_sufficient([query],question,frame))
query['result']['view']='product'
self.assertTrue(finalizer.coverage_sufficient([query],question,frame))
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,135 @@
"""Regressions for calendar-bound defect TOP and evidence-bound model contracts."""
import copy
import io
import json
import sys
import threading
import unittest
from datetime import datetime
from pathlib import Path
from zoneinfo import ZoneInfo
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_agent_task import normalize_task, bind_task, plan_task, TaskUnavailable
from nao_agent_report import build_agent_report
from nao_analysis_coverage import coverage_gaps
from nao_evidence_finalizer import coverage_sufficient, eligible, _compose_answer_blocks, _agent_task_summary, CompositionError
from test_nao_agent_task import task_value
Q = '帮我看下上月黑片缺陷TOP'
def fixture():
task = normalize_task(task_value(dimensions=['defect'], dimensionStages={'defect':['black']},
stages=['black'], fields=['scrapWeight'], period={'year':2026,'month':9}, comparisonRequired=False), Q)
ctx = bind_task({'releaseId':'release-a'}, task)
doc = {'evidenceRef':'Q12345678','mode':'defects','result':{
'kind':'quality_fine_defects','releaseId':'release-a','period':task['period'],
'filters':{},'dimension':'defect','stage':'black','category':None,'defect':None,
'coverage':{'available':True,'hasValidWeights':True}, 'rankingComplete':True,
'sourceRows':43172,'rowsReturned':2,'rows':[
{'key':'责废缺角重量','label':'责废缺角重量','category':'责废','weight':17513.162},
{'key':'料废缺角重量','label':'料废缺角重量','category':'料废','weight':13949.63}]}}
return task,ctx,[doc]
class ContractRegressions(unittest.TestCase):
def test_single_period_defect_top_has_verified_answer_without_formal_comparison(self):
task,ctx,docs=fixture()
self.assertEqual(coverage_gaps(docs,Q,ctx),[])
self.assertTrue(coverage_sufficient(docs,Q,ctx))
report=build_agent_report(docs,ctx,Q)
self.assertEqual(len(report['facts']),2)
self.assertTrue(report['coverageSufficient'])
answer,ids=_compose_answer_blocks([{'factIds':['F1','F2']}],report['facts'])
self.assertEqual(ids,['F1','F2'])
self.assertIn('17,513.16',answer)
self.assertIn('[Q12345678]',answer)
self.assertNotIn('八成',answer)
prefetched={'mode':'query','evidenceRef':'Q87654321','result':{
'kind':'quality_metrics','releaseId':'release-a','period':task['period'],
'filters':{},'summary':{'black':{'scrapWeight':999}}}}
self.assertEqual({fact['kind'] for fact in build_agent_report([prefetched,*docs],ctx,Q)['facts']},{'defect'})
ctx['_agentTask']['fields']=['totalWeight']
self.assertFalse(coverage_sufficient(docs,Q,ctx))
def test_wrong_or_unusable_evidence_never_satisfies_defect_top(self):
for mutation in ('release','month','filters','stage','coverage','weight','ranking','source'):
with self.subTest(mutation=mutation):
_,ctx,docs=fixture();r=docs[0]['result']
if mutation=='release':r['releaseId']='other'
if mutation=='month':r['period']={'year':2026,'month':8}
if mutation=='filters':r['filters']={'group':'OTHER'}
if mutation=='stage':r['stage']='bright'
if mutation=='coverage':r['coverage']['available']=False
if mutation=='weight':r['rows'][0]['weight']=None
if mutation=='ranking':r['rankingComplete']=False
if mutation=='source':r['sourceRows']=0
self.assertFalse(coverage_sufficient(docs,Q,ctx))
def test_failed_supplement_cannot_discard_facts_or_promote_missing_stage(self):
task,ctx,docs=fixture()
calls=[{'status':'completed'},{'status':'failed'}]
self.assertFalse(eligible('m','PARTIAL','evidence_incomplete',calls,False,None))
self.assertTrue(eligible('m','PARTIAL','evidence_incomplete',calls,False,None,True))
self.assertTrue(eligible('m','FAILED','mcp_error',calls,False,None,True))
for cancelled,stopped in ((True,None),(False,False)):
self.assertFalse(eligible('m','PARTIAL','evidence_incomplete',calls,cancelled,stopped,True))
self.assertFalse(eligible('m','PARTIAL','evidence_incomplete',[{'status':'started'}],False,None,True))
ctx['_agentTask']['dimensionStages']['defect'].append('bright')
self.assertFalse(coverage_sufficient(docs,Q,ctx))
self.assertTrue(build_agent_report(docs,ctx,Q)['missingEvidence'])
def test_null_nonclarification_and_independent_review_use_trusted_calendar(self):
payloads=[]
value=task_value(dimensions=['defect'],dimensionStages={'defect':['black']},stages=['black'],fields=[],
period={'year':2026,'month':9},clarification=None,defectScope=None,defectProductCount=None)
def opener(req,timeout):
payloads.append(json.loads(req.data))
result=value if len(payloads)==1 else {'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,'requiresAllSourceRows':False,'requiresTrend':False}
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(result)}}]}).encode())
result=plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',question=Q,
context={'_serverHistory':[{'question':'你好,你是谁,能做什么','evidenceRefs':[]}]},
previous={'kind':'explain','goal':'问候','rootGoal':'问候'},cancel_event=threading.Event(),opener=opener)
self.assertEqual(len(payloads),2)
self.assertEqual(result['clarification'],'')
self.assertEqual(result['rootGoal'],result['goal'])
payload=json.loads(payloads[0]['messages'][1]['content'])
self.assertEqual(payload['serverCalendar']['date'],datetime.now(ZoneInfo('Asia/Shanghai')).date().isoformat())
self.assertFalse(payload['platformCapabilities']['charts']['drawingToolRequired'])
with self.assertRaises(TaskUnavailable):normalize_task({**value,'needsClarification':True},Q)
def test_fact_ids_keep_all_numeric_and_claim_guards(self):
facts=build_agent_report(fixture()[2],fixture()[1],Q)['facts']
self.assertIn('[Q12345678]',_compose_answer_blocks([{'text':'{{F1}}'}],facts)[0])
for block in ({'factIds':['F999']},{'factIds':['F1'],'lead':'上升20%'},
{'factIds':['F1'],'advice':'证明工艺是根因'},
{'factIds':['F1'],'lead':'占八成以上'},
{'factIds':['F1'],'text':'额外正文'}):
with self.subTest(block=block),self.assertRaises(CompositionError) as caught:
_compose_answer_blocks([block],facts)
self.assertIn('answerBlocks[0]',caught.exception.field)
self.assertIn('safeRepair',caught.exception.details())
def test_structured_focus_does_not_repeat_facts_across_answer_blocks(self):
_,ctx,docs=fixture();facts=build_agent_report(docs,ctx,Q)['facts']
answer,selected=_compose_answer_blocks([
{'factIds':['F1']}, {'type':'bullet','factIds':['F1','F2']},
{'type':'bullet','factIds':['F1']}],facts)
self.assertEqual(selected,['F1','F2'])
self.assertEqual(answer.count('17,513.16'),1)
self.assertEqual(answer.count('13,949.63'),1)
def test_finalizer_retains_dimension_stage_and_exact_period(self):
task,ctx,_=fixture();summary=_agent_task_summary(ctx)
self.assertEqual(summary['dimensionStages'],{'defect':['black']})
self.assertEqual(summary['period'],task['period'])
self.assertEqual(summary['filters'],{})
self.assertFalse(summary['comparisonRequired'])
def test_source_defect_top_does_not_prefetch_unrequested_formal_kpis(self):
from nao_agent_contract import primary_query_required
task,_,_=fixture()
self.assertFalse(primary_query_required(task))
self.assertTrue(primary_query_required({**task,'comparisonRequired':True}))
self.assertTrue(primary_query_required({**task,'kind':'metrics'}))
self.assertTrue(primary_query_required({**task,'dimensions':['defect','product']}))
if __name__=='__main__':unittest.main()
@@ -0,0 +1,37 @@
import sys,json,io,tempfile,uuid,unittest
from pathlib import Path
from unittest.mock import patch,MagicMock
from contextlib import ExitStack
ROOT=Path(__file__).resolve().parents[1];sys.path[:0]=[str(ROOT/'scripts'),str(ROOT/'verification')]
import nao_quality_runtime as runtime
from test_agent_contract_regressions import fixture,Q
class RuntimeFinalizationRegression(unittest.TestCase):
def test_verified_defect_answer_survives_failed_auxiliary_renderer(self):
with tempfile.TemporaryDirectory() as folder,ExitStack() as stack:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
request={key:str(uuid.uuid4()) for key in ('turnId','sessionId','actorId','releaseId')};request.update(question=Q,context={},accessToken='synthetic')
task,ctx,docs=fixture();docs[0]['result']['releaseId']=request['releaseId'];doc={**docs[0],'tool':'quality_data','releaseId':request['releaseId']}
manager=MagicMock();manager.acquire_actor.return_value=(root,None);manager.ensure_instance.return_value=('http://127.0.0.1:49191/v1','synthetic','m','http://127.0.0.1:5000')
gate=MagicMock();gate.base_url='http://127.0.0.1:49191/v1';gate.request_audit=[]
def native(**kwargs):
p=root/'evidence'/('nao-quality-'+'a'*32+'.json');p.write_text(json.dumps(doc));p.chmod(0o600)
stream=runtime.ConversationAccumulator();stream.blocks['draft']='原始草稿';stream.block_order.append('draft');stream.record_event('assistant',text='原始草稿')
stream.tool_calls=[{'tool':'quality_data','status':'completed'},{'tool':'display_chart','status':'failed'}]
return stream,'PARTIAL','mcp_error',None
def synthesize(**kwargs):return {'answer':'责废缺角来源重量17,513.16 kg。[Q12345678]','model':'m','usage':{},'finishReason':'stop','evidenceSufficient':True,'missingEvidence':[]}
replacements={'RuntimeManager':MagicMock(return_value=manager),'write_access_file':MagicMock(return_value=root/'access.json'),
'_authenticate':MagicMock(),'_input_cancel_listener':MagicMock(),'SessionState':MagicMock(),
'_sse_request':native,'_story_files':MagicMock(return_value=(None,None)),
'_model_settings':MagicMock(return_value=('https://model.test/v1','synthetic','m',{'enabled':True,'effort':'medium','protocol':'thinking'})),
'plan_agent_task':MagicMock(return_value=task),'_model_gate_settings':MagicMock(return_value=(49191,'synthetic')),
'DelegatedApi':MagicMock(),'read_primary_evidence':MagicMock(return_value=None)}
for name,value in replacements.items():stack.enter_context(patch.object(runtime,name,value))
stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate));stack.enter_context(patch('nao_evidence_finalizer.synthesize',side_effect=synthesize))
result=runtime.run_turn(request,stdout=io.StringIO(),stdin=io.StringIO())
error=root/'private/last-runtime-error.json'
if error.exists():raise AssertionError(json.loads(error.read_text())['trace'])
assert result['status']=='COMPLETED'
assert result['timeline'][-1]['kind']=='assistant'
assert not any(entry.get('text')=='原始草稿' for entry in result['timeline'])
if __name__=='__main__':unittest.main()
+445
View File
@@ -0,0 +1,445 @@
"""Single-period and mixed historical report composition tests."""
import io
import json
import tempfile
import sys
import threading
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_agent_report import build_agent_report, build_noncomparison_report
import nao_evidence_finalizer as finalizer
RELEASE = 'release-verified'
def task(question, *, kind='metrics', period=None, filters=None, stages=None, fields=None,
history_mode='none'):
return {'version': 1, 'question': question, 'kind': kind,
'historyMode': history_mode, 'filters': filters or {}, 'period': period,
'comparisonPeriod': None, 'dimensions': [], 'stages': stages or [],
'fields': fields or [], 'verifySource': False, 'fullDetails': False}
def metric_doc(*, month=9, product='产品甲', rate=.81, black_rate=.84,
bright_rate=.78, black_rows=2, bright_rows=2, evidence='A'):
period = {'year': 2026, 'month': month}
filters = {'product': product} if product else {}
coverage = {'sourceMonths': [{'year': 2026, 'month': month,
'blackRows': black_rows, 'brightRows': bright_rows}]}
summary = {'compositeRate': rate,
'black': {'rate': black_rate, 'totalWeight': 123.25, 'scrapWeight': 3.5},
'bright': {'rate': bright_rate, 'totalWeight': 88.75, 'scrapWeight': 2.25}}
return {'evidenceRef': 'Q' + evidence * 8, 'mode': 'query',
'result': {'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': period,
'filters': filters, 'view': 'overall', 'summary': summary,
'coverage': coverage}}
def comparison_doc(*, month=9, product='产品甲', evidence='B'):
def snapshot(selected_month, rate):
return {'period': {'year': 2026, 'month': selected_month}, 'compositeRate': rate,
'black': {'rate': .9, 'totalWeight': 100, 'scrapWeight': 2},
'bright': {'rate': .9, 'totalWeight': 100, 'scrapWeight': 2},
'coverage': {'sourceMonths': [{'year': 2026, 'month': selected_month,
'blackRows': 2, 'brightRows': 2}]}}
return {'evidenceRef': 'Q' + evidence * 8, 'mode': 'compare',
'result': {'kind': 'quality_comparison', 'releaseId': RELEASE,
'filterBinding': {'product': product},
'current': snapshot(month, .92), 'comparison': snapshot(month - 1, .9),
'delta': {'available': True, 'percentagePoints': 2},
'stageRateChangesPP': {'black': 0, 'bright': 0}}}
class NoncomparisonReportTest(unittest.TestCase):
def test_leaf_only_report_has_a_real_core_conclusion(self):
facts=[{'id':'F1','kind':'defect','text':'缺陷甲来源重量12.50 kg。','evidenceRef':'QAAAAAAAA'},
{'id':'F2','kind':'defect','text':'缺陷乙来源重量8.00 kg。','evidenceRef':'QAAAAAAAA'}]
answer,selected=finalizer._focused_report_answer(facts,['F1','F2'])
self.assertIn('## 核心结论\n\n- 缺陷甲',answer)
self.assertIn('## 优先关注\n\n- 缺陷乙',answer)
self.assertEqual(selected,['F1','F2'])
def context(self, question, **updates):
value = {'releaseId': RELEASE, 'period': {'year': 2026, 'month': 9},
'filters': {'product': '产品甲'},
'_agentTask': task(question, period={'year': 2026, 'month': 9},
filters={'product': '产品甲'}, fields=['rate'])}
value['_agentTask'].update(updates)
return value
def test_single_query_uses_bound_quality_metrics_not_a_stale_generic_query(self):
question = '查询产品甲本月合格率'
fresh = metric_doc(rate=.81)
stale = {'evidenceRef': 'Q' + 'C' * 8, 'mode': 'query',
'result': {'kind': 'quality_query', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9},
'filters': {'product': '产品甲'}, 'summary': {'compositeRate': .99}}}
report = build_noncomparison_report([stale, fresh], self.context(question), question)
rendered = report['headline'] + '\n' + '\n'.join(report['sections'])
self.assertIn('81.00%', rendered)
self.assertNotIn('99.00%', rendered)
self.assertEqual(report['facts'][0]['evidenceRef'], fresh['evidenceRef'])
self.assertEqual(report['coverageSufficient'], True)
def test_explicit_stage_and_fields_do_not_add_unrequested_numeric_facts(self):
question = '只看黑片投入重量'
context = self.context(question)
context['_agentTask'].update({'stages': ['black'], 'fields': ['totalWeight']})
report = build_noncomparison_report([metric_doc()], context, question)
texts = [fact['text'] for fact in report['facts']]
self.assertEqual(len(texts), 1)
self.assertIn('黑片投入重量:123.25 kg', texts[0])
self.assertNotIn('亮片', texts[0])
self.assertNotIn('综合', texts[0])
def test_fieldless_report_includes_composite_and_stage_input_scrap(self):
question = '生成质量指标报告'
context = {'releaseId': RELEASE, 'filters': {'product': '产品甲'},
'_agentTask': task(question, kind='report', history_mode='all')}
report = build_noncomparison_report([metric_doc()], context, question)
labels = [fact['label'] for fact in report['facts']]
self.assertEqual(labels, ['综合合格率', '黑片合格率', '黑片投入重量', '黑片报废重量',
'亮片合格率', '亮片投入重量', '亮片报废重量'])
self.assertIn('81.00%', report['sections'][0])
self.assertIn('123.25 kg', report['sections'][0])
def test_quarter_frame_uses_all_three_server_coverage_months(self):
question = '查询产品甲第三季度合格率'
doc = metric_doc(month=9, rate=.81)
doc['result']['period'] = {'year': 2026, 'quarter': 3}
doc['result']['coverage']['sourceMonths'] = [
{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}
for month in (7, 8, 9)]
context = {'releaseId': RELEASE, 'period': {'year': 2026, 'quarter': 3},
'filters': {'product': '产品甲'},
'_agentTask': task(question, period={'year': 2026, 'quarter': 3},
filters={'product': '产品甲'}, fields=['rate'])}
report = build_noncomparison_report([doc], context, question)
self.assertIn('2026年第3季度', report['sections'][0])
self.assertIn('81.00%', report['facts'][0]['text'])
self.assertTrue(report['coverageSufficient'])
def test_multiple_noncomparison_frames_remain_separately_scoped(self):
question = '整合历史统计生成报告'
context = {'releaseId': RELEASE,
'_agentTask': task(question, kind='report', history_mode='all')}
first = metric_doc(month=9, product='产品甲', rate=.81, evidence='1')
second = metric_doc(month=7, product='产品乙', rate=.76, evidence='2')
report = build_noncomparison_report([first, second], context, question)
self.assertEqual(len(report['sections']), 2)
text = '\n'.join(report['sections'])
self.assertIn('产品甲', text)
self.assertIn('产品乙', text)
self.assertIn('81.00%', text)
self.assertIn('76.00%', text)
def test_missing_stage_coverage_remains_partial_and_does_not_show_rate(self):
question = '查询黑片合格率'
context = self.context(question)
context['_agentTask'].update({'stages': ['black'], 'fields': ['rate']})
doc = metric_doc(black_rows=0)
report = build_noncomparison_report([doc], context, question)
self.assertFalse(report['coverageSufficient'])
self.assertTrue(report['missingEvidence'])
self.assertIn('黑片合格率:未提供', report['facts'][0]['text'])
self.assertNotIn('84.00%', report['facts'][0]['text'])
def test_period_and_scope_mismatch_is_not_replaced_with_another_frame(self):
question = '查询产品甲本月合格率'
context = self.context(question)
with self.assertRaises(ValueError):
build_noncomparison_report([metric_doc(month=8)], context, question)
class MixedHistoryReportTest(unittest.TestCase):
def test_comparison_history_appends_a_distinct_single_period_scope(self):
question = '整合前述分析生成报告'
context = {'releaseId': RELEASE,
'_agentTask': task(question, kind='report', history_mode='all')}
comparison = comparison_doc(product='产品甲')
historic_metric = metric_doc(month=7, product='产品乙', rate=.81, evidence='D')
report = build_agent_report([comparison, historic_metric], context, question)
text = report['headline'] + '\n' + '\n'.join(report['sections'])
self.assertIn('产品甲', text)
self.assertIn('产品乙', text)
self.assertIn('81.00%', text)
self.assertTrue(any(fact['evidenceRef'] == historic_metric['evidenceRef'] for fact in report['facts']))
def test_metrics_already_inside_comparison_are_not_duplicated(self):
question = '整合前述分析生成报告'
context = {'releaseId': RELEASE,
'_agentTask': task(question, kind='report', history_mode='all')}
comparison = comparison_doc(product='产品甲')
duplicate = metric_doc(month=9, product='产品甲', rate=.92, evidence='E')
report = build_agent_report([comparison, duplicate], context, question)
refs = [fact['evidenceRef'] for fact in report['facts']]
self.assertNotIn(duplicate['evidenceRef'], refs)
self.assertEqual(report['coverageSufficient'], True)
def test_verified_source_and_rows_summaries_are_appended_without_raw_paths(self):
question = '整合前述分析生成报告'
context = {'releaseId': RELEASE,
'_agentTask': task(question, kind='report', history_mode='all')}
comparison = comparison_doc()
investigations = [
{'evidenceRef': 'Q' + 'F' * 8, 'mode': 'investigate',
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9}, 'stage': 'black',
'filters': {}, 'action': 'source',
'current': {'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
'stage': 'black'},
'totalRows': 205, 'aggregateComplete': True,
'sourceVerification': {'complete': True,
'allSelectedValuesMatchStored': True,
'expectedRows': 205, 'verifiedRows': 205,
'missingRows': 0, 'mismatchRows': 0,
'files': [{'sourceFile': '/private/raw.xlsx'}]}}}},
{'evidenceRef': 'Q' + 'A' * 8, 'mode': 'investigate',
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9}, 'stage': 'bright',
'filters': {}, 'action': 'rows',
'current': {'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
'stage': 'bright'},
'matchingRows': 12, 'offset': 0, 'returnedRows': 10,
'hasMore': True, 'recordFilter': 'differences',
'rows': [{'sensitive': 'must not render'}]}}}
]
report = build_agent_report([comparison, *investigations], context, question)
rendered = '\n'.join(report['sections'])
self.assertIn('原表值全部匹配且核验完整', rendered)
self.assertIn('已核验205行/应核验205行', rendered)
self.assertIn('共12条', rendered)
self.assertIn('差异记录', rendered)
self.assertNotIn('/private/raw.xlsx', rendered)
self.assertNotIn('must not render', rendered)
self.assertIn('QFFFFFFFF', rendered)
self.assertIn('QAAAAAAAA', rendered)
table = rendered.split('### 原表核验与明细读取', 1)[1].split('\n### ', 1)[0]
table_rows = [line for line in table.splitlines() if line.startswith('|')]
self.assertTrue(table_rows)
self.assertEqual(len(table_rows[0].split('|')), 7)
self.assertTrue(all(len(line.split('|')) == 7 for line in table_rows[1:]))
def test_full_details_preserve_all_four_rows_and_blank_values_without_paths(self):
question = '读取9月黑片全部明细'
detail_task = task(question, kind='details', period={'year': 2026, 'month': 9},
filters={}, stages=['black'], fields=[])
detail_task['fullDetails'] = True
context = {'releaseId': RELEASE, 'period': {'year': 2026, 'month': 9},
'filters': {}, '_agentTask': detail_task}
records = [
{'rowNumber': 11, 'product': '规格甲', 'sourceFile': '/private/source.xlsx',
'values': {'数值字段': 0}},
{'rowNumber': 12, 'product': '规格乙', 'sourceFile': '/private/source.xlsx',
'values': {'空值字段': None}},
{'rowNumber': 13, 'product': '规格丙', 'sourceFile': '/private/source.xlsx',
'values': {'文本字段': '已保留'}},
{'rowNumber': 14, 'product': '规格丁', 'sourceFile': '/private/source.xlsx',
'values': {'重量字段': 2.5}},
]
document = {'evidenceRef': 'Q' + '6' * 8, 'mode': 'investigate',
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9}, 'filters': {},
'stage': 'black', 'category': None, 'action': 'rows',
'current': {'context': {'releaseId': RELEASE, 'year': 2026,
'month': 9, 'stage': 'black',
'category': None, 'includeUnassigned': False},
'matchingRows': 4, 'offset': 0, 'returnedRows': 4,
'hasMore': False, 'recordFilter': 'all',
'rows': records}}}
with tempfile.TemporaryDirectory() as temp_dir:
evidence_path = Path(temp_dir) / ('nao-quality-' + '6' * 32 + '.json')
evidence_path.write_text(json.dumps({'tool': 'quality_data', 'mode': 'investigate',
'releaseId': RELEASE, 'evidenceRef': document['evidenceRef'],
'apiSource': {'base': '/private/source.xlsx'},
'result': document['result']}, ensure_ascii=False))
evidence_path.chmod(0o600)
summaries = finalizer.evidence_summaries(Path(temp_dir), RELEASE)
page = summaries[0]['result']['current']
self.assertEqual(len(page['differenceRecords']), 4)
self.assertEqual(page['differenceRecords'][0]['values'], {'数值字段': 0})
self.assertEqual(page['differenceRecords'][1]['values'], {'空值字段': None})
self.assertNotIn('sourceFile', json.dumps(summaries, ensure_ascii=False))
self.assertNotIn('/private/source.xlsx', json.dumps(summaries, ensure_ascii=False))
summarized_doc = {**document, 'result': summaries[0]['result']}
report = build_agent_report([summarized_doc], context, question)
rendered = '\n'.join(report['sections'])
detail_table = rendered.split('### 2026年9月 · 黑片原表行明细', 1)[1]
table_lines = [line for line in detail_table.splitlines() if line.startswith('|')]
self.assertEqual(len(table_lines[0].split('|')), 6)
self.assertEqual(len(table_lines), 6)
for row_number in ('11', '12', '13', '14'):
self.assertIn(row_number + ' [Q66666666]', detail_table)
self.assertIn('| 0 |', detail_table)
self.assertIn('| 未提供 |', detail_table)
self.assertIn('已保留', detail_table)
self.assertIn('2.5', detail_table)
self.assertNotIn('/private/source.xlsx', rendered)
self.assertTrue(report['coverageSufficient'])
def test_source_only_request_can_render_exact_verified_proof_without_kpi(self):
question = '核对本月黑片原表'
source_task = task(question, kind='source_verify', period={'year': 2026, 'month': 9},
filters={}, stages=['black'], fields=[], history_mode='none')
source_task['verifySource'] = True
context = {'releaseId': RELEASE, 'period': {'year': 2026, 'month': 9},
'filters': {}, '_agentTask': source_task}
proof = {'complete': True, 'allSelectedValuesMatchStored': True,
'expectedRows': 12, 'verifiedRows': 12, 'missingRows': 0, 'mismatchRows': 0}
evidence = [{'evidenceRef': 'Q' + '9' * 8, 'mode': 'investigate',
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9}, 'filters': {},
'stage': 'black', 'action': 'source',
'current': {'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
'stage': 'black'},
'totalRows': 12, 'aggregateComplete': True,
'sourceVerification': proof}}}]
report = build_agent_report(evidence, context, question)
self.assertTrue(report['coverageSufficient'])
self.assertIn('已核验12行/应核验12行', report['sections'][0])
self.assertFalse(report['rawMetrics'])
def test_source_only_history_with_mismatched_proof_stays_partial(self):
question = '整合前述分析生成报告'
context = {'releaseId': RELEASE,
'_agentTask': task(question, kind='report', history_mode='all')}
evidence = [{'evidenceRef': 'Q' + '8' * 8, 'mode': 'investigate',
'result': {'kind': 'quality_investigation', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9}, 'filters': {},
'stage': 'black', 'action': 'source',
'current': {'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
'stage': 'black'},
'totalRows': 12, 'aggregateComplete': True,
'sourceVerification': {'complete': True,
'allSelectedValuesMatchStored': True,
'expectedRows': 12, 'verifiedRows': 11,
'missingRows': 1, 'mismatchRows': 0}}}}]
report = build_agent_report(evidence, context, question)
self.assertFalse(report['coverageSufficient'])
self.assertIn('原表核验未完整通过', report['sections'][0])
self.assertTrue(report['missingEvidence'])
class ReportTitleTest(unittest.TestCase):
question = '整合前述分析生成报告'
def run_synthesis(self, parsed=None, *, timed_out=False, capture=None):
context = {'releaseId': RELEASE,
'_agentTask': task(self.question, kind='report', history_mode='all')}
def opener(request, timeout):
if timed_out:
raise TimeoutError('synthetic timeout')
if capture is not None:
capture['body'] = json.loads(request.data)
body = {'choices': [{'finish_reason': 'stop',
'message': {'content': json.dumps(parsed or {}, ensure_ascii=False)}}]}
return io.BytesIO(json.dumps(body, ensure_ascii=False).encode())
return finalizer.synthesize(base_url='https://offline.invalid/v1', api_key='synthetic',
model='compatible/test', question=self.question,
evidence=[comparison_doc()], context=context,
cancel_event=threading.Event(), opener=opener)
@staticmethod
def composed(title):
return {'reportTitle': title,
'answerBlocks': [{'type': 'paragraph', 'text': '{{F1}}'}],
'evidenceSufficient': True, 'missingEvidence': []}
def test_model_title_is_accepted_when_bound_to_scope_period_and_topic(self):
title = '2026年8月至9月 产品甲 合格率变化'
captured = {}
result = self.run_synthesis(self.composed(title), capture=captured)
self.assertEqual(result['reportTitle'], title)
self.assertTrue(result['titleGeneratedByModel'])
selector = json.loads(captured['body']['messages'][1]['content'])
self.assertIn('2026年8月', selector['reportTitleGuide']['actualPeriods'])
self.assertIn('规格=产品甲', selector['reportTitleGuide']['actualScopes'])
self.assertIn('合格率', selector['reportTitleGuide']['topics'])
self.assertIn('reportTitle', captured['body']['messages'][0]['content'])
def test_question_as_title_is_replaced_with_neutral_evidence_title(self):
result = self.run_synthesis(self.composed(self.question))
self.assertNotEqual(result['reportTitle'], self.question)
self.assertFalse(result['titleGeneratedByModel'])
self.assertIn('2026年', result['reportTitle'])
def test_natural_title_paraphrase_does_not_require_fixed_topic_words(self):
title = '2026年8—9月质量变化与重点规格报告'
result = self.run_synthesis(self.composed(title))
self.assertEqual(result['reportTitle'], title)
self.assertTrue(result['titleGeneratedByModel'])
def test_malformed_unsafe_or_unbound_numeric_titles_fall_back(self):
for title in ('<script>2026年8月产品甲合格率</script>',
'https://example.test 2026年产品甲合格率',
'sk-abcdef0123456789 2026年产品甲合格率',
'2026年8月\n产品甲合格率',
'2027年8月产品甲合格率'):
with self.subTest(title=title):
result = self.run_synthesis(self.composed(title))
self.assertFalse(result['titleGeneratedByModel'])
self.assertNotIn('<', result['reportTitle'])
self.assertNotIn('https://', result['reportTitle'])
self.assertNotIn('2027', result['reportTitle'])
def test_valid_model_title_survives_answer_block_fallback(self):
title = '2026年8月至9月 产品甲 合格率变化'
result = self.run_synthesis({
**self.composed(title),
'answerBlocks': [{'type': 'paragraph', 'text': '未绑定的99%结论{{F1}}'}],
})
self.assertTrue(result['selectionFallback'])
self.assertEqual(result['reportTitle'], title)
self.assertTrue(result['titleGeneratedByModel'])
def test_brief_metric_fallback_renders_only_requested_bound_fact(self):
question = '查询产品甲本月综合合格率'
context = {'releaseId': RELEASE, 'period': {'year': 2026, 'month': 9},
'filters': {'product': '产品甲'},
'_agentTask': task(question, period={'year': 2026, 'month': 9},
filters={'product': '产品甲'}, fields=['rate'])}
context['_agentTask'].update({'kind': 'metrics', 'responseStyle': 'brief'})
evidence = [metric_doc(rate=.81, evidence='7')]
parsed = {'reportTitle': '2026年9月 产品甲 合格率',
'answerBlocks': [{'type': 'paragraph', 'text': '未绑定结论99%:{{F1}}'}],
'evidenceSufficient': True, 'missingEvidence': []}
def opener(request, timeout):
body = {'choices': [{'finish_reason': 'stop',
'message': {'content': json.dumps(parsed, ensure_ascii=False)}}]}
return io.BytesIO(json.dumps(body, ensure_ascii=False).encode())
result = finalizer.synthesize(base_url='https://offline.invalid/v1', api_key='synthetic',
model='compatible/test', question=question, evidence=evidence,
context=context, cancel_event=threading.Event(), opener=opener)
self.assertTrue(result['selectionFallback'])
self.assertEqual(result['reportTitle'], parsed['reportTitle'])
self.assertTrue(result['titleGeneratedByModel'])
self.assertIn('81.00%', result['answer'])
self.assertIn('[Q77777777]', result['answer'])
self.assertNotIn('| 指标 |', result['answer'])
self.assertNotIn('重点关注', result['answer'])
self.assertNotIn('重量单位', result['answer'])
def test_timeout_and_legacy_response_have_neutral_nonquestion_titles(self):
timeout_result = self.run_synthesis(timed_out=True)
self.assertFalse(timeout_result['titleGeneratedByModel'])
self.assertNotIn(self.question, timeout_result['reportTitle'])
self.assertIn('产品甲', timeout_result['reportTitle'])
legacy = self.run_synthesis({'otherFactIds': ['F1'], 'groupFactIds': [],
'productFactIds': [], 'defectFactIds': [],
'evidenceSufficient': True, 'missingEvidence': []})
self.assertFalse(legacy['titleGeneratedByModel'])
self.assertIn('合格率', legacy['reportTitle'])
self.assertNotIn(self.question, legacy['reportTitle'])
if __name__ == '__main__':
unittest.main()
+571
View File
@@ -0,0 +1,571 @@
"""Runtime/task integration regressions; all model, native and delegated I/O is mocked."""
import io
import json
import sys
import tempfile
import threading
import unittest
from contextlib import ExitStack
from pathlib import Path
from unittest.mock import MagicMock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_evidence_finalizer as finalizer
import nao_quality_runtime as runtime
from nao_agent_task import TaskUnavailable, normalize_task, plan_repairs
from nao_quality_tools import QualityTools
ACTOR = '11111111-1111-4111-8111-111111111111'
SESSION = '22222222-2222-4222-8222-222222222222'
RELEASE = '33333333-3333-4333-8333-333333333333'
OTHER_RELEASE = '44444444-4444-4444-8444-444444444444'
TURN = '55555555-5555-4555-8555-555555555555'
def task(question='查询本月综合合格率', **updates):
value = {'kind': 'metrics', 'goal': question, 'freshQuery': True,
'historyMode': 'none', 'dimensions': [], 'stages': [], 'fields': ['rate'],
'verifySource': False, 'fullDetails': False, 'visualPreference': 'none',
'visualType': None, 'responseStyle': 'standard', 'needsClarification': False,
'clarification': '', 'filterUpdates': {}, 'period': {'year': 2026, 'month': 9}}
value.update(updates)
return normalize_task(value, question, {'releaseId': RELEASE})
def snapshot(month, *, rate=.95):
return {'period': {'year': 2026, 'month': month}, 'compositeRate': rate,
'black': {'rate': .97, 'totalWeight': 100, 'scrapWeight': 3},
'bright': {'rate': .98, 'totalWeight': 100, 'scrapWeight': 2},
'coverage': {'sourceMonths': [{'year': 2026, 'month': month,
'blackRows': 2, 'brightRows': 2}]}}
def metrics(*, month=9, product=None, rate=.95):
side = snapshot(month, rate=rate)
return {'kind': 'quality_metrics', 'releaseId': RELEASE,
'period': side['period'], 'filters': {'product': product} if product else {},
'view': 'overall', 'summary': {k: v for k, v in side.items() if k != 'period'},
'coverage': side['coverage'], 'rows': [], 'trend': []}
def comparison(*, month=9, product=None, rate=.95):
return {'kind': 'quality_comparison', 'releaseId': RELEASE,
'filterBinding': {'product': product} if product else {},
'current': snapshot(month, rate=rate), 'comparison': snapshot(month - 1, rate=.96),
'delta': {'available': True, 'percentagePoints': (rate - .96) * 100},
'stageRateChangesPP': {'black': 0, 'bright': 0}}
def write_doc(root, result, identifier='a' * 32, *, mode=None):
mode = mode or ('compare' if result['kind'] == 'quality_comparison' else 'query')
ref = 'Q' + identifier[:8].upper()
document = {'tool': 'quality_data', 'mode': mode, 'releaseId': result['releaseId'],
'evidenceRef': ref, 'createdAt': '2026-10-25T00:00:00Z', 'result': result}
path = root / 'evidence' / ('nao-quality-' + identifier + '.json')
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(document), encoding='utf-8')
path.chmod(0o600)
return document, path
def model_response(request, timeout):
"""A provider response, not an evidence/coverage mock."""
selector = json.loads(json.loads(request.data)['messages'][1]['content'])
facts = selector.get('facts', [])
payload = {'answerBlocks': [{'type': 'paragraph', 'text': '{{' + f['id'] + '}}'}
for f in facts[:12]],
'evidenceSufficient': True, 'missingEvidence': []}
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop',
'message': {'content': json.dumps(payload)}}]}).encode())
class AgentRuntimeIntegrationTest(unittest.TestCase):
def test_repeated_question_does_not_inherit_previous_wrong_answer_or_goal(self):
context={'_serverQuestions':[{'turnId':'first','question':'9月合格率为什么下降','status':'PARTIAL'},
{'turnId':'bad1','question':'主要是国内还是国外','status':'COMPLETED'},
{'turnId':'bad2','question':'主要是国内还是国外','status':'COMPLETED'}],
'_serverHistory':[{'turnId':'bad1','answer':'单期报废构成','evidenceRefs':['QAAAAAAAA']},
{'turnId':'bad2','answer':'本期总体','evidenceRefs':['QBBBBBBBB']}],
'lastVisibleAnswer':'错误的单期总体','filters':{'department':'新能源汽车磁钢'}}
clean,repeated=runtime._drop_repeated_followup_context('主要是国内还是国外',context)
self.assertTrue(repeated)
self.assertEqual([q['turnId'] for q in clean['_serverQuestions']],['first'])
self.assertEqual(clean['_serverHistory'],[])
self.assertNotIn('lastVisibleAnswer',clean)
self.assertEqual(clean['filters'],context['filters'])
self.assertEqual(len(context['_serverQuestions']),3)
single,reset=runtime._drop_repeated_followup_context('同一问题',{'_serverQuestions':[{'turnId':'one','question':'同一问题'}],
'_serverHistory':[{'turnId':'one','answer':'旧结论'}]})
self.assertTrue(reset);self.assertEqual(single['_serverQuestions'],[]);self.assertEqual(single['_serverHistory'],[])
def test_repeat_context_handles_truncation_and_older_same_question_answers(self):
question='比较某产品质量变化及具体分组差异'*24
truncated=question[:219]+'…'
context={'_serverQuestions':[{'turnId':'root','question':'原来的总体分析','status':'PARTIAL'},
{'turnId':'recent','question':truncated,'status':'COMPLETED','truncated':True}],
'_serverHistory':[{'turnId':'older','question':truncated,'answer':'过早的错误结论'},
{'turnId':'recent','question':truncated,'answer':'最近的错误结论'}]}
clean,repeated=runtime._drop_repeated_followup_context(question,context)
self.assertTrue(repeated);self.assertEqual(clean['_serverHistory'],[])
self.assertEqual([x['turnId'] for x in clean['_serverQuestions']],['root'])
def run_mocked(self, root, planned, native, *, repair=None, primary=None, history=None):
root.mkdir(parents=True, exist_ok=True)
(root / 'evidence').mkdir(exist_ok=True)
(root / 'private').mkdir(exist_ok=True)
payload = {'turnId': TURN, 'sessionId': SESSION, 'actorId': ACTOR,
'releaseId': RELEASE, 'accessToken': 'synthetic-delegation',
'question': planned['question'], 'context': {'releaseId': RELEASE}}
if history is not None: payload['context']['_serverHistory'] = history
manager = MagicMock()
manager.acquire_actor.return_value = (root, None)
manager.ensure_instance.return_value = ('http://127.0.0.1:1/v1', 'synthetic-key',
'synthetic-model', 'http://127.0.0.1:2')
state = MagicMock()
state.get_chat_id.return_value = None
gate = MagicMock()
gate.base_url = 'http://127.0.0.1:1/v1'
with ExitStack() as stack:
replacements = {
'RuntimeManager': MagicMock(return_value=manager),
'SessionState': MagicMock(return_value=state),
'write_access_file': MagicMock(return_value=root / 'access.json'),
'_authenticate': MagicMock(), '_input_cancel_listener': MagicMock(),
'_model_settings': MagicMock(return_value=('https://offline.invalid/v1', 'fake-key', 'synthetic-model',{'enabled':False,'effort':'auto','protocol':'thinking'})),
'_model_gate_settings': MagicMock(return_value=(1, 'synthetic-key')),
'DelegatedApi': MagicMock(), '_sse_request': native,
'_story_files': MagicMock(return_value=(None, None)),
'plan_agent_task': MagicMock(return_value=planned),
'read_primary_evidence': primary or MagicMock(return_value=None),
}
for name, replacement in replacements.items():
stack.enter_context(patch.object(runtime, name, replacement))
stack.enter_context(patch('nao_model_gate.ModelGate', return_value=gate))
stack.enter_context(patch.object(finalizer, '_open_model', side_effect=model_response))
if repair is not None:
stack.enter_context(patch.object(QualityTools, 'quality_data', side_effect=repair))
result = runtime.run_turn(payload, stdout=io.StringIO(), stdin=io.StringIO())
gate.close.assert_called_once()
return result
@staticmethod
def accumulator(answer):
value = runtime.ConversationAccumulator()
value.blocks['native'] = answer
value.block_order.append('native')
value.tool_calls.append({'tool': 'quality_data', 'status': 'completed'})
return value
def test_real_scalar_query_kind_is_fresh_and_completes_with_matching_coverage(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
write_doc(root, metrics())
return self.accumulator('综合合格率95.00% [QAAAAAAAA]'), 'COMPLETED', 'stop', None
result = self.run_mocked(root, task(), native)
self.assertEqual(result['status'], 'COMPLETED')
self.assertNotIn('未取得新的查询证据', result['answer'])
def test_scalar_wrong_product_period_and_missing_requested_field_remain_partial(self):
for defect in ('product', 'period', 'field'):
with self.subTest(defect=defect), tempfile.TemporaryDirectory() as folder:
root = Path(folder)
result_doc = metrics()
if defect == 'product': result_doc['filters'] = {'product': 'other'}
if defect == 'period': result_doc = metrics(month=8)
if defect == 'field': result_doc['summary'].pop('compositeRate')
def native(**kwargs):
write_doc(root, result_doc)
return self.accumulator('原生回答'), 'COMPLETED', 'stop', None
with patch('nao_agent_task.plan_repairs', return_value=[]):
result = self.run_mocked(root, task(), native)
self.assertEqual(result['status'], 'PARTIAL')
def test_no_initial_evidence_enters_repair_and_recovers_comparison(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
planned = task(kind='analysis', fields=[], comparisonPeriod={'year': 2026, 'month': 8})
def native(**kwargs):
return self.accumulator('尚未取得数据'), 'COMPLETED', 'stop', None
def repair(arguments):
document, _ = write_doc(root, comparison())
return {**document['result'], 'evidenceRef': document['evidenceRef']}
with patch('nao_agent_task.plan_repairs', return_value=[{'mode': 'compare', 'releaseId': RELEASE,
'period': planned['period'], 'comparisonPeriod': planned['comparisonPeriod'], 'filters': {}}]) as planner:
result = self.run_mocked(root, planned, native, repair=repair)
self.assertTrue(planner.called)
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['completionMethod'], 'bounded_evidence_finalization')
self.assertNotIn('尚未取得数据', result['answer'])
self.assertIn('[QAAAAAAAA]', result['answer'])
def test_scalar_repair_must_replace_the_native_no_data_answer(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
return self.accumulator('尚未取得数据,无法给出合格率。'), 'COMPLETED', 'stop', None
def repair(arguments):
document, _ = write_doc(root, metrics())
return {**document['result'], 'evidenceRef': document['evidenceRef']}
with patch('nao_agent_task.plan_repairs', return_value=[{'mode': 'query', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9}, 'filters': {}}]):
result = self.run_mocked(root, task(), native, repair=repair)
self.assertEqual(result['status'], 'COMPLETED')
self.assertNotIn('尚未取得数据', result['answer'])
self.assertIn('95.00%', result['answer'])
def test_scalar_native_claim_must_agree_with_queried_value(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
write_doc(root, metrics(rate=.95))
return self.accumulator('综合合格率99.00% [QAAAAAAAA]'), 'COMPLETED', 'stop', None
result = self.run_mocked(root, task(), native)
self.assertEqual(result['status'], 'COMPLETED')
self.assertNotIn('99.00%', result['answer'])
self.assertIn('95.00%', result['answer'])
def test_ai_selected_primary_frame_is_queried_before_local_investigation(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
planned = task(filterUpdates={'product': 'primary-product'})
def primary(**kwargs):
self.assertEqual(kwargs['arguments']['mode'], 'query')
self.assertEqual(kwargs['arguments']['filters'], {'product': 'primary-product'})
self.assertEqual(kwargs['arguments']['period'], {'year': 2026, 'month': 9})
result = metrics(product='primary-product', rate=.95)
document, _ = write_doc(root, result)
return {**result, 'evidenceRef': document['evidenceRef']}
def native(**kwargs):
self.assertEqual(kwargs['context']['primaryEvidence']['summary']['compositeRate'], .95)
write_doc(root, metrics(product='unrelated-child', rate=.81), identifier='b' * 32)
return self.accumulator('错误局部81%'), 'COMPLETED', 'stop', None
result = self.run_mocked(root, planned, native, primary=primary)
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('95.00%', result['answer'])
self.assertNotIn('81.00%', result['answer'])
def test_single_period_analysis_without_breakdowns_still_requires_matching_frame(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
write_doc(root, metrics(month=8))
return self.accumulator('来自八月的旧期间结论'), 'COMPLETED', 'stop', None
with patch('nao_agent_task.plan_repairs', return_value=[]):
result = self.run_mocked(root, task(kind='analysis', dimensions=[]), native)
self.assertEqual(result['status'], 'PARTIAL')
def test_inconsistent_nonfresh_metrics_plan_cannot_complete_without_any_proof(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
def native(**kwargs):
accumulator = self.accumulator('未经查询的综合合格率99.00%')
accumulator.tool_calls.clear()
return accumulator, 'COMPLETED', 'stop', None
with patch('nao_agent_task.plan_repairs', return_value=[]):
result = self.run_mocked(root, task(freshQuery=False), native)
self.assertEqual(result['status'], 'PARTIAL')
def test_off_target_native_answer_is_not_completed_when_final_synthesis_fails(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
planned=task(question='主要是国内还是国外的',kind='analysis',dimensions=['group'],
dimensionStages={'group':[]},comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
def native(**kwargs):
write_doc(root,comparison())
changes={'kind':'quality_changes','releaseId':RELEASE,'dimension':'group','stage':None,'filters':{},
'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
'sort':'scrap_delta','totalRows':2,'returnedRows':2,'truncated':False,
'rows':[{'key':'国内','label':'国内','current':{'rate':.95},'comparison':{'rate':.96},'rateDrop':.01,'scrapDelta':5},
{'key':'海外','label':'海外','current':{'rate':.96},'comparison':{'rate':.96},'rateDrop':0,'scrapDelta':0}]}
write_doc(root,changes,identifier='b'*32,mode='changes')
return self.accumulator('仅有总体指标,没有回答国内或海外。'), 'COMPLETED','stop',None
with patch.object(finalizer,'synthesize',side_effect=finalizer.FinalizationUnavailable('服务不可用')):
result=self.run_mocked(root,planned,native)
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('答复尚未完成核验',result['answer'])
def test_wrong_native_metric_is_not_completed_when_final_synthesis_fails(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
def native(**kwargs):
write_doc(root,metrics(rate=.95))
return self.accumulator('综合合格率99.00% [QAAAAAAAA]'),'COMPLETED','stop',None
with patch.object(finalizer,'synthesize',side_effect=finalizer.FinalizationUnavailable('服务不可用')):
result=self.run_mocked(root,task(),native)
self.assertEqual(result['status'],'PARTIAL')
self.assertNotIn('99.00%',result['answer'])
def test_incomplete_tool_call_cannot_skip_answer_validation(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
def native(**kwargs):
write_doc(root,metrics(rate=.95))
value=self.accumulator('综合合格率99.00% [QAAAAAAAA]')
value.tool_calls[0]['status']='started'
return value,'COMPLETED','stop',None
result=self.run_mocked(root,task(),native)
self.assertEqual(result['status'],'PARTIAL')
self.assertNotIn('99.00%',result['answer'])
def test_nonfresh_business_restatement_without_verified_history_is_partial(self):
for kind in ('explain','restate'):
with self.subTest(kind=kind),tempfile.TemporaryDirectory() as folder:
root=Path(folder)
planned=task(question='主要是国内还是国外的',kind=kind,freshQuery=False,
historyMode='last',comparisonRequired=True,
comparisonPeriod={'year':2026,'month':8})
def native(**kwargs):
result=self.accumulator('主要是国外,沿用上一轮未完成的结论。')
result.tool_calls.clear()
return result,'COMPLETED','stop',None
with patch('nao_agent_task.plan_repairs',return_value=[]):
result=self.run_mocked(root,planned,native)
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('未找到同范围、同期间的已核实历史依据',result['answer'])
def test_nonfresh_restatement_rejects_answer_that_does_not_match_question(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);write_doc(root,comparison())
planned=task(question='主要是国内还是国外的',kind='explain',freshQuery=False,
historyMode='last',comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
history=[{'turnId':'prior','question':'为什么下降','answer':'综合率下降','evidenceRefs':['QAAAAAAAA'],
'context':{'releaseId':RELEASE,'year':2026,'month':9,'comparisonPeriod':{'year':2026,'month':8}}}]
def native(**kwargs):
value=self.accumulator('只是总体下降,没有回答分组。');value.tool_calls.clear()
return value,'COMPLETED','stop',None
with patch('nao_agent_task.plan_repairs',return_value=[]),patch('nao_agent_task.verify_history_answer',return_value=False,create=True):
result=self.run_mocked(root,planned,native,history=history)
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('答复尚未完成核验',result['answer'])
def test_nonfresh_restatement_can_use_matching_verified_history(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);write_doc(root,comparison())
planned=task(question='综合合格率变化多少',kind='explain',freshQuery=False,
historyMode='last',comparisonRequired=True,comparisonPeriod={'year':2026,'month':8})
history=[{'turnId':'prior','question':'比较两期','answer':'综合率96%到95%','evidenceRefs':['QAAAAAAAA'],
'context':{'releaseId':RELEASE,'year':2026,'month':9,'comparisonPeriod':{'year':2026,'month':8}}}]
def native(**kwargs):
value=self.accumulator('综合合格率由96%降到95%。');value.tool_calls.clear()
return value,'COMPLETED','stop',None
with patch('nao_agent_task.plan_repairs',return_value=[]),patch('nao_agent_task.verify_history_answer',return_value=True):
result=self.run_mocked(root,planned,native,history=history)
self.assertEqual(result['status'],'COMPLETED')
class RepairScopeBindingTest(unittest.TestCase):
def plan(self, query, planned):
with patch('nao_agent_task._model_json', return_value={'queries': [query]}):
return plan_repairs(base_url='https://offline.invalid/v1', api_key='synthetic', model='synthetic',
task=planned, evidence=[], gaps=['missing'], failures=[], release_id=RELEASE,
cancel_event=threading.Event())
def test_changes_inherits_the_only_requested_stage(self):
planned = task(kind='analysis', dimensions=['product'], stages=['black'],
comparisonPeriod={'year': 2026, 'month': 8})
accepted = self.plan({'mode': 'changes', 'dimension': 'product', 'filters': {}}, planned)
self.assertEqual(accepted[0]['stage'], 'black')
self.assertEqual(accepted[0]['period'], planned['period'])
self.assertEqual(accepted[0]['comparisonPeriod'], planned['comparisonPeriod'])
def test_changes_cannot_switch_to_unrequested_stage(self):
planned = task(kind='analysis', dimensions=['product'], stages=['black'],
comparisonPeriod={'year': 2026, 'month': 8})
with self.assertRaises(TaskUnavailable):
self.plan({'mode': 'changes', 'dimension': 'product', 'stage': 'bright', 'filters': {}}, planned)
class TrustedHistoricalReportTest(unittest.TestCase):
def test_verified_full_report_fallback_is_not_appended_twice(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
document, _ = write_doc(root, comparison())
with patch.object(finalizer, 'synthesize', return_value={
'answer': '正式完整统计及明细', 'evidenceSufficient': True,
'reportTitle': '质量变化分析', 'titleGeneratedByModel': True,
'selectionFallback': True}):
result = self.run_report(root, [self.entry(document['evidenceRef'])])
self.assertTrue(result['reportMarkdown'].startswith('正式完整统计及明细\n\n## 数据明细'))
self.assertEqual(result['reportMarkdown'].count('正式完整统计及明细'), 1)
def run_report(self, root, entries, *, last=False):
question = '整合前面分析生成报告'
planned = task(question, kind='report', freshQuery=False, historyMode='last' if last else 'all',
fields=[], period=None)
request = {'question': question, 'releaseId': RELEASE, 'turnId': TURN,
'actorId': ACTOR, 'sessionId': SESSION,
'context': {'_serverHistory': entries}}
with patch.object(finalizer, '_open_model', side_effect=model_response):
return runtime.historical_report(request=request, task=planned, actor_root=root,
connection=('https://offline.invalid/v1', 'synthetic-only', 'synthetic-model'),
cancel_event=threading.Event(), emit=lambda value: None)
@staticmethod
def entry(ref, turn=TURN):
return {'turnId': turn, 'evidenceRefs': [ref]}
def test_all_reopens_both_trusted_frames_and_excludes_unreferenced_files(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
a, _ = write_doc(root, comparison(product='产品甲'), 'a' * 32)
b, _ = write_doc(root, comparison(month=7, product='产品乙'), 'b' * 32)
write_doc(root, comparison(product='未授权引用产品'), 'c' * 32)
# An unselected corrupt file must not poison the selected report.
bad = root / 'evidence' / ('nao-quality-' + 'd' * 32 + '.json')
bad.write_text('not-json')
result = self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('产品甲', result['reportMarkdown'])
self.assertEqual(result['agentContext']['filters'], {}, 'all-history report must clear the previous local-only display scope')
self.assertIn('产品乙', result['reportMarkdown'])
self.assertNotIn('未授权引用产品', result['reportMarkdown'])
def test_missing_other_session_ref_cannot_be_reopened(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder) / 'current'
other = Path(folder) / 'other'
a, _ = write_doc(root, comparison(), 'a' * 32)
b, _ = write_doc(other, comparison(product='另一会话'), 'b' * 32)
with self.assertRaises(runtime.RuntimeErrorSafe):
self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
def test_report_rejects_wrong_release_public_permissions_and_symlink(self):
for defect in ('release', 'permissions', 'symlink'):
with self.subTest(defect=defect), tempfile.TemporaryDirectory() as folder:
root = Path(folder) / 'current'
result = comparison()
if defect == 'release': result['releaseId'] = OTHER_RELEASE
doc, path = write_doc(root, result)
if defect == 'permissions': path.chmod(0o644)
if defect == 'symlink':
target = Path(folder) / 'external.json'
path.rename(target)
path.symlink_to(target)
with self.assertRaises((runtime.RuntimeErrorSafe, finalizer.FinalizationUnavailable)):
self.run_report(root, [self.entry(doc['evidenceRef'])])
def test_last_does_not_reopen_an_older_unavailable_entry(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
a, _ = write_doc(root, comparison(product='最新范围'))
result = self.run_report(root, [self.entry('QFFFFFFFF'), self.entry(a['evidenceRef'])], last=True)
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('最新范围', result['reportMarkdown'])
def test_all_must_not_claim_a_reference_free_turn_was_verified(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
a, _ = write_doc(root, comparison())
entries = [self.entry(a['evidenceRef']), {'turnId': 'unverified-turn', 'evidenceRefs': []}]
result=self.run_report(root, entries)
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('未纳入本次报告统计',result['answer'])
self.assertTrue(result['reportMarkdown'])
self.assertEqual(result['unverifiedHistoryTurnIds'],['unverified-turn'])
def test_recorded_non_query_turns_do_not_block_all_or_last_analysis_reports(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);a,_=write_doc(root,comparison())
chat_turn='66666666-6666-4666-8666-666666666666'
runtime._record_non_query_turn(root,{'turnId':chat_turn,'actorId':ACTOR,'sessionId':SESSION,'releaseId':RELEASE})
entries=[{'turnId':chat_turn,'evidenceRefs':[]},self.entry(a['evidenceRef']),{'turnId':chat_turn,'evidenceRefs':[]}]
for last in (False,True):
result=self.run_report(root,entries,last=last)
self.assertEqual(result['status'],'COMPLETED')
self.assertEqual(result['unverifiedHistoryTurnIds'],[])
def test_non_query_marker_from_another_scope_cannot_hide_unverified_analysis(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);a,_=write_doc(root,comparison())
chat_turn='66666666-6666-4666-8666-666666666666'
runtime._record_non_query_turn(root,{'turnId':chat_turn,'actorId':ACTOR,'sessionId':SESSION,'releaseId':OTHER_RELEASE})
result=self.run_report(root,[self.entry(a['evidenceRef']),{'turnId':chat_turn,'evidenceRefs':[]}])
self.assertEqual(result['status'],'PARTIAL')
def test_all_includes_single_period_turn_in_addition_to_comparison_turn(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
a, _ = write_doc(root, comparison(product='比较范围'), 'a' * 32)
b, _ = write_doc(root, metrics(month=7, product='单期范围', rate=.81), 'b' * 32)
result = self.run_report(root, [self.entry(a['evidenceRef']), self.entry(b['evidenceRef'])])
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('单期范围', result['reportMarkdown'])
self.assertIn('81.00%', result['reportMarkdown'])
def test_history_can_report_a_verified_source_turn_without_unrelated_metric_query(self):
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
source = {'kind': 'quality_investigation', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 9}, 'filters': {}, 'stage': 'black',
'action': 'source', 'current': {
'context': {'releaseId': RELEASE, 'year': 2026, 'month': 9,
'stage': 'black', 'includeUnassigned': False},
'aggregateComplete': True, 'totalRows': 2,
'sourceVerification': {'complete': True, 'allSelectedValuesMatchStored': True,
'expectedRows': 2, 'verifiedRows': 2,
'missingRows': 0, 'mismatchRows': 0}}}
doc, _ = write_doc(root, source, mode='investigate')
result = self.run_report(root, [self.entry(doc['evidenceRef'])])
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('原表', result['reportMarkdown'])
self.assertIn('[QAAAAAAAA]', result['answer'])
@staticmethod
def defect(month=9, product=None):
return {'kind':'quality_fine_defects','releaseId':RELEASE,'dimension':'defect',
'period':{'year':2026,'month':month},'comparisonPeriod':None,
'filters':{'product':product} if product else {},'stage':'black',
'sourceRows':2,'rankingComplete':True,'coverage':{'available':True,'hasValidWeights':True},
'rows':[{'key':'corner','label':'责废缺角','category':'责废','weight':12.5}]}
def test_history_reports_single_period_defects_without_parent_metrics(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);doc,_=write_doc(root,self.defect(),mode='defects')
result=self.run_report(root,[self.entry(doc['evidenceRef'])])
self.assertEqual(result['status'],'COMPLETED')
self.assertIn('责废缺角',result['reportMarkdown'])
self.assertIn('12.50',result['reportMarkdown'])
self.assertNotIn('综合合格率',result['answer'])
def test_history_keeps_defect_periods_scopes_and_mixed_metrics(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
a,_=write_doc(root,self.defect(product='范围甲'),'a'*32,mode='defects')
b,_=write_doc(root,self.defect(month=8,product='范围乙'),'b'*32,mode='defects')
c,_=write_doc(root,metrics(product='指标范围'),'c'*32)
result=self.run_report(root,[self.entry(d['evidenceRef']) for d in (a,b,c)])
self.assertEqual(result['status'],'COMPLETED')
for text in ('范围甲','范围乙','指标范围','2026年9月','2026年8月','12.50'):
self.assertIn(text,result['reportMarkdown'])
def test_incomplete_historical_defects_are_not_marked_complete(self):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);good=self.defect();bad=self.defect(month=8);bad['rankingComplete']=False
a,_=write_doc(root,good,'a'*32,mode='defects');b,_=write_doc(root,bad,'b'*32,mode='defects')
result=self.run_report(root,[self.entry(d['evidenceRef']) for d in (a,b)])
self.assertEqual(result['status'],'PARTIAL')
self.assertIn('12.50',result['reportMarkdown'])
def test_history_reports_weight_ranking_without_parent_metrics(self):
from nao_weight_ranking import rank_response
response={'rows':[{'key':'规格甲','black':{'totalWeight':60}},{'key':'规格乙','black':{'totalWeight':40}}],
'summary':{'black':{'totalWeight':100}},'context':{'rowCount':2},
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':2}]}}
for period in ({'year':2026,'month':9},{'year':2026}):
response['coverage']['sourceMonths']=[{'year':2026,'month':month,'blackRows':2}
for month in (range(1,10) if 'month' not in period else [9])]
ranking=rank_response(response,{'id':RELEASE},period,{},'product','black','totalWeight',1)
with self.subTest(period=period),tempfile.TemporaryDirectory() as folder:
root=Path(folder);doc,_=write_doc(root,ranking,mode='rank')
result=self.run_report(root,[self.entry(doc['evidenceRef'])])
self.assertEqual(result['status'],'COMPLETED')
self.assertIn('规格甲',result['reportMarkdown'])
self.assertIn('60.00%',result['reportMarkdown'])
self.assertIn('100.00',result['reportMarkdown'])
if 'month' not in period:self.assertIn('1—9月已发布数据',result['reportMarkdown'])
if __name__ == '__main__':
unittest.main()
+238
View File
@@ -0,0 +1,238 @@
"""Evidence-bound monthly answers must not turn an annual snapshot into a trend."""
import copy
import io
import json
import sys
import threading
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_agent_report import build_noncomparison_report
from nao_evidence_finalizer import coverage_sufficient, synthesize
from nao_analysis_coverage import coverage_gaps, plan_missing_queries
QUESTION = '查看客户甲今年的综合合格率变化趋势'
RELEASE = 'release-trend'
def task(**updates):
value = {'version': 1, 'kind': 'metrics', 'question': QUESTION, 'goal': QUESTION,
'period': {'year': 2026}, 'comparisonPeriod': None, 'comparisonRequired': False,
'filters': {'extraFilters': {'customerCode': ['客户甲']}},
'freshQuery': True, 'dimensions': [], 'dimensionStages': {}, 'stages': [],
'fields': ['rate'], 'historyMode': 'none', 'trendRequired': True,
'periodCoverage': 'available', 'visualPreference': 'auto', 'visualType': None, 'responseStyle': 'brief',
'verifySource': False, 'fullDetails': False}
value.update(updates)
return value
def context(**updates):
t = task(**updates)
return {'releaseId': RELEASE, 'period': t['period'], 'filters': t['filters'], '_agentTask': t}
def doc(months=(1, 3, 4), customer='客户甲', release=RELEASE, ref='Q12345678'):
rows = [{'period': f'2026-{m:02d}', 'compositeRate': value,
'black': {'rate': value - .01, 'totalWeight': 100},
'bright': {'rate': value + .01, 'totalWeight': 90},
'coverage': {'blackRows': 2, 'brightRows': 2}}
for m, value in zip(months, (.75, .85, .80, .78))]
return {'evidenceRef': ref, 'tool': 'quality_data', 'mode': 'query', 'releaseId': release, 'result': {
'kind': 'quality_metrics', 'releaseId': release, 'view': 'overall',
'period': {'year': 2026}, 'filters': {'extraFilters': {'customerCode': [customer]}},
'summary': {'compositeRate': .81}, 'trend': rows, 'trendCount': len(rows),
'trendTruncated': False, 'coverage': {'sourceMonths': [
{'year': 2026, 'month': m, 'blackRows': 2, 'brightRows': 2} for m in months]}}}
class TrendEvidenceTests(unittest.TestCase):
def test_annual_trend_uses_formal_months_and_keeps_missing_month_empty(self):
report = build_noncomparison_report([doc()], context(), QUESTION)
self.assertTrue(report['coverageSufficient'])
self.assertTrue(coverage_sufficient([doc()], QUESTION, context()))
text = '\n'.join(f['text'] for f in report['facts'])
self.assertIn('1月75.00%', text)
self.assertIn('2月未提供', text)
self.assertIn('3月85.00%', text)
self.assertIn('4月80.00%', text)
self.assertIn('首末对比,4月较1月上升', text)
self.assertNotIn('81.00%', text)
self.assertEqual(report['facts'][0]['evidenceRef'], 'Q12345678')
def test_explicit_comparison_still_requires_matching_comparison_evidence(self):
comparison = context(comparisonRequired=True, comparisonPeriod={'year': 2025})
self.assertFalse(coverage_sufficient([doc()], QUESTION, comparison))
self.assertTrue(coverage_gaps([doc()], QUESTION, comparison))
def test_client_filter_and_release_must_match_exactly(self):
wrong = doc(customer='客户乙')
right = doc(ref='Q87654321')
stale = doc(release='old', ref='QABCDEF12')
for docs in ([wrong], [stale]):
report = build_noncomparison_report(docs, context(), QUESTION)
self.assertFalse(report['coverageSufficient'])
self.assertFalse(coverage_sufficient(docs, QUESTION, context()))
self.assertFalse(report['facts'])
report = build_noncomparison_report([wrong, stale, right], context(), QUESTION)
self.assertTrue(report['coverageSufficient'])
self.assertEqual([f['evidenceRef'] for f in report['facts']], ['Q87654321'])
def test_history_report_never_relabels_another_customer_as_the_current_scope(self):
historical = context(kind='report', historyMode='all', freshQuery=False)
wrong = doc(customer='客户乙')
report = build_noncomparison_report([wrong], historical, QUESTION)
self.assertFalse(report['coverageSufficient'])
self.assertFalse(report['facts'])
self.assertFalse(coverage_sufficient([wrong], QUESTION, historical))
def test_empty_truncated_wrong_year_and_duplicate_month_are_incomplete(self):
cases = []
empty = doc(months=()); cases.append(empty)
truncated = doc(); truncated['result']['trendTruncated'] = True; cases.append(truncated)
count = doc(); count['result']['trendCount'] += 1; cases.append(count)
wrong_year = doc(); wrong_year['result']['trend'][0]['period'] = '2025-01'; cases.append(wrong_year)
duplicate = doc(); duplicate['result']['trend'][1]['period'] = '2026-01'; cases.append(duplicate)
for item in cases:
with self.subTest(item=item):
self.assertFalse(build_noncomparison_report([item], context(), QUESTION)['coverageSufficient'])
self.assertFalse(coverage_sufficient([item], QUESTION, context()))
def test_no_trend_data_returns_an_explanation_without_asking_model_to_invent(self):
empty = doc(months=())
def opener(request, timeout):
raise AssertionError('No factual content should be sent for narration')
result = synthesize(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question=QUESTION, evidence=[empty], cancel_event=threading.Event(),
context=context(), opener=opener)
self.assertFalse(result['evidenceSufficient'])
self.assertIn('未取得符合本轮范围和期间的可用月度趋势数据', result['answer'])
self.assertIn('不足以判断趋势', result['answer'])
def test_single_stage_composite_retains_formal_business_rate(self):
item = doc()
for row in item['result']['trend'][:2]: row['coverage']['brightRows'] = 0
report = build_noncomparison_report([item], context(), QUESTION)
self.assertTrue(report['coverageSufficient'])
self.assertTrue(coverage_sufficient([item], QUESTION, context()))
self.assertIn('75.00%', str(report['facts']))
self.assertIn('85.00%', str(report['facts']))
def test_stage_reference_both_directions_matches_chart_and_customer_scope(self):
from analysis_visuals import build_visuals
item = doc(months=(1, 2, 3, 4))
rows = item['result']['trend']
rows[0]['coverage']['blackRows'] = 0
rows[1]['coverage']['blackRows'] = 0
rows[1]['coverage']['brightRows'] = 0
rows[2]['coverage']['brightRows'] = 0
report = build_noncomparison_report([item], context(), QUESTION)
self.assertTrue(report['coverageSufficient'])
text = report['facts'][0]['text']
for value in ('1月75.00%', '2月未提供', '3月80.00%', '4月78.00%'):
self.assertIn(value, text)
visual = build_visuals([item], QUESTION, 'turn-customer-stage', context=context())
self.assertEqual(visual['visualState'], 'ready')
chart = next(c for c in visual['visuals'] if c.get('metric') == 'composite_rate')
self.assertEqual(chart['series'][0]['values'], [.75, None, .80, .78])
self.assertEqual(chart['scope']['extraFilters'], {'customerCode': ['客户甲']})
self.assertEqual(chart['evidenceRefs'], ['Q12345678'])
self.assertEqual(build_visuals([doc(customer='客户乙')], QUESTION, 'turn-wrong-client',
context=context())['visuals'], [])
def test_both_stages_missing_does_not_create_observations(self):
item = doc()
for row in item['result']['trend'][:2]:
row['coverage']['blackRows'] = 0
row['coverage']['brightRows'] = 0
report = build_noncomparison_report([item], context(), QUESTION)
self.assertFalse(report['coverageSufficient'])
self.assertFalse(coverage_sufficient([item], QUESTION, context()))
self.assertTrue(any('不足以判断趋势' in value for value in report['missingEvidence']))
self.assertNotIn('75.00%', str(report['facts']))
self.assertNotIn('85.00%', str(report['facts']))
def test_two_valid_months_explain_chart_limit_and_required_chart_is_partial(self):
item = doc(months=(1, 2))
report = build_noncomparison_report([item], context(), QUESTION)
self.assertTrue(report['coverageSufficient'])
self.assertIn('不足3个有效月份', report['facts'][0]['text'])
required = context(visualPreference='required')
self.assertFalse(build_noncomparison_report([item], required, QUESTION)['coverageSufficient'])
self.assertFalse(coverage_sufficient([item], QUESTION, required))
def test_missing_trend_is_requeried_only_in_exact_requested_frame(self):
ctx = context()
wrong = doc(customer='客户乙')
query = {'mode': 'query', 'releaseId': RELEASE, 'period': {'year': 2026},
'filters': {'extraFilters': {'customerCode': ['客户甲']}}, 'view': 'overall'}
self.assertEqual(plan_missing_queries([wrong], QUESTION, ctx), [query])
self.assertTrue(coverage_gaps([wrong], QUESTION, ctx))
self.assertEqual(plan_missing_queries([doc()], QUESTION, ctx), [])
self.assertEqual(coverage_gaps([doc()], QUESTION, ctx), [])
truncated = doc(); truncated['result']['trendTruncated'] = True
self.assertEqual(plan_missing_queries([truncated], QUESTION, ctx), [])
self.assertTrue(coverage_gaps([truncated], QUESTION, ctx))
def test_missing_trend_does_not_guess_period_or_drop_scope(self):
for ctx in (context(period=None), context(period={'year': 2026, 'month': 9}),
context(filters={'extraFilters': {'customerCode': ['客户甲']},
'exclusions': {'product': ['X']}}),
context(trendRequired=False), context(needsClarification=True)):
with self.subTest(ctx=ctx):
self.assertEqual(plan_missing_queries([], QUESTION, ctx), [])
no_release = context(); no_release.pop('releaseId')
self.assertEqual(plan_missing_queries([], QUESTION, no_release), [])
def test_runtime_fills_missing_trend_and_keeps_failed_reads_incomplete(self):
from unittest.mock import MagicMock, patch
import nao_quality_runtime as runtime
ctx = context()
request = {'actorId': 'actor', 'sessionId': 'session', 'releaseId': RELEASE,
'question': QUESTION, 'context': ctx}
for fails in (False, True):
with self.subTest(fails=fails):
docs = [doc(customer='客户乙')]
seen = []
def read(arguments):
seen.append(arguments)
if fails:
raise RuntimeError('no data')
evidence = doc()
docs.append(evidence)
return evidence
with patch('nao_evidence_finalizer.evidence_summaries', side_effect=lambda *a, **k: list(docs)), \
patch('nao_agent_task.plan_repairs', return_value=[]), \
patch.object(runtime, 'DelegatedApi') as delegated, \
patch('nao_quality_tools.QualityTools') as quality:
delegated.return_value.login.return_value = MagicMock()
quality.return_value.quality_data.side_effect = read
runtime.complete_agent_evidence(task=ctx['_agentTask'], request=request,
actor_root=Path('/offline/session'), access_path=Path('/offline/access'),
manager=MagicMock(), connection=('offline', 'synthetic', 'm'),
accumulator=runtime.ConversationAccumulator(), prior_files=frozenset(),
cancel_event=threading.Event(), emit=lambda event: None)
self.assertEqual(len(seen), 1)
self.assertEqual(seen[0]['filters'], {'extraFilters': {'customerCode': ['客户甲']}})
self.assertEqual(coverage_sufficient(docs, QUESTION, ctx), not fails)
def test_finalizer_includes_both_requested_stage_trends_even_when_model_omits_one(self):
ctx = context(stages=['black', 'bright'])
item = doc()
def opener(request, timeout):
answer = {'answerBlocks': [{'type': 'paragraph', 'factIds': ['F1']}],
'evidenceSufficient': True, 'missingEvidence': []}
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop',
'message': {'content': json.dumps(answer)}}]}).encode())
result = synthesize(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question=QUESTION, evidence=[item], cancel_event=threading.Event(),
context=ctx, opener=opener)
self.assertIn('黑片合格率月度趋势', result['answer'])
self.assertIn('亮片合格率月度趋势', result['answer'])
self.assertEqual(result['selectedFactIds'], ['F1', 'F2'])
if __name__ == '__main__':
unittest.main()
+179
View File
@@ -0,0 +1,179 @@
import sys
from pathlib import Path
import unittest
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
from analysis_visual_evidence import visual_result
from analysis_visuals import build_visuals
def evidence(result=None):
result = result or {
"kind": "quality_comparison",
"releaseId": "r-1",
"filterBinding": {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
"current": {
"period": {"year": 2026, "month": 9},
"compositeRate": 0.98,
"black": {"rate": 0.97, "scrapWeight": 12.0},
"bright": {"rate": 0.99, "scrapWeight": 5.0},
},
"comparison": {
"period": {"year": 2026, "month": 8},
"compositeRate": 0.96,
"black": {"rate": 0.95, "scrapWeight": 10.0},
"bright": {"rate": 0.98, "scrapWeight": 7.0},
},
"delta": {"available": True, "percentagePoints": 2.0},
"stageRateChangesPP": {"black": 2.0, "bright": 1.0},
}
return {
"tool": "quality_data",
"mode": "compare",
"releaseId": "r-1",
"evidenceRef": "Q1234ABCD",
"result": {"release": {"id": "r-1"}, **result},
}
def task(question, *, preference="auto", chart_type=None, kind="analysis", dimensions=None,
stages=None, fields=None):
return {
"version": 1,
"question": question,
"goal": "compare formal quality evidence",
"kind": kind,
"dimensions": dimensions or [],
"stages": stages or [],
"fields": fields or [],
"freshQuery": True,
"visualPreference": preference,
"visualType": chart_type,
"responseStyle": "concise",
}
def context_for(question, **kwargs):
return {
"releaseId": "r-1",
"filters": {"department": "电机部"},
"_agentTask": task(question, **kwargs),
}
class AgentVisualTests(unittest.TestCase):
def test_semantic_task_makes_chart_choice_independent_of_question_wording(self):
questions = ["电机部近期情况如何", "帮我看看这段时间的表现"]
outputs = [build_visuals([evidence()], question, "turn-semantic", context=context_for(
question, dimensions=["department"], fields=["rate"])) for question in questions]
self.assertEqual(outputs[0]["visualState"], "ready")
self.assertEqual(outputs[0]["visuals"], outputs[1]["visuals"])
self.assertEqual({chart["metric"] for chart in outputs[0]["visuals"]},
{"composite_rate", "black_rate", "bright_rate"})
def test_none_overrides_explicit_chart_words(self):
question = "画图比较两期综合合格率"
output = build_visuals([evidence()], question, "turn-none",
context=context_for(question, preference="none", fields=["rate"]))
self.assertEqual(output["visualState"], "not_requested")
self.assertEqual(output["visuals"], [])
def test_required_without_evidence_reports_unavailable_and_invents_nothing(self):
question = "检查重点产品"
output = build_visuals([], question, "turn-required-empty",
context=context_for(question, preference="required", fields=["rate"]))
self.assertEqual(output["visualState"], "unavailable")
self.assertEqual(output["visuals"], [])
def test_wrong_question_task_is_ignored(self):
question = "比较两个期间变化"
stale = task("不要图,只要文字", preference="none")
output = build_visuals([evidence()], question, "turn-stale",
context={"releaseId": "r-1", "filters": {"department": "电机部"},
"_agentTask": stale})
self.assertEqual(output["visualState"], "ready")
self.assertTrue(output["visuals"])
def test_semantic_scope_keeps_evidence_binding_and_requested_stage(self):
question = "只看本部门黑片"
output = build_visuals([evidence()], question, "turn-bound",
context=context_for(question, dimensions=["department"], stages=["black"],
fields=["rate"]))
self.assertEqual(output["visualState"], "ready")
self.assertEqual(len(output["visuals"]), 1)
chart = output["visuals"][0]
self.assertEqual(chart["stage"], "black")
self.assertEqual(chart["scope"]["department"], "电机部")
self.assertEqual(chart["evidenceRefs"], ["Q1234ABCD"])
def test_supported_semantic_style_is_applied_without_style_words(self):
question = "请查看两期质量表现"
output = build_visuals([evidence()], question, "turn-style",
context=context_for(question, chart_type="column", fields=["rate"]))
self.assertEqual(output["visualState"], "ready")
self.assertTrue(output["visuals"])
self.assertTrue(all(chart["type"] == "column" for chart in output["visuals"]))
def test_unsupported_for_evidence_style_has_no_fallback(self):
question = "请查看两期质量表现"
output = build_visuals([evidence()], question, "turn-style-unsafe",
context=context_for(question, preference="required", chart_type="heatmap", fields=["rate"]))
self.assertEqual(output["visualState"], "unavailable")
self.assertEqual(output["visuals"], [])
def test_auto_style_hint_cannot_hide_an_evidence_supported_chart(self):
question='请查看两期质量表现'
output=build_visuals([evidence()],question,'turn-auto-style',
context=context_for(question,chart_type='heatmap',fields=['rate']))
self.assertEqual(output['visualState'],'ready')
self.assertTrue(output['visuals'])
self.assertTrue(all(chart['type'] in ('bar','column') for chart in output['visuals']))
def test_auto_pareto_for_source_defects_preserves_weights_without_inventing_shares(self):
from test_analysis_visuals import fine_field
question='帮我看下上月黑片缺陷TOP'
keys=['责废缺角重量','料废缺角重量']
result={'kind':'quality_fine_defects','releaseId':'r-1','dimension':'defect',
'grain':'source_defect_weight','stage':'black','filters':{},'period':{'year':2026,'month':9},
'coverage':{'available':True,'sourceRows':2},'sourceRows':2,
'sourceFields':[fine_field(key,'责废' if key.startswith('责废') else '料废') for key in keys],
'rows':[{'key':key,'label':key,'category':'责废' if key.startswith('责废') else '料废','weight':weight}
for key,weight in zip(keys,[17513.162,13949.63])],
'totalRows':2,'truncated':False,'rankingComplete':True}
context={'releaseId':'r-1','filters':{},'_agentTask':task(question,dimensions=['defect'],stages=['black'],chart_type='pareto')}
output=build_visuals([evidence(result)],question,'turn-auto-defect',context=context)
self.assertEqual(output['visualState'],'ready')
chart=output['visuals'][0]
self.assertEqual(chart['type'],'bar')
self.assertEqual(chart['series'][0]['values'],[17513.162,13949.63])
self.assertFalse(any(series.get('unit')=='%' for series in chart['series']))
context['_agentTask']['visualPreference']='required'
self.assertEqual(build_visuals([evidence(result)],question,'turn-required-defect',context=context)['visuals'],[])
def test_auto_does_not_chart_a_single_number_metric(self):
question = "本期综合合格率是多少"
metric = evidence({
"kind": "quality_metrics", "releaseId": "r-1", "view": "overall",
"filters": {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
"period": {"year": 2026, "month": 9},
"summary": {"compositeRate": 0.98},
})
for kind in ("metrics", "restate", "explain"):
with self.subTest(kind=kind):
output = build_visuals([metric], question, f"turn-single-metric-{kind}",
context=context_for(question, kind=kind, fields=["rate"]))
self.assertEqual(output["visualState"], "not_requested")
self.assertEqual(output["visuals"], [])
def test_evidence_adapter_forwards_the_validated_task_unchanged(self):
question = "看两期表现"
request_context = context_for(question, fields=["rate"])
with patch("analysis_visual_evidence.load_visual_evidence", return_value=[evidence()]), \
patch("analysis_visuals.build_visuals", return_value={"visuals": []}) as builder:
visual_result(Path("."), "r-1", question, "turn-forward", context=request_context)
self.assertIs(builder.call_args.kwargs["context"]["_agentTask"], request_context["_agentTask"])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,76 @@
import json
import os
from pathlib import Path
import sys
import tempfile
import threading
import unittest
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from analysis_visual_evidence import load_visual_evidence, visual_result
class VisualEvidenceTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.root = Path(self.temp.name)
self.path = self.root / ('nao-quality-' + 'a' * 32 + '.json')
self.doc = {'tool': 'quality_data', 'mode': 'compare', 'releaseId': 'release',
'evidenceRef': 'QAAAAAAAA', 'result': {'releaseId': 'release', 'kind': 'quality_comparison'}}
self.save()
def tearDown(self):
self.temp.cleanup()
def save(self):
self.path.write_text(json.dumps(self.doc))
self.path.chmod(0o600)
def test_full_document_and_only_this_turn(self):
self.assertEqual(load_visual_evidence(self.root, 'release'), [self.doc])
self.assertEqual(load_visual_evidence(self.root, 'release', exclude_files={self.path.name}), [])
def test_release_tool_and_reference_mismatch_fail_closed(self):
for key, value in [('releaseId', 'other'), ('tool', 'arbitrary'), ('evidenceRef', 'forged')]:
with self.subTest(key=key):
original = self.doc[key]
self.doc[key] = value
self.save()
with self.assertRaises(ValueError): load_visual_evidence(self.root, 'release')
self.doc[key] = original
def test_symlinks_and_shared_files_rejected(self):
self.path.chmod(0o644)
with self.assertRaises(ValueError): load_visual_evidence(self.root, 'release')
self.path.unlink()
outside = self.root / 'outside.json'
outside.write_text(json.dumps(self.doc))
self.path.symlink_to(outside)
with self.assertRaises(OSError): load_visual_evidence(self.root, 'release')
def test_cancel_and_deadline_are_bounded(self):
cancel = threading.Event(); cancel.set()
self.assertEqual(load_visual_evidence(self.root, 'release', cancel_event=cancel), [])
with self.assertRaises(ValueError): load_visual_evidence(self.root, 'release', budget_seconds=-1)
def test_large_file_and_failure_never_change_business_status(self):
with patch('analysis_visual_evidence.FILE_LIMIT', 10):
self.assertEqual(visual_result(self.root, 'release', '比较', 'turn'),
{'visualSchemaVersion': 1, 'visualState': 'failed', 'visuals': []})
def test_feature_switch_does_not_read_data(self):
with patch.dict(os.environ, {'QUALITY_ANALYSIS_VISUALS_ENABLED': 'false'}):
result = visual_result(Path('/does-not-exist'), 'release', '比较', 'turn')
self.assertEqual(result['visualState'], 'not_requested')
def test_turn_context_is_forwarded_without_reinterpreting_question(self):
import inspect
context={'year':2026,'month':9,'group':'组A'}
kwargs={'context':context} if 'context' in inspect.signature(visual_result).parameters else {}
with patch('analysis_visuals.build_visuals',return_value={'visuals':[]}) as build:
visual_result(self.root,'release','比较','turn',**kwargs)
self.assertEqual(build.call_args.kwargs.get('context'),context)
if __name__ == '__main__': unittest.main()
+911
View File
@@ -0,0 +1,911 @@
import json
import unittest
from scripts.analysis_visuals import MAX_BODY_BYTES, build_visuals
def evidence(result, *, ref="Q1234ABCD", release_id="r-1"):
return {
"tool": "quality_data",
"mode": "compare",
"releaseId": release_id,
"evidenceRef": ref,
"result": {
"release": {"id": release_id},
"releaseId": release_id,
**result,
},
}
def comparison_result(**overrides):
result = {
"kind": "quality_comparison",
"releaseId": "r-1",
"filterBinding": {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
"current": {
"period": {"year": 2026, "month": 9},
"compositeRate": 0.98,
"black": {"rate": 0.97, "scrapWeight": 12.0},
"bright": {"rate": 0.99, "scrapWeight": 5.0},
},
"comparison": {
"period": {"year": 2026, "month": 8},
"compositeRate": 0.96,
"black": {"rate": 0.95, "scrapWeight": 10.0},
"bright": {"rate": 0.98, "scrapWeight": 7.0},
},
"delta": {"available": True, "percentagePoints": 2.0},
"stageRateChangesPP": {"black": 2.0, "bright": 1.0},
}
result.update(overrides)
return result
def decomposed_comparison_result(**overrides):
result = comparison_result(
current={
"period": {"year": 2026, "month": 9}, "compositeRate": 0.9556,
"black": {"rate": 0.947, "scrapWeight": 12.0,
"defects": [{"name": "责废", "weight": 12.0}, {"name": "料废", "weight": 8.0}, {"name": "外废", "weight": 2.0}]},
"bright": {"rate": 0.978, "scrapWeight": 5.0,
"defects": [{"name": "镀废", "weight": 5.0}, {"name": "材料废", "weight": 3.0},
{"name": "加工废", "weight": 2.0}, {"name": "扩散废", "weight": 1.0}]},
},
comparison={
"period": {"year": 2026, "month": 8}, "compositeRate": 0.96,
"black": {"rate": 0.95, "scrapWeight": 10.0,
"defects": [{"name": "责废", "weight": 10.0}, {"name": "料废", "weight": 7.0}, {"name": "外废", "weight": 1.0}]},
"bright": {"rate": 0.98, "scrapWeight": 7.0,
"defects": [{"name": "镀废", "weight": 6.0}, {"name": "材料废", "weight": 3.5},
{"name": "加工废", "weight": 2.0}, {"name": "扩散废", "weight": 1.5}]},
},
delta={"available": True, "rate": -0.0044, "percentagePoints": -0.44},
stageRateChangesPP={"black": -0.3, "bright": -0.2},
stageDecomposition={"status": "available", "blackContributionToCompositePP": -0.3,
"brightContributionToCompositePP": -0.14, "sumPP": -0.44},
)
result.update(overrides)
return result
def changes_result(**overrides):
result = {
"kind": "quality_changes",
"releaseId": "r-1",
"filters": {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
"dimension": "product",
"stage": "black",
"category": None,
"currentPeriod": {"year": 2026, "month": 9},
"comparisonPeriod": {"year": 2026, "month": 8},
"sort": "scrap_delta",
"rows": [
{"rank": 1, "key": "p-1", "label": "规格一", "scrapDelta": 4.0, "current": {"scrapWeight": 8.0}},
{"rank": 2, "key": "p-2", "label": "规格二", "scrapDelta": -2.0, "current": {"scrapWeight": 3.0}},
],
"totalRows": 8,
"returnedRows": 2,
"truncated": True,
}
result.update(overrides)
return result
def metric_result(period, composite_rate, *, release_id="r-1", filters=None):
return {
"kind": "quality_metrics",
"releaseId": release_id,
"period": period,
"filters": filters or {"department": "电机部", "includeUnassigned": False, "extraFilters": {}},
"view": "overall",
"summary": {"compositeRate": composite_rate, "black": {"rate": composite_rate - 0.01},
"bright": {"rate": composite_rate + 0.01}},
}
def fine_field(key, category):
return {"key": key, "label": key, "category": category, "sourceField": key}
class AnalysisVisualsTest(unittest.TestCase):
def test_frozen_followup_scope_selects_same_comparison_as_text(self):
import inspect
global_result=comparison_result(filterBinding={})
local=comparison_result(filterBinding={'group':'组A'})
local['current']['compositeRate']=.80
docs=[evidence(global_result,ref='Q00000001'),evidence(local,ref='Q00000002')]
kwargs={'context':{'year':2026,'month':9,'group':'组A'}} if 'context' in inspect.signature(build_visuals).parameters else {}
charts=build_visuals(docs,'比较本期综合率变化','turn-bound-followup',**kwargs)['visuals']
overview=[c for c in charts if c.get('metric')=='composite_rate']
self.assertEqual(len(overview),1)
self.assertEqual(overview[0]['scope'].get('group'),'组A')
self.assertEqual(overview[0]['series'][0]['values'],[.96,.80])
def test_frozen_period_excludes_other_period_comparison_and_defects(self):
import inspect
selected=comparison_result(filterBinding={})
older=comparison_result(filterBinding={})
older['current']['period']={'year':2026,'month':7};older['comparison']['period']={'year':2026,'month':6}
kwargs={'context':{'year':2026,'month':9}} if 'context' in inspect.signature(build_visuals).parameters else {}
charts=build_visuals([evidence(selected,ref='Q00000001'),evidence(older,ref='Q00000002')],
'比较综合率变化','turn-bound-period',**kwargs)['visuals']
self.assertTrue(charts)
self.assertTrue(all(c.get('period',{}).get('month')==9 for c in charts))
def test_available_requested_stage_is_drawn_when_other_stage_blocks_composite(self):
result=comparison_result(delta={'available':False,'reason':'缺少黑片来源'},stageRateChangesPP={'black':None,'bright':1})
result['current']['black']['rate']=1
charts=build_visuals([evidence(result)],'比较亮片合格率变化','turn-stage-only')['visuals']
self.assertEqual([c['metric'] for c in charts],['bright_rate'])
scoped=build_visuals([evidence(result)],'比较本期变化','turn-stage-context',context={'year':2026,'month':9,'stage':'bright'})['visuals']
self.assertEqual([c['metric'] for c in scoped],['bright_rate'])
def test_formal_comparison_emits_safe_single_metric_chart_with_zero_and_null(self):
result = comparison_result(
current={"period": {"year": 2026, "month": 9}, "compositeRate": 0.0,
"black": {"rate": None}, "bright": {"rate": 0.99}},
comparison={"period": {"year": 2026, "month": 8}, "compositeRate": None,
"black": {"rate": 0.95}, "bright": {"rate": 0.98}},
stageRateChangesPP={"black": None, "bright": 1.0},
)
output = build_visuals([evidence(result)], "请比较8月和9月综合合格率", "turn-01")
self.assertEqual(output["visualState"], "ready")
chart = output["visuals"][0]
self.assertEqual(chart["metric"], "composite_rate")
self.assertEqual(chart["unit"], "ratio")
self.assertEqual(chart["series"][0]["values"], [None, 0.0])
self.assertNotIn("stage", chart)
self.assertEqual(chart["scope"]["releaseId"], "r-1")
self.assertEqual(chart["originTurnId"], "turn-01")
self.assertEqual(chart["type"], "column")
def test_comparison_skips_mismatched_period_or_release(self):
bad_period = comparison_result()
bad_period["current"]["period"] = {"year": 2026, "month": 8}
self.assertEqual(build_visuals([evidence(bad_period)], "比较两个期间变化", "turn-01")["visualState"], "not_requested")
self.assertEqual(build_visuals([evidence(comparison_result(), release_id="other")], "比较两个期间变化", "turn-01")["visualState"], "not_requested")
def test_invalid_rate_is_never_clamped_or_drawn(self):
result = comparison_result()
result["current"]["compositeRate"] = 1.01
output = build_visuals([evidence(result)], "比较综合合格率变化", "turn-01")
self.assertEqual(output["visualState"], "not_requested")
self.assertEqual(output["visuals"], [])
def test_broad_cause_analysis_adds_three_rates_math_waterfall_and_both_category_deltas(self):
output = build_visuals([evidence(decomposed_comparison_result())], "分析整体合格率下降原因", "turn-broad-chapters")
charts = output["visuals"]
self.assertEqual(output["visualState"], "ready")
by_section = {}
for chart in charts:
by_section.setdefault(chart["sectionKey"], []).append(chart)
self.assertEqual(len(by_section["overview"]), 2)
self.assertEqual({chart["metric"] for chart in by_section["stage"]}, {"black_rate", "bright_rate"})
waterfall = next(chart for chart in charts if chart["type"] == "waterfall")
self.assertEqual(waterfall["metric"], "composite_impact_pp")
self.assertEqual(waterfall["series"][0]["values"], [-0.3, -0.14, -0.44])
self.assertEqual(waterfall["requiredFields"], [
"black_good_weight", "black_total_weight", "bright_good_weight", "bright_total_weight"
])
self.assertIn("拆解", waterfall["note"])
self.assertNotIn("数学", waterfall["note"])
self.assertNotIn("生产根因", waterfall["title"])
self.assertEqual({chart["sectionKey"] for chart in charts if chart["sectionKey"].startswith("category_")},
{"category_black", "category_bright"})
black_categories = next(chart for chart in charts if chart["sectionKey"] == "category_black")
self.assertEqual(black_categories["series"][0]["values"], [2.0, 1.0, 1.0])
self.assertEqual(black_categories["dimension"], "category")
def test_followup_product_comparisons_do_not_crowd_out_main_chapters(self):
docs = [evidence(decomposed_comparison_result(filterBinding={}), ref="Q00000001")]
for index in range(6):
docs.append(evidence(decomposed_comparison_result(filterBinding={"product": f"规格-{index}"}), ref=f"Q{index+2:08X}"))
for index, dimension in enumerate(("department", "group", "product")):
docs.append(evidence(changes_result(filters={}, dimension=dimension), ref=f"Q{index+20:08X}"))
for index, stage in enumerate(("black", "bright")):
docs.append(evidence(changes_result(filters={}, dimension="department", stage=stage), ref=f"Q{index+30:08X}"))
field, category = ("责废其他重量", "责废") if stage == "black" else ("加工磕边重量", "加工废")
docs.append(evidence({"kind":"quality_fine_defects","dimension":"defect","grain":"source_defect_weight","stage":stage,"filters":{},
"period":{"year":2026,"month":9},"comparisonPeriod":{"year":2026,"month":8},
"coverage":{"available":True},"comparisonCoverage":{"available":True},
"sourceFields":[fine_field(field,category)],"comparisonSourceFields":[fine_field(field,category)],
"pairedRows":[{"key":field,"current":{"label":field,"category":category,"weight":3},"comparison":{"label":field,"category":category,"weight":2},"deltaWeight":1,"deltaAvailable":True}],
"pairedTotalRows":1,"pairedTruncated":False,"pairedRankingComplete":True,"sort":"scrap_delta"},ref=f"Q{index+40:08X}"))
charts = build_visuals(docs, "分析整体变化,覆盖产业部、业务分组、重点产品与报废类别", "turn-balanced")["visuals"]
self.assertEqual(len(charts), 12)
self.assertTrue({"department", "group", "product", "category_black", "category_bright", "defect_black", "defect_bright"}.issubset({c["sectionKey"] for c in charts}))
self.assertFalse(charts[0]["scope"].get("product"))
self.assertEqual({c["metric"] for c in charts if c["sectionKey"] == "stage" and not c["scope"].get("product")}, {"black_rate", "bright_rate"})
def test_waterfall_requires_exact_finite_decomposition_that_matches_service_delta(self):
valid = decomposed_comparison_result()
bad_cases = []
mismatch_delta = json.loads(json.dumps(valid))
mismatch_delta["delta"]["percentagePoints"] = -0.5
bad_cases.append(mismatch_delta)
mismatch_sum = json.loads(json.dumps(valid))
mismatch_sum["stageDecomposition"]["sumPP"] = -0.5
bad_cases.append(mismatch_sum)
missing_stage = json.loads(json.dumps(valid))
missing_stage["stageDecomposition"]["brightContributionToCompositePP"] = None
bad_cases.append(missing_stage)
non_finite = json.loads(json.dumps(valid))
non_finite["stageDecomposition"]["blackContributionToCompositePP"] = float("inf")
bad_cases.append(non_finite)
for index, result in enumerate(bad_cases):
output = build_visuals([evidence(result)], "画瀑布图解释整体合格率下降原因", f"turn-waterfall-invalid-{index}")
self.assertEqual(output["visualState"], "unavailable")
self.assertEqual(output["visuals"], [])
def test_formal_category_delta_requires_matching_complete_category_sets(self):
result = decomposed_comparison_result()
chart = next(chart for chart in build_visuals(
[evidence(result)], "分析两阶段正式报废类别增减", "turn-category-delta"
)["visuals"] if chart["sectionKey"] == "category_bright")
self.assertEqual(chart["metric"], "source_defect_delta_weight")
self.assertEqual(chart["stage"], "bright")
self.assertEqual(chart["series"][0]["values"], [-1.0, -0.5, 0.0, -0.5])
self.assertEqual(chart["coverage"], {"totalRows": 4, "returnedRows": 4, "truncated": False})
partial = json.loads(json.dumps(result))
partial["current"]["bright"]["defects"].pop()
output = build_visuals([evidence(partial)], "分析两阶段正式报废类别增减", "turn-category-delta-partial")
self.assertFalse(any(chart.get("sectionKey") == "category_bright" for chart in output["visuals"]))
zero_only = json.loads(json.dumps(result))
for side in ("current", "comparison"):
for item in zero_only[side]["black"]["defects"]:
item["weight"] = 0
output = build_visuals([evidence(zero_only)], "分析黑片正式报废类别增减", "turn-category-delta-zero")
self.assertFalse(any(chart.get("sectionKey") == "category_black" for chart in output["visuals"]))
def test_scoped_defect_charts_keep_separate_chapter_scope_from_global_chart(self):
def fine_result(filters):
return {
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
"grain": "source_defect_weight", "stage": "black", "filters": filters,
"period": {"year": 2026, "month": 9}, "comparisonPeriod": {"year": 2026, "month": 8},
"coverage": {"available": True}, "comparisonCoverage": {"available": True},
"sourceFields": [fine_field("责废其他重量", "责废")],
"comparisonSourceFields": [fine_field("责废其他重量", "责废")],
"pairedRows": [{
"key": "责废其他重量",
"current": {"label": "责废其他重量", "category": "责废", "weight": 3.0},
"comparison": {"label": "责废其他重量", "category": "责废", "weight": 2.0},
"deltaWeight": 1.0, "deltaAvailable": True,
}],
"sort": "scrap_delta", "pairedTotalRows": 1, "pairedTruncated": False,
"pairedRankingComplete": True,
}
docs = [
evidence(fine_result({"extraFilters": {}}), ref="Q00000031"),
evidence(fine_result({"group": "汽车交通-国内", "extraFilters": {}}), ref="Q00000032"),
]
charts = build_visuals(docs, "画图看黑片来源具体缺陷增减", "turn-scoped-defects")["visuals"]
self.assertEqual(len(charts), 2)
self.assertEqual({chart["sectionKey"] for chart in charts}, {"defect_black"})
self.assertEqual({chart["dimension"] for chart in charts}, {"defect"})
self.assertEqual({chart["scope"].get("group") for chart in charts}, {None, "汽车交通-国内"})
self.assertEqual(len({chart["id"] for chart in charts}), 2)
scoped_chart = next(chart for chart in charts if chart["scope"].get("group"))
self.assertIn("范围 业务分组:汽车交通-国内", scoped_chart["subtitle"])
many_scopes = [evidence(fine_result({"group": f"业务组-{index}", "extraFilters": {}}),
ref=f"Q{index:08X}") for index in range(13)]
bounded = build_visuals(many_scopes, "画图看黑片来源具体缺陷增减", "turn-12-limit")
self.assertEqual(len(bounded["visuals"]), 12)
self.assertEqual(len({chart["id"] for chart in bounded["visuals"]}), 12)
self.assertLess(len(json.dumps(bounded, ensure_ascii=False).encode("utf-8")), MAX_BODY_BYTES)
def test_change_chart_preserves_signed_values_and_returned_order(self):
rows = changes_result()["rows"]
rows[0]["scrapDelta"] = -1.0
rows[1]["scrapDelta"] = 5.0
output = build_visuals([evidence(changes_result(rows=rows))], "画图比较产品报废增量", "turn-02")
chart = output["visuals"][0]
self.assertEqual(chart["metric"], "scrap_delta")
self.assertEqual(chart["stage"], "black")
self.assertEqual(chart["series"][0]["values"], [-1.0, 5.0])
self.assertEqual([row["key"] for row in chart["categories"]], ["p-1", "p-2"])
self.assertTrue(chart["coverage"]["truncated"])
self.assertEqual(chart["coverage"]["totalRows"], 8)
def test_rate_drop_is_signed_and_converted_to_percentage_points(self):
result = changes_result(sort="rate_drop", rows=[
{"rank": 1, "key": "p-1", "label": "规格一", "rateDrop": -0.025},
{"rank": 2, "key": "p-2", "label": "规格二", "rateDrop": 0.04},
])
chart = build_visuals([evidence(result)], "画图比较产品合格率降幅", "turn-03")["visuals"][0]
self.assertEqual(chart["metric"], "rate_drop_pp")
self.assertEqual(chart["unit"], "pp")
self.assertEqual(chart["series"][0]["values"], [-2.5, 4.0])
self.assertIn("率降幅线索", chart["title"])
self.assertIn("未纳入投入规模", chart["note"])
def test_broad_product_focus_omits_rate_only_ranking_chart(self):
rate = changes_result(stage=None, sort="rate_drop", returnedRows=1, rows=[
{"rank": 1, "key": "tiny", "label": "小量规格", "rateDrop": 0.9943,
"scrapDelta": 6.74, "current": {"blackInputWeight": 7.1, "brightInputWeight": 39.66}},
])
scrap = changes_result(stage=None, sort="scrap_delta", returnedRows=1, rows=[
{"rank": 1, "key": "large", "label": "报废增量规格", "scrapDelta": 758.76},
])
charts = build_visuals(
[evidence(rate, ref="Q00000001"), evidence(scrap, ref="Q00000002")],
"分析下近期要关注的重点产品", "turn-product-focus",
)["visuals"]
self.assertEqual([chart["metric"] for chart in charts], ["scrap_delta"])
raw_only = build_visuals(
[evidence(rate)], "分析下近期要关注的重点产品", "turn-product-focus-raw-only",
)
self.assertEqual(raw_only["visualState"], "not_requested")
self.assertEqual(raw_only["visuals"], [])
mixed = build_visuals(
[evidence(rate)], "分析重点产品的合格率下降和报废占比", "turn-product-focus-mixed",
)
self.assertEqual(mixed["visuals"], [])
def test_rank_metadata_must_match_order_and_zero_is_preserved(self):
result = changes_result(rows=[
{"rank": 2, "key": "p-1", "label": "规格一", "scrapDelta": 0.0},
{"rank": 1, "key": "p-2", "label": "规格二", "scrapDelta": 3.0},
])
self.assertEqual(build_visuals([evidence(result)], "画图比较产品报废增量", "turn-04")["visualState"], "unavailable")
def test_change_chart_requires_stage_and_matching_filter_shape(self):
no_stage = changes_result(stage=None)
output = build_visuals([evidence(no_stage)], "画图比较产品报废增量", "turn-05")
chart = output["visuals"][0]
self.assertEqual(chart["metric"], "scrap_delta")
self.assertNotIn("stage", chart)
self.assertNotIn("stage", chart["scope"])
self.assertEqual(chart["requiredFields"], ["black_scrap_weight", "bright_scrap_weight"])
secret_filter = changes_result(filters={"department": "电机部", "apiKey": "secret-value"})
self.assertEqual(build_visuals([evidence(secret_filter)], "画图比较产品报废增量", "turn-05")["visualState"], "unavailable")
def test_overall_rate_drop_uses_canonical_two_stage_fields(self):
result = changes_result(stage=None, sort="rate_drop", rows=[
{"rank": 1, "key": "p-1", "label": "规格一", "rateDrop": 0.02},
{"rank": 2, "key": "p-2", "label": "规格二", "rateDrop": -0.01},
])
chart = build_visuals([evidence(result)], "画产品综合合格率变化", "turn-05b")["visuals"][0]
self.assertEqual(chart["metric"], "rate_drop_pp")
self.assertEqual(chart["requiredFields"], [
"black_good_weight", "black_total_weight", "bright_good_weight", "bright_total_weight"
])
self.assertEqual(chart["series"][0]["values"], [2.0, -1.0])
def test_quality_top_uses_ranked_weights_and_never_share(self):
result = {
"kind": "quality_top", "releaseId": "r-1", "stage": "black", "dimension": "product",
"period": {"year": 2026, "month": 9},
"filters": {"department": "电机部", "extraFilters": {}},
"rankingCoverage": {"groups": 12, "complete": False},
"rows": [
{"rank": index + 1, "key": f"p-{index}", "label": f"产品{index}",
"weight": index + 0.5, "share": 0.99 - index * 0.01}
for index in range(12)
],
}
chart = build_visuals([evidence(result)], "哪些产品报废最多", "turn-top") ["visuals"][0]
self.assertEqual(chart["metric"], "current_scrap_weight")
self.assertEqual(chart["requiredFields"], ["black_scrap_weight"])
self.assertEqual(chart["series"][0]["values"], [index + 0.5 for index in range(10)])
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 10, "truncated": True})
self.assertNotIn("share", json.dumps(chart, ensure_ascii=False))
def test_quality_top_categories_require_stage_canonical_source_fields(self):
result = {
"kind": "quality_top", "releaseId": "r-1", "stage": "bright", "dimension": "category",
"period": {"year": 2026, "month": 9}, "filters": {"extraFilters": {}},
"rankingCoverage": {"groups": 3, "complete": True},
"cumulativeThresholdApplied": False,
"rows": [
{"rank": 1, "key": "加工废", "label": "加工废", "weight": 3.0},
{"rank": 2, "key": "材料废", "label": "材料废", "weight": 2.0},
{"rank": 3, "key": "扩散废", "label": "扩散废", "weight": 1.0},
],
}
chart = build_visuals([evidence(result)], "分析亮片报废类别排名", "turn-cat")["visuals"][0]
self.assertEqual(chart["metric"], "source_defect_weight")
self.assertEqual([item["key"] for item in chart["categories"]], [
"bright_processing_scrap", "bright_material_scrap", "bright_diffusion_scrap"
])
self.assertEqual(chart["requiredFields"], [
"bright_diffusion_scrap", "bright_material_scrap", "bright_processing_scrap"
])
result["rows"][0]["label"] = "VCM总计"
self.assertEqual(build_visuals([evidence(result)], "分析亮片报废类别排名", "turn-cat")["visualState"], "not_requested")
def test_real_category_top_shape_without_ranking_coverage_requires_endpoint_denominator_proof(self):
denominator = 7118.885021
weights = [3421.516018, 2812.668772, 868.241231, 16.459]
labels = ["加工废", "镀废", "材料废", "扩散废"]
result = {
"kind": "quality_top", "releaseId": "r-1", "stage": "bright", "dimension": "category",
"period": {"year": 2026, "month": 9},
"filters": {"department": "工业电机产业部", "includeUnassigned": False},
"rankingCoverage": None,
"denominator": denominator,
"shareBasis": "所选筛选条件下选中原报废分类的重量总和",
"cumulativeThresholdApplied": False,
"rows": [
{"rank": index + 1, "key": label, "label": label, "weight": weight,
"denominator": denominator, "share": weight / denominator}
for index, (label, weight) in enumerate(zip(labels, weights))
],
}
doc = evidence(result)
doc["apiSource"] = {"calls": [
{"method": "GET", "path": "/api/releases", "params": {}},
{"method": "GET", "path": "/api/analytics/top", "params": {
"releaseId": "r-1", "stage": "bright", "dimension": "category", "limit": 10,
"year": 2026, "month": 9, "department": "工业电机产业部", "includeUnassigned": False,
}},
]}
donut = build_visuals([doc], "亮片已列报废类别用环状图展示构成", "turn-real-category-donut")["visuals"][0]
self.assertEqual(donut["type"], "donut")
self.assertEqual(donut["series"][0]["values"], weights)
self.assertEqual(donut["coverage"], {
"totalRows": 4, "returnedRows": 4, "truncated": False, "shareBasis": "displayed_categories"
})
pareto = build_visuals([doc], "亮片报废类别画柏拉图", "turn-real-category-pareto")["visuals"][0]
self.assertEqual(pareto["type"], "pareto")
self.assertEqual(pareto["series"][0]["values"], weights)
subset = json.loads(json.dumps(result))
subset["rows"] = result["rows"][:3]
subset["denominator"] = sum(weights[:3])
for row in subset["rows"]:
row["denominator"] = subset["denominator"]
row["share"] = row["weight"] / subset["denominator"]
subset_doc = evidence(subset)
subset_doc["apiSource"] = doc["apiSource"]
self.assertEqual(build_visuals([subset_doc], "亮片类别环状构成", "turn-authorized-subset")["visualState"], "ready")
invalid_cases = []
low_limit = json.loads(json.dumps(doc))
low_limit["apiSource"]["calls"][1]["params"]["limit"] = 3
invalid_cases.append(low_limit)
wrong_denominator = json.loads(json.dumps(doc))
wrong_denominator["result"]["denominator"] += 1
invalid_cases.append(wrong_denominator)
threshold = json.loads(json.dumps(doc))
threshold["result"]["cumulativeThresholdApplied"] = True
invalid_cases.append(threshold)
narrowed = json.loads(json.dumps(doc))
narrowed["apiSource"]["calls"][1]["params"]["category"] = "加工废"
invalid_cases.append(narrowed)
mismatched_filter = json.loads(json.dumps(doc))
mismatched_filter["apiSource"]["calls"][1]["params"]["department"] = "其他部门"
invalid_cases.append(mismatched_filter)
for index, invalid_doc in enumerate(invalid_cases):
self.assertEqual(build_visuals([invalid_doc], "亮片类别环状构成", f"turn-category-invalid-{index}")["visualState"], "unavailable")
def test_candidate_priority_keeps_comparison_product_and_fine_defect_charts_separate(self):
comparison = evidence(comparison_result(), ref="Q00000011")
changes = evidence(changes_result(stage=None), ref="Q00000012")
fine_result = {
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
"grain": "source_defect_weight", "stage": "bright", "filters": {"department": "其他部门"},
"period": {"year": 2026, "month": 9}, "comparisonPeriod": {"year": 2026, "month": 8},
"coverage": {"available": True}, "comparisonCoverage": {"available": True},
"sourceFields": [fine_field("晶界扩散其他缺陷重量", "扩散废"), fine_field("材料隐裂重量", "材料废")],
"comparisonSourceFields": [fine_field("晶界扩散其他缺陷重量", "扩散废"), fine_field("材料隐裂重量", "材料废")],
"pairedRows": [
{"key": "晶界扩散其他缺陷重量", "current": {"label": "晶界扩散其他缺陷重量", "category": "扩散废", "weight": 3},
"comparison": {"label": "晶界扩散其他缺陷重量", "category": "扩散废", "weight": 2}, "deltaWeight": 1, "deltaAvailable": True},
{"key": "材料隐裂重量", "current": {"label": "材料隐裂重量", "category": "材料废", "weight": 2},
"comparison": {"label": "材料隐裂重量", "category": "材料废", "weight": 1}, "deltaWeight": 1, "deltaAvailable": True},
],
"sort": "scrap_delta", "pairedTotalRows": 2, "pairedTruncated": False,
"pairedRankingComplete": True,
}
fine = evidence(fine_result, ref="Q00000013")
output = build_visuals([fine, changes, comparison], "分析本期变化并绘制综合对比、产品报废增量和来源缺陷图", "turn-priority")
self.assertEqual([chart["metric"] for chart in output["visuals"]], [
"composite_rate", "scrap_delta", "source_defect_delta_weight"
])
self.assertEqual(output["visuals"][2]["scope"]["department"], "其他部门")
self.assertEqual(output["visuals"][2]["requiredFields"], ["晶界扩散其他缺陷重量", "材料隐裂重量"])
scoped = build_visuals([fine, changes, comparison],
"展示正式综合合格率、重点产品报废增量和亮片具体缺陷增量", "turn-priority-stage")
self.assertEqual([chart["metric"] for chart in scoped["visuals"]], [
"composite_rate", "scrap_delta", "source_defect_delta_weight"
])
def test_fine_defects_use_paired_rows_and_do_not_turn_totals_only_into_zero(self):
result = {
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
"grain": "source_defect_weight", "stage": "bright",
"filters": {"department": "电机部", "extraFilters": {}},
"period": {"year": 2026, "month": 9}, "comparisonPeriod": {"year": 2026, "month": 8},
"coverage": {"available": True}, "comparisonCoverage": {"available": True},
"sourceRows": 4,
"sourceFields": [fine_field("晶界扩散其他缺陷重量", "扩散废")],
"comparisonSourceFields": [fine_field("晶界扩散其他缺陷重量", "扩散废")],
"pairedRows": [
{"key": "晶界扩散其他缺陷重量", "current": {"label": "晶界扩散其他缺陷重量", "category": "扩散废", "weight": 0.0},
"comparison": {"label": "晶界扩散其他缺陷重量", "category": "扩散废", "weight": 1.0}, "deltaWeight": -1.0, "deltaAvailable": True},
],
"sort": "scrap_delta", "pairedTotalRows": 2, "pairedTruncated": False,
"pairedRankingComplete": True,
"categoryReconciliation": [{"category": "扩散废", "sourceRows": 4,
"expectedTotalsOnly": {"rows": 1}}],
"comparisonCategoryReconciliation": [{"category": "扩散废", "sourceRows": 4,
"expectedTotalsOnly": {"rows": 0}}],
}
output = build_visuals([evidence(result)], "画图比较亮片来源具体缺陷增量", "turn-06")
chart = output["visuals"][0]
self.assertEqual(chart["metric"], "source_defect_delta_weight")
self.assertEqual(chart["stage"], "bright")
self.assertEqual(chart["categories"], [{"key": "晶界扩散其他缺陷重量", "label": "晶界扩散其他缺陷重量"}])
self.assertEqual(chart["series"][0]["values"], [-1.0])
self.assertNotIn("VCM", chart["note"])
self.assertEqual(chart["requiredFields"], ["晶界扩散其他缺陷重量"])
def test_fine_defect_missing_delta_stays_null_and_unavailable_coverage_skips(self):
result = {
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
"grain": "source_defect_weight", "stage": "black", "filters": {},
"period": {"year": 2026, "month": 9}, "comparisonPeriod": {"year": 2026, "month": 8},
"coverage": {"available": True}, "comparisonCoverage": {"available": True},
"sourceFields": [fine_field("责废尺寸重量", "责废"), fine_field("责废其他重量", "责废")],
"comparisonSourceFields": [fine_field("责废尺寸重量", "责废"), fine_field("责废其他重量", "责废")],
"pairedRows": [
{"key": "责废尺寸重量", "current": {"label": "责废尺寸重量", "category": "责废", "weight": 2},
"comparison": None, "deltaWeight": None, "deltaAvailable": False},
{"key": "责废其他重量", "current": {"label": "责废其他重量", "category": "责废", "weight": 3},
"comparison": {"label": "责废其他重量", "category": "责废", "weight": 1}, "deltaWeight": 2, "deltaAvailable": True},
],
"sort": "scrap_delta", "pairedTotalRows": 2, "pairedTruncated": False,
"pairedRankingComplete": False,
}
output = build_visuals([evidence(result)], "画图比较黑片来源缺陷增量", "turn-07")
self.assertEqual(output["visuals"][0]["series"][0]["values"], [None, 2.0])
result["coverage"] = {"available": False}
self.assertEqual(build_visuals([evidence(result)], "画图比较黑片来源缺陷增量", "turn-07")["visualState"], "unavailable")
def test_fine_defects_authorize_only_real_source_fields_and_bind_category_scope(self):
result = {
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
"grain": "source_defect_weight", "stage": "bright",
"filters": {"department": "电机部", "extraFilters": {"项目类型": ["VCM"]}},
"category": "加工废",
"period": {"year": 2026, "month": 9}, "coverage": {"available": True, "sourceRows": 3},
"sourceRows": 3, "sourceFields": [fine_field("加工尺寸超差重量", "加工废")],
"rows": [{"key": "加工尺寸超差重量", "label": "加工尺寸超差重量", "category": "加工废", "weight": 0}],
"totalRows": 1, "truncated": False, "rankingComplete": True,
}
chart = build_visuals([evidence(result)], "画图看亮片来源具体缺陷重量", "turn-real-shape")["visuals"][0]
self.assertEqual(chart["categories"], [{"key": "加工尺寸超差重量", "label": "加工尺寸超差重量"}])
self.assertEqual(chart["requiredFields"], ["加工尺寸超差重量"])
self.assertEqual(chart["series"][0]["values"], [0.0])
self.assertEqual(chart["scope"]["category"], "加工废")
self.assertEqual(chart["scope"]["extraFilters"], {"项目类型": ["VCM"]})
result["rows"][0]["key"] = "未授权缺陷重量"
result["rows"][0]["label"] = "未授权缺陷重量"
self.assertEqual(build_visuals([evidence(result)], "画图看亮片来源具体缺陷重量", "turn-real-shape")["visualState"], "unavailable")
def test_fine_defect_partial_totals_only_keeps_valid_mixed_rows_and_full_totals_only_is_not_zero(self):
mixed = {
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
"grain": "source_defect_weight", "stage": "bright", "filters": {},
"period": {"year": 2026, "month": 9}, "coverage": {"available": True, "sourceRows": 4},
"sourceRows": 4, "sourceFields": [fine_field("晶界扩散外观异常重量", "扩散废")],
"rows": [{"key": "晶界扩散外观异常重量", "label": "晶界扩散外观异常重量",
"category": "扩散废", "weight": 2.5}],
"categoryReconciliation": [{"category": "扩散废", "sourceRows": 4,
"expectedTotalsOnly": {"rows": 1},
"reconciliationStatus": "DIFF"}],
"totalRows": 1, "truncated": False, "rankingComplete": True,
}
chart = build_visuals([evidence(mixed)], "画图看亮片来源具体缺陷排名", "turn-mixed")["visuals"][0]
self.assertEqual(chart["series"][0]["values"], [2.5])
self.assertNotIn("VCM", chart.get("note", ""))
all_totals = dict(mixed)
all_totals["rows"] = [{"key": "晶界扩散外观异常重量", "label": "晶界扩散外观异常重量",
"category": "扩散废", "weight": None}]
all_totals["categoryReconciliation"] = [{"category": "扩散废", "sourceRows": 4,
"expectedTotalsOnly": {"rows": 4},
"reconciliationStatus": "EXPECTED_TOTALS_ONLY"}]
self.assertEqual(build_visuals([evidence(all_totals)], "画图看亮片来源具体缺陷排名", "turn-mixed")["visualState"], "unavailable")
paired_totals = dict(all_totals)
paired_totals["comparisonPeriod"] = {"year": 2026, "month": 8}
paired_totals["comparisonCoverage"] = {"available": True, "sourceRows": 4}
paired_totals["comparisonSourceFields"] = [fine_field("晶界扩散外观异常重量", "扩散废")]
paired_totals["comparisonCategoryReconciliation"] = list(all_totals["categoryReconciliation"])
paired_totals["sort"] = "scrap_delta"
paired_totals["pairedRows"] = [{
"key": "晶界扩散外观异常重量",
"current": {"label": "晶界扩散外观异常重量", "category": "扩散废", "weight": None},
"comparison": {"label": "晶界扩散外观异常重量", "category": "扩散废", "weight": None},
"deltaWeight": None, "deltaAvailable": False,
}]
paired_totals["pairedTotalRows"] = 1
paired_totals["pairedTruncated"] = False
paired_totals["pairedRankingComplete"] = True
self.assertEqual(build_visuals([evidence(paired_totals)], "画图比较亮片来源具体缺陷增量", "turn-mixed")["visualState"], "unavailable")
def test_fine_defect_chart_limits_to_ten_and_authorizes_only_displayed_fields(self):
keys = ["责废尺寸重量", "责废缺角重量", "责废垂直度重量", "责废刀丝重量", "责废烂毛重量",
"责废开裂重量", "责废其他重量", "料废尺寸重量", "料废缺角重量", "料废开裂重量",
"料废烂毛重量", "料废其他重量"]
result = {
"kind": "quality_fine_defects", "releaseId": "r-1", "dimension": "defect",
"grain": "source_defect_weight", "stage": "black", "filters": {},
"period": {"year": 2026, "month": 9}, "coverage": {"available": True, "sourceRows": 2},
"sourceRows": 2, "sourceFields": [fine_field(key, "责废" if key.startswith("责废") else "料废") for key in keys],
"rows": [{"key": key, "label": key, "category": "责废" if key.startswith("责废") else "料废",
"weight": float(index + 1)} for index, key in enumerate(keys)],
"totalRows": len(keys), "truncated": False, "rankingComplete": True,
}
chart = build_visuals([evidence(result)], "画图看黑片来源具体缺陷排名", "turn-ten")["visuals"][0]
self.assertEqual(len(chart["categories"]), 10)
self.assertEqual(chart["coverage"], {"totalRows": 12, "returnedRows": 10, "truncated": True})
self.assertEqual(chart["requiredFields"], [item["key"] for item in chart["categories"]])
def test_multi_query_trend_requires_same_release_and_filters_and_keeps_gaps_null(self):
docs = [
evidence(metric_result({"year": 2026, "month": 1}, 0.91), ref="Q00000001"),
evidence(metric_result({"year": 2026, "month": 3}, 0.93), ref="Q00000002"),
evidence(metric_result({"year": 2026, "month": 4}, 0.94), ref="Q00000003"),
]
charts = build_visuals(docs, "画综合合格率趋势图", "turn-08")["visuals"]
chart = next(item for item in charts if item["metric"] == "composite_rate")
self.assertEqual([item["key"] for item in chart["categories"]], ["2026-01", "2026-02", "2026-03", "2026-04"])
self.assertEqual(chart["series"][0]["values"], [0.91, None, 0.93, 0.94])
annual = {'year': 2026, 'department': '电机部', 'trendGranularity': 'month'}
bounded = build_visuals(docs, '画综合合格率趋势图', 'turn-annual', context=annual)['visuals']
self.assertEqual(bounded[0]['series'][0]['values'], [0.91, None, 0.93, 0.94])
unrelated = [evidence(metric_result({'year':2025,'month':12},0.50),ref='Q00000009'),
evidence(metric_result({'year':2026,'month':2},0.10,filters={'department':'其他'}),ref='Q00000010')]
filtered = build_visuals(docs+unrelated, '画综合合格率趋势图', 'turn-filtered-year', context=annual)['visuals']
self.assertEqual(filtered[0]['series'][0]['values'], [0.91, None, 0.93, 0.94])
quarter_docs = docs + [evidence(metric_result({'year':2026,'month':2},0.92),ref='Q00000011')]
quarter = build_visuals(quarter_docs, '画综合合格率趋势图', 'turn-quarter', context={**annual,'quarter':1})['visuals']
self.assertEqual(quarter[0]['series'][0]['values'], [0.91, 0.92, 0.93])
single_month = build_visuals(docs, '画综合合格率趋势图', 'turn-month', context={**annual,'month':3})
self.assertEqual(single_month['visualState'], 'unavailable')
mismatch = [docs[0], docs[1], evidence(metric_result({"year": 2026, "month": 4}, 0.94, filters={"department": "其他"}), ref="Q00000004")]
self.assertEqual(build_visuals(mismatch, "画综合合格率趋势图", "turn-08")["visualState"], "unavailable")
mixed_release = [docs[0], docs[1], evidence(metric_result({"year": 2026, "month": 4}, 0.94, release_id="r-2"), ref="Q00000005", release_id="r-2")]
self.assertEqual(build_visuals(mixed_release, "画综合合格率趋势图", "turn-08")["visualState"], "unavailable")
def test_trend_uses_the_same_unique_comparison_scope_as_the_text(self):
scoped = comparison_result(filterBinding={'group':'A'},
current={'period':{'year':2026},'compositeRate':.98,'black':{'rate':.99},'bright':{'rate':.99}},
comparison={'period':{'year':2025},'compositeRate':.97,'black':{'rate':.98},'bright':{'rate':.99}})
global_docs = [evidence(metric_result({'year':2026,'month':m},.90+m/100,filters={}),ref=f'Q0000000{m}') for m in (1,3,4)]
charts = build_visuals([evidence(scoped)]+global_docs,'比较今年变化并画趋势图','turn-root-trend',context={'year':2026})['visuals']
self.assertFalse(any(chart.get('sectionKey')=='trend' for chart in charts))
local_docs = [evidence(metric_result({'year':2026,'month':m},.95+m/100,filters={'group':'A'}),ref=f'Q0000001{m}') for m in (1,3,4)]
charts = build_visuals([evidence(scoped)]+global_docs+local_docs,'比较今年变化并画趋势图','turn-root-trend-ok',context={'year':2026})['visuals']
trend = next(chart for chart in charts if chart.get('sectionKey')=='trend')
self.assertEqual(trend['scope']['group'], 'A')
self.assertEqual(trend['series'][0]['values'], [.96, None, .98, .99])
def test_native_trend_is_sorted_and_bounded(self):
result = metric_result({"year": 2026, "month": 9}, 0.93)
result.update({
"trend": [
{"period": "2026-03", "compositeRate": 0.93, "black": {"rate": 0.92}, "bright": {"rate": 0.94}},
{"period": "2026-01", "compositeRate": 0.91, "black": {"rate": 0.90}, "bright": {"rate": 0.92}},
{"period": "2026-04", "compositeRate": 0.94, "black": {"rate": 0.93}, "bright": {"rate": 0.95}},
],
"trendCount": 3,
"trendTruncated": False,
})
chart = build_visuals([evidence(result)], "画合格率趋势图", "turn-09")["visuals"][0]
self.assertEqual(chart["metric"], "composite_rate")
self.assertEqual([item["key"] for item in chart["categories"]], ["2026-01", "2026-02", "2026-03", "2026-04"])
self.assertEqual(chart["series"][0]["values"], [0.91, None, 0.93, 0.94])
result["trendTruncated"] = True
self.assertEqual(build_visuals([evidence(result)], "画合格率趋势图", "turn-09")["visualState"], "unavailable")
def test_explicit_heatmap_and_area_are_real_period_trends_only(self):
result = metric_result({"year": 2026, "month": 12}, 0.93)
result.update({
"trend": [
{"period": f"2026-{month:02d}", "compositeRate": 0.8 + month / 100,
"black": {"rate": 0.79 + month / 100, "scrapWeight": month},
"bright": {"rate": 0.81 + month / 100, "scrapWeight": month / 2}}
for month in range(1, 13)
],
"trendCount": 12,
"trendTruncated": False,
})
ordinary = build_visuals([evidence(result)], "查看综合合格率月度趋势", "turn-usual-trend")["visuals"][0]
self.assertEqual(ordinary["type"], "line")
heatmap = build_visuals([evidence(result)], "用热力图展示综合合格率趋势", "turn-heatmap")["visuals"][0]
self.assertEqual(heatmap["type"], "heatmap")
self.assertEqual(len(heatmap["categories"]), 12)
self.assertEqual(len(heatmap["series"]), 1)
result["trend"] = result["trend"][:4]
result["trendCount"] = 4
area = build_visuals([evidence(result)], "画亮片报废重量面积趋势图", "turn-area")["visuals"][0]
self.assertEqual(area["type"], "area")
self.assertEqual(area["stage"], "bright")
self.assertEqual(area["metric"], "current_scrap_weight")
self.assertTrue(all(value is None or value >= 0 for value in area["series"][0]["values"]))
result["trend"] = result["trend"][:2]
result["trendCount"] = 2
self.assertEqual(build_visuals([evidence(result)], "画面积图", "turn-area-invalid")["visualState"], "unavailable")
def test_formal_summary_produces_guarded_donut_and_pareto(self):
defects = [
{"name": "镀废", "weight": 3.0},
{"name": "材料废", "weight": 5.0},
{"name": "加工废", "weight": 0.0},
{"name": "扩散废", "weight": 2.0},
]
result = metric_result({"year": 2026, "month": 9}, 0.93)
result["summary"]["bright"] = {"defects": defects}
doc = evidence(result)
donut = build_visuals([doc], "用环状图展示亮片已列正式报废类别构成", "turn-donut")["visuals"][0]
self.assertEqual(donut["type"], "donut")
self.assertEqual(donut["metric"], "source_defect_weight")
self.assertEqual(donut["series"][0]["values"], [3.0, 5.0, 0.0, 2.0])
self.assertEqual(donut["coverage"], {
"totalRows": 4, "returnedRows": 4, "truncated": False, "shareBasis": "displayed_categories"
})
self.assertEqual(donut["requiredFields"], [
"bright_plating_scrap", "bright_material_scrap", "bright_processing_scrap", "bright_diffusion_scrap"
])
self.assertIn("占比按本图已列类别重量合计", donut["note"])
pareto = build_visuals([doc], "绘制亮片柏拉图并标注报废类别", "turn-pareto")["visuals"][0]
self.assertEqual(pareto["type"], "pareto")
self.assertEqual(pareto["series"][0]["values"], [5.0, 3.0, 2.0, 0.0])
self.assertEqual([category["key"] for category in pareto["categories"]], [
"bright_material_scrap", "bright_plating_scrap", "bright_diffusion_scrap", "bright_processing_scrap"
])
self.assertNotEqual(donut["id"], pareto["id"])
comparison = comparison_result()
comparison["current"]["bright"] = {"defects": defects, "rate": 0.99, "scrapWeight": 5.0}
current_donut = build_visuals([evidence(comparison)], "亮片当前已列类别构成环状图", "turn-comparison-donut")["visuals"][0]
self.assertEqual(current_donut["type"], "donut")
self.assertEqual(current_donut["period"], {"year": 2026, "month": 9})
self.assertNotIn("comparisonPeriod", current_donut)
def test_summary_and_complete_category_top_only_make_share_charts_when_complete(self):
result = metric_result({"year": 2026, "month": 9}, 0.93)
result["summary"]["black"] = {"defects": [
{"name": "责废", "weight": 1.0}, {"name": "料废", "weight": 2.0}, {"name": "外废", "weight": 3.0}
]}
result["summary"]["black"]["defects"][1]["weight"] = -2.0
self.assertEqual(build_visuals([evidence(result)], "黑片环状图展示构成", "turn-negative")["visualState"], "unavailable")
result["summary"]["black"]["defects"][1]["weight"] = 0.0
result["summary"]["black"]["defects"][2]["weight"] = 0.0
result["summary"]["black"]["defects"][0]["weight"] = 0.0
self.assertEqual(build_visuals([evidence(result)], "黑片环状图展示构成", "turn-zero")["visualState"], "unavailable")
result["summary"]["black"]["defects"][0]["weight"] = 1.0
filtered = dict(result)
filtered["filters"] = {"category": "责废", "extraFilters": {}}
self.assertEqual(build_visuals([evidence(filtered)], "黑片环状图展示构成", "turn-filtered-category")["visualState"], "unavailable")
top = {
"kind": "quality_top", "releaseId": "r-1", "stage": "black", "dimension": "category",
"period": {"year": 2026, "month": 9}, "filters": {"extraFilters": {}},
"rankingCoverage": {"groups": 3, "complete": True},
"cumulativeThresholdApplied": False,
"rows": [
{"rank": 1, "key": "责废", "label": "责废", "weight": 8.0},
{"rank": 2, "key": "料废", "label": "料废", "weight": 4.0},
{"rank": 3, "key": "外废", "label": "外废", "weight": 1.0},
],
}
chart = build_visuals([evidence(top)], "黑片柏拉图按正式类别", "turn-pareto-top")["visuals"][0]
self.assertEqual(chart["type"], "pareto")
self.assertEqual(chart["series"][0]["values"], [8.0, 4.0, 1.0])
top["rankingCoverage"]["complete"] = False
self.assertEqual(build_visuals([evidence(top)], "黑片柏拉图按正式类别", "turn-pareto-incomplete")["visualState"], "unavailable")
def test_colloquial_quality_analysis_requests_generate_bound_charts(self):
questions = [
'查一下消费电子美洲近期质量下滑严重的产品',
'看看电机部合格率下跌的规格', '电机部质量走低的产品',
'电机部合格率回落的产品', '电机部合格率降低的产品',
'查查电机部质量变差的产品', '查查电机部质量变坏的产品',
'查查电机部质量变糟的产品', '找出电机部降幅大的规格',
'查电机部跌幅大的产品', '电机部质量最差的产品',
'电机部质量较差的产品', '电机部合格率偏低的规格',
'查查电机部质量不理想的产品', '电机部表现不佳的产品',
'电机部质量不稳定的产品', '电机部质量波动大的产品',
'电机部报废突然变多的产品', '电机部报废突增的规格',
'电机部报废激增的产品', '近期值得关注的规格',
'近期重点产品', '电机部质量好转的规格',
'电机部合格率提升的产品', '电机部合格率回升的产品',
'电机部报废减少的产品', '电机部质量环比怎么样',
'电机部与上月差异', '电机部质量异常的产品',
'电机部质量不好的产品', '电机部质量不佳的产品',
'电机部质量差的产品', '电机部质量糟糕的产品',
'电机部合格率变低的规格', '电机部质量退步的产品',
'电机部质量不达标的产品', '电机部合格率低于目标的规格',
'电机部质量降得厉害的产品', '电机部合格率掉得多的规格',
]
doc = evidence(comparison_result())
context = {'releaseId': 'r-1', 'filters': {'department': '电机部'}}
for question in questions:
with self.subTest(question=question):
output = build_visuals([doc], question, 'turn-colloquial', context=context)
self.assertEqual(output['visualState'], 'ready')
self.assertTrue(output['visuals'])
self.assertTrue(all(v['scope'].get('department') == '电机部' for v in output['visuals']))
self.assertTrue(all(v['evidenceRefs'] == ['Q1234ABCD'] for v in output['visuals']))
def test_colloquial_requests_keep_text_only_and_no_evidence_guards(self):
question = '查一下近期质量下滑严重的产品'
doc = evidence(comparison_result())
for suffix in ['只要文字', '仅需文本', '仅用文字', '只返回数值', '不用画图', '不需要配图', '不要图表', '只需要文字', '仅输出数字', '仅文字', '无需图', '不必画图']:
with self.subTest(suffix=suffix):
self.assertEqual(build_visuals([doc], question + ',' + suffix, 'turn-text-only')['visuals'], [])
for simple in ['综合合格率是多少', '黑片合格率多少', '产品名称是什么', '查一下数据源文件名']:
with self.subTest(simple=simple):
self.assertEqual(build_visuals([doc], simple, 'turn-simple')['visuals'], [])
self.assertEqual(build_visuals([], question, 'turn-no-evidence')['visuals'], [])
wrong_scope = {'releaseId': 'r-1', 'filters': {'department': '无关部门'}}
labelled = build_visuals([doc], question, 'turn-wrong-scope', context=wrong_scope)['visuals']
self.assertTrue(all(v['scope'].get('department') == '电机部' for v in labelled))
def test_simple_text_and_negative_visual_request_override(self):
doc = evidence(comparison_result())
self.assertEqual(build_visuals([doc], "综合合格率是多少", "turn-10")["visualState"], "not_requested")
self.assertEqual(build_visuals([doc], "不要图,只要文字,比较两个期间", "turn-10")["visualState"], "not_requested")
self.assertEqual(build_visuals([doc], "不要绘图,只要文本,比较两个期间", "turn-10")["visualState"], "not_requested")
self.assertEqual(build_visuals([], "重新分析重点产品并写出降幅", "turn-followup", context={})["visualState"], "not_requested")
self.assertEqual(build_visuals([], "画图比较重点产品降幅", "turn-followup", context={})["visualState"], "unavailable")
column = build_visuals([doc], "不要环状图,用柱状图比较两期综合合格率", "turn-negative-style-column")["visuals"][0]
self.assertEqual(column["type"], "column")
self.assertEqual(build_visuals([doc], "不用柏拉图,显示条形图比较两期综合合格率", "turn-negative-style-bar")["visuals"][0]["type"], "bar")
def test_explicit_composite_trend_does_not_add_unrequested_stages_or_plot_missing_as_perfect(self):
result = metric_result({"year": 2026}, 0.93)
result.update({"trend": [
{"period": "2026-01", "compositeRate": 0.91},
{"period": "2026-02", "compositeRate": 1.0, "coverage": {"blackRows": 0, "brightRows": 0}},
{"period": "2026-03", "compositeRate": 0.93},
{"period": "2026-04", "compositeRate": 0.94},
{"period": "2026-05", "compositeRate": 1.0, "monthStatus": "future"},
], "trendCount": 5, "trendTruncated": False})
charts = build_visuals([evidence(result)], "只展示综合合格率趋势图", "turn-nodata")["visuals"]
self.assertEqual([chart["metric"] for chart in charts], ["composite_rate"])
self.assertEqual(charts[0]["series"][0]["values"], [0.91, None, 0.93, 0.94, None])
def test_ids_are_stable_per_semantics_and_output_is_isolated(self):
doc = evidence(comparison_result())
doc["absoluteEvidencePath"] = "/Users/alice/private/source.json"
first = build_visuals([doc], "比较两个期间变化", "turn-11")
second = build_visuals([doc], "比较两个期间变化", "turn-11")
self.assertEqual(first["visuals"][0]["id"], second["visuals"][0]["id"])
self.assertNotIn("absoluteEvidencePath", json.dumps(first, ensure_ascii=False))
self.assertNotIn("/Users/alice", json.dumps(first, ensure_ascii=False))
self.assertLess(len(json.dumps(first, ensure_ascii=False).encode("utf-8")), MAX_BODY_BYTES)
changed_turn = build_visuals([doc], "比较两个期间变化", "turn-12")
self.assertNotEqual(first["visuals"][0]["id"], changed_turn["visuals"][0]["id"])
def test_malformed_evidence_and_large_rankings_fail_softly(self):
malformed = evidence(changes_result(rows=[{"rank": True, "key": "p-1", "label": "x", "scrapDelta": 1}]))
self.assertEqual(build_visuals([malformed], "画产品报废增量排名", "turn-13")["visualState"], "unavailable")
result = changes_result(rows=[
{"rank": index + 1, "key": f"p-{index}", "label": f"规格{index}", "scrapDelta": index}
for index in range(15)
], totalRows=15, returnedRows=15, truncated=False)
chart = build_visuals([evidence(result)], "画产品报废增量排名", "turn-13")["visuals"][0]
self.assertEqual(len(chart["categories"]), 10)
self.assertTrue(chart["coverage"]["truncated"])
self.assertEqual(chart["series"][0]["values"], list(map(float, range(10))))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,47 @@
import copy
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from analysis_visuals import _scope, build_visuals
from test_month_range_evidence import QUESTION, context, metric_doc
class ExclusionVisualScopeTests(unittest.TestCase):
def setup_frame(self):
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
frame = context(period)
frame['filters'] = {**frame['filters'], 'exclusions': {'product': ['SPEC-A']}}
frame['_agentTask']['filters'] = copy.deepcopy(frame['filters'])
doc = metric_doc(period, range(1, 10), filters=frame['filters'])
return frame, doc
def test_excluded_product_is_preserved_in_the_complete_monthly_chart(self):
frame, doc = self.setup_frame()
result = build_visuals([doc], QUESTION, 'turn-test', context=frame)
self.assertEqual(result['visualState'], 'ready')
chart = result['visuals'][0]
self.assertEqual(chart['scope']['exclusions'], {'product': ['SPEC-A']})
self.assertEqual(len(chart['series'][0]['values']), 9)
self.assertEqual(chart['scope']['monthEnd'], 9)
def test_scope_mismatch_never_uses_a_chart_from_other_exclusions(self):
for wrong in ({}, {'product': ['SPEC-B']}, {'product': ['SPEC-A'], 'group': ['OTHER']}):
frame, doc = self.setup_frame()
doc['result']['filters']['exclusions'] = wrong
# Separate the source receipt from the immutable expected frame.
frame['filters']['exclusions'] = {'product': ['SPEC-A']}
frame['_agentTask']['filters'] = copy.deepcopy(frame['filters'])
doc['result']['filters'] = {**doc['result']['filters'], 'exclusions': wrong}
result = build_visuals([doc], QUESTION, 'turn-test', context=frame)
self.assertEqual(result['visuals'], [])
def test_malformed_or_unsupported_exclusions_are_rejected(self):
for wrong in ({'product': []}, {'product': [None]}, {'customerCode': ['001']},
{'product': 'SPEC-A'}, {'product': [' ']}):
self.assertEqual(_scope({'exclusions': wrong}), (None, None))
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,270 @@
"""Regression: a grouped comparison must not expand into per-object reads."""
import copy
import json
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
str(Path(__file__).resolve().parent)]
from nao_agent_task import normalize_task
from nao_protocol_execution import _scoped_arguments, _validate_protocol_scope, execute
from test_nao_agent_task import task_value
from test_nao_protocol_execution import BoundApi, RELEASE, TURN, identity
def changes_node(**updates):
return {'nodeId': 'groups', 'purpose': 'changes', 'dimension': 'group',
'fields': ['rate'], 'stages': [], 'sort': 'rate_drop', 'limit': 1,
'selector': {'kind': 'base'}, 'comparisonRequired': True,
'visualPreference': 'none', 'visualType': None, **updates}
def normalized(nodes):
dimensions = list(dict.fromkeys(n.get('dimension') for n in nodes if n.get('dimension')))
stages = list(dict.fromkeys(s for n in nodes for s in n['stages']))
fields = list(dict.fromkeys(f for n in nodes for f in n['fields']))
return normalize_task(task_value(kind='analysis', queryPlan=nodes,
dimensions=dimensions, dimensionStages={}, fields=fields, stages=stages,
trendRequired=False, comparisonRequired=True,
period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2026, 'month': 8},
visualPreference='none'), '9月份哪个业务分组综合合格率下滑最多',
{'_protocol': {'version': 1}})
class MinimalQueryTests(unittest.TestCase):
def test_auto_batch_style_hint_does_not_report_missing_chart(self):
from analysis_visual_evidence import protocol_visual_result
task=normalized([changes_node(visualPreference='auto',visualType='bar')])
task.pop('queryPlan');task['visualPreference']='auto';task['visualType']='bar'
scope={'nodeId':'groups','scopeId':'batch','purpose':'changes','task':task}
with patch('analysis_visuals.build_visuals',return_value={'visuals':[]}):
result=protocol_visual_result([], [scope],task['question'],TURN,release_id=RELEASE)
self.assertEqual(result['visualState'],'not_requested',result)
self.assertEqual(result['missingEvidence'],[])
task['visualPreference']='required'
result=protocol_visual_result([], [scope],task['question'],TURN,release_id=RELEASE)
self.assertEqual(result['visualState'],'unavailable',result)
self.assertTrue(result['missingEvidence'])
task['visualPreference']='auto'
with patch('analysis_visuals.build_visuals',return_value={'visuals':[{'metric':'rate_drop_pp'}]}):
result=protocol_visual_result([], [scope],task['question'],TURN,release_id=RELEASE)
self.assertEqual(result['visualState'],'ready',result)
def test_requested_weights_cannot_be_completed_when_hidden(self):
from nao_changes import validate_changes
task=normalized([changes_node(fields=['rate','totalWeight','scrapWeight'])])
task['changeQuery']={'dimension':'group','sort':'rate_drop','limit':1}
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
'dimension':'group','stage':None,'sort':'rate_drop','completeRanking':True,'totalRows':1,
'rows':[{'rank':1,'key':'甲','current':{'rate':0.7},'comparison':{'rate':0.9}}]}
with self.assertRaisesRegex(ValueError,'正式重量'):
validate_changes(result,task,RELEASE)
def test_group_rate_answer_does_not_force_weight_details(self):
from nao_changes import changes_report
task=normalized([changes_node()]);task['changeQuery']={'dimension':'group','sort':'rate_drop','limit':1}
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
'dimension':'group','stage':None,'sort':'rate_drop','completeRanking':True,'totalRows':10,
'rows':[{'rank':1,'key':'甲','rateDrop':0.2,
'current':{'rate':0.7},'comparison':{'rate':0.9}}]}
report=changes_report([{'evidenceRef':'Q12345678','result':result}],task,RELEASE)
self.assertEqual(len(report['requiredFactIds']),1)
self.assertNotIn('投入',report['facts'][0]['text'])
self.assertIn('下降20',report['facts'][0]['text'])
self.assertNotIn('报废 kg','\n'.join(report['sections']))
self.assertNotIn('投入 kg','\n'.join(report['sections']))
def test_batch_comparison_chart_keeps_both_periods_without_extra_metric_read(self):
from analysis_visuals import build_visuals
from nao_agent_task import bind_task
task=normalized([changes_node(visualPreference='required')])
task.pop('queryPlan');task['visualPreference']='required'
task['changeQuery']={'dimension':'group','sort':'rate_drop','limit':1}
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
'dimension':'group','stage':None,'sort':'rate_drop','completeRanking':True,
'totalRows':10,'returnedRows':1,'truncated':True,
'rows':[{'rank':1,'key':'甲','label':'甲','rateDrop':0.2,
'current':{'rate':0.7},'comparison':{'rate':0.9}}]}
doc={'evidenceRef':'Q12345678','releaseId':RELEASE,'mode':'changes','result':result}
visual=build_visuals([doc],task['question'],TURN,
context=bind_task({'releaseId':RELEASE},task))
self.assertEqual(visual['visualState'],'ready',visual)
self.assertEqual(visual['visuals'][0]['metric'],'rate_drop_pp')
self.assertEqual(visual['visuals'][0]['series'][0]['values'],[20.0])
result['sort']='current_rate';task['changeQuery']['sort']='current_rate'
visual=build_visuals([doc],task['question'],TURN,
context=bind_task({'releaseId':RELEASE},task))
self.assertEqual(visual['visualState'],'ready',visual)
self.assertEqual(visual['visuals'][0]['metric'],'composite_rate')
self.assertEqual(visual['visuals'][0]['series'][0]['values'],[0.7])
def test_whole_group_comparison_is_one_batch_node(self):
task = normalized([changes_node()])
child = copy.deepcopy(task); child.pop('queryPlan')
child['changeQuery'] = {'dimension': 'group', 'sort': 'rate_drop', 'limit': 1}
scope = {'nodeId': 'groups', 'scopeId': 'batch', 'purpose': 'changes',
'label': '业务分组批量对比', 'task': child, 'limit': 1}
_validate_protocol_scope(scope, task['queryPlan'][0], task, 'release')
args = _scoped_arguments({'releaseId': 'release'}, scope, task['queryPlan'][0])
self.assertEqual(len(args), 1)
self.assertEqual(args[0]['mode'], 'changes')
self.assertEqual(args[0]['limit'], 1)
self.assertNotIn('group', args[0]['filters'])
def test_optional_drilldown_depends_on_actual_leading_result(self):
task = normalized([changes_node(limit=3), changes_node(nodeId='products', dimension='product',
optional=True, selector={'kind':'changes_top','sourceNodeId':'groups',
'count':1,'stage':'dominant'})])
self.assertEqual(task['queryPlan'][1]['selector']['sourceNodeId'], 'groups')
self.assertTrue(task['queryPlan'][1]['optional'])
def test_cannot_select_unqueried_or_future_result(self):
with self.assertRaises(ValueError):
normalized([changes_node(selector={'kind':'changes_top','sourceNodeId':'future',
'count':1,'stage':'overall'})])
def test_overall_and_stage_rate_nodes_cannot_reread_the_same_baseline(self):
rate={'nodeId':'overall','purpose':'metrics','fields':['rate'],'stages':[],
'selector':{'kind':'base'},'comparisonRequired':True,'visualPreference':'auto','visualType':None}
with self.assertRaisesRegex(ValueError,'无需重复读取'):
normalized([rate,{**rate,'nodeId':'stage_rate','stages':['black','bright']}])
def test_cause_does_not_rank_tiny_products_by_rate_alone(self):
task=normalized([changes_node(dimension='product')])
from nao_query_plan import normalize_query_plan
task['question']='9月份合格率下降原因'
with self.assertRaisesRegex(ValueError,'小批量'):
normalize_query_plan(task['queryPlan'],root_task=task)
def test_performance_core_keeps_rate_ranking_and_scale_is_only_supporting(self):
from nao_changes import changes_report
task=normalized([changes_node(dimension='product',sort='current_rate',fields=['rate','totalWeight','scrapWeight'])])
task['question']='3C哪些产品近期表现最差';task['changeQuery']={'dimension':'product','sort':'current_rate','limit':1}
side=lambda rate,weight:{'rate':rate,'scrapWeight':weight,
'black':{'rate':rate,'totalWeight':1000,'scrapWeight':weight},
'bright':{'rate':1,'totalWeight':1000,'scrapWeight':0},
'blackInputWeight':1000,'brightInputWeight':1000}
tiny={'rank':1,'key':'小样','label':'小样','rateDrop':1,'current':side(0,1),'comparison':side(1,0)}
# A large scrap volume can coexist with an improving rate. It cannot
# supplant the actual lowest-rate product in the requested ranking.
large={'key':'批量产品','label':'批量产品','scrapDelta':200,'current':side(.9,300),'comparison':side(.8,100)}
result={'kind':'quality_changes','releaseId':RELEASE,'filters':task['filters'],
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
'dimension':'product','stage':None,'sort':'current_rate','completeRanking':True,'totalRows':10,
'rows':[tiny],'prioritySignals':{'currentScrap':[large],'scrapIncrease':[large]}}
report=changes_report([{'evidenceRef':'Q12345678','result':result}],task,RELEASE)
core=[f['text'] for f in report['facts'] if f['id'] in report['requiredFactIds']]
self.assertTrue(core)
self.assertIn('小样',core[0]);self.assertIn('0.00%',core[0])
self.assertFalse(any('批量产品' in text for text in core))
signals=[f for f in report['facts'] if '批量产品' in f['text']]
self.assertTrue(signals)
self.assertTrue(all(f['kind']=='supporting' for f in signals))
self.assertIn('小样','\n'.join(report['sections']))
def test_real_executor_reads_only_batch_and_honors_conditional_stop(self):
task = normalized([changes_node(), changes_node(nodeId='products', dimension='product',
optional=True, selector={'kind':'changes_top','sourceNodeId':'groups',
'count':1,'stage':'overall'})])
child = copy.deepcopy(task);child.pop('queryPlan')
child.update(dimensions=['group'],dimensionStages={'group':[]},
changeQuery={'dimension':'group','sort':'rate_drop','limit':1})
scope = {'nodeId':'groups','scopeId':'batch','purpose':'changes','label':'业务分组',
'task':child,'limit':1}
class Api(BoundApi):
def __init__(self):
super().__init__();self.reads=[]
def call(self,path,payload=None,method=None,*,node_id=None,scope_id=None):
if path == '/api/analysis/protocol/queries':
return {'version':1,'scopes':[scope], 'pending':[], 'gaps':[],
'skipped':[{'nodeId':'products','reason':'no_adverse_change'}]}
if path == '/api/analysis/changes':
self.reads.append((path,copy.deepcopy(payload)))
return {'kind':'quality_changes','releaseId':RELEASE,
'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
'filters':task['filters'],'dimension':'group','stage':None,'sort':'rate_drop',
'totalRows':10,'returnedRows':1,'truncated':True,'completeRanking':True,
'coverage':{'rateComparable':True},
'rows':[{'rank':1,'key':'组甲','label':'组甲','rateDrop':-0.01,'scrapDelta':-10,
'current':{'rate':0.97,'scrapWeight':20},
'comparison':{'rate':0.96,'scrapWeight':30}}],
'protocolEvidence':{**identity(),'queryId':'batch-query','responseHash':'a'*64,
'nodeId':node_id,'scopeId':scope_id}}
if path.startswith('/api/metrics') or path.endswith('/defects'):
raise AssertionError('排名问题不能附加逐对象指标或缺陷读取')
return super().call(path,payload,method)
api=Api();api.task=task
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
with patch('nao_evidence_finalizer._open_model', side_effect=OSError('offline synthesis')):
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':task['question']},
task=task,authority={**identity(),'task':task},actor_root=root,delegated=api,
connection=('http://offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
self.assertEqual(result['status'],'COMPLETED',result.get('evidenceCompletion'))
self.assertEqual(len(api.reads),1)
self.assertEqual(len(result['protocolEvidence']),1)
self.assertNotIn('没有取得', result['answer'])
self.assertIn('## 统计数据附录',result['answer'])
self.assertIn('| 序号 | 对象 |',result['answer'])
class ConfirmedScalarSelectionTests(unittest.TestCase):
def scalar(self):
original='2026年9月18X15X4规格的综合合格率是多少?不要包含18X15X4.1。'
task=normalize_task(task_value(goal=original,kind='analysis',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9},comparisonRequired=False,
trendRequired=False,visualPreference='none',filterUpdates={
'product':'18X15X4+T','exclusions':{'product':['18X15X4.1+Z']}}),original,{})
task['rootGoal']=original
task['productResolution']={'originalQuestion':original,'selectedName':'18X15X4+T',
'productMatch':{'status':'ambiguous','candidates':[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]}}
return task
def test_only_signed_scalar_choice_dispatches_same_metrics_and_keeps_goal_review(self):
from nao_protocol_execution import _confirmed_scalar_selection
task=self.scalar();before=copy.deepcopy(task);api=BoundApi()
request={'releaseId':RELEASE,'turnId':TURN,'question':'选择18X15X4+T。','_autonomousAnswer':True}
self.assertTrue(_confirmed_scalar_selection(request,task))
for sufficient in (True,False):
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
with patch('nao_evidence_finalizer.synthesize',return_value={
'answer':'已核实本期正式率','evidenceSufficient':sufficient,
'missingEvidence':[] if sufficient else ['原目标仍有未回答内容']}) as synth:
result=execute(request=copy.deepcopy(request),task=task,authority=identity(),actor_root=root,
delegated=api,connection=('offline','unused','model'),cancel_event=threading.Event(),emit=lambda _:None)
context=synth.call_args.kwargs['context']
self.assertTrue(context['_autonomousAnswer']);self.assertTrue(context['_confirmedScalarSelection'])
self.assertEqual(context['_agentTask']['rootGoal'],before['rootGoal'])
self.assertEqual(result['status'],'COMPLETED' if sufficient else 'PARTIAL')
self.assertEqual(result['agentTask'],before)
self.assertEqual(len(result['protocolEvidence']),1)
self.assertEqual(task,before)
self.assertTrue(all('exclusions=' in path and 'product=18X15X4%2BT' in path
for _,path,_ in api.calls if path.startswith('/api/metrics')))
def test_old_unselected_investigative_and_non_scalar_tasks_stay_rejected(self):
from nao_protocol_execution import _confirmed_scalar_selection
request={'_autonomousAnswer':True};task=self.scalar()
self.assertFalse(_confirmed_scalar_selection({},task))
for changes in ({'stages':['black']},{'trendRequired':True},{'comparisonRequired':True},
{'verifySource':True},{'fullDetails':True},{'executionMode':'native_agent'},
{'dimensions':['product']},{'fields':['rate','totalWeight']},
{'period':{'year':2026}},{'productResolution':{}},{'rootGoal':'找出重点规格'}):
with self.subTest(changes=changes):
self.assertFalse(_confirmed_scalar_selection(request,{**task,**changes}))
for original in ('找重点规格的综合合格率是多少?','合格率为什么下降?','最低合格率前五项是多少?'):
altered=copy.deepcopy(task);altered['rootGoal']=original;altered['productResolution']['originalQuestion']=original
self.assertFalse(_confirmed_scalar_selection(request,altered))
if __name__ == '__main__': unittest.main()
+217
View File
@@ -0,0 +1,217 @@
"""Month-range evidence stays bound to its exact selected calendar window."""
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from analysis_visuals import build_visuals
from nao_agent_report import _build_trend_report
from nao_evidence_finalizer import _coverage_period, coverage_sufficient
from nao_evidence_report import _frame_key, _same_periods, period, select_comparison_frames
from nao_period import expected_months, normalize_period
QUESTION = '查看客户甲2026年3至5月的综合合格率趋势图'
RELEASE = 'release-range'
FILTERS = {'extraFilters': {'customerCode': ['客户甲']}}
def selected_task(period_value, *, period_coverage='available'):
return {
'version': 1, 'kind': 'metrics', 'question': QUESTION, 'goal': QUESTION,
'period': period_value, 'comparisonPeriod': None, 'comparisonRequired': False,
'filters': FILTERS, 'freshQuery': True, 'dimensions': [], 'dimensionStages': {},
'stages': [], 'fields': ['rate'], 'historyMode': 'none', 'trendRequired': True,
'periodCoverage': period_coverage, 'visualPreference': 'auto', 'visualType': None,
'responseStyle': 'brief', 'verifySource': False, 'fullDetails': False,
}
def context(period_value, *, period_coverage='available'):
task = selected_task(period_value, period_coverage=period_coverage)
return {'releaseId': RELEASE, 'period': period_value, 'filters': FILTERS, '_agentTask': task}
def metric_doc(selected_period, months, *, release=RELEASE, filters=None, ref='Q1234ABCD'):
rates = {1: .71, 2: .72, 3: .73, 4: .74, 5: .75, 6: .76, 7: .77, 8: .78, 9: .79, 10: .80}
trend = [
{'period': f"2026-{month:02d}", 'compositeRate': rates[month],
'black': {'rate': rates[month] - .01}, 'bright': {'rate': rates[month] + .01},
'coverage': {'blackRows': 5, 'brightRows': 5}}
for month in months
]
return {
'evidenceRef': ref, 'mode': 'query', 'releaseId': release,
'result': {
'kind': 'quality_metrics', 'releaseId': release, 'view': 'overall',
'period': selected_period, 'filters': filters if filters is not None else FILTERS,
'summary': {'compositeRate': .74}, 'trend': trend, 'trendCount': len(trend),
'trendTruncated': False,
'coverage': {'sourceMonths': [
{'year': 2026, 'month': month, 'blackRows': 5, 'brightRows': 5} for month in months
]},
},
}
class MonthRangeEvidenceTests(unittest.TestCase):
def test_period_helper_supports_ranges_and_preserves_single_month_quarter_year(self):
self.assertIsNone(normalize_period(None))
self.assertEqual(expected_months({'year': 2026, 'monthStart': 1, 'monthEnd': 9}), list(range(1, 10)))
self.assertEqual(expected_months({'year': 2026, 'monthStart': 3, 'monthEnd': 5}), [3, 4, 5])
self.assertEqual(expected_months({'year': 2026}), list(range(1, 13)))
self.assertEqual(expected_months({'year': 2026, 'quarter': 2}), [4, 5, 6])
self.assertEqual(expected_months({'year': 2026, 'month': 9}), [9])
def test_range_requires_paired_valid_ordered_endpoints(self):
invalid = (
{'year': 2026, 'monthStart': 3},
{'year': 2026, 'monthEnd': 5},
{'year': 2026, 'monthStart': 5, 'monthEnd': 3},
{'year': 2026, 'monthStart': 0, 'monthEnd': 5},
{'year': 2026, 'monthStart': 3, 'monthEnd': 13},
{'year': 2026, 'monthStart': 3, 'monthEnd': 5, 'month': 4},
{'year': 2026, 'monthStart': 3, 'monthEnd': 5, 'quarter': 2},
)
for value in invalid:
with self.subTest(period=value), self.assertRaises(ValueError):
normalize_period(value)
def test_report_renders_range_and_does_not_fill_an_absent_month_with_zero(self):
selected = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
item = metric_doc(selected, [1, 2, 3, 5, 6, 7, 8, 9])
report = _build_trend_report([item], context(selected), QUESTION)
self.assertTrue(report['coverageSufficient'])
text = report['facts'][0]['text']
self.assertIn('2026年1–9月', text)
self.assertIn('1月71.00%', text)
self.assertIn('4月未提供', text)
self.assertIn('9月79.00%', text)
self.assertNotIn('4月0.00%', text)
def test_short_range_includes_only_selected_months_and_rejects_outer_data(self):
selected = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
item = metric_doc(selected, [3, 4, 5])
report = _build_trend_report([item], context(selected), QUESTION)
self.assertTrue(report['coverageSufficient'])
self.assertIn('3月73.00%', report['facts'][0]['text'])
self.assertIn('5月75.00%', report['facts'][0]['text'])
self.assertNotIn('2月', report['facts'][0]['text'])
self.assertNotIn('6月', report['facts'][0]['text'])
out_of_range = metric_doc(selected, [2, 3, 4, 5])
rejected = _build_trend_report([out_of_range], context(selected), QUESTION)
self.assertFalse(rejected['coverageSufficient'])
self.assertFalse(rejected['facts'])
self.assertIn('期间以外', rejected['missingEvidence'][0])
def test_unknown_stage_rows_do_not_become_known_zero_in_a_composite_trend_point(self):
selected = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
item = metric_doc(selected, [1, 2, 3, 4, 5])
february = next(row for row in item['result']['trend'] if row['period'] == '2026-02')
february['coverage']['brightRows'] = None
parent_february = next(row for row in item['result']['coverage']['sourceMonths']
if row['month'] == 2)
parent_february['brightRows'] = None
report = _build_trend_report([item], context(selected), QUESTION)
text = report['facts'][0]['text']
self.assertIn('2月未提供', text)
self.assertNotIn('2月72.00%', text)
def test_period_identity_and_release_scope_reject_other_range_evidence(self):
large = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
short = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
self.assertNotEqual(_frame_key({'current': {'period': large}, 'comparison': {'period': {'year': 2025}},
'filterBinding': FILTERS}),
_frame_key({'current': {'period': short}, 'comparison': {'period': {'year': 2025}},
'filterBinding': FILTERS}))
self.assertEqual(period(short), '2026年3–5月')
def comparison(ref, current_period):
return {
'evidenceRef': ref, 'mode': 'compare',
'result': {'kind': 'quality_comparison', 'releaseId': RELEASE,
'filterBinding': FILTERS,
'current': {'period': current_period},
'comparison': {'period': {'year': 2025}},
'delta': {'available': True, 'percentagePoints': 1.0}},
}
roots, limitations = select_comparison_frames(
[comparison('Q11111111', large), comparison('Q22222222', short)], context(short))
self.assertEqual([root['evidenceRef'] for root in roots], ['Q22222222'])
self.assertEqual(limitations, [])
roots, limitations = select_comparison_frames([comparison('Q11111111', large)], context(short))
self.assertTrue(limitations)
wrong = metric_doc(large, [3, 4, 5])
report = _build_trend_report([wrong], context(short), QUESTION)
self.assertFalse(report['coverageSufficient'])
self.assertFalse(report['facts'])
self.assertFalse(coverage_sufficient([wrong], QUESTION, context(short)))
expected = {'kind': 'quality_changes', 'currentPeriod': short,
'comparisonPeriod': {'year': 2026, 'monthStart': 1, 'monthEnd': 2}}
wrong_frame = {'kind': 'quality_changes', 'currentPeriod': large,
'comparisonPeriod': {'year': 2026, 'monthStart': 1, 'monthEnd': 2}}
self.assertTrue(_same_periods(expected, {
'kind': 'quality_comparison',
'current': {'period': short},
'comparison': {'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 2}},
'filterBinding': FILTERS,
}))
self.assertFalse(_same_periods(wrong_frame, {
'kind': 'quality_comparison',
'current': {'period': short},
'comparison': {'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 2}},
'filterBinding': FILTERS,
}))
def test_complete_range_requires_every_selected_month_and_source_coverage(self):
selected = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
item = metric_doc(selected, [3, 4, 5])
item['result']['coverage']['sourceMonths'].pop(1)
report = _build_trend_report([item], context(selected, period_coverage='complete'), QUESTION)
self.assertFalse(report['coverageSufficient'])
self.assertIn('全部月份', ' '.join(report['missingEvidence']))
self.assertFalse(coverage_sufficient([item], QUESTION,
context(selected, period_coverage='complete')))
missing_month = metric_doc(selected, [3, 5])
report = _build_trend_report([missing_month], context(selected, period_coverage='complete'), QUESTION)
self.assertFalse(report['coverageSufficient'])
self.assertIn('4月未提供', report['facts'][0]['text'])
def test_visual_scope_and_points_stay_inside_selected_range(self):
selected = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
item = metric_doc(selected, [1, 2, 3, 5, 6, 7, 8, 9])
visual_context = {
**context(selected),
'_agentTask': {
'version': 1, 'kind': 'metrics', 'question': QUESTION,
'visualPreference': 'required', 'visualType': 'line',
'dimensions': [], 'stages': [], 'fields': ['rate'],
},
}
result = build_visuals([item], QUESTION, 'range-visual', context=visual_context)
self.assertEqual(result['visualState'], 'ready')
chart = next(chart for chart in result['visuals'] if chart.get('metric') == 'composite_rate')
self.assertEqual(chart['scope']['year'], 2026)
self.assertEqual(chart['scope']['monthStart'], 1)
self.assertEqual(chart['scope']['monthEnd'], 9)
self.assertEqual(chart['series'][0]['values'], [.71, .72, .73, None, .75, .76, .77, .78, .79])
outside = metric_doc(selected, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
rejected = build_visuals([outside], QUESTION, 'range-visual-outside', context=visual_context)
self.assertEqual(rejected['visuals'], [])
def test_finalizer_period_identity_accepts_range_exactly(self):
selected = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
self.assertEqual(_coverage_period(selected), selected)
self.assertIsNone(_coverage_period({'year': 2026, 'monthStart': 3}))
self.assertIsNone(_coverage_period({'year': 2026, 'monthStart': 6, 'monthEnd': 5}))
if __name__ == '__main__':
unittest.main()
+76
View File
@@ -0,0 +1,76 @@
import sys
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_agent_task import TaskUnavailable, bind_task, normalize_task, plan_task
from nao_metric_request import frozen_metric_frame
from nao_quality_tools import QualityToolError, QualityTools, normalize_period
from nao_query_contract import bind_query
from test_nao_agent_task import task_value
class MonthRangeTaskTests(unittest.TestCase):
def test_range_survives_condition_changes_and_exact_query_binding(self):
period = {'year': 2026, 'monthStart': 3, 'monthEnd': 5}
context = {'_protocol': {'version': 1}, 'releaseId': 'fixed'}
original = normalize_task(task_value(kind='metrics', dimensions=[], stages=[], fields=['rate'],
period=period, trendRequired=True, filterUpdates={'extraFilters': {'customerCode': ['800001']}}),
'客户三至五月趋势', context)
inherited = bind_task(context, original)
updated = normalize_task(task_value(kind='metrics', dimensions=[], stages=[], fields=['rate'],
trendRequired=True, filterUpdates={'extraFilters': None}), '取消客户,期间不变', inherited)
self.assertEqual(updated['period'], period)
self.assertNotIn('extraFilters', updated['filters'])
self.assertEqual(frozen_metric_frame(bind_task(context, updated))['period'], period)
contract = {'task': updated, 'releaseId': 'fixed'}
self.assertEqual(bind_query({'mode': 'query'}, contract)['period'], period)
for wrong in ({'year': 2026}, {'year': 2026, 'month': 5},
{'year': 2026, 'monthStart': 1, 'monthEnd': 5}):
with self.assertRaises(ValueError):
bind_query({'mode': 'query', 'period': wrong}, contract)
def test_invalid_ranges_are_rejected_by_planner_and_tools(self):
for period in ({'year': 2026, 'monthStart': 1}, {'year': 2026, 'monthEnd': 9},
{'year': 2026, 'monthStart': 9, 'monthEnd': 1},
{'year': 2026, 'monthStart': True, 'monthEnd': 9},
{'year': 2026, 'monthStart': 1, 'monthEnd': 13},
{'year': 2026, 'monthStart': 1, 'monthEnd': 9, 'month': 9},
{'year': 2026, 'monthStart': 1, 'monthEnd': 9, 'quarter': 3}):
with self.subTest(period=period):
with self.assertRaises(TaskUnavailable):
normalize_task(task_value(period=period), '月度趋势', {})
with self.assertRaises(QualityToolError):
normalize_period(period)
def test_metric_receipt_cannot_omit_or_expand_month_range(self):
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 9}
response = {'summary': {}, 'coverage': {'rawRows': 1, 'includedRows': 1}, 'release': {'id': 'fixed'}, 'context': {**period, 'view': 'overall', 'includeUnassigned': False}}
QualityTools._validate_source_snapshot(response, 'fixed', period, {}, 'overall')
for changed in ({'year': 2026, 'view': 'overall'},
{**period, 'monthEnd': 12, 'view': 'overall'}):
with self.assertRaises(QualityToolError):
QualityTools._validate_source_snapshot({**response, 'context': changed}, 'fixed', period, {}, 'overall')
def test_new_task_resets_root_goal_and_prompt_describes_ranges(self):
previous = normalize_task(task_value(period={'year': 2025}), '旧目标', {})
previous['rootGoal'] = '旧根目标'
context = bind_task({'_protocol': {'version': 1}}, previous)
proposal = task_value(kind='metrics', dimensions=[], stages=[], fields=['rate'],
goal='客户月度新目标', newTask=True, trendRequired=True,
period={'year': 2026, 'monthStart': 1, 'monthEnd': 9})
semantic = {'aligned': True, 'requiresEvidence': True, 'requiresTrend': True,
'requiresComparison': False, 'requiresSourceVerification': False, 'requiresAllSourceRows': False}
with patch('nao_agent_task._model_json', side_effect=[proposal, semantic]) as model:
planned = plan_task(base_url='https://offline.invalid', api_key='synthetic', model='m',
question='新任务:客户1至9月综合合格率趋势', context=context, cancel_event=threading.Event())
self.assertEqual(planned['rootGoal'], '客户月度新目标')
prompt = model.call_args_list[0].kwargs['system']
self.assertIn('monthStart', prompt)
self.assertIn('单独month只代表该月', prompt)
if __name__ == '__main__':
unittest.main()
+919
View File
@@ -0,0 +1,919 @@
import io
import json
import sys
import threading
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_agent_task import _explicit_read_safeguards, normalize_task, bind_task, plan_task, plan_repairs, TaskUnavailable
from nao_analysis_coverage import coverage_gaps, plan_missing_queries, formal_metric_coverage, semantic_metric_gaps
from nao_metric_request import metric_request
from nao_evidence_finalizer import coverage_sufficient
from test_nao_analysis_coverage import evidence
from test_nao_metric_requests import metric_evidence
def task_value(**changes):
return {'goal': '完整理解本轮质量问题', 'kind': 'analysis', 'freshQuery': True,
'dimensions': ['department', 'group', 'product', 'defect'], 'stages': ['black', 'bright'],
'fields': ['rate', 'totalWeight', 'scrapWeight'], 'verifySource': False,
'fullDetails': False, 'visualPreference': 'auto', 'visualType': None,
'responseStyle': 'standard', 'needsClarification': False, 'clarification': '',
'filterUpdates': {}, **changes}
def semantic_ok():
return {'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,'requiresAllSourceRows':False,'requiresTrend':False}
class AgentTaskTest(unittest.TestCase):
def context(self, question, **changes):
return bind_task({}, normalize_task(task_value(**changes), question, {}))
def test_negated_lists_do_not_request_raw_rows_but_positive_siblings_still_do(self):
for question in (
'新任务:查询2026年9月客户800001在黑片和亮片中的全部成品规格清单,去重并显示各自来源阶段。这里只要成员清单,不要正式指标、排名或逐行原始记录。',
'只要清单,不需要指标及全部原始记录',
'不要图表或核验原件',
):
with self.subTest(question=question):
self.assertEqual(_explicit_read_safeguards(question),{'requiresSourceVerification':False,'requiresAllSourceRows':False})
for question in ('不要排名,但需要逐行原始记录','不要排名,逐行读取源记录','不要去重且逐行读取全部原始记录','不要图表或排名,请逐行读取原始记录'):
with self.subTest(question=question):
self.assertTrue(_explicit_read_safeguards(question)['requiresAllSourceRows'])
def test_current_read_negations_do_not_force_or_cancel_a_different_purpose(self):
for text in ('不读逐行原始记录', '不读取全部原始记录', '不做逐条原始记录读取', '原始记录不需要逐行读取'):
with self.subTest(text=text):
self.assertFalse(_explicit_read_safeguards(text)['requiresAllSourceRows'])
self.assertEqual(_explicit_read_safeguards('不要核验原件,但请逐行读取原始记录'),
{'requiresSourceVerification': False, 'requiresAllSourceRows': True})
self.assertEqual(_explicit_read_safeguards('不读逐行原始记录,但请核验归档原件'),
{'requiresSourceVerification': True, 'requiresAllSourceRows': False})
self.assertTrue(_explicit_read_safeguards('不要去重且逐行读取全部原始记录')['requiresAllSourceRows'])
def test_defect_only_metrics_plan_is_repaired_instead_of_answering_parent_input(self):
replies=[task_value(kind='metrics',dimensions=['defect'],stages=['black'],fields=['totalWeight'],dimensionStages={'defect':['black']}),
task_value(kind='analysis',dimensions=['defect'],stages=['black'],fields=['scrapWeight'],dimensionStages={'defect':['black']}),semantic_ok()]
payloads=[]
def opener(request,timeout):
payloads.append(json.loads(json.loads(request.data)['messages'][1]['content']))
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(replies.pop(0))}}]}).encode())
planned=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='查看具体缺陷排名',context={},cancel_event=threading.Event(),opener=opener)
self.assertEqual(planned['kind'],'analysis')
self.assertEqual(planned['fields'],['scrapWeight'])
self.assertEqual(payloads[1]['validationDetails']['field'],'kind')
def test_semantic_coverage_is_independent_of_user_wording(self):
for question in ['分析整体质量变化', '帮我全面看看最近的质量表现', '最近生产表现还好吗']:
context = self.context(question)
self.assertEqual(len(plan_missing_queries([evidence('quality_comparison')], question, context)), 5)
self.assertEqual(len(coverage_gaps([evidence('quality_comparison')], question, context)), 5)
question = '哪些产品恶化最明显'
context = self.context(question, dimensions=['product'])
self.assertEqual(len(plan_missing_queries([evidence('quality_comparison')], question, context)), 1)
def test_all_source_defect_fields_are_not_misread_as_all_raw_records(self):
question = ('改为只看黑片,客户800001不变,期间仍是2026年8至9月与7月比较。'
'新增加工单位等于机加工部,并排除规格18X15X4+Z;'
'按缺陷维度列出全部来源细项重量和变化,保留柱状图。')
requirements = _explicit_read_safeguards(question)
self.assertFalse(requirements['requiresAllSourceRows'])
self.assertFalse(requirements['requiresSourceVerification'])
for aggregate_request in ('汇总全部来源细项重量', '列出所有源字段汇总',
'比较全部缺陷类别重量', '逐条列出全部来源缺陷字段汇总'):
with self.subTest(aggregate_request=aggregate_request):
self.assertFalse(_explicit_read_safeguards(aggregate_request)['requiresAllSourceRows'])
def test_explicit_row_level_source_requests_still_require_full_details(self):
for question in ('逐行读取原始记录', '读取全部源记录', '列出所有原始数据行',
'输出全部原始行字段', '每条记录列出所有字段'):
with self.subTest(question=question):
self.assertTrue(_explicit_read_safeguards(question)['requiresAllSourceRows'])
self.assertFalse(_explicit_read_safeguards('不需要逐行读取全部源记录')['requiresAllSourceRows'])
def test_semantic_metrics_accept_aliases_and_date_notation(self):
question = '只看SYNTH-16X15+T,2026-09对2026-08,黑片亮片良率和投入、报废'
value = task_value(kind='metrics', dimensions=[], filterUpdates={'product': 'SYNTH-16X15+T'},
period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2026, 'month': 8})
context = bind_task({}, normalize_task(value, question))
request = metric_request(question, context)
self.assertIsNotNone(request)
self.assertEqual(request['fields'], ['rate', 'totalWeight', 'scrapWeight'])
contract = formal_metric_coverage(metric_evidence(), question, context)
self.assertEqual(contract['missingEvidence'], [])
self.assertEqual(len(contract['metrics']), 12)
def test_semantic_full_records_and_source_verification_cannot_pass_on_query_only(self):
docs = [{'mode': 'query', 'result': {'kind': 'quality_query'}}]
for question, updates in [('把每一条记录都列出来', {'kind': 'details', 'fullDetails': True}),
('对照原始Excel核实数值', {'kind': 'source_verify', 'verifySource': True})]:
self.assertFalse(coverage_sufficient(docs, question, self.context(question, dimensions=[], **updates)))
def test_scope_updates_keep_other_filters_and_release(self):
context = {'releaseId': 'fixed-release', 'period': {'year': 2026, 'month': 9},
'group': '汽车交通-海外', 'filters': {'group': '汽车交通-海外', 'product': 'A'}}
task = normalize_task(task_value(filterUpdates={'group': '汽车交通-国内', 'product': None}), '那国内呢', context)
bound = bind_task(context, task)
self.assertEqual(bound['filters'], {'group': '汽车交通-国内'})
self.assertNotIn('group', bound)
self.assertEqual(bound['releaseId'], 'fixed-release')
self.assertEqual(bound['period'], context['period'])
for bad in ({'filterUpdates': {'releaseId': 'evil'}}, {'freshQuery': 'true'}, {'period': {'year': 2026, 'month': 13}}):
with self.assertRaises(TaskUnavailable): normalize_task(task_value(**bad), 'q', context)
def test_recoverable_filter_shapes_preserve_existing_scope(self):
context={'filters':{'department':'消费电子','product':'OLD'}}
value=task_value(filterUpdates={'product':[' 18X15X4+T '], 'department':' '})
self.assertEqual(normalize_task(value,'q',context)['filters'],{'department':'消费电子','product':'18X15X4+T'})
value['filterUpdates']={'product':[]}
self.assertEqual(normalize_task(value,'q',context)['filters'],context['filters'])
value['filterUpdates']={'product':None}
self.assertNotIn('product',normalize_task(value,'q',context)['filters'])
value['filterUpdates']={'product':['A','B']}
with self.assertRaises(TaskUnavailable) as err: normalize_task(value,'q',context)
self.assertEqual(err.exception.field,'filterUpdates.product')
def test_model_plan_is_validated_and_retried_without_keyword_parser(self):
replies = [{}, task_value(dimensions=['product'], freshQuery=True, dimensionStages={'product': []}),semantic_ok()]
def opener(request, timeout):
self.assertNotIn('apiKey', request.data.decode())
value = replies.pop(0)
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(value)}}]}).encode())
task = plan_task(base_url='https://example.test/v1', api_key='test-only', model='any-compatible-model',
question='按最新数据再跑一遍', context={}, previous=None,
cancel_event=threading.Event(), opener=opener)
self.assertTrue(task['freshQuery'])
self.assertEqual(task['dimensions'], ['product'])
self.assertFalse(replies)
def test_retry_semantics_retain_prior_question_without_using_partial_answer(self):
from unittest.mock import patch
context={'releaseId':'fixed-release',
'_serverQuestions':[{'turnId':'previous','question':'9月汽车交通的合格率为什么下降?','status':'PARTIAL'}]}
wrong=task_value(goal='比较9月报废构成',dimensions=['group'],stages=['black'],fields=['scrapWeight'],
comparisonRequired=False,dimensionStages={'group':['black']},period={'year':2026,'month':9})
correct=task_value(goal='比较国内与海外对9月合格率下降的变化线索',dimensions=['group'],
fields=['rate','totalWeight','scrapWeight'],comparisonRequired=True,dimensionStages={'group':[]},
period={'year':2026,'month':9},comparisonPeriod={'year':2026,'month':8})
rejection={'aligned':False,'reason':'前文问合格率变化,不是单期报废构成'}
replies=[wrong,rejection,rejection,correct,semantic_ok()]
payloads=[]
def response(**kwargs):
payloads.append(kwargs['payload']);return replies.pop(0)
with patch('nao_agent_task._model_json',side_effect=response):
planned=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='主要是国内还是国外的',context=context,cancel_event=threading.Event(),semantic_review=True)
self.assertTrue(planned['comparisonRequired'])
self.assertEqual(planned['comparisonPeriod'],{'year':2026,'month':8})
self.assertEqual(planned['dimensionStages'],{'group':[]})
self.assertEqual(planned['rootGoal'],'9月汽车交通的合格率为什么下降?')
self.assertEqual(payloads[0]['previousUserQuestions'],context['_serverQuestions'])
self.assertNotIn('answer',json.dumps(payloads[0]['previousUserQuestions']))
self.assertFalse(replies)
def test_last_valid_plan_survives_false_rejection_only_with_two_consistent_reviews(self):
from unittest.mock import patch
first=task_value(dimensions='invalid')
second=task_value(fields=['invalid'])
final=task_value(goal='查询今年产品合格率趋势',kind='metrics',dimensions=[],
fields=['rate'],stages=[],dimensionStages={},filterUpdates={'product':['18X15X4+T']},
period={'year':2026})
reject={'aligned':False,'reason':'产品分析缺少group'}
replies=[first,second,final,reject,semantic_ok(),semantic_ok()]
with patch('nao_agent_task._model_json',side_effect=replies) as call:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年18X15X4+T合格率变化趋势',context={},cancel_event=threading.Event())
self.assertEqual(task['filters']['product'],'18X15X4+T')
self.assertEqual(task['period'],{'year':2026})
self.assertEqual(task['fields'],['rate'])
def test_filter_only_followup_reuses_prior_query_plan_and_recomputes_child_scopes(self):
from unittest.mock import patch
plan = [
{'nodeId':'rank','purpose':'ranking','dimension':'product','limit':5,
'fields':['totalWeight'],'stages':['black'],'selector':{'kind':'base'},
'visualType':'pareto','visualPreference':'auto'},
{'nodeId':'trend','purpose':'trend','fields':['scrapRate'],'stages':['black'],
'selector':{'kind':'ranking_top','sourceNodeId':'rank','stage':'black',
'metric':'totalWeight','count':2},
'visualType':'line','visualPreference':'auto'},
]
previous_value = task_value(kind='analysis',queryPlan=plan,dimensions=['product'],
dimensionStages={'product':['black']},stages=['black'],fields=['totalWeight','scrapRate'],
trendRequired=True,comparisonRequired=False,period={'year':2026},periodCoverage='available')
previous = normalize_task(previous_value,'按产品重量排名后查看前两项趋势',{'_protocol':{'version':1}})
context = {'_protocol':{'version':1},'_agentTask':previous,'_serverQuestions':[]}
proposal = {'goal':'保持其他分析不变,只改部门筛选','kind':'analysis','freshQuery':True,
'verifySource':False,'fullDetails':False,'needsClarification':False,
'filterUpdates':{'department':'部门甲'},'periodCoverage':'available'}
reviewed = {'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,
'requiresAllSourceRows':False,'requiresTrend':True,'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[proposal,reviewed]):
planned = plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='其他不变,只把部门改为部门甲',context=context,
cancel_event=threading.Event())
self.assertEqual(planned['filters']['department'],'部门甲')
self.assertEqual(planned['queryPlan'],previous['queryPlan'])
self.assertEqual(planned['fields'],['totalWeight','scrapRate'])
self.assertTrue(planned['trendRequired'])
def test_query_plan_prompt_explains_selector_objects_and_root_stage_union(self):
from unittest.mock import patch
nodes = [
{'nodeId':'rank','purpose':'ranking','dimension':'product','limit':10,
'fields':['totalWeight'],'stages':['black'],'selector':{'kind':'base'},
'visualType':'pareto','visualPreference':'auto','comparisonRequired':False},
{'nodeId':'trend','purpose':'trend','fields':['rate'],'stages':[],
'selector':{'kind':'ranking_top','sourceNodeId':'rank','stage':'black',
'metric':'totalWeight','count':2},
'visualType':'line','visualPreference':'auto','comparisonRequired':False},
]
invalid = task_value(kind='analysis',queryPlan=nodes,dimensions=['product'],
dimensionStages={'product':['black']},stages=['black','bright'],
fields=['rate','totalWeight'],trendRequired=True,comparisonRequired=False,
period={'year':2026,'monthStart':1,'monthEnd':9})
valid = {**invalid,'stages':['black']}
review = {**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[invalid,valid,review]) as call:
planned = plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='按黑片投入重量排名,再看前两项综合合格率走势',
context={'_protocol':{'version':1}},cancel_event=threading.Event())
self.assertEqual(planned['stages'],['black'])
prompt = call.call_args_list[0].kwargs['system']
self.assertIn('selector始终是带kind的JSON对象',prompt)
self.assertIn('{"kind":"base"}',prompt)
self.assertIn('{"kind":"catalog_members","dimension":"group","names":["部门甲-国内","部门甲-海外"]}',prompt)
self.assertIn('{"kind":"ranking_top","sourceNodeId":"rank","stage":"black","metric":"totalWeight","count":2}',prompt)
self.assertIn('{"kind":"previous_ranking","queryId":"trusted-query-id","stage":"black","metric":"totalWeight","ranks":[1,2]}',prompt)
self.assertIn('root stages必须是所有节点stages数组的字面去重并集',prompt)
self.assertIn('"stages":[]',prompt)
self.assertIn('nodeId必须符合[a-z][a-z0-9_-]{0,31}',prompt)
self.assertIn('root dimensionStages记录ranking/changes节点实际排名的维度和阶段',prompt)
self.assertIn('没有ranking节点时必须为{}',prompt)
self.assertIn('root dimensions来自ranking节点的dimension及selector的dimension',prompt)
self.assertIn('父filters中的department不会自动成为分析维度',prompt)
self.assertIn('产业部甲内比较国内和海外组:root dimensions=[group]、stages=[black]、dimensionStages={}',prompt)
self.assertIn('"needsClarification":false',prompt)
self.assertIn('多指定目录对象分别查询指标、比较或趋势时必须用queryPlan和catalog_members',prompt)
self.assertIn('上轮排名对象只能用previous_ranking选择',prompt)
self.assertIn('同比追问在root保留comparisonMode="yoy"',prompt)
self.assertIn('若去年来源缺失,保留同比期间并如实返回部分结果',prompt)
shapes = call.call_args_list[1].kwargs['payload']['requiredShapes']
self.assertEqual(shapes['queryPlanSelectorExamples'][0],{'kind':'base'})
self.assertEqual(shapes['queryPlanSelectorExamples'][2]['kind'],'ranking_top')
self.assertIn('root stages=[black]',shapes['queryPlanRootExample'])
self.assertIn('trend node fields=[rate], stages=[]',shapes['queryPlanRootExample'])
self.assertIn('expands to two product scopes',shapes['queryPlanRootExample'])
self.assertEqual(shapes['queryPlanNodeIdPattern'],
'[a-z][a-z0-9_-]{0,31}; lowercase ASCII initial, 1..32 chars')
self.assertIn('with no ranking nodes use {}',shapes['queryPlanDimensionStages'])
self.assertIn('comparison, trend, or any named/product/mixed defect request requires a bounded queryPlan',
shapes['queryPlanRequiredWhenGroupedMetrics'])
group_example = shapes['queryPlanGroupComparisonExample']
self.assertFalse(group_example['needsClarification'])
self.assertEqual(group_example['dimensions'],['group'])
self.assertEqual(group_example['stages'],['black'])
self.assertEqual(group_example['dimensionStages'],{})
self.assertEqual(group_example['queryPlan'][0]['stages'],[])
self.assertTrue(all(node['comparisonRequired'] for node in group_example['queryPlan']))
self.assertEqual(call.call_count,3)
def test_protocol_v1_grouped_metrics_without_query_plan_enters_planner_repair(self):
from unittest.mock import patch
groups=['部门甲-国内','部门甲-海外']
context={'_protocol':{'version':1},'_businessCatalog':{
'departments':['部门甲'],'groups':groups,
'groupDepartments':{name:'部门甲' for name in groups}}}
question='比较部门甲国内和海外的综合合格率及黑片投入重量同比'
legacy=task_value(goal=question,kind='metrics',dimensions=['group'],
dimensionStages={'group':['black']},fields=['rate','totalWeight'],stages=['black'],
trendRequired=False,comparisonRequired=True,comparisonMode='yoy',
period={'year':2026,'month':9},filterUpdates={'department':'部门甲'},
targetEntities={'group':groups})
with self.assertRaises(TaskUnavailable) as err:
normalize_task(legacy,question,context)
self.assertEqual(err.exception.field,'queryPlan')
self.assertEqual(err.exception.expected,'bounded node plan')
# Non-protocol callers retain the legacy normalization behavior.
self.assertTrue(normalize_task(legacy,question)['comparisonRequired'])
selector={'kind':'catalog_members','dimension':'group','names':groups}
valid={**legacy,'dimensionStages':{},'queryPlan':[
{'nodeId':'group_rate','purpose':'metrics','fields':['rate'],'stages':[],
'selector':selector,'visualType':'column','visualPreference':'auto','comparisonRequired':True},
{'nodeId':'group_weight','purpose':'metrics','fields':['totalWeight'],'stages':['black'],
'selector':selector,'visualType':'column','visualPreference':'auto','comparisonRequired':True},
]}
review={**semantic_ok(),'requiresTrend':False,'requiresComparison':True,
'targetEntities':{'group':groups}}
with patch('nao_agent_task._model_json',side_effect=[legacy,valid,review]) as call:
planned=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context=context,cancel_event=threading.Event())
self.assertEqual(len(planned['queryPlan']),2)
self.assertEqual(planned['comparisonMode'],'yoy')
self.assertEqual(planned['comparisonPeriod'],{'year':2025,'month':9})
self.assertEqual(call.call_args_list[1].kwargs['payload']['requiredShapes']['queryPlanRequiredWhenGroupedMetrics'],
'protocol v1: nonempty dimensions plus rate/scrapRate fields, comparison, trend, or any named/product/mixed defect request requires a bounded queryPlan; keep the full user requirement')
self.assertEqual(call.call_count,3)
def test_last_valid_plan_survives_transient_review_protocol_failure(self):
from unittest.mock import patch
bad_shape=task_value(dimensions='invalid')
bad_fields=task_value(fields=['invalid'])
valid=task_value(goal='查看今年合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
good={**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
responses=[bad_shape,bad_fields,valid,{'aligned':'true'},good]
events=[]
with patch('nao_agent_task._model_json',side_effect=responses) as call:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年合格率走势',context={},cancel_event=threading.Event(),on_validation_error=events.append)
self.assertTrue(task['trendRequired']);self.assertEqual(task['period'],{'year':2026})
self.assertEqual(call.call_count,5)
self.assertEqual(call.call_args_list[3].kwargs['payload']['plannedTask'],call.call_args_list[4].kwargs['payload']['plannedTask'])
self.assertEqual([(e['phase'],e['reviewAttempt'],e['field'],e['receivedType']) for e in events if e.get('phase')=='task_review'],
[('task_review',1,'aligned','str')])
def test_review_protocol_retry_is_bounded_and_keeps_valid_plan(self):
from unittest.mock import patch
plan=task_value(kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[])
malformed={'aligned':True,'requiresEvidence':'true','requiresSourceVerification':False,'requiresAllSourceRows':False}
with patch('nao_agent_task._model_json',side_effect=[plan,malformed,semantic_ok()]) as call:
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='查9月合格率',context={},cancel_event=threading.Event())
self.assertEqual(result['kind'],'metrics');self.assertEqual(call.call_count,3)
with patch('nao_agent_task._model_json',side_effect=[plan,TaskUnavailable('模型超时'),malformed]) as call:
with self.assertRaises(TaskUnavailable) as err:
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='查9月合格率',context={},cancel_event=threading.Event())
self.assertEqual(err.exception.field,'review.protocol');self.assertEqual(call.call_count,3)
def test_review_disagreement_never_bypasses_evidence_or_scope(self):
from unittest.mock import patch
value=task_value(dimensions=['group'],dimensionStages={'group':[]})
rejection={'aligned':False,'reason':'范围错误'}
contradictory={**semantic_ok(),'requiresSourceVerification':True}
with patch('nao_agent_task._model_json',side_effect=[value,rejection,semantic_ok(),contradictory]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='比较各业务分组',context={},cancel_event=threading.Event())
def test_monthly_trend_is_structured_and_semantic_review_rejects_annual_total(self):
from unittest.mock import patch
annual=task_value(goal='查询年度合格率',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026})
trend={**annual,'goal':'查询今年逐月合格率走势','trendRequired':True}
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[annual,review,trend,review]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='看看今年良率怎么变化',context={},cancel_event=threading.Event())
self.assertTrue(task['trendRequired']);self.assertEqual(task['periodCoverage'],'available')
with self.assertRaises(TaskUnavailable):normalize_task({**annual,'trendRequired':'true'},'q')
self.assertFalse(normalize_task(annual,'q')['trendRequired'])
def test_missing_trend_review_gets_bounded_independent_check(self):
from unittest.mock import patch
plan=task_value(goal='查询月度合格率走势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
incomplete={**{k:v for k,v in semantic_ok().items() if k!='requiresTrend'},'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[plan,dict(incomplete),{}, {'requiresTrend':True}]) as call:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年的变化趋势',context={},cancel_event=threading.Event())
self.assertTrue(task['trendRequired']);self.assertEqual(call.call_count,4)
self.assertEqual(call.call_args_list[2].kwargs['payload']['plannedTask']['goal'],plan['goal'])
with patch('nao_agent_task._model_json',side_effect=[plan,dict(incomplete),{},{}]) as call:
with self.assertRaises(TaskUnavailable) as err:
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年的变化趋势',context={},cancel_event=threading.Event())
self.assertEqual(err.exception.field,'requiresTrend.review');self.assertEqual(call.call_count,4)
current={**plan,'goal':'查询9月合格率','period':{'year':2026,'month':9},'trendRequired':False}
with patch('nao_agent_task._model_json',side_effect=[current,dict(incomplete),{'requiresTrend':False}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='9月合格率',context={},cancel_event=threading.Event())
self.assertFalse(task['trendRequired'])
def test_pure_trend_does_not_add_a_comparison_period_but_yoy_keeps_it(self):
from unittest.mock import patch
plan=task_value(goal='今年月度合格率趋势',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],
stages=[],period={'year':2026},trendRequired=True,comparisonRequired=True)
pure={**plan,'comparisonRequired':False}
verdict={**semantic_ok(),'requiresTrend':True,'requiresComparison':False}
with patch('nao_agent_task._model_json',side_effect=[plan,verdict,pure,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年合格率每月走势',context={},cancel_event=threading.Event())
self.assertTrue(task['trendRequired']);self.assertFalse(task['comparisonRequired'])
self.assertIsNone(task['comparisonPeriod'])
yoy={**plan,'goal':'今年月度合格率同比趋势','comparisonPeriod':{'year':2025}}
yoy_review={**semantic_ok(),'requiresTrend':True,'requiresComparison':True}
with patch('nao_agent_task._model_json',side_effect=[yoy,yoy_review]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年合格率趋势和去年同比',context={},cancel_event=threading.Event())
self.assertTrue(task['comparisonRequired']);self.assertEqual(task['comparisonPeriod'],{'year':2025})
def test_ambiguous_catalogue_requires_choice_and_keeps_original_trend_for_followup(self):
from unittest.mock import patch
candidates=[{'name':'18X15X4+T','departments':[],'groups':[]},
{'name':'18X15X4+Z','departments':[],'groups':[]}]
product_match={'status':'ambiguous','query':'18x15x4','candidates':candidates,'candidateCount':2,'truncated':False}
context={'_businessCatalog':{'productMatch':product_match},
'_serverQuestions':[{'question':'你好','status':'COMPLETED'}]}
value=task_value(goal='今年18x15x4合格率变化趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True,
filterUpdates={'product':'18X15X4+T'})
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'productScope':'single'}
with patch('nao_agent_task._model_json',side_effect=[value,review]):
pending=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年18x15x4合格率变化趋势',context=context,cancel_event=threading.Event())
self.assertTrue(pending['needsClarification']);self.assertNotIn('product',pending['filters'])
self.assertIn('18X15X4+T',pending['clarification']);self.assertIn('18X15X4+Z',pending['clarification'])
self.assertEqual(pending['productResolution']['originalQuestion'],'今年18x15x4合格率变化趋势')
selected={'status':'exact','candidates':[candidates[0]],'candidateCount':1,'truncated':False,
'selectedFromClarification':True,'previousQuestion':'今年18x15x4合格率变化趋势'}
context['_businessCatalog']['productMatch']=selected
# Simulate a planner forgetting the prior year and trend and the only
# server history being an unrelated greeting.
short=task_value(goal='确认选项',kind='metrics',dimensions=['product'],dimensionStages={'product':[]},
fields=[],stages=[],filterUpdates={})
with patch('nao_agent_task._model_json',side_effect=[short,review]) as model:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选+T',context=context,previous=pending,cancel_event=threading.Event())
self.assertFalse(task['needsClarification']);self.assertEqual(task['filters']['product'],'18X15X4+T')
self.assertEqual(task['period'],{'year':2026});self.assertEqual(task['fields'],['rate'])
self.assertTrue(task['trendRequired']);self.assertEqual(task['dimensions'],[])
self.assertEqual(task['rootGoal'],'今年18x15x4合格率变化趋势')
self.assertEqual(task['productResolution']['selectedName'],'18X15X4+T')
self.assertEqual(model.call_args_list[1].kwargs['payload']['pendingProductQuestion'],task['rootGoal'])
selected['candidates']=[{'name':'18X15X4+OTHER'}]
with patch('nao_agent_task._model_json',side_effect=[short]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选+T',context=context,previous=pending,cancel_event=threading.Event())
def test_selected_product_is_bound_even_with_static_base_query_plan(self):
from unittest.mock import patch
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
previous=normalize_task(task_value(goal='今年18x15x4合格率变化趋势',kind='metrics',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026},
trendRequired=True,needsClarification=True,clarification='请选择规格'), '今年18x15x4合格率变化趋势', {})
previous['filters']={'exclusions':{'product':['18X15X4.1+Z']}}
previous['productResolution']={'originalQuestion':'今年18x15x4合格率变化趋势',
'productMatch':{'status':'ambiguous','candidates':candidates}}
context={'_protocol':{'version':1,'autonomousEnabled':True},'_businessCatalog':{'productMatch':{
'status':'exact','candidates':[candidates[0]],'selectedFromClarification':True,
'previousQuestion':'今年18x15x4合格率变化趋势'}}}
value=task_value(goal='查询已选规格的综合合格率',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9},queryPlan=[{
'nodeId':'value','purpose':'metrics','fields':['rate'],'stages':[],
'selector':{'kind':'base'},'visualPreference':'none','visualType':None}])
repaired={**value,'trendRequired':True,'period':{'year':2026},'queryPlan':[{
**value['queryPlan'][0],'purpose':'trend','visualType':'line'}]}
with patch('nao_agent_task._model_json',side_effect=[value,repaired,{**semantic_ok(),'requiresTrend':True}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选择18X15X4+T。',context=context,previous=previous,cancel_event=threading.Event())
self.assertEqual(task['filters']['product'],'18X15X4+T')
self.assertEqual(task['period'],{'year':2026})
self.assertTrue(task['trendRequired'])
self.assertEqual(task['queryPlan'][0]['selector'],{'kind':'base'})
self.assertEqual(task['queryPlan'][0]['purpose'],'trend')
self.assertEqual(task['filters']['exclusions']['product'],['18X15X4.1+Z'])
def test_new_analysis_without_supported_execution_route_is_repaired_before_confirmation(self):
from unittest.mock import patch
value=task_value(goal='查询9月综合合格率',kind='analysis',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9})
context={'_protocol':{'version':1,'autonomousEnabled':True}}
with patch('nao_agent_task._model_json',side_effect=[value,{**value,'kind':'metrics'}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='查2026年9月综合合格率',context=context,cancel_event=threading.Event())
self.assertEqual(task['kind'],'metrics');self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_explicit_history_only_report_cannot_restart_prior_investigation(self):
from unittest.mock import patch
from nao_native_planning import history_only_requested
question='请用本会话第一轮已核实证据生成分析报告,不重新查询数据。'
self.assertTrue(history_only_requested(question))
self.assertFalse(history_only_requested('不用报告,请重新查询2026年9月数据'))
self.assertFalse(history_only_requested('不重新查询历史数据,但请重新查询最新数据'))
bad=task_value(goal='重新分析9月下降原因',kind='analysis',executionMode='native_agent',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026,'month':9})
good=task_value(goal='复用第一轮证据整理报告',kind='report',historyMode='first',freshQuery=False,
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False,trendRequired=False)
context={'_protocol':{'version':1,'autonomousEnabled':True}}
with patch('nao_agent_task._model_json',side_effect=[bad,good]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context=context,cancel_event=threading.Event())
self.assertEqual(task['kind'],'report');self.assertFalse(task['freshQuery'])
self.assertEqual(task['historyMode'],'first');self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_first_history_mode_requires_new_autonomous_report(self):
value=task_value(kind='report',historyMode='first',freshQuery=False,
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False)
for context in ({}, {'_protocol':{'version':1,'autonomousEnabled':False}}):
with self.subTest(context=context), self.assertRaises(TaskUnavailable):
normalize_task(value,'整理首轮结果',context)
with self.assertRaises(TaskUnavailable):
normalize_task({**value,'kind':'restate'},'整理首轮结果',
{'_protocol':{'version':1,'autonomousEnabled':True}})
def test_history_no_read_phrasings_repair_before_any_fresh_query(self):
from unittest.mock import patch
from nao_native_planning import history_only_requested
questions=('仅用本会话第一轮已核实结果生成报告,不要开展新查询',
'基于第一轮已核实结果,不做新的查询,生成报告',
'只整理第一轮已核实结果,不需要重新读取数据')
bad=task_value(goal='重新查数',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9})
good=task_value(goal='整理第一轮已核实结果',kind='report',historyMode='first',freshQuery=False,
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False)
for question in questions:
with self.subTest(question=question):
self.assertTrue(history_only_requested(question))
with patch('nao_agent_task._model_json',side_effect=[bad,good]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context={'_protocol':{'version':1,'autonomousEnabled':True}},
cancel_event=threading.Event())
self.assertEqual(task['historyMode'],'first')
self.assertFalse(task['freshQuery'])
self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_first_history_request_cannot_silently_select_latest(self):
from unittest.mock import patch
value=task_value(goal='整理首轮已核实证据',kind='report',historyMode='last',freshQuery=False,
dimensions=[],dimensionStages={},fields=[],stages=[],comparisonRequired=False)
with patch('nao_agent_task._model_json',side_effect=[value,{**value,'historyMode':'first'}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='用本会话第一轮已核实证据整理报告,不重新查询数据。',
context={'_protocol':{'version':1,'autonomousEnabled':True}},cancel_event=threading.Event())
self.assertEqual(task['historyMode'],'first')
self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_selection_completes_pending_empty_metric_intent_without_erasing_scope(self):
from unittest.mock import patch
original='2026年9月18X15X4合格率是多少,不含18X15X4.1'
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
previous=normalize_task(task_value(goal=original,kind='analysis',dimensions=[],dimensionStages={},
fields=[],stages=[],period={'year':2026,'month':9},needsClarification=True,
clarification='请选择规格',filterUpdates={'exclusions':{'product':['18X15X4.1+Z']}}),original,{})
previous['productResolution']={'originalQuestion':original,'productMatch':{
'status':'ambiguous','candidates':candidates}}
context={'_protocol':{'version':1,'autonomousEnabled':True},'_businessCatalog':{'productMatch':{
'status':'exact','candidates':[candidates[0]],'selectedFromClarification':True,
'previousQuestion':original}}}
value=task_value(goal='查询选定规格本期综合率',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9})
with patch('nao_agent_task._model_json',side_effect=[{**value,'kind':'analysis'}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选择18X15X4+T。',context=context,previous=previous,cancel_event=threading.Event())
self.assertEqual(task['fields'],['rate']);self.assertEqual(task['kind'],'analysis')
from nao_protocol_execution import _confirmed_scalar_selection
self.assertTrue(_confirmed_scalar_selection({'_autonomousAnswer':True},task))
self.assertEqual(task['filters'],{'product':'18X15X4+T','exclusions':{'product':['18X15X4.1+Z']}})
self.assertEqual(task['rootGoal'],original)
def test_ambiguous_candidates_remain_clarifiable_when_planner_fails(self):
from unittest.mock import patch
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
match={'status':'ambiguous','query':'18x15x4','candidates':candidates,'candidateCount':2,'truncated':False}
context={'_businessCatalog':{'productMatch':match}}
with patch('nao_agent_task._model_json',side_effect=[{}, {}, {}]):
pending=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年18x15x4合格率变化趋势',context=context,cancel_event=threading.Event())
self.assertTrue(pending['needsClarification']);self.assertTrue(pending['productResolution']['plannerDeferred'])
self.assertIn('18X15X4+T',pending['clarification']);self.assertNotIn('product',pending['filters'])
context['_businessCatalog']['productMatch']={'status':'exact','candidates':[candidates[0]],
'selectedFromClarification':True,'previousQuestion':'今年18x15x4合格率变化趋势'}
planned=task_value(goal='今年18X15X4+T合格率变化趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
with patch('nao_agent_task._model_json',side_effect=[planned,{**semantic_ok(),'requiresTrend':True,'requiresComparison':False}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='选+T',context=context,previous=pending,cancel_event=threading.Event())
self.assertEqual(task['filters']['product'],'18X15X4+T');self.assertTrue(task['trendRequired'])
self.assertEqual(task['period'],{'year':2026});self.assertEqual(task['fields'],['rate'])
self.assertEqual(task['rootGoal'],'今年18x15x4合格率变化趋势')
def test_customer_code_trend_stays_authorized_and_bound_in_queries_and_followup(self):
from unittest.mock import patch
from nao_query_contract import bind_query
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
context={'_businessCatalog':catalog,'filters':{'department':'消费电子','extraFilters':{'blankProducer':['Y']}}}
value=task_value(goal='查看客户代码 C-2026 的今年月度合格率走势',kind='metrics',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026},
trendRequired=True,filterUpdates={'extraFilters':{'customerCode':'C-2026'}})
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'customerCodes':['C-2026']}
with patch('nao_agent_task._model_json',side_effect=[value,review]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码 C-2026 今年的合格率趋势',context=context,cancel_event=threading.Event())
self.assertEqual(task['filters'],{'department':'消费电子','extraFilters':{'blankProducer':['Y'],'customerCode':['C-2026']}})
self.assertEqual(task['period'],{'year':2026});self.assertTrue(task['trendRequired'])
query=bind_query({'mode':'query'}, {'task':task,'releaseId':'fixed'})
self.assertEqual(query['filters']['extraFilters']['customerCode'],['C-2026'])
with self.assertRaises(ValueError):
bind_query({'mode':'query','filters':{'extraFilters':{'customerCode':['OTHER']}}},
{'task':task,'releaseId':'fixed'})
followup=task_value(goal='继续看客户 C-2026 的今年月度合格率趋势',kind='metrics',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],trendRequired=True,
filterUpdates={})
next_context=bind_task(context,task)
with patch('nao_agent_task._model_json',side_effect=[followup,review]):
next_task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='换成折线图',context=next_context,previous=task,cancel_event=threading.Event())
self.assertEqual(next_task['filters']['extraFilters']['customerCode'],['C-2026'])
self.assertEqual(next_task['period'],{'year':2026});self.assertTrue(next_task['trendRequired'])
denied={'_businessCatalog':{'extraFields':[]}}
with self.assertRaises(TaskUnavailable):normalize_task(value,'q',denied)
def test_customer_code_cannot_be_lost_when_planner_and_review_disagree(self):
from unittest.mock import patch
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
value=task_value(goal='查看客户代码 C-2026 今年月度合格率趋势',kind='metrics',
dimensions=[],dimensionStages={},fields=['rate'],stages=[],period={'year':2026},
trendRequired=True,filterUpdates={})
verdict={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'customerCodes':['C-2026']}
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C-2026今年合格率趋势',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertEqual(task['filters']['extraFilters']['customerCode'],['C-2026'])
with patch('nao_agent_task._model_json',side_effect=[value,verdict]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C-2026今年合格率趋势',context={'_businessCatalog':{'extraFields':[]}},cancel_event=threading.Event())
invented={**verdict,'customerCodes':['80099999']}
with patch('nao_agent_task._model_json',side_effect=[value,invented]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C-2026今年合格率趋势',context={'_businessCatalog':catalog},cancel_event=threading.Event())
def test_customer_filter_is_not_invented_or_retained_after_cancellation(self):
from unittest.mock import patch
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
context={'_businessCatalog':catalog,'filters':{'extraFilters':{'customerCode':['C-2026']}}}
keep=task_value(goal='今年整体合格率',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[],
period={'year':2026},filterUpdates={})
clear={**keep,'filterUpdates':{'extraFilters':{'customerCode':None}}}
verdict={**semantic_ok(),'customerCodes':[]}
with patch('nao_agent_task._model_json',side_effect=[keep,verdict,clear,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='不按客户筛选,看今年整体合格率',context=context,cancel_event=threading.Event())
self.assertNotIn('customerCode',task['filters'].get('extraFilters',{}))
invented={**keep,'filterUpdates':{'extraFilters':{'customerCode':['INVENTED']}}}
with patch('nao_agent_task._model_json',side_effect=[invented,verdict]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年整体合格率',context={'_businessCatalog':catalog},cancel_event=threading.Event())
def test_customer_code_switch_preserves_exact_new_case(self):
from unittest.mock import patch
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
context={'_businessCatalog':catalog,'filters':{'extraFilters':{'customerCode':['C1']}}}
value=task_value(goal='客户代码 c1 今年合格率',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[],
period={'year':2026},filterUpdates={'extraFilters':{'customerCode':['c1']}})
verdict={**semantic_ok(),'customerCodes':['c1']}
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码 C1 换成 c1,看今年合格率',context=context,cancel_event=threading.Event())
self.assertEqual(task['filters']['extraFilters']['customerCode'],['c1'])
def test_missing_customer_subject_is_rechecked_without_replanning(self):
from unittest.mock import patch
value=task_value(goal='客户代码C1合格率',kind='metrics',dimensions=[],dimensionStages={},fields=['rate'],stages=[],filterUpdates={})
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}]}
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok(),{}, {'customerCodes':['C1']}]) as call:
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C1合格率',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertEqual(task['filters']['extraFilters']['customerCode'],['C1']);self.assertEqual(call.call_count,4)
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok(),{},{}]) as call:
with self.assertRaises(TaskUnavailable) as err:
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='客户代码C1合格率',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertEqual(err.exception.field,'customerCodes.review');self.assertEqual(call.call_count,4)
def test_product_shaped_customer_code_is_not_automatically_a_product(self):
from unittest.mock import patch
from nao_product_identification import _normalized
question='客户代码“18x15x4”今年合格率趋势'
token='18x15x4';at=_normalized(question).index(token)
candidates=[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]
mention={'status':'ambiguous','query':token,'normalizedSpan':[at,at+len(token)],
'candidates':candidates,'candidateCount':2,'truncated':False}
catalog={'extraFields':[{'key':'customerCode','label':'客户代码'}],
'productMatch':{**mention,'mentions':[mention]}}
value=task_value(goal='客户代码18x15x4今年按月合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True,
filterUpdates={'extraFilters':{'customerCode':[token]}})
verdict={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'customerCodes':[token],'productScope':'single'}
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertFalse(task['needsClarification']);self.assertNotIn('product',task['filters'])
self.assertEqual(task['filters']['extraFilters']['customerCode'],[token])
double='客户代码18x15x4和产品18x15x4今年合格率趋势'
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=double,context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertTrue(task['needsClarification']);self.assertIn('18X15X4+T',task['clarification'])
def test_single_product_analysis_cannot_lose_product_filter(self):
from unittest.mock import patch
match={'status':'exact','query':'18x15x4','candidates':[{'name':'18X15X4+T'}],
'candidateCount':1,'truncated':False}
catalog={'productMatch':match}
value=task_value(goal='为什么18X15X4+T合格率下降',kind='analysis',dimensions=['product'],
dimensionStages={'product':[]},fields=['rate'],stages=[],filterUpdates={},comparisonRequired=True)
verdict={**semantic_ok(),'productScope':'single'}
with patch('nao_agent_task._model_json',side_effect=[value,verdict]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='为什么18X15X4+T合格率下降',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertEqual(task['filters']['product'],'18X15X4+T');self.assertNotIn('product',task['dimensions'])
comparison={**verdict,'productScope':'compare'}
with patch('nao_agent_task._model_json',side_effect=[value,comparison]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='把18X15X4+T与其他产品比较',context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertNotIn('product',task['filters']);self.assertIn('product',task['dimensions'])
def test_product_mentioned_as_background_is_not_automatically_applied(self):
from unittest.mock import patch
name='18X15X4+T';match={'status':'exact','query':name,'candidates':[{'name':name}],'candidateCount':1,'truncated':False}
catalog={'productMatch':match,'mentionedProducts':[name]}
value=task_value(goal='先不看18X15X4+T,查看今年总体合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True,filterUpdates={})
review={**semantic_ok(),'requiresTrend':True,'requiresComparison':False,'productScope':'other'}
with patch('nao_agent_task._model_json',side_effect=[value,review]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=value['goal'],context={'_businessCatalog':catalog},cancel_event=threading.Event())
self.assertNotIn('product',task['filters'])
def test_greeting_does_not_become_the_business_root_question(self):
from unittest.mock import patch
value=task_value(goal='今年客户代码C1合格率趋势',kind='metrics',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026},trendRequired=True)
context={'_serverQuestions':[{'question':'你好','status':'COMPLETED'}]}
with patch('nao_agent_task._model_json',side_effect=[value,{**semantic_ok(),'requiresTrend':True,'requiresComparison':False}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='今年客户代码C1合格率趋势',context=context,cancel_event=threading.Event())
self.assertNotEqual(task['rootGoal'],'你好')
def test_retry_does_not_accept_an_unverified_semantic_review(self):
from unittest.mock import patch
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
proposal=task_value(dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[proposal,{'aligned':'yes'},{'aligned':'yes'}]) as call:
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='主要是哪一组',context=context,cancel_event=threading.Event(),semantic_review=True)
self.assertEqual(call.call_count,3)
def test_retry_comparison_replans_a_history_only_answer_as_a_fresh_analysis(self):
from unittest.mock import patch
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
prior=task_value(kind='explain',freshQuery=False,historyMode='last',dimensions=[],fields=[],comparisonRequired=True)
analysis=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[prior,analysis,semantic_ok()]) as call:
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='哪个分组',
context=context,cancel_event=threading.Event(),semantic_review=True,fresh_retry=True)
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery']);self.assertEqual(call.call_count,3)
def test_unverified_business_claim_cannot_use_direct_conversation(self):
from unittest.mock import patch
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
bad={'kind':'explain','goal':'判断合格率下降主要来自哪个分组',
'directResponse':{'purpose':'acknowledgement','requiresMemory':False,
'answer':'主要是国内分组所致;国内合格率降幅明显更大,海外基本持平。'}}
good=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[bad,good,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='主要是国内还是国外',
context=context,cancel_event=threading.Event(),semantic_review=True,fresh_retry=True)
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery'])
def test_direct_response_requiring_evidence_is_replanned(self):
from unittest.mock import patch
context={'_serverQuestions':[{'turnId':'previous','question':'9月合格率为何下降','status':'PARTIAL'}]}
bad={'kind':'explain','goal':'判断哪个分组需要关注',
'directResponse':{'purpose':'acknowledgement','requiresMemory':False,
'answer':'国内是重点关注对象,海外情况稳定。'}}
good=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[bad,{'aligned':True,'requiresEvidence':True},
good,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='主要是国内还是国外',
context=context,cancel_event=threading.Event(),semantic_review=True,fresh_retry=True)
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery'])
def test_first_question_direct_response_is_reviewed_for_evidence(self):
from unittest.mock import patch
bad={'kind':'explain','goal':'判断哪个分组需要关注',
'directResponse':{'purpose':'acknowledgement','requiresMemory':False,
'answer':'国内是重点关注对象,海外情况稳定。'}}
good=task_value(kind='analysis',dimensions=['group'],dimensionStages={'group':[]},comparisonRequired=True)
with patch('nao_agent_task._model_json',side_effect=[bad,{'aligned':True,'requiresEvidence':True},good,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='9月合格率下降主要是国内还是国外',context={},cancel_event=threading.Event())
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery'])
def test_direct_response_review_without_evidence_decision_fails_closed(self):
from unittest.mock import patch
direct={'kind':'explain','goal':'回应问候',
'directResponse':{'purpose':'greeting','requiresMemory':False,'answer':'你好。'}}
with patch('nao_agent_task._model_json',side_effect=[direct,{'aligned':True}]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='你好',context={},cancel_event=threading.Event())
def test_group_targets_are_bound_from_semantic_plan_and_scoped_catalog(self):
from unittest.mock import patch
groups=['汽车交通-国内','汽车交通-海外','其他部门-国内']
context={'_businessCatalog':{'groups':groups,'groupDepartments':{'汽车交通-国内':'新能源汽车磁钢',
'汽车交通-海外':'新能源汽车磁钢','其他部门-国内':'其他部门'}}}
value=task_value(goal='比较国内与海外的合格率变化',dimensions=['group'],dimensionStages={'group':[]},
filterUpdates={'department':'新能源汽车磁钢'})
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='主要是国内还是国外的',context=context,cancel_event=threading.Event())
self.assertEqual(result['targetEntities']['group'],groups[:2])
value.update(goal='比较这两个地区',targetEntities={'group':groups[:2]})
with patch('nao_agent_task._model_json',side_effect=[value,semantic_ok()]):
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='请比较这两个地区',context=context,cancel_event=threading.Event())
self.assertEqual(result['targetEntities']['group'],groups[:2])
def test_planner_repairs_with_invalid_plan_and_field_contract_then_continues(self):
replies=[task_value(dimensions='unknown'),task_value(fields=['unsupported']),task_value(dimensions=['product'],dimensionStages={'product': []}),semantic_ok()]
payloads=[]
def opener(request,timeout):
payloads.append(json.loads(json.loads(request.data)['messages'][1]['content']))
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(replies.pop(0))}}]}).encode())
task=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',question='绘制产品图',
context={},cancel_event=threading.Event(),opener=opener)
self.assertEqual(task['dimensions'],['product'])
self.assertEqual(len(payloads),4)
self.assertEqual(payloads[1]['invalidPlan']['dimensions'],'unknown')
self.assertEqual(payloads[1]['requiredShapes']['dimensions']['type'],'array')
self.assertIn('validationFeedback',payloads[2])
self.assertFalse(replies)
def test_repair_plan_cannot_widen_pinned_scope_or_period(self):
task = normalize_task(task_value(filterUpdates={'group': '消费电子-美洲'},
period={'year': 2026, 'month': 9}), 'q')
def run(query):
def opener(request, timeout):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({'queries': [query]})}}]}).encode())
return plan_repairs(base_url='https://example.test/v1', api_key='test-only', model='m', task=task,
evidence=[], gaps=['missing'], failures=[], release_id='fixed',
cancel_event=threading.Event(), opener=opener)
self.assertEqual(run({'mode': 'compare', 'filters': {}})[0]['filters']['group'],'消费电子-美洲')
with self.assertRaises(TaskUnavailable): run({'mode':'compare','filters':{'group':'其他分组'}})
with self.assertRaises(TaskUnavailable): run({'mode': 'execute_sql', 'sql': 'SELECT 1'})
with self.assertRaises(TaskUnavailable): run({'mode': 'query', 'filters': task['filters'], 'period': {'year': 2026, 'month': 8}})
self.assertEqual(run({'mode': 'query', 'filters': task['filters'], 'period': task['period'], 'releaseId': 'other'})[0]['releaseId'], 'fixed')
self.assertEqual(run({'mode': 'query', 'filters': task['filters']})[0]['period'], task['period'])
def test_single_period_metrics_bind_scope_stage_fields_and_source_months(self):
import copy
question = '九月黑片良率告诉我一下'
value = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 9}, filterUpdates={'product': 'SYNTH-16X15+T'})
context = bind_task({'releaseId': 'release-a'}, normalize_task(value, question))
side = metric_evidence()[0]['result']['current']
docs = [{'mode': 'query', 'result': {'kind': 'quality_metrics', 'releaseId': 'release-a',
'period': side['period'], 'filters': context['filters'], 'summary': side, 'coverage': side['coverage']}}]
self.assertEqual(semantic_metric_gaps(docs, question, context), [])
for mutation in ('month', 'product', 'rate', 'coverage'):
bad = copy.deepcopy(docs)
result = bad[0]['result']
if mutation == 'month': result['period']['month'] = 8
if mutation == 'product': result['filters']['product'] = 'OTHER'
if mutation == 'rate': result['summary']['black']['rate'] = None
if mutation == 'coverage': result['coverage']['sourceMonths'][0]['blackRows'] = 0
with self.subTest(mutation=mutation):
self.assertTrue(semantic_metric_gaps(bad, question, context))
def test_client_task_cannot_bypass_server_planner(self):
from nao_quality_runtime import _normalize_request
import uuid
payload = {key: str(uuid.uuid4()) for key in ('turnId', 'sessionId', 'actorId', 'releaseId')}
payload.update(question='q', accessToken='test-only', context={'_agentTask': {'freshQuery': False},
'agentContext': {'filters': {}}, '_resolvedMetricRequest': {}})
normalized = _normalize_request(payload)
self.assertEqual(normalized['context'], {})
def test_overall_task_keeps_primary_frame_when_native_drills_into_a_group(self):
from nao_evidence_report import select_comparison_frames
question = '9月份合格率下降原因'
context = self.context(question, dimensions=[], period={'year': 2026, 'month': 9}, comparisonRequired=True)
overall = evidence('quality_comparison')
local = evidence('quality_comparison', filters={'group': '汽车交通-国内'}, ref='Q00000002')
roots, limitations = select_comparison_frames([overall, local], context)
self.assertEqual(roots, [overall])
self.assertEqual(limitations, [])
class WeightRankingDefaultCoverageTests(unittest.TestCase):
def test_weight_ranking_without_explicit_chart_type_defaults_to_published_months(self):
value=task_value(kind='analysis', dimensions=['product'], stages=['bright'], fields=['totalWeight'],
trendRequired=False, comparisonRequired=False, visualType=None)
self.assertEqual(normalize_task(value,'今年亮片投入重量按产品排名')['periodCoverage'],'available')
value['periodCoverage']='complete'
self.assertEqual(normalize_task(value,'要求完整全年齐全')['periodCoverage'],'complete')
if __name__ == '__main__': unittest.main()
@@ -0,0 +1,86 @@
"""Regression cases for complete, release-bound broad analysis."""
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_analysis_coverage import coverage_gaps, plan_missing_queries
PERIOD = {'year': 2026, 'month': 9}
PRIOR = {'year': 2026, 'month': 8}
def evidence(kind, *, dimension=None, stage=None, filters=None, sort=None, ref='Q00000001', available=True):
result = {'kind': kind, 'releaseId': 'release-a'}
if kind == 'quality_comparison':
result.update(filterBinding=filters or {}, current={'period': PERIOD, 'compositeRate': .95},
comparison={'period': PRIOR, 'compositeRate': .96}, delta={'available': True, 'percentagePoints': -1})
else:
result.update(filters=filters or {}, currentPeriod=PERIOD, comparisonPeriod=PRIOR,
dimension=dimension, stage=stage, sort=sort)
if kind == 'quality_fine_defects':
result['period'] = PERIOD
result['coverage'] = {'available': available}
result['comparisonCoverage'] = {'available': available}
result['category'] = None
result['defect'] = None
return {'evidenceRef': ref, 'result': result}
class AnalysisCoverageTest(unittest.TestCase):
def test_broad_question_plans_only_missing_dimensions_in_same_frame(self):
docs = [evidence('quality_comparison', filters={'group': '汽车交通-国内'}),
evidence('quality_changes', dimension='department', filters={'group': '汽车交通-国内'}, sort='scrap_delta')]
planned = plan_missing_queries(docs, '分析整体质量变化', {'year': 2026, 'month': 9, 'group': '汽车交通-国内'})
self.assertEqual([(item['mode'], item.get('dimension'), item.get('stage')) for item in planned],
[('changes', 'group', None), ('changes', 'product', None),
('defects', 'defect', 'black'), ('defects', 'defect', 'bright')])
self.assertTrue(all(item['filters'] == {'group': '汽车交通-国内'} and item['releaseId'] == 'release-a'
and item['period'] == PERIOD and item['comparisonPeriod'] == PRIOR for item in planned))
self.assertEqual(len(coverage_gaps(docs, '分析整体质量变化', {'year': 2026, 'month': 9, 'group': '汽车交通-国内'})), 4)
def test_complete_broad_question_and_narrow_number(self):
docs = [evidence('quality_comparison')]
docs += [evidence('quality_changes', dimension=dimension, sort='scrap_delta') for dimension in ('department', 'group', 'product')]
docs += [evidence('quality_fine_defects', dimension='defect', stage=stage) for stage in ('black', 'bright')]
self.assertEqual(coverage_gaps(docs, '整体质量变化如何'), [])
self.assertEqual(plan_missing_queries(docs, '综合合格率是多少'), [])
self.assertEqual(coverage_gaps([docs[0]], '综合合格率是多少'), [])
def test_rate_only_and_unavailable_defects_cannot_complete_broad_analysis(self):
docs = [evidence('quality_comparison')]
docs += [evidence('quality_changes', dimension=dimension, sort='scrap_delta') for dimension in ('department', 'group')]
docs.append(evidence('quality_changes', dimension='product', sort='rate_drop'))
docs.append(evidence('quality_fine_defects', dimension='defect', stage='black', available=False))
docs.append(evidence('quality_fine_defects', dimension='defect', stage='bright'))
gaps = coverage_gaps(docs, '整体质量变化如何')
self.assertTrue(any('产品' in gap for gap in gaps))
self.assertTrue(any('黑片' in gap for gap in gaps))
self.assertEqual([(item['mode'], item.get('dimension'), item.get('stage')) for item in plan_missing_queries(docs, '整体质量变化如何')],
[('changes', 'product', None)])
def test_product_stage_pair_is_required_only_for_actual_product_drilldown(self):
docs = [evidence('quality_comparison')]
docs.append(evidence('quality_fine_defects', dimension='defect', stage='black', filters={'product': '规格A'}, sort='current_scrap'))
self.assertTrue(any('亮片' in item for item in coverage_gaps(docs, '分析重点产品')))
self.assertEqual(coverage_gaps(docs, '只看黑片规格A的缺陷'), [])
def test_actual_focus_product_question_requires_product_ranking_evidence(self):
docs = [evidence('quality_comparison')]
question = '分析下近期要关注的重点产品'
self.assertTrue(any('产品多维重点排序' in gap for gap in coverage_gaps(docs, question)))
self.assertEqual([(item['mode'], item['dimension']) for item in plan_missing_queries(docs, question)],
[('changes', 'product')])
docs.append(evidence('quality_changes', dimension='product', sort='scrap_delta'))
self.assertEqual(coverage_gaps(docs, question), [])
def test_stage_scoped_broad_question_uses_same_stage_for_changes(self):
docs = [evidence('quality_comparison')]
context = {'stage': 'black'}
plans = plan_missing_queries(docs, '分析整体质量变化', context)
self.assertEqual([(item['mode'], item.get('stage')) for item in plans],
[('changes', 'black')] * 3 + [('defects', 'black')])
if __name__ == '__main__': unittest.main()
+79
View File
@@ -0,0 +1,79 @@
"""Stage rate support must not hide official zero-weight comparison values."""
import copy
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_changes import validate_changes, changes_report
def fixture():
period, prior = {'year': 2026, 'month': 9}, {'year': 2026, 'month': 8}
task = {'period': period, 'comparisonPeriod': prior, 'filters': {'department': '测试部'}, 'stages': [],
'fields': ['rate', 'totalWeight', 'scrapWeight'], 'question': '本期合格率最低的产品',
'changeQuery': {'dimension': 'product', 'sort': 'current_rate', 'limit': 1}}
absent = {'rate': None, 'qualifiedWeight': None, 'totalWeight': None, 'scrapWeight': None}
result = {'kind': 'quality_changes', 'releaseId': 'fixture-release', 'currentPeriod': period,
'comparisonPeriod': prior, 'filters': task['filters'], 'dimension': 'product', 'sort': 'current_rate',
'stage': None, 'completeRanking': True, 'totalRows': 898, 'rows': [{
'rank': 1, 'key': '规格甲', 'label': '规格甲', 'changeType': 'continuing',
'present': {'current': True, 'comparison': True}, 'rateDrop': None, 'scrapDelta': None,
'current': {'rate': 0, 'rateSupport': {'blackMeasured': True, 'brightMeasured': False,
'compositeReferenceAvailable': True, 'missingStages': ['bright']},
'black': {'rate': 0, 'qualifiedWeight': None, 'totalWeight': 10.438, 'scrapWeight': 10.438},
'bright': copy.deepcopy(absent), 'blackInputWeight': 10.438, 'brightInputWeight': None, 'scrapWeight': None},
'comparison': {'rate': None, 'rateSupport': {'blackMeasured': False, 'brightMeasured': False,
'compositeReferenceAvailable': False, 'missingStages': ['black', 'bright']},
'black': {'rate': None, 'qualifiedWeight': None, 'totalWeight': 0, 'scrapWeight': 0},
'bright': copy.deepcopy(absent), 'blackInputWeight': 0, 'brightInputWeight': None, 'scrapWeight': None}}]}
return result, task
class ChangesWeightReferenceTests(unittest.TestCase):
def test_current_rate_accepts_official_zero_baseline_with_unavailable_rate(self):
result, task = fixture()
self.assertIs(validate_changes(result, task, 'fixture-release'), result)
report = changes_report([{'result': result, 'evidenceRef': 'Q00000001'}], task, 'fixture-release')
self.assertTrue(report['coverageSufficient'])
self.assertIn('两期合格率不可比', report['facts'][0]['text'])
self.assertIn('0.00%', report['facts'][0]['text'])
self.assertNotIn('下降100', report['facts'][0]['text'])
self.assertIn('本期亮片阶段率未实测', report['facts'][0]['text'])
self.assertIn('基期黑片、亮片阶段率未实测', report['facts'][0]['text'])
self.assertIsNone(result['rows'][0]['comparison']['rate'])
self.assertIsNone(result['rows'][0]['current']['bright']['totalWeight'])
self.assertIn('黑片报废重量 0.00→10.44 kg', report['facts'][1]['text'])
def test_new_product_absent_baseline_does_not_require_or_invent_weights(self):
result, task = fixture()
result['rows'][0]['present']['comparison'] = False
result['rows'][0]['comparison']['black'] = None
result['rows'][0]['changeType'] = 'new'
self.assertIs(validate_changes(result, task, 'fixture-release'), result)
def test_rate_missing_stage_cannot_hide_missing_requested_weight(self):
result, task = fixture()
result['rows'][0]['comparison']['black']['totalWeight'] = None
with self.assertRaisesRegex(ValueError, r'black.totalWeight'):
validate_changes(result, task, 'fixture-release')
def test_null_stages_and_hidden_measured_fields_remain_incomplete(self):
for measured in (False, True):
result, task = fixture()
side = result['rows'][0]['comparison']
side['black']['totalWeight'] = side['black']['scrapWeight'] = None
side['rateSupport']['blackMeasured'] = measured
with self.subTest(measured=measured), self.assertRaises(ValueError):
validate_changes(result, task, 'fixture-release')
def test_invalid_weight_is_rejected_even_when_stage_rate_unavailable(self):
for value in (-1, float('nan'), True):
result, task = fixture()
result['rows'][0]['comparison']['black']['scrapWeight'] = value
with self.subTest(value=value), self.assertRaises(ValueError):
validate_changes(result, task, 'fixture-release')
if __name__ == '__main__':
unittest.main()
+430
View File
@@ -0,0 +1,430 @@
"""Offline isolation checks for concurrent NAO actor sessions; never calls a model."""
from __future__ import annotations
import contextlib
import json
import os
import sqlite3
import sys
import tempfile
import threading
import unittest
import uuid
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
import nao_quality_runtime as runtime
import nao_quality_runtime_mcp as runtime_mcp
class WaitingResponse:
def __init__(self):
self.closed = threading.Event()
def __enter__(self):
return self
def __exit__(self, *_):
self.close()
def __iter__(self):
self.closed.wait(0.5)
return iter(())
def close(self):
self.closed.set()
class WaitingClient:
def open(self, *_args, **_kwargs):
return WaitingResponse()
class SessionIsolationTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix="test-nao-users-")
self.users = Path(self.temp.name).resolve()
self.manager = runtime.RuntimeManager(users_root=self.users)
self.actor_id = str(uuid.uuid4())
self.actor_b = str(uuid.uuid4())
self.session_a = str(uuid.uuid4())
self.session_b = str(uuid.uuid4())
self.release_a = str(uuid.uuid4())
self.release_b = str(uuid.uuid4())
def tearDown(self):
self.temp.cleanup()
def test_same_actor_sessions_have_independent_roots_locks_credentials_and_mcp_paths(self):
root_a, lock_a = self.manager.acquire_actor(self.actor_id, self.session_a)
root_b, lock_b = self.manager.acquire_actor(self.actor_id, self.session_b)
try:
self.assertEqual(root_a, self.users / self.actor_id / "sessions" / self.session_a)
self.assertEqual(root_b, self.users / self.actor_id / "sessions" / self.session_b)
self.assertNotEqual(root_a, root_b)
self.assertNotEqual(root_a / "active.lock", root_b / "active.lock")
access_a = runtime.write_access_file(root_a, "token-session-A", self.release_a,
expected_actor_id=self.actor_id,
expected_session_id=self.session_a)
access_b = runtime.write_access_file(root_b, "token-session-B", self.release_b,
expected_actor_id=self.actor_id,
expected_session_id=self.session_b)
snapshot_a = runtime.load_access_snapshot(
access_a, runtime_root=self.users, expected_actor_id=self.actor_id,
expected_session_id=self.session_a,
)
snapshot_b = runtime.load_access_snapshot(
access_b, runtime_root=self.users, expected_actor_id=self.actor_id,
expected_session_id=self.session_b,
)
self.assertEqual((snapshot_a.actor_id, snapshot_a.token, snapshot_a.release_id),
(self.actor_id, "token-session-A", self.release_a))
self.assertEqual((snapshot_b.actor_id, snapshot_b.token, snapshot_b.release_id),
(self.actor_id, "token-session-B", self.release_b))
self.assertNotEqual(access_a, access_b)
runtime._write_private_state(root_a)
runtime._write_private_state(root_b)
runtime._write_project(root_a, self.session_a, "test-model", refresh_templates=True,
refresh_mcp=True, backup_schema=False,
expected_actor_id=self.actor_id)
runtime._write_project(root_b, self.session_b, "test-model", refresh_templates=True,
refresh_mcp=True, backup_schema=False,
expected_actor_id=self.actor_id)
mcp_a = json.loads((root_a / "project/agent/mcps/mcp.json").read_text())
mcp_b = json.loads((root_b / "project/agent/mcps/mcp.json").read_text())
env_a = mcp_a["mcpServers"]["quality"]["env"]
env_b = mcp_b["mcpServers"]["quality"]["env"]
self.assertEqual(env_a["QUALITY_AGENT_ACTOR_ID"], self.actor_id)
self.assertEqual(env_b["QUALITY_AGENT_ACTOR_ID"], self.actor_id)
self.assertEqual(env_a["QUALITY_AGENT_SESSION_ID"], self.session_a)
self.assertEqual(env_b["QUALITY_AGENT_SESSION_ID"], self.session_b)
self.assertEqual(env_a["QUALITY_AGENT_ACCESS_FILE"], str(access_a))
self.assertEqual(env_b["QUALITY_AGENT_ACCESS_FILE"], str(access_b))
self.assertNotEqual(env_a["QUALITY_AGENT_SESSION_STATE_FILE"],
env_b["QUALITY_AGENT_SESSION_STATE_FILE"])
self.assertNotEqual(runtime._write_private_state(root_a), runtime._write_private_state(root_b))
finally:
self.manager.release_actor(lock_a)
self.manager.release_actor(lock_b)
# flock files are stable coordination objects and are never unlinked.
self.assertTrue((root_a / "active.lock").is_file())
self.assertTrue((root_b / "active.lock").is_file())
def test_same_session_is_mutually_exclusive_but_other_session_and_actor_can_run(self):
_root_a, lock_a = self.manager.acquire_actor(self.actor_id, self.session_a)
root_b, lock_b = self.manager.acquire_actor(self.actor_id, self.session_b)
root_c, lock_c = self.manager.acquire_actor(self.actor_b, self.session_a)
try:
self.assertNotEqual(root_b, root_c)
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "另一轮分析"):
self.manager.acquire_actor(self.actor_id, self.session_a)
finally:
self.manager.release_actor(lock_c)
self.manager.release_actor(lock_b)
self.manager.release_actor(lock_a)
_again, lock_again = self.manager.acquire_actor(self.actor_id, self.session_a)
self.manager.release_actor(lock_again)
def test_access_snapshot_rejects_actor_session_mismatch_and_symlink(self):
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
try:
access = runtime.write_access_file(root_a, "strict-token", self.release_a,
expected_actor_id=self.actor_id,
expected_session_id=self.session_a)
with self.assertRaisesRegex(ValueError, "当前会话"):
runtime.load_access_snapshot(access, runtime_root=self.users,
expected_actor_id=self.actor_b,
expected_session_id=self.session_a)
with self.assertRaisesRegex(ValueError, "路径与当前会话"):
runtime.load_access_snapshot(access, runtime_root=self.users,
expected_actor_id=self.actor_id,
expected_session_id=self.session_b)
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "actor/session"):
runtime.write_access_file(root_a, "wrong-owner", self.release_a,
expected_actor_id=self.actor_b,
expected_session_id=self.session_a)
link = root_a / "linked-access.json"
link.symlink_to(access)
with self.assertRaisesRegex(ValueError, "路径无效"):
runtime.load_access_snapshot(link, runtime_root=self.users,
expected_actor_id=self.actor_id,
expected_session_id=self.session_a)
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "actor/session"):
runtime._write_project(root_a, self.session_a, "test-model", refresh_templates=False,
refresh_mcp=False, backup_schema=False,
expected_actor_id=self.actor_b)
finally:
self.manager.release_actor(fd_a)
def test_mcp_shim_accepts_only_fixed_actor_session_paths(self):
fake_root = self.users / "fake-project"
users_root = fake_root / ".runtime" / "nao-users"
session_root = users_root / self.actor_id / "sessions" / self.session_a
session_root.mkdir(parents=True)
environment = {
"QUALITY_AGENT_USERS_ROOT": str(users_root),
"QUALITY_AGENT_ACCESS_FILE": str(session_root / "access.json"),
"QUALITY_AGENT_EVIDENCE_DIR": str(session_root / "evidence"),
"QUALITY_AGENT_SESSION_STATE_FILE": str(session_root / "private" / "current-session.json"),
"QUALITY_AGENT_ACTOR_ID": self.actor_id,
"QUALITY_AGENT_SESSION_ID": self.session_a,
}
with patch.object(runtime_mcp, "ROOT", fake_root), patch.dict(os.environ, environment, clear=False):
access, evidence, state, actor, session, root = runtime_mcp._paths()
self.assertEqual(access, session_root / "access.json")
self.assertEqual(evidence, session_root / "evidence")
self.assertEqual(state, session_root / "private" / "current-session.json")
self.assertEqual((actor, session), (self.actor_id, self.session_a))
self.assertEqual(root, users_root.resolve())
os.environ["QUALITY_AGENT_ACCESS_FILE"] = str(users_root / self.actor_id / "access.json")
with self.assertRaisesRegex(ValueError, "路径无效"):
runtime_mcp._paths()
def test_cancel_stops_only_the_current_session_chat(self):
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
try:
state_a = runtime.SessionState(root_a, self.actor_id, self.session_a)
state_b = runtime.SessionState(root_b, self.actor_id, self.session_b)
state_a.set_chat_id(self.session_a, "native-chat-A")
state_b.set_chat_id(self.session_b, "native-chat-B")
cancel = threading.Event()
cancel.set()
stopped = []
def stop_native(_client, _base, chat_id):
stopped.append(chat_id)
return True
with patch.object(runtime, "_stop_native",
side_effect=lambda _client, _base, chat_id: stopped.append(chat_id) or True):
result = runtime._sse_request(
client=WaitingClient(), base_url="http://127.0.0.1:10000", model="offline-test",
question="q", release_id=self.release_a, context=None,
chat_id=state_a.get_chat_id(self.session_a), session_state=state_a,
session_id=self.session_a, actor_root=root_a, emit=lambda _event: None,
cancel_event=cancel, timeout=5,
)
self.assertEqual(result[1], "CANCELLED")
self.assertEqual(stopped, ["native-chat-A"])
self.assertEqual(state_b.get_chat_id(self.session_b), "native-chat-B")
self.assertFalse((root_b / "instance.json").exists())
finally:
self.manager.release_actor(fd_b)
self.manager.release_actor(fd_a)
def test_stop_instance_only_targets_processes_recorded_in_that_session(self):
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
try:
for root, session_id, pids, port in (
(root_a, self.session_a, [10101, 10102], 41001),
(root_b, self.session_b, [20201, 20202], 42001),
):
runtime._atomic_json(root / "instance.json", {
"actorId": self.actor_id, "sessionId": session_id,
"processes": [{"pid": pid, "pgid": pid, "lstart": "simulated",
"kind": "server", "port": port + index}
for index, pid in enumerate(pids)],
})
killed = []
def matches(record, runtime_root):
return runtime_root == root_a and record.get("pid") in (10101, 10102)
with patch.object(runtime, "_process_matches", side_effect=matches), \
patch.object(runtime, "_ps_value", return_value="simulated process"), \
patch.object(runtime.os, "killpg", side_effect=lambda pgid, _sig: killed.append(pgid)), \
patch.object(runtime.time, "sleep"):
runtime._stop_instance(root_a, grace_seconds=0)
self.assertEqual(set(killed), {10101, 10102})
self.assertTrue((root_b / "instance.json").is_file())
finally:
self.manager.release_actor(fd_b)
self.manager.release_actor(fd_a)
def test_stop_all_skips_active_session_and_scans_legacy_and_idle_sessions(self):
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
legacy = self.users / self.actor_b
legacy.mkdir(parents=True)
runtime._atomic_json(legacy / "instance.json", {"actorId": self.actor_b, "processes": []})
for root in (root_a, root_b, legacy):
runtime._atomic_json(root / "access.json", {"token": "private"})
self.manager.release_actor(fd_b)
try:
stopped = []
manager = self.manager
with patch.object(runtime, "_stop_instance", side_effect=lambda root: stopped.append(root) or True):
count = manager.stop_idle_all()
self.assertEqual(count, 2)
self.assertEqual(set(stopped), {root_b, legacy})
self.assertTrue((root_a / "access.json").exists())
self.assertFalse((root_b / "access.json").exists())
self.assertFalse((legacy / "access.json").exists())
self.assertIn(legacy, runtime._runtime_roots(self.users))
self.assertIn(root_a, runtime._runtime_roots(self.users))
finally:
self.manager.release_actor(fd_a)
def test_project_template_symlink_cannot_write_into_another_session(self):
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
try:
project_a = root_a / "project"
project_a.mkdir()
project_b = root_b / "project"
project_b.mkdir()
protected_config = project_b / "nao_config.yaml"
protected_config.write_text("session B config", encoding="utf-8")
(project_a / "nao_config.yaml").symlink_to(protected_config)
with self.assertRaisesRegex(ValueError, "运行路径无效"):
runtime._write_project(root_a, self.session_a, "test-model", refresh_templates=True,
refresh_mcp=True, backup_schema=False,
expected_actor_id=self.actor_id)
self.assertEqual(protected_config.read_text(encoding="utf-8"), "session B config")
finally:
self.manager.release_actor(fd_b)
self.manager.release_actor(fd_a)
def test_server_log_open_rejects_symlink_and_non_regular_files(self):
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
try:
logs_a = root_a / "logs"
logs_b = root_b / "logs"
logs_a.mkdir()
logs_b.mkdir()
protected_log = logs_b / "server.log"
protected_log.write_text("session B log", encoding="utf-8")
(logs_a / "server.log").symlink_to(protected_log)
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "日志文件路径无效"):
runtime._open_server_log(logs_a / "server.log", root_a)
self.assertEqual(protected_log.read_text(encoding="utf-8"), "session B log")
(logs_a / "server.log").unlink()
fifo = logs_a / "server.log"
os.mkfifo(fifo)
with self.assertRaisesRegex(runtime.RuntimeErrorSafe, "日志文件路径无效"):
runtime._open_server_log(fifo, root_a)
finally:
self.manager.release_actor(fd_b)
self.manager.release_actor(fd_a)
def test_legacy_session_history_is_copied_to_private_root_without_changing_source(self):
actor_root = self.users / self.actor_id
legacy_private = actor_root / "private"
legacy_private.mkdir(parents=True)
mapping = legacy_private / "sessions.json"
runtime._atomic_json(mapping, {
"actorId": self.actor_id,
"sessions": {self.session_a: "old-native-chat-A", self.session_b: "old-native-chat-B"},
})
mapping_before = mapping.read_bytes()
runtime._atomic_json(legacy_private / "credentials.json", {
"authSecret": "legacy-auth", "email": "legacy@example.invalid", "password": "legacy-password",
})
legacy_project = actor_root / "project"
legacy_project.mkdir(parents=True)
source_db = legacy_private / "nao.sqlite"
with contextlib.closing(sqlite3.connect(source_db)) as database:
with database:
# Mirrors NAO's actual SQLite project.path and chat.project_id
# relationship used by agent routes and MCP project lookup.
database.executescript("""
CREATE TABLE project (
id TEXT PRIMARY KEY NOT NULL,
org_id TEXT,
name TEXT NOT NULL,
type TEXT NOT NULL,
path TEXT,
agent_settings TEXT,
enabled_tools TEXT NOT NULL DEFAULT '[]',
known_mcp_servers TEXT NOT NULL DEFAULT '[]',
disabled_mcp_servers TEXT NOT NULL DEFAULT '[]',
disabled_mcp_tools TEXT NOT NULL DEFAULT '[]',
env_vars TEXT NOT NULL DEFAULT '{}',
CONSTRAINT local_project_path_required
CHECK(CASE WHEN type = 'local' THEN path IS NOT NULL ELSE TRUE END)
);
CREATE TABLE chat (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
project_id TEXT NOT NULL REFERENCES project(id),
title TEXT DEFAULT 'New Conversation' NOT NULL,
is_starred INTEGER DEFAULT false NOT NULL,
deleted_at INTEGER,
slack_thread_id TEXT,
teams_thread_id TEXT,
telegram_thread_id TEXT,
mattermost_thread_id TEXT,
discord_thread_id TEXT,
whatsapp_thread_id TEXT,
fork_metadata TEXT,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
""")
database.execute(
"INSERT INTO project(id,name,type,path) VALUES (?,?,?,?)",
("legacy-project-id", "Quality", "local", str(legacy_project)),
)
database.executemany(
"INSERT INTO chat(id,user_id,project_id,created_at,updated_at) VALUES (?,?,?,?,?)",
(("old-native-chat-A", "legacy-user", "legacy-project-id", 1, 1),
("old-native-chat-B", "legacy-user", "legacy-project-id", 2, 2)),
)
source_db.chmod(0o600)
root_a, fd_a = self.manager.acquire_actor(self.actor_id, self.session_a)
root_b, fd_b = self.manager.acquire_actor(self.actor_id, self.session_b)
try:
self.assertTrue(self.manager._migrate_legacy_session(self.actor_id, self.session_a, root_a))
self.assertTrue(self.manager._migrate_legacy_session(self.actor_id, self.session_b, root_b))
state = runtime.SessionState(root_a, self.actor_id, self.session_a)
other_state = runtime.SessionState(root_b, self.actor_id, self.session_b)
self.assertEqual(state.get_chat_id(self.session_a), "old-native-chat-A")
self.assertEqual(other_state.get_chat_id(self.session_b), "old-native-chat-B")
self.assertNotEqual(root_a / "private" / "nao.sqlite", source_db)
with contextlib.closing(sqlite3.connect(root_a / "private" / "nao.sqlite")) as database:
self.assertEqual(database.execute(
"SELECT id,path FROM project WHERE id='legacy-project-id'"
).fetchone(), ("legacy-project-id", str(root_a / "project")))
self.assertEqual(database.execute(
"SELECT id,project_id FROM chat WHERE id='old-native-chat-A'"
).fetchone(), ("old-native-chat-A", "legacy-project-id"))
self.assertEqual(database.execute(
"SELECT id,project_id FROM chat WHERE id='old-native-chat-B'"
).fetchone(), ("old-native-chat-B", "legacy-project-id"))
with contextlib.closing(sqlite3.connect(root_b / "private" / "nao.sqlite")) as database:
with database:
database.execute(
"INSERT INTO chat(id,user_id,project_id,created_at,updated_at) VALUES (?,?,?,?,?)",
("session-B-only", "legacy-user", "legacy-project-id", 3, 3),
)
self.assertEqual(database.execute(
"SELECT id,path FROM project WHERE id='legacy-project-id'"
).fetchone(), ("legacy-project-id", str(root_b / "project")))
with contextlib.closing(sqlite3.connect(root_a / "private" / "nao.sqlite")) as database:
rows = database.execute("SELECT id FROM chat ORDER BY id").fetchall()
self.assertEqual(rows, [("old-native-chat-A",), ("old-native-chat-B",)])
with contextlib.closing(sqlite3.connect(source_db)) as database:
self.assertEqual(database.execute(
"SELECT id,path FROM project WHERE id='legacy-project-id'"
).fetchone(), ("legacy-project-id", str(legacy_project)))
self.assertEqual(database.execute(
"SELECT id,project_id FROM chat WHERE id='old-native-chat-A'"
).fetchone(), ("old-native-chat-A", "legacy-project-id"))
self.assertEqual(mapping.read_bytes(), mapping_before)
self.assertTrue(source_db.is_file())
self.assertEqual(runtime._read_json(root_a / "private" / "credentials.json")["authSecret"],
"legacy-auth")
finally:
self.manager.release_actor(fd_b)
self.manager.release_actor(fd_a)
if __name__ == "__main__":
unittest.main()
+102
View File
@@ -0,0 +1,102 @@
"""Semantic conversational replies must avoid agent overhead without bypassing evidence work."""
import io,json,sys,threading,unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
from nao_agent_task import normalize_task,plan_task,TaskUnavailable
def direct(**changes):
return {'kind':'explain','goal':'回应开场问候并介绍质量分析能力',
'directResponse':{'purpose':'capabilities','answer':'你好,我可以帮你查询质量指标、比较期间变化并分析重点规格。','requiresMemory':False},**changes}
class ConversationTests(unittest.TestCase):
def test_minimal_semantic_reply_needs_no_full_business_plan(self):
task=normalize_task(direct(),'请介绍一下你能做哪些事')
self.assertEqual(task['directResponse']['answer'],direct()['directResponse']['answer'])
self.assertFalse(task['freshQuery']);self.assertEqual(task['visualPreference'],'none')
self.assertEqual(task['dimensions'],[]);self.assertEqual(task['fields'],[])
def test_direct_reply_cannot_request_business_actions_or_hide_memory_writes(self):
cases=[{'kind':'metrics'},{'freshQuery':True},{'comparisonRequired':True},
{'verifySource':True},{'fullDetails':True},{'needsClarification':True},
{'fields':['rate']},{'stages':['black']},{'dimensions':['product']},
{'filterUpdates':{'group':'OTHER'}},{'period':{'year':2026,'month':9}},
{'historyMode':'all'},{'visualPreference':'required'}]
for changes in cases:
with self.subTest(changes=changes),self.assertRaises(TaskUnavailable):normalize_task(direct(**changes),'q')
for changes in ({'requiresMemory':True},{'purpose':'business_analysis'},
{'answer':'本月合格率为95.5%'},{'answer':'已完成统计 [Q12345678]'},
{'answer':'https://example.test'},{'answer':'Bearer secretvalue'}):
value=direct();value['directResponse'].update(changes)
with self.subTest(changes=changes),self.assertRaises(TaskUnavailable):normalize_task(value,'q')
def test_plain_chat_preserves_active_business_frame_without_declaring_new_analysis(self):
task=normalize_task(direct(),'谢谢你',{'releaseId':'r','period':{'year':2026,'month':9},'filters':{'group':'G'}})
self.assertEqual(task['period'],{'year':2026,'month':9});self.assertEqual(task['filters'],{'group':'G'})
self.assertFalse(task['freshQuery'])
def test_rephrased_capability_reply_is_still_plain_conversation(self):
value=direct(kind='restate')
result=normalize_task(value,'换一种说法介绍一下你能做什么')
self.assertEqual(result['kind'],'explain');self.assertIn('directResponse',result)
def test_optional_null_direct_reply_does_not_block_real_business_planning(self):
from test_nao_agent_task import task_value
value=task_value(dimensions=['product'],dimensionStages={'product':['black']},directResponse=None)
result=normalize_task(value,'查黑片重点规格')
self.assertEqual(result['kind'],'analysis');self.assertTrue(result['freshQuery']);self.assertNotIn('directResponse',result)
def test_numbered_capability_lists_do_not_trigger_replanning_or_allow_metric_values(self):
value=direct();value['directResponse']['answer']='我能帮助你:\n1. 查询质量指标\n2、比较期间变化\n3) 分析重点规格'
task=normalize_task(value,'介绍一下能力')
self.assertNotIn('1.',task['directResponse']['answer']);self.assertEqual(task['directResponse']['answer'].count('- '),3)
value['directResponse']['answer']='1. 合格率为95.5%'
with self.assertRaises(TaskUnavailable):normalize_task(value,'q')
value['directResponse']['answer']='我可以帮你筛选TOP5重点规格,也可比较两期指标。'
self.assertIn('TOP5',normalize_task(value,'介绍能力')['directResponse']['answer'])
for text in ('合格率百分之九十五','报废重量17513','我已记住你的偏好','已经保存了你的设置'):
value['directResponse']['answer']=text
with self.subTest(text=text),self.assertRaises(TaskUnavailable):normalize_task(value,'q')
def test_capability_units_are_not_claimed_measurements(self):
value=direct();value['directResponse']['answer']='可查询投入重量(kg)、合格率(%),并按个百分点比较变化。'
self.assertIn('kg',normalize_task(value,'介绍能力')['directResponse']['answer'])
def test_capabilities_can_describe_query_results_without_claiming_work_completed(self):
value=direct();value['directResponse']['answer']='我可以解释查询结果,并把当前数据整理成图表和报告。'
self.assertIn('查询结果',normalize_task(value,'能做什么')['directResponse']['answer'])
value['directResponse']['answer']='我已经查到最新数据,合格率为95%'
with self.assertRaises(TaskUnavailable):normalize_task(value,'q')
def test_direct_repair_keeps_minimal_reply_and_points_to_the_actual_field(self):
bad=direct();bad['directResponse']['answer']='合格率95%'
replies=[bad,direct(),{'aligned':True,'requiresEvidence':False}];payloads=[];errors=[]
def opener(req,timeout):
payloads.append(json.loads(json.loads(req.data)['messages'][1]['content']))
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(replies.pop(0))}}]}).encode())
result=plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',question='你好',context={},cancel_event=threading.Event(),opener=opener,on_validation_error=errors.append)
self.assertIn('directResponse',result);self.assertEqual(errors[0]['field'],'directResponse.answer')
self.assertEqual(errors[0]['receivedType'],'dict')
self.assertIn('directResponse',payloads[1]['invalidPlan']);self.assertIn('无需补全',payloads[1]['repairInstructions'])
def test_planner_reviews_model_authored_reply(self):
bodies=[]
def opener(request,timeout):
bodies.append(json.loads(request.data));reply=direct() if len(bodies)==1 else {'aligned':True,'requiresEvidence':False}
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(reply)}}]}).encode())
task=plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',question='嗨,先认识一下',
context={},cancel_event=threading.Event(),opener=opener)
self.assertEqual(len(bodies),2);self.assertIn('directResponse',task)
self.assertIn('directResponse',bodies[0]['messages'][0]['content'])
def test_exact_json_code_fence_does_not_require_replanning_or_skip_validation(self):
calls=[]
def opener(request,timeout):
calls.append(request)
reply=direct() if len(calls)==1 else {'aligned':True,'requiresEvidence':False}
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':'```json\n'+json.dumps(reply)+'\n```'}}]}).encode())
result=plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',question='你好',context={},cancel_event=threading.Event(),opener=opener)
self.assertEqual(len(calls),2);self.assertFalse(result['freshQuery'])
def test_internal_turn_correlation_marker_is_not_user_visible(self):
from nao_quality_runtime import public_progress_text
from nao_thinking_stream import public_thinking_text
text='原文 quality-turn:12345678-1234-1234-1234-123456789abc 继续答复'
for sanitizer in (public_progress_text,public_thinking_text):
output=sanitizer(text);self.assertNotIn('quality-turn:',output);self.assertIn('继续答复',output)
if __name__=='__main__':unittest.main()
@@ -0,0 +1,56 @@
import io,json,sys,tempfile,unittest,uuid
from pathlib import Path
from contextlib import ExitStack
from unittest.mock import MagicMock,patch
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
import nao_quality_runtime as runtime
from nao_agent_task import normalize_task
from test_nao_conversation import direct
from test_nao_agent_task import task_value
class ConversationRuntimeTests(unittest.TestCase):
def run_case(self,case):
with tempfile.TemporaryDirectory() as folder,ExitStack() as stack:
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
req={key:str(uuid.uuid4()) for key in ('turnId','actorId','sessionId','releaseId')};req.update(question='谢谢你',context={},accessToken='synthetic')
prior=root/'private/last-agent-task.json';business={'releaseId':req['releaseId'],'sessionId':req['sessionId'],
'baseMetricFrame':{},'task':normalize_task(task_value(dimensions=[],dimensionStages={}), '分析质量变化')}
prior.write_text(json.dumps(business));prior.chmod(0o600);saved=prior.read_bytes()
controls=root/'private/last-model-controls.json';controls.write_text('{}');controls.chmod(0o600);control_before=controls.read_bytes()
visible=root/'private/last-visible-answer.json';visible.write_text('{"answer":"原业务结果"}');visible.chmod(0o600)
task=normalize_task(direct(),'谢谢你') if case!='memory' else normalize_task(task_value(kind='explain',freshQuery=False,dimensions=[],stages=[],fields=[]),'请记住我偏好简洁')
manager=MagicMock();manager.acquire_actor.return_value=(root,None);manager.ensure_instance.return_value=('http://127.0.0.1:49191/v1','synthetic','m','http://127.0.0.1:5000')
state=MagicMock();state.get_chat_id.return_value=None;gate=MagicMock();gate.base_url='http://127.0.0.1:49191/v1';gate.request_audit=[]
def planned(**kw):
if case=='cancel':kw['cancel_event'].set()
return task
conversation=runtime.ConversationAccumulator();conversation.blocks['a']='已记录你的表达偏好。';conversation.block_order=['a'];conversation.record_event('assistant',text=conversation.answer)
native=MagicMock(return_value=(conversation,'COMPLETED','stop',None));auth=MagicMock();primary=MagicMock()
mocks={'RuntimeManager':MagicMock(return_value=manager),'write_access_file':MagicMock(return_value=root/'access.json'),
'_model_settings':MagicMock(return_value=('https://offline.invalid','synthetic','m',{'enabled':True,'effort':'medium','protocol':'thinking'})),
'_model_gate_settings':MagicMock(return_value=(49191,'synthetic')),'SessionState':MagicMock(return_value=state),
'_input_cancel_listener':MagicMock(),'DelegatedApi':MagicMock(),'plan_agent_task':planned,
'_sse_request':native,'_authenticate':auth,'read_primary_evidence':primary,'_story_files':MagicMock(return_value=(None,None))}
for key,value in mocks.items():stack.enter_context(patch.object(runtime,key,value))
stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate))
result=runtime.run_turn(req,stdout=io.StringIO(),stdin=io.StringIO())
if case=='memory':
self.assertEqual(result['status'],'COMPLETED');manager.ensure_instance.assert_called_once();native.assert_called_once();auth.assert_called_once()
else:
manager.ensure_instance.assert_not_called();native.assert_not_called();auth.assert_not_called();primary.assert_not_called()
self.assertEqual(prior.read_bytes(),saved);self.assertEqual(controls.read_bytes(),control_before)
self.assertEqual(visible.read_text(),'{"answer":"原业务结果"}')
self.assertEqual(result['toolCalls'],[]);self.assertEqual(result['queryEvidenceRefs'],[])
if case=='cancel':
self.assertEqual(result['status'],'CANCELLED');self.assertEqual(result['answer'],'');self.assertFalse((root/'private/last-conversation-answer.json').exists())
self.assertFalse((root/'private/non-query-turns'/ (req['turnId']+'.json')).exists())
else:
self.assertEqual(result['status'],'COMPLETED');self.assertEqual(result['answer'],task['directResponse']['answer'])
self.assertEqual(result['timeline'][-1]['kind'],'assistant')
self.assertTrue((root/'private/non-query-turns'/ (req['turnId']+'.json')).exists())
gate.close.assert_called_once()
def test_conversation_skips_native_title_and_memory_without_replacing_business_state(self):self.run_case('plain')
def test_cancelled_conversation_does_not_emit_an_answer_or_change_business_state(self):self.run_case('cancel')
def test_explicit_memory_request_still_starts_the_native_agent(self):self.run_case('memory')
if __name__=='__main__':unittest.main()
+29
View File
@@ -0,0 +1,29 @@
import sys
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_analysis_coverage import coverage_gaps
class DefectScopeTest(unittest.TestCase):
def test_three_selected_products_do_not_require_parent_defects(self):
task = {'version':1,'kind':'analysis','question':'fixture','dimensions':['defect'],
'stages':['black','bright'],'defectScope':'products','defectProductCount':3}
root = {'releaseId':'r','filterBinding':{'group':'g'},
'current':{'period':{'year':2026,'month':9}},
'comparison':{'period':{'year':2026,'month':8}}}
docs = [{'result':{'kind':'quality_fine_defects','releaseId':'r','dimension':'defect','stage':stage,
'filters':{'group':'g','product':product},'period':root['current']['period'],
'comparisonPeriod':root['comparison']['period'],'sort':'scrap_delta',
'coverage':{'available':True},'comparisonCoverage':{'available':True}}}
for product in ('a','b','c') for stage in ('black','bright')]
with patch('nao_analysis_coverage._root', return_value=root):
self.assertEqual(coverage_gaps(docs,'fixture',{'_agentTask':task}), [])
self.assertTrue(coverage_gaps(docs[:4],'fixture',{'_agentTask':task}))
wrong = [{**doc,'result':{**doc['result'],'filters':{'group':'other','product':'c'}}} for doc in docs[4:]]
self.assertTrue(coverage_gaps(docs[:4]+wrong,'fixture',{'_agentTask':task}))
task['defectScope']='both'
self.assertTrue(any('来源具体缺陷' in gap for gap in coverage_gaps(docs,'fixture',{'_agentTask':task})))
if __name__=='__main__': unittest.main()
+116
View File
@@ -0,0 +1,116 @@
"""Mixed overall metrics and dimension-specific stages must not share a coverage scope."""
import copy
import io
import json
from pathlib import Path
import sys
import threading
import unittest
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT / 'scripts'), str(ROOT / 'verification')]
from nao_agent_task import normalize_task, bind_task, plan_task, plan_repairs, TaskUnavailable
from nao_analysis_coverage import coverage_gaps, plan_missing_queries
from nao_product_defect_pairing import plan_missing_product_defect_stages
from test_nao_agent_task import task_value
from test_nao_analysis_coverage import evidence, PERIOD, PRIOR
from test_nao_product_defect_pairing import evidence as defect_evidence
QUESTION = '对比两期综合合格率,再查询黑片报废增量最大的3个规格'
def mixed_task(**changes):
return normalize_task(task_value(dimensions=['product'], stages=['black', 'bright'],
dimensionStages={'product': ['black']}, comparisonRequired=True,
period=PERIOD, comparisonPeriod=PRIOR, **changes), QUESTION)
def documents(stage='black'):
return [evidence('quality_comparison'),
evidence('quality_changes', dimension='product', stage=stage, sort='scrap_delta')]
class DimensionStageTests(unittest.TestCase):
def test_overall_comparison_and_black_product_rank_keep_independent_stages(self):
task = mixed_task()
context = bind_task({'releaseId': 'release-a'}, task)
self.assertEqual(task['stages'], ['black', 'bright'])
self.assertNotIn('stage', context) # Do not turn composite rate into black-sheet rate.
self.assertEqual(coverage_gaps(documents(), QUESTION, context), [])
self.assertEqual(plan_missing_queries(documents(), QUESTION, context), [])
self.assertTrue(coverage_gaps(documents('bright'), QUESTION, context))
planned = plan_missing_queries([documents()[0]], QUESTION, context)
self.assertEqual([(p['dimension'], p['stage']) for p in planned], [('product', 'black')])
def test_wrong_period_scope_release_and_stage_still_block_completion(self):
context = bind_task({'releaseId': 'release-a'}, mixed_task())
for field, value in [('stage', 'bright'), ('currentPeriod', {'year': 2026, 'month': 10}),
('comparisonPeriod', {'year': 2026, 'month': 7}),
('releaseId', 'other-release'), ('filters', {'group': 'other-scope'})]:
docs = copy.deepcopy(documents()); docs[1]['result'][field] = value
with self.subTest(field=field): self.assertTrue(coverage_gaps(docs, QUESTION, context))
self.assertTrue(coverage_gaps(documents()[1:], QUESTION, context))
def test_separate_black_and_bright_ranks_require_both_and_never_accept_a_combined_rank(self):
task = mixed_task(); task['dimensionStages']['product'] = ['black', 'bright']
context = bind_task({}, task)
self.assertEqual(len(coverage_gaps(documents(), QUESTION, context)), 1)
self.assertEqual(plan_missing_queries(documents(), QUESTION, context)[0]['stage'], 'bright')
both = documents() + [documents('bright')[1]]
self.assertEqual(coverage_gaps(both, QUESTION, context), [])
self.assertTrue(coverage_gaps(documents(None), QUESTION, context))
def test_combined_ranking_cannot_be_proven_by_a_single_stage(self):
task = mixed_task(); task['dimensionStages']['product'] = []
context = bind_task({}, task)
self.assertTrue(coverage_gaps(documents(), QUESTION, context))
self.assertEqual(coverage_gaps(documents(None), QUESTION, context), [])
def test_legacy_plans_are_not_silently_reinterpreted_from_the_evidence(self):
task = mixed_task(); task.pop('dimensionStages')
self.assertTrue(coverage_gaps(documents(), QUESTION, bind_task({}, task)))
def test_dimension_stage_schema_rejects_untrusted_or_ambiguous_values(self):
for scopes in ({'product': 'black'}, {'product': ['unknown']}, {'product': [True]},
{'secret': ['black']}, {'defect': []}):
with self.subTest(scopes=scopes), self.assertRaises(TaskUnavailable):
normalize_task(task_value(dimensionStages=scopes), 'q')
def test_missing_dimension_scope_is_returned_to_the_model_for_repair(self):
replies = [task_value(dimensions=['product']),
task_value(dimensions=['product'], dimensionStages={'product': ['black']}),
{'aligned':True,'requiresEvidence':True,'requiresSourceVerification':False,'requiresAllSourceRows':False,'requiresTrend':False}]
payloads = []
def opener(request, timeout):
payloads.append(json.loads(json.loads(request.data)['messages'][1]['content']))
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {
'content': json.dumps(replies.pop(0))}}]}).encode())
task = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='fixture',
question=QUESTION, context={}, cancel_event=threading.Event(), opener=opener)
self.assertEqual(task['dimensionStages'], {'product': ['black']})
self.assertIn('dimensionStages', payloads[1]['requiredShapes'])
self.assertIn('dimensionStages', payloads[1]['validationFeedback'])
def test_repair_queries_cannot_switch_a_dimension_to_the_other_stage(self):
def run(stage):
query = {'mode': 'changes', 'dimension': 'product', 'filters': {}, 'stage': stage}
def opener(request, timeout):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {
'content': json.dumps({'queries': [query]})}}]}).encode())
return plan_repairs(base_url='https://offline.invalid/v1', api_key='synthetic', model='fixture',
task=mixed_task(), evidence=[], gaps=['missing'], failures=[], release_id='release-a',
cancel_event=threading.Event(), opener=opener)
self.assertEqual(run('black')[0]['stage'], 'black')
with self.assertRaises(TaskUnavailable): run('bright')
self.assertEqual(run(None)[0]['stage'], 'black')
def test_defect_drilldown_keeps_its_own_stage_in_a_mixed_task(self):
task = normalize_task(task_value(dimensions=['defect'], stages=['black', 'bright'],
dimensionStages={'defect': ['black']}), QUESTION)
context = bind_task({}, task)
self.assertEqual(plan_missing_product_defect_stages([defect_evidence('A', 'black')], QUESTION, context), [])
missing = plan_missing_product_defect_stages([defect_evidence('A', 'bright')], QUESTION, context)
self.assertEqual([q['stage'] for q in missing], ['black'])
if __name__ == '__main__': unittest.main()
+481
View File
@@ -0,0 +1,481 @@
"""Evidence-size, completeness and bounded synthesis regressions; no model calls."""
import io
import copy
import json
import sys
import tempfile
import threading
import ssl
import unittest
import uuid
from unittest.mock import MagicMock, patch
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_evidence_finalizer as finalizer
import nao_quality_runtime as runtime
FORMAL_PRODUCT_QUESTION = '只看SYNTH-16X15+T,请重新查询2026年9月与8月黑片、亮片的合格率、报废重量和两期投入,附正式数据依据。'
FORMAL_PRODUCT_GAPS = ['缺少仅针对SYNTH-16X15+T规格的两期分规格投入重量正式数据',
'缺少仅针对该规格的黑片、亮片报废重量分规格正式数据']
def formal_product_evidence():
result = {'kind': 'quality_comparison', 'releaseId': 'release-a',
'filterBinding': {'product': 'SYNTH-16X15+T', 'includeUnassigned': False},
'delta': {'available': True, 'percentagePoints': -2},
'stageRateChangesPP': {'black': -1, 'bright': -1}}
for side, month, inputs, scraps in [('current', 9, (1200.5, 2400.5), (36.1, 72.1)),
('comparison', 8, (1000.5, 2000.5), (20.1, 40.1))]:
result[side] = {'period': {'year': 2026, 'month': month}, 'compositeRate': .95,
'coverage': {'sourceMonths': [{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}]}}
for index, stage in enumerate(('black', 'bright')):
result[side][stage] = {'rate': .97, 'totalWeight': inputs[index], 'scrapWeight': scraps[index]}
return [{'evidenceRef': 'Q00000001', 'mode': 'compare', 'result': result}]
def unpack(packed):
def decode(value):
if isinstance(value, list): return [decode(item) for item in value]
if not isinstance(value, dict): return value
if set(value) == {'$shared'}: return decode(packed['shared'][value['$shared']])
if set(value) == {'$columns', '$values'}:
return [dict(zip(decode(value['$columns']), decode(row))) for row in decode(value['$values'])]
return {key: decode(item) for key,item in value.items()}
return decode(packed['evidence'])
class EvidenceFinalizerTest(unittest.TestCase):
def test_real_connection_keeps_system_certificate_and_hostname_verification(self):
with patch.object(finalizer,'urlopen',return_value=io.BytesIO(b'{}')) as request:
finalizer._open_model('request',35)
context=request.call_args.kwargs['context']
self.assertEqual(context.verify_mode,ssl.CERT_REQUIRED)
self.assertTrue(context.check_hostname)
def test_worker_finalizes_only_this_turn_and_preserves_partial_and_cancelled_states(self):
for outcome in ('complete','retry','retry_no_queries','fresh_no_queries','restatement','native_complete','native_query','insufficient','contradictory','server_incomplete','finalizer_error','cancel','cancel_after_stop','stop_failed'):
with self.subTest(outcome=outcome),tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir()
old=root/'evidence'/('nao-quality-'+'b'*32+'.json')
old.write_text('{}');old.chmod(0o600)
payload={key:str(uuid.uuid4()) for key in ('turnId','sessionId','actorId','releaseId')}
payload.update(question='全部明细' if outcome=='server_incomplete' else '比较质量',context={},accessToken='test-delegation')
if outcome == 'fresh_no_queries': payload['question'] = '请重新查询刚才的数据'
if outcome == 'restatement': payload['question'] = '请复述上一轮结论'
if outcome in ('retry','retry_no_queries'):payload['context']={'_restartConversation':True}
manager=MagicMock();manager.acquire_actor.return_value=(root,None)
manager.ensure_instance.return_value=('http://127.0.0.1:49191/v1','test-gate-key','mimo-v2.6-flash','http://127.0.0.1:5000')
gate=MagicMock();gate.base_url='http://127.0.0.1:49191/v1'
stream=runtime.ConversationAccumulator();stream.blocks['first']='查询已返回';stream.block_order.append('first')
stream.record_event('assistant',text='查询已返回')
stream.tool_calls=[{'tool':'mcp_call','status':'completed'}]
cancellation=[]
def sse(**kwargs):
if outcome in ('retry','retry_no_queries'):
self.assertIsNone(kwargs['chat_id'])
self.assertEqual(kwargs['context'],{})
if outcome in ('retry_no_queries', 'fresh_no_queries', 'restatement'):
stream.tool_calls=[]
return stream,'COMPLETED','stop',None
cancellation.append(kwargs['cancel_event'])
document={'tool':'quality_data','releaseId':payload['releaseId'],'evidenceRef':'Q00000001',
'mode':'compare','result':{'kind':'quality_query' if outcome=='native_query' else 'quality_comparison','releaseId':payload['releaseId'],
'current':{'period':{'year':2026,'month':9},'compositeRate':.95},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':.96},
'delta':{'available':True,'percentagePoints':-1}}}
path=root/'evidence'/('nao-quality-'+'a'*32+'.json');path.write_text(json.dumps(document));path.chmod(0o600)
if outcome == 'cancel_after_stop':
kwargs['cancel_event'].set()
return stream, 'PARTIAL', 'coverage_ready', True
if outcome in ('native_complete','native_query','finalizer_error'):return stream,'COMPLETED','stop',None
return stream,'PARTIAL','timeout',outcome!='stop_failed'
def synthesize(**kwargs):
self.assertEqual([d['evidenceRef'] for d in kwargs['evidence']],['Q00000001'])
self.assertEqual(kwargs['base_url'],'http://127.0.0.1:49191/v1')
self.assertEqual(kwargs['api_key'],'test-gate-key')
self.assertEqual(kwargs['context'],{})
if outcome=='finalizer_error':raise finalizer.FinalizationUnavailable('整理失败')
if outcome=='cancel':cancellation[0].set()
return {'answer':'最终统计 [Q00000001]','model':'mimo-v2.6-flash','usage':{},'finishReason':'stop',
'evidenceSufficient':outcome!='insufficient','missingEvidence':['缺少用户要求的对照期'] if outcome=='insufficient' else [],
'modelDeclaredEvidenceSufficient':outcome not in ('insufficient','contradictory'),
'completionAdjusted':outcome=='contradictory'}
with patch.object(runtime,'RuntimeManager',return_value=manager),patch.object(runtime,'write_access_file',return_value=root/'access.json'), \
patch.object(runtime,'_authenticate'),patch.object(runtime,'_input_cancel_listener'),patch.object(runtime,'SessionState'), \
patch.object(runtime,'_sse_request',side_effect=sse),patch.object(runtime,'_story_files',return_value=(None,None)), \
patch.object(runtime,'_model_settings',return_value=('https://model.test/v1','test-key','mimo-v2.6-flash',{'enabled':False,'effort':'auto','protocol':'thinking'})) as selected, \
patch.object(runtime,'plan_agent_task',return_value=None,create=True), \
patch.object(runtime,'_model_gate_settings',return_value=(49191,'test-gate-key')),patch.object(runtime,'DelegatedApi'), \
patch('nao_model_gate.ModelGate',return_value=gate) as gate_factory, \
patch.object(finalizer,'synthesize',side_effect=synthesize) as request, \
patch.object(finalizer,'coverage_sufficient',return_value=outcome not in ('finalizer_error','server_incomplete')):
result=runtime.run_turn(payload,stdout=io.StringIO(),stdin=io.StringIO())
selected.assert_called_once()
self.assertEqual(gate_factory.call_args.kwargs['upstream_key'],'test-key')
gate.close.assert_called_once()
expected={'complete':'COMPLETED','retry':'COMPLETED','retry_no_queries':'PARTIAL','fresh_no_queries':'PARTIAL','restatement':'COMPLETED','native_complete':'COMPLETED','native_query':'COMPLETED','insufficient':'PARTIAL','contradictory':'COMPLETED','server_incomplete':'PARTIAL','finalizer_error':'PARTIAL','cancel':'CANCELLED','cancel_after_stop':'CANCELLED','stop_failed':'PARTIAL'}[outcome]
self.assertEqual(result['status'],expected)
self.assertEqual(request.call_count,0 if outcome in ('stop_failed','native_query','retry_no_queries','fresh_no_queries','restatement','cancel_after_stop') else 1)
if outcome in ('retry_no_queries', 'fresh_no_queries'):
self.assertEqual(result['stopReason'],'evidence_incomplete')
self.assertIn('未取得新的查询证据',result['answer'])
if outcome=='native_query':
self.assertEqual(result['stopReason'],'stop')
self.assertEqual(result['answer'],'查询已返回')
self.assertFalse(any(call['tool']=='model.finalize' for call in result['toolCalls']))
self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],'COMPLETED')
if outcome in ('complete','native_complete'):self.assertEqual(result['stopReason'],'evidence_finalized')
if outcome in ('complete','native_complete'):
self.assertEqual(len([entry for entry in result['timeline'] if entry['kind']=='assistant']),1)
self.assertEqual(result['timeline'][-1]['text'],'最终统计 [Q00000001]')
self.assertEqual(json.loads((root/'private/last-visible-answer.json').read_text())['answer'],'最终统计 [Q00000001]')
if outcome in ('complete','native_complete','insufficient','contradictory','server_incomplete','finalizer_error'):
self.assertEqual(json.loads((root/'finalization.json').read_text())['status'],expected)
if outcome=='contradictory':
self.assertFalse(result['evidenceCompletion']['modelDeclaredEvidenceSufficient'])
self.assertTrue(result['evidenceCompletion']['completionAdjusted'])
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
if outcome=='server_incomplete':
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
def test_columnar_and_shared_encoding_roundtrips_without_rounding_or_null_loss(self):
evidence=[{'rows':[{'field':'电镀磕边重量','weight':None,'count':4120,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}},
{'field':'镀废合计','weight':855.801072,'count':431,'coverage':{'notice':'字段为空,不代表零缺陷。'*20}}]}]*4
self.assertEqual(unpack(finalizer.packed_evidence(evidence)),evidence)
def test_oversize_evidence_fails_closed_without_discarding_statistics(self):
with self.assertRaises(finalizer.FinalizationUnavailable):
finalizer.packed_evidence([{'unique': 'a'*finalizer.MAX_INPUT_CHARS}])
def test_real_second_turn_replay_fits_and_preserves_every_summary_value(self):
root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime'
directories=list(root.glob('.runtime/nao-users/*/sessions/57676d5a-5138-48f4-9343-bae9731aae5a/evidence'))
if not directories: self.skipTest('Private captured acceptance evidence not present')
with patch.object(finalizer,'packed_evidence',side_effect=AssertionError('Local archives do not consume model input budget')):
docs=finalizer.evidence_summaries(directories[0],'194b0287-9044-4d80-9138-ecd3c481015c')
self.assertEqual(len(docs),15)
self.assertEqual(unpack(finalizer.packed_evidence(docs)),docs)
# This captured run predates paired black/bright defect reads for each
# highlighted product, so it is now correctly labelled incomplete.
self.assertFalse(finalizer.coverage_sufficient(docs,'分析质量变化'))
# Ordinary analysis must not be blocked by minor source reconciliation issues.
missing=[d for d in docs if not(d['result'].get('action')=='source' and d['result'].get('stage')=='bright')]
self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化'))
# Full raw difference pagination is optional for an ordinary statistics question.
missing=[d for d in docs if not(d['result'].get('action')=='rows' and d['result'].get('stage')=='bright')]
self.assertFalse(finalizer.coverage_sufficient(missing,'分析质量变化'))
def test_previous_turn_evidence_is_excluded(self):
with tempfile.TemporaryDirectory() as directory:
root=Path(directory)
for index in range(2):
path=root/('nao-quality-'+str(index)*32+'.json')
path.write_text(json.dumps({'tool':'quality_data','releaseId':'release','evidenceRef':'Q0000000'+str(index),
'mode':'compare','result':{'releaseId':'release'},'createdAt':str(index)}))
path.chmod(0o600)
docs=finalizer.evidence_summaries(root,'release',exclude_files={'nao-quality-'+'0'*32+'.json'})
self.assertEqual([d['evidenceRef'] for d in docs],['Q00000001'])
def test_identity_permissions_and_duplicate_refs_are_rejected(self):
with tempfile.TemporaryDirectory() as directory:
root=Path(directory);path=root/('nao-quality-'+'a'*32+'.json')
doc={'tool':'quality_data','releaseId':'r','evidenceRef':'Q00000001','mode':'query','result':{'releaseId':'r'}}
path.write_text(json.dumps(doc));path.chmod(0o644)
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r')
path.chmod(0o600)
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'other')
twin=root/('nao-quality-'+'b'*32+'.json');twin.write_text(json.dumps(doc));twin.chmod(0o600)
with self.assertRaises(finalizer.FinalizationUnavailable):finalizer.evidence_summaries(root,'r')
def test_cancel_failed_stop_and_incomplete_tools_never_start_finalization(self):
calls=[{'status':'completed'}]
args=['mimo-v2.6-flash','PARTIAL','timeout',calls,False,True]
self.assertTrue(finalizer.eligible(*args))
for position,value in ((4,True),(5,False),(5,None),(3,[{'status':'started'}]),(2,'transport_or_stream_error')):
changed=list(args);changed[position]=value
self.assertFalse(finalizer.eligible(*changed))
self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','timeout',calls,False,True))
self.assertTrue(finalizer.eligible('mimo-v2.6-flash','FAILED','coverage_ready',calls,False,True))
def test_full_detail_requires_all_pages_but_accepts_contiguous_completed_pagination(self):
evidence=[{'mode':'compare','result':{}}]
for offset,count,more in ((0,2,True),(2,1,False)):
evidence.append({'mode':'investigate','result':{'action':'rows','current':{
'context':{'month':9},'matchingRows':3,'offset':offset,'returnedRows':count,
'hasMore':more,'differenceRecords':[{}]*count}}})
self.assertTrue(finalizer.coverage_sufficient(evidence,'全部明细'))
self.assertFalse(finalizer.coverage_sufficient(evidence[:-1],'全部明细'))
def test_synthesis_is_one_bounded_openai_compatible_request_with_verified_citations(self):
calls=[]
def opener(request,timeout):
calls.append((json.loads(request.data),timeout))
return io.BytesIO(json.dumps({'model':'mimo-v2.6-flash','choices':[{'finish_reason':'stop','message':{
'content':json.dumps({'otherFactIds':['F1'],'groupFactIds':[],'productFactIds':[],'defectFactIds':[],'answer':'伪造上升方向和QDEADBEEF引用必须忽略','evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
'delta':{'percentagePoints':-1}}}]
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
self.assertTrue(result['evidenceSufficient']);self.assertEqual(len(calls),1)
self.assertIn('下降1.00个百分点',result['answer']);self.assertNotIn('QDEADBEEF',result['answer']);self.assertNotIn('伪造',result['answer'])
self.assertEqual(calls[0][0]['thinking'],{'type':'disabled'})
from nao_runtime_policy import MAX_OUTPUT_TOKENS
self.assertEqual(calls[0][0]['max_completion_tokens'],MAX_OUTPUT_TOKENS)
self.assertEqual(calls[0][1],finalizer.FINAL_TIMEOUT_SECONDS)
cancelled=threading.Event();cancelled.set()
with self.assertRaises(finalizer.FinalizationUnavailable):
finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='质量如何',evidence=evidence,cancel_event=cancelled,opener=opener)
self.assertEqual(len(calls),1)
def test_selector_timeout_uses_verified_report_without_claiming_missing_coverage(self):
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{
'kind':'quality_comparison',
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
'delta':{'percentagePoints':-1}}}]
def timeout(request, timeout):
self.assertEqual(timeout, finalizer.FINAL_TIMEOUT_SECONDS)
raise TimeoutError('model selector timed out')
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',
model='mimo-v2.6-flash',question='质量如何',evidence=evidence,
cancel_event=threading.Event(),opener=timeout)
self.assertTrue(result['selectionFallback'])
self.assertTrue(result['evidenceSufficient'])
self.assertIn('下降1.00个百分点',result['answer'])
def test_model_cannot_override_missing_stage_or_ambiguous_frozen_frame(self):
from test_nao_evidence_report import coverage_comparison, comparison
def opener(request,timeout):
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
missing=coverage_comparison()
missing['result']['current']['coverage']['sourceMonths'][0].update(blackRows=0,brightRows=0)
result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
question='比较8月相对7月',evidence=[missing],cancel_event=threading.Event(),opener=opener)
self.assertTrue(result['modelDeclaredEvidenceSufficient'])
self.assertFalse(result['evidenceSufficient'])
self.assertFalse(finalizer.coverage_sufficient([missing],'比较8月相对7月'))
self.assertIn('没有可引用的正式综合合格率',result['answer'])
self.assertNotIn('100.00%',result['answer'])
docs=[comparison('Q00000001',{},.97),comparison('Q00000002',{'group':'组A'},.80)]
unresolved=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
question='只看组A',evidence=docs,cancel_event=threading.Event(),opener=opener)
self.assertFalse(unresolved['evidenceSufficient'])
self.assertIn('组A',unresolved['answer']);self.assertIn('80.00%',unresolved['answer'])
context={'year':2026,'month':7,'group':'组A'}
bound=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='mimo-v2.6-flash',
question='只看组A',evidence=docs,context=context,cancel_event=threading.Event(),opener=opener)
self.assertTrue(bound['evidenceSufficient'])
self.assertTrue(finalizer.coverage_sufficient(docs,'只看组A',context))
self.assertIn('80.00%',bound['answer']);self.assertNotIn('97.00%',bound['answer'])
def test_comparison_required_task_does_not_get_rechecked_as_single_period(self):
question = '全面分析质量变化'
current = {'year': 2026, 'month': 9}
comparison = {'year': 2026, 'month': 8}
context = {'releaseId': 'release-a', '_agentTask': {
'version': 1, 'question': question, 'kind': 'analysis', 'freshQuery': True,
'dimensions': ['department', 'group', 'product', 'defect'],
'stages': ['black', 'bright'], 'fields': ['rate'],
'period': current, 'comparisonPeriod': None, 'comparisonRequired': True,
'filters': {}, 'verifySource': False, 'fullDetails': False,
}}
evidence = [{'evidenceRef': 'Q' + '4' * 8, 'mode': 'compare',
'result': {'kind': 'quality_comparison', 'releaseId': 'release-a',
'filterBinding': {},
'current': {'period': current},
'comparison': {'period': comparison}}}]
with patch('nao_analysis_coverage.coverage_gaps', return_value=[]), \
patch('nao_analysis_coverage.semantic_single_period_gaps',
side_effect=AssertionError('comparison task was treated as single period')), \
patch('nao_analysis_coverage.formal_metric_coverage', return_value=None), \
patch('nao_evidence_report.build_report', return_value={'coverageSufficient': True}):
self.assertTrue(finalizer.coverage_sufficient(evidence, question, context))
def test_optional_focus_metadata_cannot_discard_or_invent_verified_statistics(self):
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
'delta':{'percentagePoints':-1}}}]
def opener(request,timeout):
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
'otherFactIds':['invented','F1','F1','F2','F3'], 'groupFactIds':['F1'],
'evidenceSufficient':True,'missingEvidence':[]})}}]}).encode())
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
self.assertTrue(result['selectionAdjusted'])
self.assertTrue(result['evidenceSufficient'])
self.assertEqual(result['selectedFactIds'],['F1','F2'])
self.assertNotIn('invented',result['answer'])
self.assertIn('下降1.00个百分点',result['answer'])
def test_completion_requires_concrete_missing_items_and_keeps_model_diagnostics(self):
evidence=[{'evidenceRef':'Q00000001','mode':'compare','result':{'kind':'quality_comparison',
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':1,'brightRows':1}]}},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':1,'brightRows':1}]}},
'delta':{'percentagePoints':-1}}}]
for declared,missing,expected in [(False,[],True),(False,[' ',''],True),
(False,[' 缺少对照期 ','缺少对照期'],False),(True,['缺少对照期'],False),(True,[],True)]:
with self.subTest(declared=declared,missing=missing):
def opener(request,timeout):
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
'evidenceSufficient':declared,'missingEvidence':missing})}}]}).encode())
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='质量如何',evidence=evidence,cancel_event=threading.Event(),opener=opener)
self.assertEqual(result['evidenceSufficient'],expected)
self.assertEqual(result['modelDeclaredEvidenceSufficient'],declared)
self.assertEqual(result['modelMissingEvidence'],missing)
self.assertEqual(result['completionAdjusted'],not declared and expected)
self.assertEqual(result['missingEvidence'],[] if expected else ['缺少对照期'])
if not expected:self.assertIn('缺少对照期',result['answer'])
def test_captured_3c_with_missing_paired_stage_remains_partial(self):
root=Path(__file__).resolve().parents[1]
paths=list((root/'.runtime/nao-users').glob('*/sessions/98829d63-0158-4e70-89db-ef0d58fe546a'))
if not paths:self.skipTest('Private captured session not present')
original=json.loads((paths[0]/'finalization.json').read_text())
self.assertFalse(original['modelDeclaredEvidenceSufficient'])
self.assertEqual(original['missingEvidence'],[])
self.assertTrue(original['serverCoverageSufficient'])
release=json.loads((root/'output/20261023-incomplete-reason/session.json').read_text())['releaseId']
docs=finalizer.evidence_summaries(paths[0]/'evidence',release)
def opener(request,timeout):
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps({
'evidenceSufficient':False,'missingEvidence':[]})}}]}).encode())
result=finalizer.synthesize(base_url='https://service.test/v1',api_key='not-real',model='mimo-v2.6-flash',
question='3C哪些产品近期表现最差',evidence=docs,cancel_event=threading.Event(),opener=opener)
self.assertFalse(result['evidenceSufficient'])
self.assertFalse(finalizer.coverage_sufficient(docs,'3C哪些产品近期表现最差'))
self.assertFalse(result['completionAdjusted'])
self.assertTrue(any('缺少' in item for item in result['missingEvidence']))
self.assertIn('Q039D5FA7',result['answer'])
self.assertIn('90.11%',result['answer'])
def test_exact_formal_metrics_refute_only_the_model_claims_they_prove_false(self):
captured = []
def opener(request, timeout):
captured.append(json.loads(request.data))
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
'evidenceSufficient': False, 'missingEvidence': FORMAL_PRODUCT_GAPS})}}]}).encode())
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=opener)
self.assertTrue(result['evidenceSufficient'])
self.assertEqual(result['missingEvidence'], [])
self.assertEqual(result['modelMissingEvidence'], FORMAL_PRODUCT_GAPS)
self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS)
self.assertTrue(result['completionAdjusted'])
selector = json.loads(captured[0]['messages'][1]['content'])
contract = selector['formalMetricCoverage']
self.assertEqual(contract['filters'], {'product': 'SYNTH-16X15+T'})
self.assertEqual(len(contract['metrics']), 12)
self.assertTrue(all(row['covered'] for row in contract['metrics']))
self.assertIn(1200.5, [row['value'] for row in contract['metrics']])
self.assertNotIn('未覆盖内容', result['answer'])
def test_formal_metric_contract_rejects_wrong_scope_period_release_and_incomplete_values(self):
mutations = {
'product': lambda r: r['filterBinding'].update(product='SYNTH-16X15+T-OTHER'),
'extra_filter': lambda r: r['filterBinding'].update(group='其他分组'),
'period': lambda r: r['comparison']['period'].update(month=7),
'release': lambda r: r.update(releaseId='release-other'),
'input': lambda r: r['current']['black'].pop('totalWeight'),
'scrap': lambda r: r['comparison']['bright'].pop('scrapWeight'),
'rate': lambda r: r['current']['bright'].update(rate=None),
'nonfinite': lambda r: r['current']['black'].update(totalWeight='NaN'),
'source_months': lambda r: r['comparison'].pop('coverage'),
'source_stage': lambda r: r['current']['coverage']['sourceMonths'][0].update(brightRows=0),
}
for name, mutate in mutations.items():
with self.subTest(name=name):
docs = copy.deepcopy(formal_product_evidence()); mutate(docs[0]['result'])
def opener(request, timeout):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
'evidenceSufficient': True, 'missingEvidence': []})}}]}).encode())
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
question=FORMAL_PRODUCT_QUESTION, evidence=docs, context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=opener)
self.assertFalse(result['evidenceSufficient'])
self.assertTrue(result['missingEvidence'])
self.assertFalse(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, {'releaseId': 'release-a'}))
def test_unknown_mixed_and_other_scope_model_gaps_still_block_completion(self):
for missing in ['缺少该规格投入重量和现场原因证据', '缺少该规格投入重量原始明细',
'缺少其他规格的投入重量正式数据', '缺少2026年7月投入重量正式数据',
'缺少对照期', '缺少生产原因证据']:
with self.subTest(missing=missing):
def opener(request, timeout):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
'evidenceSufficient': False, 'missingEvidence': [*FORMAL_PRODUCT_GAPS, missing]})}}]}).encode())
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=opener)
self.assertFalse(result['evidenceSufficient'])
self.assertEqual(result['missingEvidence'], [missing])
self.assertEqual(result['resolvedModelMissingEvidence'], FORMAL_PRODUCT_GAPS)
def test_metric_reconciliation_does_not_infer_unspecified_scope_or_hide_later_missing_items(self):
from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps
for question in [FORMAL_PRODUCT_QUESTION.replace('只看SYNTH-16X15+T,', ''),
FORMAL_PRODUCT_QUESTION.replace('2026年', ''),
FORMAL_PRODUCT_QUESTION.replace('黑片、亮片', ''),
FORMAL_PRODUCT_QUESTION.replace('9月与8月', '13月与8月')]:
with self.subTest(question=question):
contract = formal_metric_coverage(formal_product_evidence(), question, {'releaseId': 'release-a'})
self.assertIsNone(contract)
self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, []))
contract = formal_metric_coverage(formal_product_evidence(), FORMAL_PRODUCT_QUESTION.replace('报废重量和', ''), {'releaseId': 'release-a'})
self.assertEqual(resolve_proven_metric_gaps(contract, [FORMAL_PRODUCT_GAPS[1]]), ([FORMAL_PRODUCT_GAPS[1]], []))
gaps = ['缺少' + '的' * index + '该规格投入重量正式数据' for index in range(12)] + ['缺少生产原因证据']
def opener(request, timeout):
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps({
'evidenceSufficient': False, 'missingEvidence': gaps})}}]}).encode())
result = finalizer.synthesize(base_url='https://service.test/v1', api_key='not-real', model='mimo-v2.6-flash',
question=FORMAL_PRODUCT_QUESTION, evidence=formal_product_evidence(), context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=opener)
self.assertFalse(result['evidenceSufficient'])
self.assertEqual(result['missingEvidence'], ['缺少生产原因证据'])
self.assertEqual(result['modelMissingEvidence'], gaps)
def test_formal_contract_accepts_the_same_explicit_frozen_filters(self):
from nao_analysis_coverage import formal_metric_coverage
for filters in [{'group': '合成分组'}, {'department': '合成部门'},
{'group': '合成分组', 'extraFilters': {'region': '合成地区'}, 'includeUnassigned': True}]:
with self.subTest(filters=filters):
docs = formal_product_evidence()
docs[0]['result']['filterBinding'].update(filters)
expected = {'product': 'SYNTH-16X15+T', **filters}
context = {'releaseId': 'release-a', 'filters': expected}
contract = formal_metric_coverage(docs, FORMAL_PRODUCT_QUESTION, context)
self.assertEqual(contract['filters'], expected)
self.assertEqual(contract['missingEvidence'], [])
self.assertTrue(finalizer.coverage_sufficient(docs, FORMAL_PRODUCT_QUESTION, context))
def test_unknown_extra_scope_or_analysis_language_does_not_activate_the_contract(self):
from nao_analysis_coverage import formal_metric_coverage, resolve_proven_metric_gaps, coverage_gaps
docs = formal_product_evidence()
docs[0]['result']['filterBinding']['group'] = '合成分组'
for question in [FORMAL_PRODUCT_QUESTION + '仅统计合成分组。',
FORMAL_PRODUCT_QUESTION.replace(',请', ',合成部门,分析人员因素,请'),
'合成分组中,' + FORMAL_PRODUCT_QUESTION,
FORMAL_PRODUCT_QUESTION + '并提供现场生产原因。']:
with self.subTest(question=question):
contract = formal_metric_coverage(docs, question, {'releaseId': 'release-a'})
self.assertIsNone(contract)
self.assertEqual(coverage_gaps(docs, question, {'releaseId': 'release-a'}), [])
self.assertEqual(resolve_proven_metric_gaps(contract, FORMAL_PRODUCT_GAPS), (FORMAL_PRODUCT_GAPS, []))
if __name__=='__main__':unittest.main()
+211
View File
@@ -0,0 +1,211 @@
import sys
import unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
from nao_evidence_report import build_report,render_report,product_priority_summary
from nao_evidence_finalizer import evidence_summaries
def comparison(ref='Q00000001', filters=None, rate=.95, month=7):
def coverage():
return {'sourceMonths': [{'year': 2026, 'month': selected,
'blackRows': 2, 'brightRows': 2}
for selected in range(1, 13)]}
return {'evidenceRef':ref,'mode':'compare','result':{'kind':'quality_comparison','filterBinding':filters or {},
'current':{'period':{'year':2026,'month':month},'compositeRate':rate,'coverage':coverage()},
'comparison':{'period':{'year':2026,'month':month-1},'compositeRate':.96,'coverage':coverage()},
'delta':{'available':True,'percentagePoints':(rate-.96)*100}}}
def coverage_comparison():
from nao_quality_tools import QualityTools
def metrics(month):
missing=month==8
return {'summary':{'compositeRate':.9 if missing else .95,
'black':{'rate':1 if missing else .97,'qualifiedWeight':None if missing else 97,'totalWeight':None if missing else 100,'scrapWeight':None if missing else 3},
'bright':{'rate':.9 if missing else .98,'qualifiedWeight':90 if missing else 98,'totalWeight':100,'scrapWeight':10 if missing else 2}},
'coverage':{'rawRows':50 if missing else 100,'includedRows':50 if missing else 100,
'sourceMonths':[{'year':2026,'month':month,'blackRows':0 if missing else 50,'brightRows':50}]}}
tools=QualityTools.__new__(QualityTools)
tools._metrics=lambda release,period,filters,view:metrics(period['month'])
return {'mode':'compare','evidenceRef':'Q00000004','result':tools._compare({'id':'synthetic'},
{'year':2026,'month':8},{'year':2026,'month':7},{},None,None)}
class BoundReportTest(unittest.TestCase):
def test_priority_summary_keeps_three_signals_and_rate_denominators_separate(self):
signals={'scrapIncrease':[{'label':'放量规格','scrapDelta':120}],
'currentScrap':[{'label':'存量规格','current':{'scrapWeight':500}}],
'rateDrop':[{'label':'小量规格','rateDrop':0.8,
'comparison':{'blackInputWeight':400,'brightInputWeight':300},
'current':{'blackInputWeight':4,'brightInputWeight':3}}]}
text=product_priority_summary(signals,None)
self.assertIn('放量规格(+120.00 kg)',text)
self.assertIn('存量规格(500.00 kg)',text)
self.assertIn('小量规格(下降80.00个百分点',text)
self.assertIn('黑片投入400.00→4.00 kg',text)
self.assertIn('不能相加为总体贡献',text)
def test_frozen_group_period_anchor_wins_over_global_regardless_of_order(self):
global_doc=comparison('Q00000001',{},.97)
local=comparison('Q00000002',{'group':'组A'},.80)
another_month=comparison('Q00000003',{'group':'组A'},.70,6)
for docs in ([global_doc,local,another_month],[another_month,local,global_doc]):
report=build_report(docs,{'year':2026,'month':7,'group':'组A'})
self.assertIn('80.00%',report['headline']);self.assertNotIn('97.00%',report['headline'])
self.assertIn('业务分组=组A',report['policy'])
self.assertTrue(report['coverageSufficient'])
def test_ambiguous_frames_stay_separate_and_cannot_claim_completion(self):
report=build_report([comparison('Q00000001',{},.97),comparison('Q00000002',{'group':'组A'},.80)])
self.assertFalse(report['coverageSufficient'])
text=render_report(report,[report['facts'][0]['id']],report['missingEvidence'])
self.assertIn('97.00%',text);self.assertIn('80.00%',text);self.assertIn('业务分组=组A',text)
self.assertIn('多个期间和范围',text)
mismatch=build_report([comparison()],{'year':2026,'month':7,'group':'未查组'})
self.assertFalse(mismatch['coverageSufficient'])
self.assertIn('未匹配',mismatch['missingEvidence'][0])
def test_annual_and_quarter_anchors_do_not_match_months(self):
monthly=comparison('Q00000001',{},.95)
annual=comparison('Q00000002',{},.85)
annual['result']['current']['period']={'year':2026}
annual['result']['comparison']['period']={'year':2025}
quarterly=comparison('Q00000003',{},.75)
quarterly['result']['current']['period']={'year':2026,'quarter':3}
quarterly['result']['comparison']['period']={'year':2026,'quarter':2}
docs=[monthly,annual,quarterly]
self.assertIn('85.00%',build_report(docs,{'year':2026})['headline'])
self.assertIn('75.00%',build_report(docs,{'year':2026,'quarter':3})['headline'])
def test_other_period_defects_cannot_borrow_root_column_labels(self):
docs=[comparison(),{'evidenceRef':'Q00000003','mode':'defects','result':{
'kind':'quality_fine_defects','stage':'black','dimension':'defect','filters':{},
'period':{'year':2026,'month':6},'comparisonPeriod':{'year':2026,'month':5},
'categoryReconciliation':[],
'defectColumns':['field','category','currentWeight','comparisonWeight','deltaWeight','deltaAvailable'],
'defectValues':[['合成磕边重量','料废',30,10,20,True]]}}]
report=build_report(docs)
text=render_report(report,[report['facts'][0]['id']],[])
self.assertNotIn('合成磕边重量',text)
self.assertIn('其他期间证据未合并',text)
self.assertIn('[Q00000003]',text)
def test_real_tool_missing_stage_remains_missing_and_raw_value_is_traceable(self):
doc=coverage_comparison()
self.assertTrue(doc['result']['delta']['available'])
report=build_report([doc])
text=render_report(report,[report['facts'][0]['id']],report['missingEvidence'])
self.assertTrue(report['coverageSufficient'])
self.assertIn('缺失的黑片来源月份:2026-08',text)
self.assertNotIn('100.00%',text)
self.assertIn('| 黑片合格率 | 97.00% | — | 不可比 |',text)
self.assertEqual(report['rawMetrics']['current']['black']['rate'],1)
self.assertEqual(doc['result']['current']['black']['rate'],1)
def test_unknown_comparison_coverage_stays_incomplete(self):
doc=comparison()
for side in ('current','comparison'):
doc['result'][side].pop('coverage')
report=build_report([doc])
self.assertFalse(report['coverageSufficient'])
self.assertIn('没有可引用的正式综合合格率', ';'.join(report['missingEvidence']))
self.assertIn('—→', report['headline'])
def test_frozen_stage_uses_its_formal_delta_and_other_stage_does_not_block_it(self):
doc=coverage_comparison()
# The synthetic tool result has bright coverage in both periods, even
# though no composite comparison is possible without the black source.
report=build_report([doc],{'year':2026,'month':8,'stage':'bright'})
self.assertTrue(report['coverageSufficient'])
self.assertIn('亮片合格率',report['headline'])
self.assertIn('98.00%→2026年8月90.00%',report['headline'])
self.assertIn('下降8.00个百分点',report['headline'])
self.assertIn('缺失的黑片来源月份',report['policy'])
black=build_report([doc],{'year':2026,'month':8,'stage':'black'})
self.assertFalse(black['coverageSufficient'])
self.assertNotIn('100.00%',black['headline'])
def test_chapter_leads_are_bound_facts_before_their_data_tables(self):
docs=[{'evidenceRef':'Q00000001','result':{'kind':'quality_comparison','filterBinding':{},
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95,
'black':{'rate':0.97,'totalWeight':100,'qualifiedWeight':97,'defects':[{'name':'责废','weight':12},{'name':'料废','weight':5}]},
'bright':{'rate':0.98,'totalWeight':100,'qualifiedWeight':98},
'coverage':{'sourceMonths':[{'year':2026,'month':9,'blackRows':2,'brightRows':2}]}},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96,
'black':{'rate':0.98,'totalWeight':100,'qualifiedWeight':98,'defects':[{'name':'责废','weight':8},{'name':'料废','weight':4}]},
'bright':{'rate':0.98,'totalWeight':100,'qualifiedWeight':98},
'coverage':{'sourceMonths':[{'year':2026,'month':8,'blackRows':2,'brightRows':2}]}},
'delta':{'percentagePoints':-1},'stageRateChangesPP':{'black':-1,'bright':0}}},
{'evidenceRef':'Q00000002','result':{'kind':'quality_changes','dimension':'group','filters':{},
'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
'rows':[{'label':'改善组','current':{'rate':0.93},'comparison':{'rate':0.90},'rateDrop':-0.03,'scrapDelta':-10},
{'label':'下降组','current':{'rate':0.93},'comparison':{'rate':0.95},'rateDrop':0.02,'scrapDelta':12}]}}]
report=build_report(docs)
stage=next(s for s in report['sections'] if s.startswith('### 总体与阶段'))
self.assertLess(stage.index('黑片合格率:98.00%→97.00%,下降1.00个百分点。[Q00000001]'),stage.index('| 指标'))
category=next(s for s in report['sections'] if s.startswith('### 正式报废类别'))
self.assertIn('黑片责废:8.00→12.00 kg,增减+4.00 kg。[Q00000001]',category.split('| 类别')[0])
group=next(s for s in report['sections'] if s.startswith('### 业务分组变化'))
self.assertIn('本次返回的组织中,下降组',group.split('| 范围')[0])
self.assertNotIn('生产主因', '\n'.join(report['sections']))
def test_actual_wrong_group_directions_and_product_citations_cannot_be_generated(self):
root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime'
paths=list(root.glob('.runtime/nao-users/*/sessions/4c3c5e05-3c81-4c43-acf4-3e70af3ab7c7/evidence'))
if not paths:self.skipTest('Private real acceptance evidence not present')
docs=evidence_summaries(paths[0],'194b0287-9044-4d80-9138-ecd3c481015c')
report=build_report(docs)
groups={row['label']:row for row in report['facts'] if row['kind']=='group'}
self.assertIn('下降0.38个百分点',groups['工业电机-中国区']['text'])
self.assertIn('下降0.05个百分点',groups['工业电机-VCM']['text'])
self.assertIn('上升3.12个百分点',groups['工业电机-亚太']['text'])
product=next(row for row in report['facts'] if row['kind']=='product')
self.assertEqual(product['evidenceRef'],'QE8DBF76C')
text=render_report(report,[groups['工业电机-中国区']['id'],groups['工业电机-VCM']['id'],product['id']],[])
self.assertIn('本部报废',text);self.assertIn('VCM亮片仅录合计',text)
self.assertNotIn('中国区、VCM、亚太合格率小幅上行',text)
with self.assertRaises(ValueError):render_report(report,['F999999'],[])
def test_missing_rates_are_not_imputed_to_zero(self):
docs=[{'evidenceRef':'Q00000001','result':{'kind':'quality_comparison','filterBinding':{},
'current':{'period':{'year':2026,'month':9},'compositeRate':0.95},
'comparison':{'period':{'year':2026,'month':8},'compositeRate':0.96},'delta':{'percentagePoints':-1}}},
{'evidenceRef':'Q00000002','result':{'kind':'quality_changes','dimension':'group','filters':{},
'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
'rows':[{'label':'新进组','current':{'rate':0.93},'comparison':{'rate':None},'rateDrop':None,'scrapDelta':12}]}}]
report=build_report(docs);group=next(row for row in report['facts'] if row['kind']=='group')
self.assertIn('不可比',group['text']);self.assertIn('—→93.00%',group['text']);self.assertNotIn('0.00%→',group['text'])
def test_rate_drop_product_remains_a_volume_qualified_signal_with_exact_pp(self):
root=comparison(month=9)
product={'evidenceRef':'Q00000002','result':{'kind':'quality_changes','dimension':'product','stage':None,
'filters':{},'currentPeriod':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},
'sort':'rate_drop','totalRows':3566,'rows':[{'label':'小量规格','rateDrop':0.9943484950890867,
'scrapDelta':6.737,'current':{'rate':0,'scrapWeight':9.095,'blackInputWeight':7.1,'brightInputWeight':39.656},
'comparison':{'rate':0.9943484950890867,'scrapWeight':2.358,'blackInputWeight':481.48,'brightInputWeight':417.234}}]}}
report=build_report([root,product])
section=next(s for s in report['sections'] if s.startswith('### 产品率变化线索'))
self.assertNotIn('### 重点产品',section)
self.assertIn('下降99.43个百分点',section)
self.assertIn('黑片投入481.48→7.10 kg',section)
self.assertIn('亮片投入417.23→39.66 kg',section)
self.assertIn('不能只凭率降幅确定重点',section)
self.assertIn('仅是率变化线索',next(f for f in report['facts'] if f['kind']=='product_rate_signal')['text'])
self.assertFalse(any(f['kind']=='product' for f in report['facts']))
def test_stage_rates_and_totals_only_vcm_detail_keep_their_actual_meaning(self):
root=Path(__file__).resolve().parents[1]/'output/20261023-autonomous-investigation/runtime'
paths=list(root.glob('.runtime/nao-users/*/sessions/24e16a62-0527-4bff-ba9f-535005acbfaf/evidence'))
if not paths:self.skipTest('Private real acceptance evidence not present')
report=build_report(evidence_summaries(paths[0],'194b0287-9044-4d80-9138-ecd3c481015c'))
text=render_report(report,[report['facts'][0]['id']],[])
self.assertIn('黑片业务分组变化',text)
self.assertIn('亮片业务分组变化',text)
self.assertIn('黑片合格率:比较期→本期',text)
self.assertFalse(any(row['kind']=='cross' and 'VCM607+Z' in row['label'] for row in report['facts']))
self.assertIn('VCM亮片仅录合计',text)
if __name__=='__main__':unittest.main()
+6 -2
View File
@@ -16,8 +16,12 @@ class FinalizerFallbackTests(unittest.TestCase):
def test_malformed_optional_focus_uses_verified_report_without_failure(self):
evidence = [{"evidenceRef": "Q00000001", "mode": "compare", "result": {
"kind": "quality_comparison",
"current": {"period": {"year": 2026, "month": 9}, "compositeRate": 0.95},
"comparison": {"period": {"year": 2026, "month": 8}, "compositeRate": 0.96},
"current": {"period": {"year": 2026, "month": 9}, "compositeRate": 0.95,
"coverage": {"sourceMonths": [{"year": 2026, "month": 9,
"blackRows": 1, "brightRows": 1}]}},
"comparison": {"period": {"year": 2026, "month": 8}, "compositeRate": 0.96,
"coverage": {"sourceMonths": [{"year": 2026, "month": 8,
"blackRows": 1, "brightRows": 1}]}},
"delta": {"percentagePoints": -1}}}]
bad_responses = [b"not-json",
json.dumps({"choices": [{"finish_reason": "length", "message": {
+652
View File
@@ -0,0 +1,652 @@
import copy
import json
import sys
import tempfile
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from unittest.mock import patch
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
from nao_history_report import (HistorySnapshotError, execute_history_report,
select_history_headline, validate_history_snapshot,
_validate_model_selection)
from nao_model_gate import ModelGate
from nao_protocol_execution import execute
RELEASE = 'release-2026-09'
ATTEMPT = '11111111-1111-4111-8111-111111111111'
SESSION = '22222222-2222-4222-8222-222222222222'
TURN = '33333333-3333-4333-8333-333333333333'
SOURCE_A = '44444444-4444-4444-8444-444444444444'
SOURCE_B = '55555555-5555-4555-8555-555555555555'
def report_task(source_ids=(SOURCE_A, SOURCE_B)):
return {
'version': 1, 'kind': 'report', 'historyMode': 'all', 'freshQuery': False,
'goal': '整合已确认历史质量结果', 'dimensions': [], 'stages': [], 'fields': [],
'comparisonRequired': False, 'trendRequired': False, 'verifySource': False,
'fullDetails': False, 'filters': {}, 'period': None, 'comparisonPeriod': None,
'comparisonMode': None, 'dimensionStages': {},
'historySourceTurnIds': list(source_ids), 'needsClarification': False,
}
def authority(task=None):
return {'attemptId': ATTEMPT, 'sessionId': SESSION, 'turnId': TURN,
'revision': 3, 'executionId': '66666666-6666-4666-8666-666666666666',
'task': copy.deepcopy(task if task is not None else report_task())}
def snapshot(*, unavailable_second=False, facts=None):
sources = [
{'turnId': SOURCE_A, 'revision': 2, 'executionId': 'execution-a',
'status': 'COMPLETED', 'available': True, 'queryIds': ['query-a']},
{'turnId': SOURCE_B, 'revision': 1, 'executionId': 'execution-b',
'status': 'PARTIAL' if unavailable_second else 'COMPLETED',
'available': not unavailable_second,
**({'revision': None, 'executionId': None} if unavailable_second else {'queryIds': ['query-b']}),
**({'reason': '2025年9月海外组没有可用历史来源。'} if unavailable_second else {})},
]
sections = [{
'title': '2026年9月黑片正式报废重量', 'sourceTurnId': SOURCE_A,
'period': {'year': 2026, 'month': 9},
'filters': {'extraFilters': {'customerCode': ['800001']},
'columnFilters': [{'field': 'orderNo', 'stages': ['black'],
'operator': 'in', 'values': ['WO-1']}]},
'stage': 'black',
'columns': [{'key': 'label', 'label': '对象'},
{'key': 'scrapWeight', 'label': '报废重量(kg)'}],
'rows': [{'label': '整体', 'scrapWeight': 1163.6817}],
'note': '历史正式查询结果,本轮未重新计算',
}]
if not unavailable_second:
sections.append({
'title': '2025年9月亮片正式报废重量', 'sourceTurnId': SOURCE_B,
'period': {'year': 2025, 'month': 9}, 'filters': {'group': '汽车交通-海外'},
'stage': 'bright',
'columns': [{'key': 'label', 'label': '对象'},
{'key': 'scrapWeight', 'label': '报废重量(kg)'}],
'rows': [{'label': '整体', 'scrapWeight': 17.25}],
})
return {
'kind': 'protocol_history_snapshot', 'version': 1, 'releaseId': RELEASE,
'historySources': sources, 'sections': sections,
'facts': facts if facts is not None else [
{'id': 'history-fact:F1', 'text': '2026年9月,客户800001,黑片正式报废重量1163.6817 kg。',
'sourceTurnId': SOURCE_A},
*([] if unavailable_second else [{
'id': 'history-fact:F2',
'text': '2025年9月,汽车交通-海外亮片正式报废重量17.25 kg。',
'sourceTurnId': SOURCE_B,
}]),
],
'historyCoverage': {
'complete': not unavailable_second, 'expectedSources': 2,
'importedSources': 1 if unavailable_second else 2,
'missingEvidence': ['2025年9月一条确认历史来源暂不可用。'] if unavailable_second else [],
},
'historyEvidence': {
'importId': 'import-7', 'attemptId': ATTEMPT, 'sessionId': SESSION,
'turnId': TURN, 'revision': 3,
'executionId': '66666666-6666-4666-8666-666666666666',
'snapshotHash': 'sha256:' + 'a' * 64,
},
}
class FakeDelegated:
def __init__(self, response):
self.response = response
self.calls = []
def call(self, path, payload=None, method=None):
self.calls.append((path, copy.deepcopy(payload), method))
if isinstance(self.response, Exception):
raise self.response
return copy.deepcopy(self.response)
class HistorySnapshotValidationTest(unittest.TestCase):
def test_first_mode_keeps_server_confirmed_sources_and_identity_checks(self):
task = report_task((SOURCE_A,)); task['historyMode'] = 'first'
candidate = snapshot()
candidate['historySources'] = candidate['historySources'][:1]
candidate['sections'] = candidate['sections'][:1]
candidate['facts'] = candidate['facts'][:1]
candidate['historyCoverage'].update(expectedSources=1, importedSources=1)
self.assertEqual(validate_history_snapshot(candidate, task=task, authority=authority(task),
release_id=RELEASE, request_turn_id=TURN)['historySources'][0]['turnId'], SOURCE_A)
wrong = copy.deepcopy(candidate); wrong['historySources'][0]['turnId'] = SOURCE_B
with self.assertRaises(HistorySnapshotError):
validate_history_snapshot(wrong, task=task, authority=authority(task), release_id=RELEASE, request_turn_id=TURN)
with self.assertRaises(HistorySnapshotError):
validate_history_snapshot(candidate, task=task, authority=authority(), release_id=RELEASE, request_turn_id=TURN)
def test_autonomous_helper_rejects_unvalidated_identity_before_model_request(self):
from nao_native_answer import synthesize_history_snapshot
candidate = snapshot()
candidate['historyEvidence']['turnId'] = SOURCE_A
with patch('nao_evidence_finalizer._open_model') as model, self.assertRaises(HistorySnapshotError):
synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
question='汇总', snapshot=candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event())
model.assert_not_called()
def test_all_published_source_period_is_exact_and_not_a_calendar_override(self):
candidate = snapshot()
candidate['sections'][0]['period'] = {'allPublished': True}
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN)
for period in ({'allPublished': False}, {'allPublished': 1},
{'allPublished': True, 'year': 2026}):
candidate['sections'][0]['period'] = period
with self.subTest(period=period), self.assertRaises(HistorySnapshotError):
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN)
def test_rejects_wrong_source_release_and_identity(self):
task = report_task()
auth = authority(task)
base = snapshot()
for mutate in (
lambda value: value.update(releaseId='other-release'),
lambda value: value['historySources'][0].update(turnId='unconfirmed'),
lambda value: value['historyEvidence'].update(executionId='other-execution')):
candidate = copy.deepcopy(base)
mutate(candidate)
with self.subTest(candidate=candidate['releaseId']):
with self.assertRaises(HistorySnapshotError):
validate_history_snapshot(candidate, task=task, authority=auth,
release_id=RELEASE, request_turn_id=TURN)
def test_rejects_old_protocol_evidence_even_when_nested_in_section(self):
candidate = snapshot()
candidate['sections'][0]['protocolEvidence'] = {'queryId': 'old-query'}
with self.assertRaisesRegex(HistorySnapshotError, '普通查询证明'):
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN)
def test_rejects_unavailable_source_rows_and_inconsistent_complete_coverage(self):
candidate = snapshot(unavailable_second=True)
candidate['sections'].append(copy.deepcopy(snapshot()['sections'][1]))
with self.assertRaises(HistorySnapshotError):
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN)
candidate = snapshot()
candidate['historyCoverage']['importedSources'] = 1
with self.assertRaisesRegex(HistorySnapshotError, '可用来源不一致'):
validate_history_snapshot(candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN)
def test_preserves_long_record_cells_but_enforces_global_snapshot_capacity(self):
long_cell = '原始长文本' * 600
candidate = snapshot()
candidate['sections'][0]['columns'].append({'key': 'sourceText', 'label': '原始文本'})
candidate['sections'][0]['rows'][0]['sourceText'] = long_cell
validated = validate_history_snapshot(candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN)
self.assertEqual(validated['sections'][0]['rows'][0]['sourceText'], long_cell)
oversized = snapshot()
large_value = 'x' * 3_500_000
columns = [{'key': f'cell{index}', 'label': f'文本{index}'} for index in range(5)]
oversized['sections'][0]['columns'] = columns
oversized['sections'][0]['rows'] = [{column['key']: large_value for column in columns}]
with self.assertRaisesRegex(HistorySnapshotError, '快照超出允许大小'):
validate_history_snapshot(oversized, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN)
class HistoryReportExecutionTest(unittest.TestCase):
def test_scoped_formula_context_reaches_synthesis_and_the_same_review(self):
from nao_native_answer import synthesize_history_snapshot
from test_native_answer import response
for product in (True, False):
candidate = snapshot()
task = report_task((SOURCE_A,))
candidate['historySources'] = candidate['historySources'][:1]
candidate['sections'] = candidate['sections'][:1]
candidate['facts'] = candidate['facts'][:1]
candidate['historyCoverage'].update(expectedSources=1, importedSources=1)
rules = {'supportsStageContribution': product, 'metricRules': {'compositeRate': {
'formula': 'blackRate*brightRate' if product else '(blackRate+brightRate)/2'}}}
context = {'ruleVersionId': 'formal-version', 'calculationRules': rules,
'metricFormulaDescriptions': {'compositeRate': '正式规则说明'}}
candidate['sections'][0]['metricContext'] = context
candidate['historySources'][0]['metricContexts'] = [context]
candidate['facts'][0].update(sectionIndex=0, text='2026年9月综合合格率 = 0.9531',
displayText='2026年9月综合合格率95.31%。')
phases = []
def provider(request, timeout):
body = json.loads(request.data); payload = json.loads(body['messages'][1]['content'])
semantics = payload['metricSemantics']
self.assertIs(semantics['productConfirmed'], product)
supplied = semantics['scopedMetricContexts'][0]
self.assertEqual(supplied['calculationRules'], rules)
self.assertEqual(payload['scopes'][0]['metricContextId'], supplied['id'])
if body['messages'][0]['content'].startswith('逐句核验证据'):
phases.append('review')
return response({'reviews': [{'supported': True, 'statisticalClaimsSupported': True}], 'missingGoalDescriptions': []})
phases.append('synthesis')
return response({'blocks': [{'role': 'core', 'text': '本期综合合格率95.31%,沿用本轮正式组合规则解释。', 'factIds': ['F1']}]})
with self.subTest(product=product):
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
question='汇总历史结果', snapshot=candidate, task=task, authority=authority(task),
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
self.assertTrue(result['evidenceSufficient'], result['compositionDiagnostics'])
self.assertEqual(phases, ['synthesis', 'review'])
def test_display_percent_and_short_month_use_verified_history_metadata(self):
from nao_native_answer import synthesize_history_snapshot
from test_native_answer import response
candidate = snapshot()
candidate['facts'][0].update(text='2026年9月综合合格率 = 0.9530615',
displayText='2026年9月综合合格率95.31%。', sectionIndex=0)
for text, expected in (('9月综合合格率95.31%。', True), ('10月综合合格率95.31%。', False),
('9月综合合格率95.31 kg。', False), ('9月综合合格率99.31%。', False)):
def provider(request, timeout):
if json.loads(request.data)['messages'][0]['content'].startswith('逐句核验证据'):
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
return response({'blocks': [{'text': text, 'factIds': ['F1']}]})
with self.subTest(text=text):
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
question='历史指标', snapshot=candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
self.assertEqual(result['evidenceSufficient'], expected, result['compositionDiagnostics'])
if expected:
self.assertIn('95.31%', result['answer'])
self.assertFalse(result['compositionRepairAttempted'])
def test_many_history_facts_share_compact_frames_and_keep_later_product(self):
from nao_native_answer import synthesize_history_snapshot
from test_native_answer import response
candidate = snapshot()
source = candidate['historySources'][0]['turnId']
product = '16.33X15.5X3.75+T'
candidate['sections'].append({**copy.deepcopy(candidate['sections'][0]),
'title': '重点规格指标', 'filters': {'product': product},
'columns': [{'key': 'product', 'label': '产品'}, {'key': 'weight', 'label': '报废重量'}],
'rows': [{'product': product, 'weight': 100}]})
candidate['facts'] = [{'id': f'history:{index}', 'sourceTurnId': source, 'sectionIndex': 0,
'text': f'2026年9月黑片报废重量{index}.00 kg。'} for index in range(120)]
candidate['facts'].append({'id': 'history:late-product', 'sourceTurnId': source, 'sectionIndex': 2,
'text': '2026年9月'+product+'黑片报废重量100.00 kg。'})
def provider(request, timeout):
body = json.loads(request.data)
if body['messages'][0]['content'].startswith('逐句核验证据'):
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
payload = json.loads(body['messages'][1]['content'])
self.assertEqual(len(payload['scopes']), 2)
self.assertGreaterEqual(len(payload['facts']), 60)
self.assertFalse(any('rows' in scope or 'historyFactId' in scope for scope in payload['scopes']))
late = next(fact for fact in payload['facts'] if fact['evidenceRef'] == 'history:late-product')
return response({'blocks': [{'text': product+'黑片报废重量100.00 kg,是应核查的规格线索。', 'factIds': [late['id']]}]})
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
question='汇总重点规格', snapshot=candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
self.assertTrue(result['evidenceSufficient'], result['compositionDiagnostics'])
self.assertIn(product, result['readingSummary']['lead'])
self.assertLess(result['compositionDiagnostics']['factPool']['modelEvidenceChars'], 30000)
def test_verified_row_values_remain_visible_to_synthesis_and_same_review(self):
from nao_native_answer import synthesize_history_snapshot
from test_native_answer import response
candidate = snapshot()
candidate['sections'][0].update(title='已保存阶段聚合', stage='black',
columns=[{'key': 'metric', 'label': '指标'}, {'key': 'value', 'label': '正式值'}],
rows=[{'metric': 'rate', 'value': .9531}, {'metric': 'totalWeight', 'value': 100.00}])
candidate['facts'][0].update(text='2026年9月黑片合格率 = 0.9531', displayText='2026年9月黑片合格率95.31%。', sectionIndex=0,
metricCode='rate', sourceKind='native_saved_aggregate', scope={'period': {'year': 2026, 'month': 9}, 'filters': candidate['sections'][0]['filters'], 'stage': 'black'},
values=[{'metricCode': 'currentBlackInput', 'unit': 'source_weight', 'displayValue': '100.00'}])
def provider(request, timeout):
body = json.loads(request.data); payload = json.loads(body['messages'][1]['content'])
current = next(fact for fact in payload['facts'] if fact['id'] == 'F1')
self.assertEqual(current['kind'], 'stage')
self.assertEqual(current['values'][0]['metricCode'], 'currentBlackInput')
if body['messages'][0]['content'].startswith('逐句核验证据'):
self.assertIn('来源重量100.00', payload['blocks'][0]['text'])
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
return response({'blocks': [{'text': '本期黑片合格率95.31%,黑片投入来源重量100.00,应结合阶段变化核查。', 'factIds': ['F1']}]})
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
question='汇总历史', snapshot=candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
self.assertTrue(result['evidenceSufficient'], result['compositionDiagnostics'])
self.assertNotIn('kg', result['answer'])
def test_global_product_history_keeps_server_row_identifier_and_rejects_dimension_typo(self):
from nao_native_answer import synthesize_history_snapshot
from test_native_answer import response, fixture_payload
product = '16.33X15.5X3.75+T'
candidate = snapshot()
candidate['sections'][0].update(title='Top 2 product排名', filters={},
columns=[{'key': key, 'label': key} for key in ('rank', 'key', 'label', 'weight')],
rows=[{'rank': 1, 'key': product, 'label': product, 'weight': 100.00},
{'rank': 2, 'key': '18X15X4+T', 'label': '18X15X4+T', 'weight': 90.00}])
candidate['facts'][0]['text'] = '2026年9月;全局产品排名:'+product+',黑片报废重量100.00 kg。'
for typo in (False, True):
with self.subTest(typo=typo):
calls = []
def provider(request, timeout):
body = json.loads(request.data); calls.append(body)
if body['messages'][0]['content'].startswith('逐句核验证据'):
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
payload = fixture_payload(request)
current = next(fact for fact in payload['facts'] if fact['id'] == 'F1')
scope = next(scope for scope in payload['scopes'] if scope['scopeId'] == current['scopeId'])
self.assertTrue(any(item['value'] == product and item['factId'] == 'F1' for item in payload['identifierTokens'].values()))
value_id = next(key for key, token in payload['numericTokens'].items()
if token['factId'] == 'F1' and token['value'] == '100.00')
name = product.replace('3.75', '3.76') if typo else product
return response({'blocks': [{'text': '全局产品排名中,'+name+'的黑片报废重量为{{'+value_id+'}} kg,应结合投入规模核查。',
'factIds': ['F1'], 'evidenceRefs': ['history-fact:F1'], 'claimType': 'inference'}]})
result = synthesize_history_snapshot(base_url='https://offline.invalid/v1', api_key='fixture', model='fixture',
question='解读全局产品历史排名', snapshot=candidate, task=report_task(), authority=authority(),
release_id=RELEASE, request_turn_id=TURN, cancel_event=threading.Event(), opener=provider)
self.assertEqual(result['evidenceSufficient'], not typo, result.get('compositionDiagnostics'))
if not typo:
self.assertIn(product, result['answer'])
self.assertEqual(result['readingSummary']['evidenceRefs'], ['history-fact:F1'])
self.assertTrue(result['compositionDiagnostics']['identifierBindings'])
else:
self.assertTrue(result['compositionRepairAttempted'])
self.assertNotIn('3.76', result['answer'])
def test_autonomous_report_uses_shared_free_synthesis_and_actual_history_references(self):
from test_native_answer import response
calls = []
def provider(request, timeout):
body = json.loads(request.data); calls.append(body)
if body['messages'][0]['content'].startswith('逐句核验证据'):
return response({'reviews': [{'index': 0, 'supported': True}], 'missingGoalDescriptions': []})
payload = json.loads(body['messages'][1]['content'])
self.assertEqual([fact['evidenceRef'] for fact in payload['facts']], ['history-fact:F1', 'history-fact:F2'])
scopes = {scope['scopeId']: scope for scope in payload['scopes']}
self.assertTrue(all('period' in scopes[fact['scopeId']] for fact in payload['facts']))
self.assertTrue(all('historyFactId' not in scope for scope in scopes.values()))
return response({'blocks': [{'text': '历史来源各自适用不同期间和筛选,应分别解读各自的报废重量。',
'factIds': ['F1', 'F2'], 'evidenceRefs': ['history-fact:F1', 'history-fact:F2'], 'claimType': 'inference'}]})
with tempfile.TemporaryDirectory() as folder, patch('nao_evidence_finalizer._open_model', side_effect=provider), patch(
'nao_history_report.select_history_headline', side_effect=AssertionError('must use shared synthesis')):
result = execute_history_report(request={'releaseId': RELEASE, 'turnId': TURN, 'question': '汇总历史', '_autonomousAnswer': True},
task=report_task(), authority=authority(), actor_root=Path(folder), delegated=FakeDelegated(snapshot()),
connection=('https://offline.invalid/v1', 'fixture', 'fixture'), cancel_event=threading.Event(), emit=lambda _: None, timings={})
private = json.loads((Path(folder) / 'private/native-composition.json').read_text())
self.assertEqual(result['status'], 'COMPLETED')
self.assertIn('各自适用不同期间和筛选', result['answer'])
self.assertEqual(result['readingSummary']['evidenceRefs'], ['history-fact:F1', 'history-fact:F2'])
self.assertEqual(result['historyHeadlineFactIds'], ['history-fact:F1', 'history-fact:F2'])
self.assertEqual(result['historyCoverage'], snapshot()['historyCoverage'])
self.assertEqual(len(calls), 2)
self.assertEqual(len(private['rawModelResponses']), 2)
self.assertNotIn('rawModelResponses', result)
def test_autonomous_raw_or_member_only_snapshot_has_explicit_limitation(self):
with tempfile.TemporaryDirectory() as folder, patch('nao_evidence_finalizer._open_model') as model:
result = execute_history_report(request={'releaseId': RELEASE, 'turnId': TURN, 'question': '汇总历史', '_autonomousAnswer': True},
task=report_task(), authority=authority(), actor_root=Path(folder), delegated=FakeDelegated(snapshot(facts=[])),
connection=('https://offline.invalid/v1', 'fixture', 'fixture'), cancel_event=threading.Event(), emit=lambda _: None, timings={})
self.assertEqual(result['status'], 'PARTIAL')
self.assertIn('平台附录展示', result['answer'])
self.assertTrue(result['readingSummary']['notices'])
self.assertEqual(result['historyCoverage'], snapshot()['historyCoverage'])
model.assert_not_called()
def run_report(self, response, model_result=None, *, model_results=None, cancel_after_model=False):
delegated = FakeDelegated(response)
emitted, model_payloads = [], []
cancel_event = threading.Event()
queued_results = list(model_results) if model_results is not None else []
def fake_model_json(**kwargs):
model_payloads.append(kwargs['payload'])
if cancel_after_model:
cancel_event.set()
result = queued_results.pop(0) if queued_results else model_result
if isinstance(result, Exception):
raise result
return result
with tempfile.TemporaryDirectory() as temp, patch('nao_agent_task._model_json', side_effect=fake_model_json):
result = execute(
request={'releaseId': RELEASE, 'turnId': TURN,
'question': '请汇总已确认历史质量报告'},
task=report_task(), authority=authority(), actor_root=Path(temp),
delegated=delegated, connection=('https://offline.invalid/v1', 'secret', 'model-x'),
cancel_event=cancel_event, emit=emitted.append, phase_timings={})
return result, delegated, emitted, model_payloads
def test_imports_server_snapshot_and_models_only_title_and_existing_fact_ids(self):
result, delegated, emitted, payloads = self.run_report(
snapshot(), {'reportTitle': '2025至2026年质量表现',
'headlineFactIds': ['history-fact:F1', 'history-fact:F2']})
self.assertEqual(delegated.calls, [('/api/analysis/protocol/history', {}, 'POST')])
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['completionMethod'], 'confirmed_history_report')
self.assertEqual(result['stopReason'], 'history_report_complete')
self.assertTrue(result['titleGeneratedByModel'])
self.assertEqual(result['protocolEvidence'], [])
self.assertEqual(result['historyHeadlineFactIds'], ['history-fact:F1', 'history-fact:F2'])
self.assertEqual(result['historyCoverage'], snapshot()['historyCoverage'])
self.assertEqual(result['agentTask'], authority()['task'])
self.assertNotIn('agentContext', result)
self.assertEqual(result['visualState'], 'not_requested')
self.assertEqual(result['visuals'], [])
self.assertEqual(len(emitted), 2)
self.assertEqual(emitted[-1]['type'], 'result')
self.assertEqual(len(payloads), 1)
serialized = json.dumps(payloads[0], ensure_ascii=False)
self.assertNotIn('rows', payloads[0])
self.assertNotIn('整体', serialized)
self.assertIn('history-fact:F1', serialized)
self.assertEqual(len(payloads[0]['historySources']), 2)
self.assertIn('columnFilters', serialized)
def test_incomplete_history_remains_partial_and_includes_unavailable_source_reason(self):
result, _, _, _ = self.run_report(
snapshot(unavailable_second=True),
{'reportTitle': '2026年9月客户质量结果', 'headlineFactIds': ['history-fact:F1']})
self.assertEqual(result['status'], 'PARTIAL')
self.assertEqual(result['stopReason'], 'history_report_partial')
self.assertIn('2025年9月海外组没有可用历史来源。',
result['evidenceCompletion']['missingEvidence'])
def test_available_partial_source_is_preserved_but_not_marked_complete(self):
partial = snapshot()
partial['historySources'][1]['status'] = 'PARTIAL'
partial['historyCoverage'] = {
'complete': False, 'expectedSources': 2, 'importedSources': 2,
'missingEvidence': ['历史范围中有一项来源覆盖不完整。'],
}
result, _, _, _ = self.run_report(
partial, {'reportTitle': '2025至2026年质量表现',
'headlineFactIds': ['history-fact:F1']})
self.assertEqual(result['status'], 'PARTIAL')
self.assertEqual(result['evidenceCompletion']['serverCoverageSufficient'], False)
def test_unknown_fact_invalid_title_and_model_failure_fall_back_to_partial(self):
invalid_results = [
{'reportTitle': '2026年9月客户质量结果', 'headlineFactIds': ['not-a-fact']},
{'reportTitle': '<b>质量报告</b>', 'headlineFactIds': ['history-fact:F1']},
RuntimeError('model unavailable'),
]
for invalid in invalid_results:
with self.subTest(invalid=repr(invalid)):
result, _, _, _ = self.run_report(snapshot(), invalid)
self.assertEqual(result['status'], 'PARTIAL')
self.assertFalse(result['titleGeneratedByModel'])
self.assertEqual(result['reportTitle'], '历史质量分析报告')
self.assertEqual(result['historyHeadlineFactIds'], [])
def test_mixed_customer_and_unfiltered_source_repairs_title_to_multi_scope(self):
candidate = snapshot()
candidate['sections'][1]['filters'] = {'group': '汽车交通-海外'}
bad = {'reportTitle': '2026年9月客户800001黑片亮片报废重量',
'headlineFactIds': ['history-fact:F1']}
repaired = {'reportTitle': '多范围汇总质量分析',
'headlineFactIds': ['history-fact:F1', 'history-fact:F2']}
result, delegated, _, payloads = self.run_report(candidate, model_results=[bad, repaired])
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['reportTitle'], '多范围汇总质量分析')
self.assertEqual(len(payloads), 2)
self.assertEqual(payloads[0]['facts'], payloads[1]['facts'])
self.assertEqual(payloads[0]['historySources'], payloads[1]['historySources'])
self.assertIn('repairInstructions', payloads[1])
self.assertEqual(delegated.calls, [('/api/analysis/protocol/history', {}, 'POST')])
self.assertNotIn('rows', payloads[1])
def test_mixed_product_scope_is_not_named_but_common_product_can_be_named(self):
mixed = snapshot()
mixed['sections'][0]['filters']['product'] = '磁钢规格-X'
bad = {'reportTitle': '2026年磁钢规格-X质量汇总',
'headlineFactIds': ['history-fact:F1']}
repaired = {'reportTitle': '多范围汇总质量分析',
'headlineFactIds': ['history-fact:F1']}
result, _, _, payloads = self.run_report(mixed, model_results=[bad, repaired])
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['reportTitle'], '多范围汇总质量分析')
self.assertEqual(len(payloads), 2)
common = snapshot()
common['sections'][0]['filters']['product'] = '磁钢规格-X'
common['sections'][1]['filters']['product'] = '磁钢规格-X'
common_title = {'reportTitle': '2026年磁钢规格-X质量汇总',
'headlineFactIds': ['history-fact:F1']}
result, _, _, payloads = self.run_report(common, common_title)
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['reportTitle'], common_title['reportTitle'])
self.assertEqual(len(payloads), 1)
def test_noncommon_column_filter_values_are_blocked_but_field_ids_are_not(self):
candidate = snapshot()
facts = candidate['facts']
self.assertIsNone(_validate_model_selection(
{'reportTitle': '2026年WO-1订单质量汇总', 'headlineFactIds': ['history-fact:F1']},
candidate, '整理历史报告', facts))
self.assertIsNotNone(_validate_model_selection(
{'reportTitle': 'orderNo质量汇总', 'headlineFactIds': ['history-fact:F1']},
candidate, '整理历史报告', facts))
def test_negative_filter_literals_are_never_named_even_when_common(self):
for negative_filter in (
{'columnFilters': [{'field': 'customerCode', 'operator': 'not_in',
'values': ['800001']}]},
{'exclusions': {'product': ['磁钢规格-X']}}):
candidate = snapshot()
candidate['sections'][0]['filters'] = copy.deepcopy(negative_filter)
candidate['sections'][1]['filters'] = copy.deepcopy(negative_filter)
title = ('2026年客户800001质量汇总' if 'columnFilters' in negative_filter
else '2026年磁钢规格-X质量汇总')
with self.subTest(negative_filter=negative_filter):
self.assertIsNone(_validate_model_selection(
{'reportTitle': title, 'headlineFactIds': ['history-fact:F1']},
candidate, '整理历史报告', candidate['facts']))
def test_common_positive_in_filter_literal_may_be_named(self):
candidate = snapshot()
common_filter = {'columnFilters': [{'field': 'customerCode', 'operator': 'in',
'values': ['800001']}]}
candidate['sections'][0]['filters'] = copy.deepcopy(common_filter)
candidate['sections'][1]['filters'] = copy.deepcopy(common_filter)
self.assertIsNotNone(_validate_model_selection(
{'reportTitle': '2026年客户800001质量汇总',
'headlineFactIds': ['history-fact:F1']},
candidate, '整理历史报告', candidate['facts']))
def test_title_repair_is_limited_to_one_retry(self):
candidate = snapshot()
candidate['sections'][1]['filters'] = {}
bad = {'reportTitle': '2026年9月客户800001质量汇总',
'headlineFactIds': ['history-fact:F1']}
result, _, _, payloads = self.run_report(candidate, model_results=[bad, bad])
self.assertEqual(result['status'], 'PARTIAL')
self.assertFalse(result['titleGeneratedByModel'])
self.assertEqual(len(payloads), 2)
def test_no_facts_skips_model_and_is_partial(self):
empty = snapshot(facts=[])
result, _, _, payloads = self.run_report(empty, None)
self.assertEqual(result['status'], 'PARTIAL')
self.assertEqual(payloads, [])
self.assertIn('没有可引用事实', result['evidenceCompletion']['missingEvidence'][0])
def test_cancellation_after_snapshot_keeps_partial_and_does_not_accept_late_title(self):
result, _, _, payloads = self.run_report(
snapshot(), {'reportTitle': '2025至2026年质量表现',
'headlineFactIds': ['history-fact:F1']}, cancel_after_model=True)
self.assertEqual(len(payloads), 1)
self.assertEqual(result['status'], 'PARTIAL')
self.assertFalse(result['titleGeneratedByModel'])
def test_api_failure_is_not_replaced_by_a_local_history_scan(self):
delegated = FakeDelegated(RuntimeError('history service unavailable'))
with tempfile.TemporaryDirectory() as temp:
with self.assertRaisesRegex(RuntimeError, 'history service unavailable'):
execute_history_report(
request={'releaseId': RELEASE, 'turnId': TURN, 'question': '报告'},
task=report_task(), authority=authority(), actor_root=Path(temp),
delegated=delegated, connection=('u', 'k', 'm'),
cancel_event=threading.Event(), emit=lambda _: None, timings={})
self.assertEqual(delegated.calls, [('/api/analysis/protocol/history', {}, 'POST')])
def test_headline_uses_authorized_finalizing_phase_through_real_model_gate(self):
class ProviderHandler(BaseHTTPRequestHandler):
def log_message(self, *_):
pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
owner.provider_calls.append((self.headers.get('Authorization'), body))
selection = ({'reportTitle': '2026年9月客户800001质量汇总',
'headlineFactIds': ['history-fact:F1']}
if len(owner.provider_calls) == 1 else
{'reportTitle': '多范围汇总质量表现',
'headlineFactIds': ['history-fact:F1']})
content = json.dumps(selection, ensure_ascii=False)
response = json.dumps({'model': 'history-model', 'choices': [{
'finish_reason': 'stop', 'message': {'content': content},
}]}, ensure_ascii=False).encode()
self.send_response(200)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(response)))
self.end_headers()
self.wfile.write(response)
owner = self
self.provider_calls = []
provider = ThreadingHTTPServer(('127.0.0.1', 0), ProviderHandler)
provider.daemon_threads = True
provider_thread = threading.Thread(target=provider.serve_forever, daemon=True)
provider_thread.start()
authorize_calls = []
gate = ModelGate(
upstream_base=f'http://127.0.0.1:{provider.server_port}/v1',
upstream_key='fake-provider-key', model='history-model', port=0,
client_key='fake-worker-key', authorize=lambda: authorize_calls.append(True),
native_turn_marker='quality-turn:current-history-turn')
try:
selection = select_history_headline(
snapshot=snapshot(), question='整理已确认历史报告',
connection=(f'http://127.0.0.1:{gate._server.server_port}/v1',
gate.client_key, 'history-model'),
cancel_event=threading.Event())
self.assertEqual(selection, ('多范围汇总质量表现', ['history-fact:F1']))
self.assertGreaterEqual(len(authorize_calls), 2)
self.assertEqual(gate.rejected_stale_requests, 0)
self.assertEqual(len(self.provider_calls), 2)
for authorization, body in self.provider_calls:
self.assertEqual(authorization, 'Bearer fake-provider-key')
self.assertEqual(body['model'], 'history-model')
self.assertEqual(len(body['messages']), 2)
self.assertNotIn('sections', body['messages'][1]['content'])
self.assertIn('repairInstructions', self.provider_calls[1][1]['messages'][1]['content'])
finally:
gate.close()
provider.shutdown()
provider.server_close()
provider_thread.join(timeout=1)
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,192 @@
import copy
import sys
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_agent_task import (TaskUnavailable, _explicit_history_refresh_intent,
normalize_task, plan_task)
from test_nao_agent_task import task_value
def history_plan(**changes):
value = task_value(kind='report', freshQuery=False, historyMode='all', dimensions=[],
stages=[], fields=[], comparisonRequired=False, trendRequired=False,
verifySource=False, fullDetails=False, filterUpdates={})
value.update(changes)
return value
class HistoryReportTaskTest(unittest.TestCase):
def test_refresh_intent_respects_negation_without_weakening_positive_refresh(self):
for question in ('整理历史报告,不重新查询', '不要重新查数据,只整理已完成结果',
'无需重查,汇总历史报告', '不需要重新核验数据',
'不用最新数据,整理历史', '不需要重新查最新数据,只整理历史'):
with self.subTest(question=question):
self.assertFalse(_explicit_history_refresh_intent(question))
for question in ('不要复用历史,重新查询当前数据', '重新跑数据后更新报告',
'最新数据需要重新核验', '必须重查当前数据'):
with self.subTest(question=question):
self.assertTrue(_explicit_history_refresh_intent(question))
task = normalize_task(history_plan(), '整理已完成历史结果,不重新查询',
{'_protocol': {'version': 1}})
self.assertEqual(task['historyMode'], 'all')
with self.assertRaises(TaskUnavailable):
normalize_task(history_plan(), '不要复用历史,重新查询数据',
{'_protocol': {'version': 1}})
def test_history_report_does_not_inherit_local_filters_or_period(self):
context = {
'_protocol': {'version': 1},
'filters': {'department': '新能源汽车磁钢',
'extraFilters': {'customerCode': ['800001']}},
'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 9},
'comparisonPeriod': {'year': 2025, 'monthStart': 1, 'monthEnd': 9},
}
task = normalize_task(history_plan(), '重新生成以上报告,正文突出重点', context)
self.assertEqual(task['kind'], 'report')
self.assertEqual(task['historyMode'], 'all')
self.assertFalse(task['freshQuery'])
self.assertEqual(task['filters'], {})
self.assertIsNone(task['period'])
self.assertIsNone(task['comparisonPeriod'])
self.assertEqual(task['dimensions'], [])
self.assertEqual(task['stages'], [])
self.assertEqual(task['fields'], [])
self.assertEqual(task['dimensionStages'], {})
self.assertNotIn('queryPlan', task)
self.assertFalse(task['comparisonRequired'])
self.assertFalse(task['trendRequired'])
self.assertFalse(task['verifySource'])
self.assertFalse(task['fullDetails'])
self.assertEqual(task['visualPreference'], 'none')
def test_new_filter_period_and_refresh_intent_cannot_be_silently_dropped(self):
invalid = [
(history_plan(filterUpdates={'department': '消费电子'}), '整理历史报告'),
(history_plan(period={'year': 2026, 'month': 9}), '整理历史报告'),
(history_plan(comparisonPeriod={'year': 2025, 'month': 9}), '整理历史报告'),
(history_plan(), '重新查询数据并更新报告'),
(history_plan(freshQuery=True), '整理历史报告'),
(history_plan(fields=['rate']), '整理历史报告'),
(history_plan(historyMode='none'), '整理历史报告'),
]
for value, question in invalid:
with self.subTest(value=value, question=question):
with self.assertRaises(TaskUnavailable):
normalize_task(value, question, {'_protocol': {'version': 1},
'filters': {'columnFilters': [{'field': 'code'}]}})
column_rule = {'field': 'orderNo', 'stages': ['black'],
'operator': 'in', 'values': ['WO-1']}
context = {'_protocol': {'version': 1},
'_businessCatalog': {'sourceFields': [{
'key': 'orderNo', 'label': '委托单号', 'type': 'source_value',
'stages': ['black'], 'columns': {'black': '委托单号'},
'operators': ['in'],
}]}}
with self.assertRaises(TaskUnavailable) as error:
normalize_task(history_plan(columnFilters=[column_rule]), '整理历史报告', context)
self.assertEqual(error.exception.field, 'freshQuery')
def test_existing_record_and_source_receipts_can_be_preserved_without_new_reads(self):
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False}
with patch('nao_agent_task._model_json', side_effect=[history_plan(), review]):
planned = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question='把已完成的全部逐行记录和原件核验结果整合成历史报告,不重新打开原件',
context={'_protocol': {'version': 1}}, cancel_event=threading.Event())
self.assertEqual(planned['kind'], 'report')
self.assertFalse(planned['fullDetails'])
self.assertFalse(planned['verifySource'])
def test_history_report_does_not_override_an_independent_review_requiring_new_rows(self):
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
'requiresAllSourceRows': True, 'requiresTrend': False, 'requiresComparison': False}
with patch('nao_agent_task._model_json', side_effect=[history_plan(), review] * 3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question='把该规格本月每条原始记录及全部字段列入报告',
context={'_protocol': {'version': 1}}, cancel_event=threading.Event())
def test_legacy_non_protocol_report_is_not_changed(self):
value = history_plan()
task = normalize_task(value, '整理历史报告', {})
self.assertEqual(task['kind'], 'report')
self.assertEqual(task['historyMode'], 'all')
def test_protocol_planner_prompt_exposes_bounded_history_report_contract(self):
replies = [history_plan(), {
'aligned': True, 'requiresEvidence': True,
'requiresSourceVerification': False, 'requiresAllSourceRows': False,
'requiresTrend': False, 'requiresComparison': False,
}]
calls = []
def fake_model(**kwargs):
calls.append(kwargs)
return replies.pop(0)
with patch('nao_agent_task._model_json', side_effect=fake_model):
planned = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question='整理当前会话已经完成的分析结果',
context={'_protocol': {'version': 1}},
cancel_event=threading.Event())
self.assertEqual(planned['kind'], 'report')
self.assertEqual(len(calls), 2)
self.assertIn('仅在用户要求汇总/重排已完成历史时用kind=report', calls[0]['system'])
self.assertIn('historyMode=last或all、freshQuery=false', calls[0]['system'])
self.assertIn('previousVerifiedTurns为空或缺失不表示会话没有已分析来源', calls[0]['system'])
self.assertIn('用户说不重新查询、不要重查、无需重新查时是复用历史', calls[0]['system'])
self.assertEqual(calls[0]['payload']['previousVerifiedTurns'], [])
self.assertIn('不能据此判断历史数据不存在', calls[1]['system'])
self.assertEqual(calls[1]['payload']['previousVerifiedTurns'], [])
capabilities = calls[0]['payload']['platformCapabilities']
self.assertIn('confirmed_history_report', capabilities['enabled'])
self.assertNotIn('report', capabilities['unsupported'])
def test_report_review_does_not_rebind_prior_customer_product_or_column_scope(self):
column_rule = {'field': 'orderNo', 'stages': ['black'],
'operator': 'in', 'values': ['WO-1']}
previous = {
'kind': 'analysis', 'goal': '客户800001规格SPEC-A的正式质量分析',
'question': '查询客户800001规格SPEC-A', 'period': {'year': 2026, 'month': 9},
'dimensions': ['product'], 'fields': ['rate'], 'stages': ['black'],
'filters': {'extraFilters': {'customerCode': ['800001']}, 'product': 'SPEC-A',
'columnFilters': [column_rule]},
}
context = {
'_protocol': {'version': 1}, '_agentTask': previous,
'filters': copy.deepcopy(previous['filters']), 'period': previous['period'],
'_businessCatalog': {
'extraFields': [{'key': 'customerCode', 'label': '客户代码'}],
'sourceFields': [{'key': 'orderNo', 'label': '委托单号', 'type': 'source_value',
'stages': ['black'], 'columns': {'black': '委托单号'},
'operators': ['in']}],
'productMatch': {'status': 'unique', 'query': 'SPEC-A',
'candidates': [{'name': 'SPEC-A'}]},
},
}
replies = [history_plan(), {
'aligned': True, 'requiresEvidence': True,
'requiresSourceVerification': False, 'requiresAllSourceRows': False,
'requiresTrend': False, 'requiresComparison': False,
'customerCodes': ['800001'], 'productScope': 'single',
}]
calls = []
def fake_model(**kwargs):
calls.append(kwargs)
return replies.pop(0)
with patch('nao_agent_task._model_json', side_effect=fake_model):
planned = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question='整理当前会话已完成的历史报告,沿用各轮范围',
context=context, previous=previous, cancel_event=threading.Event())
self.assertEqual(planned['kind'], 'report')
self.assertEqual(planned['filters'], {})
self.assertIsNone(planned['period'])
self.assertEqual(len(calls), 2)
self.assertFalse(calls[1]['payload']['columnFilterReviewRequired'])
if __name__ == '__main__':
unittest.main()
+44
View File
@@ -0,0 +1,44 @@
"""Strict tool scope and pagination/source completeness contracts."""
import copy,json,sys,tempfile,unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parent))
from test_nao_quality_tools import FakeApi
from nao_quality_tools import QualityTools,QualityToolError
class Api(FakeApi):
mutation=None
def call(self,path,payload=None,method=None):
if path=='/api/analysis/investigate':
context={k:payload.get(k) for k in ('releaseId','year','month','quarter','department','group','product','projectGroup','category','stage')};context['includeUnassigned']=payload.get('includeUnassigned',False);context['extraFilters']=payload.get('extraFilters',{})
r={'kind':'quality_investigation','action':payload['action'],'context':context,'aggregateComplete':True,'totalRows':205,'categories':[],'lineage':[]}
if payload['action']=='source':r['sourceVerification']={'complete':True,'expectedRows':205,'verifiedRows':205,'missingRows':0,'mismatchRows':0}
if payload['action']=='rows':r.update(rows=[{'recordId':1}],offset=payload.get('offset',0),returnedRows=1,matchingRows=205,hasMore=True,nextOffset=payload.get('offset',0)+1)
if self.mutation:self.mutation(r)
return r
return super().call(path,payload,method=method)
class TestInvestigation(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.api=Api();self.tools=QualityTools(api_factory=lambda:self.api,evidence_dir=Path(self.tmp.name));self.args={'mode':'investigate','releaseId':'release-a','period':{'year':2026,'month':9},'stage':'bright','action':'source','filters':{'department':'工业电机产业部'}}
def tearDown(self):self.tmp.cleanup()
def test_both_periods_and_evidence_are_preserved(self):
args={**self.args,'comparisonPeriod':{'year':2026,'month':8}};r=self.tools.quality_data(args);self.assertEqual(r['current']['totalRows'],205);self.assertEqual(r['comparison']['context']['month'],8);self.assertTrue(r['evidenceRef'].startswith('Q'));self.assertTrue(Path(r['evidencePath']).exists())
def test_scope_mismatch_refused(self):
self.api.mutation=lambda r:r['context'].update(department='其他');self.assertRaises(QualityToolError,self.tools.quality_data,self.args)
def test_false_source_completeness_refused(self):
self.api.mutation=lambda r:r['sourceVerification'].update(verifiedRows=5);self.assertRaises(QualityToolError,self.tools.quality_data,self.args)
def test_bad_pagination_refused(self):
self.api.mutation=lambda r:r.update(nextOffset=0);self.assertRaises(QualityToolError,self.tools.quality_data,{**self.args,'action':'rows'})
def test_extra_parameters_never_ignored(self):
for extra in ({'sql':'select 1'},{'offset':True},{'fields':['x']*33},{'sourceRevisionId':'../x'},{'sort':'rate_drop'}):
with self.subTest(extra=extra):self.assertRaises(QualityToolError,self.tools.quality_data,{**self.args,**extra})
def test_source_only_parameters_rejected_for_other_modes(self):
self.assertRaises(QualityToolError,self.tools.quality_data,{**self.args,'mode':'query'})
class CompactEvidenceTests(unittest.TestCase):
def test_compression_preserves_full_evidence_and_proof_counts(self):
full={"kind":"quality_investigation","action":"source","current":{"totalRows":205,"fieldDictionary":[{"duplicate":True}],"categories":[{"differenceFormalMinusSubitems":5.7822,"fields":[{"field":"x","coverage":{"rows":205,"numericRows":200,"nullOrBlankRows":5,"keyAbsentRows":0,"invalidNumericRows":0,"negativeRows":0,"numericSum":10}}]}],"sourceVerification":{"complete":True,"allSelectedValuesMatchStored":True,"verifiedRows":205,"files":[{"sha256":"original","fields":[{"field":"x"}]}]}}}
view=QualityTools._model_view(full)
self.assertIn("fields",full["current"]["categories"][0]);self.assertIn("fieldDictionary",full["current"])
self.assertEqual(view["current"]["categories"][0]["fieldCoverage"][0],["x",205,200,5,0,0,0,10])
self.assertEqual(view["current"]["sourceVerification"]["verifiedRows"],205)
self.assertEqual(view["current"]["sourceVerification"]["files"][0]["verifiedFields"],["x"])
self.assertEqual(view["current"]["categories"][0]["differenceFormalMinusSubitems"],5.7822)
if __name__=='__main__':unittest.main()
+196
View File
@@ -0,0 +1,196 @@
"""Synthetic regression cases for bounded metric intent and early completion."""
import copy
import json
import io
import sys
import tempfile
import threading
import unittest
import uuid
from pathlib import Path
from unittest.mock import MagicMock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_quality_runtime as runtime
import nao_evidence_finalizer as finalizer
from nao_analysis_coverage import formal_metric_coverage
from nao_metric_request import (resolve_metric_request, prepare_metric_context, frozen_metric_frame,
metric_request, requires_fresh_query, is_history_restatement)
QUESTION = '只看SYNTH-16X15+T,请重新查询2026年9月与8月黑片、亮片的合格率、报废重量和两期投入,附正式数据依据。'
def metric_evidence():
result = {'kind': 'quality_comparison', 'releaseId': 'release-a',
'filterBinding': {'product': 'SYNTH-16X15+T'},
'delta': {'available': True, 'percentagePoints': -2},
'stageRateChangesPP': {'black': -1, 'bright': -1}}
for side, month in [('current', 9), ('comparison', 8)]:
result[side] = {'period': {'year': 2026, 'month': month}, 'compositeRate': .95,
'coverage': {'sourceMonths': [{'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 2}]},
'black': {'rate': .97, 'totalWeight': 1200.5, 'scrapWeight': 36.1},
'bright': {'rate': .98, 'totalWeight': 2400.5, 'scrapWeight': 72.1}}
return [{'tool': 'quality_data', 'releaseId': 'release-a', 'evidenceRef': 'Q00000001',
'mode': 'compare', 'result': result}]
class MetricRequestRegressionTests(unittest.TestCase):
def test_common_equivalent_questions_have_the_same_formal_contract(self):
questions = [QUESTION,
'请对比规格SYNTH-16X15+T在2026年9月和8月的黑片及亮片合格率、投入重量、报废重量。',
'帮我查一下SYNTH-16X15+T的2026年9月与8月黑片、亮片合格率、投入量和报废量分别是多少?',
'SYNTH-16X15+T,2026年9月与8月,黑片和亮片的合格率、投入重量及报废重量,给我正式数据。']
for question in questions:
with self.subTest(question=question):
contract = formal_metric_coverage(metric_evidence(), question, {'releaseId': 'release-a'})
self.assertIsNotNone(contract)
self.assertEqual(contract['missingEvidence'], [])
self.assertEqual(len(contract['metrics']), 12)
def _stream_readiness(self, *, mutate=None, question=QUESTION, model='mimo-v2.6-flash',
excluded=False, cancelled=False, pending=False, failed_auxiliary=False):
clock = [0.0]
gates = []
readiness = []
class Stopper:
cause = None
stop_succeeded = None
def __init__(self, stop, timeout, ready): self.ready = ready; gates.append(self)
def set_chat_id(self, value): pass
def set_response(self, value): pass
def start(self): pass
def close(self): pass
def request_cancel(self, cause): self.cause = cause
class Response:
def __enter__(self): return self
def __exit__(self, *args): pass
def __iter__(self):
yield b'data: {"type":"tool-input-available","toolName":"quality_data","toolCallId":"q1"}\n'
if not pending:
yield b'data: {"type":"tool-output-available","toolCallId":"q1","output":{"success":true}}\n'
if failed_auxiliary:
yield b'data: {"type":"tool-input-available","toolName":"quality_data","toolCallId":"q2"}\n'
yield b'data: {"type":"tool-output-available","toolCallId":"q2","output":{"success":false,"error":"invalid auxiliary query"}}\n'
readiness.append(gates[0].ready())
clock[0] = 3.0
readiness.append(gates[0].ready())
yield b'data: {"type":"text-delta","delta":"done"}\n'
yield b'data: {"type":"finish","finishReason":"stop"}\n'
yield b'data: [DONE]\n'
class Client:
def open(self, *args, **kwargs): return Response()
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir()
evidence = root / 'evidence' / ('nao-quality-' + 'a' * 32 + '.json')
document = metric_evidence()[0]
if mutate: mutate(document['result'])
evidence.write_text(json.dumps(document)); evidence.chmod(0o600)
cancel_event = threading.Event()
if cancelled: cancel_event.set()
with patch.object(runtime, 'NativeStopController', Stopper), patch.object(runtime.time, 'monotonic', side_effect=lambda: clock[0]):
runtime._sse_request(client=Client(), base_url='http://127.0.0.1:10000', model=model,
question=question, release_id='release-a', context={'releaseId': 'release-a'}, chat_id='native',
session_state=None, session_id=str(uuid.uuid4()), actor_root=root, emit=lambda event: None, timeout=60,
cancel_event=cancel_event, prior_evidence_files={evidence.name} if excluded else frozenset())
return readiness
def test_pure_metric_stream_is_ready_within_three_seconds_after_evidence(self):
self.assertEqual(self._stream_readiness(), [False, True], 'Covered pure metrics still wait for native model completion')
def test_failed_auxiliary_does_not_block_finalizing_already_complete_evidence(self):
self.assertEqual(self._stream_readiness(failed_auxiliary=True), [False, True])
self.assertEqual(self._stream_readiness(failed_auxiliary=True, mutate=lambda r:r['current']['black'].pop('totalWeight')), [False,False])
def test_early_completion_keeps_missing_scope_pending_cancel_and_profile_guards(self):
cases = [
{'mutate': lambda r: r['current']['black'].pop('totalWeight')},
{'mutate': lambda r: r['current']['coverage']['sourceMonths'][0].update(brightRows=0)},
{'mutate': lambda r: r['filterBinding'].update(group='额外范围')},
{'mutate': lambda r: r['comparison']['period'].update(month=7)},
{'question': QUESTION + '并分析现场原因。'},
{'excluded': True}, {'pending': True}, {'cancelled': True},
]
for case in cases:
with self.subTest(case=case): self.assertEqual(self._stream_readiness(**case), [False, False])
self.assertEqual(self._stream_readiness(model='other-model'), [False, True])
def test_followups_inherit_only_verified_dimensions_in_the_same_frozen_frame(self):
base = {'releaseId': 'release-a', 'filters': {'group': '合成分组', 'extraFilters': {'region': '合成地区'}}}
bound, first = prepare_metric_context(QUESTION, base)
record = {'baseMetricFrame': frozen_metric_frame(base), 'metricRequest': first}
for question, stage, field in [('那亮片呢?', ['bright'], first['fields']),
('再看黑片投入量。', ['black'], ['totalWeight']),
('该规格报废量是多少?', first['stages'], ['scrapWeight'])]:
with self.subTest(question=question):
context, request = prepare_metric_context(question, base, record)
self.assertEqual(request['stages'], stage)
self.assertEqual(request['fields'], field)
self.assertEqual(request['product'], first['product'])
self.assertEqual(request['periods'], first['periods'])
self.assertEqual(request['filters']['group'], '合成分组')
self.assertEqual(metric_request(question, context), request)
docs = metric_evidence(); docs[0]['result']['filterBinding'].update(base['filters'])
self.assertEqual(formal_metric_coverage(docs, question, context)['missingEvidence'], [])
for changed in [{**base, 'releaseId': 'release-b'}, {**base, 'filters': {'group': '另一分组'}},
{**base, 'period': {'year': 2025, 'month': 9}}]:
with self.subTest(changed=changed):
self.assertIsNone(prepare_metric_context('那亮片呢?', changed, record)[1])
self.assertIsNone(prepare_metric_context('那亮片呢?', base)[1])
def test_explicit_new_goal_updates_product_period_stage_and_retains_other_filters(self):
base = {'releaseId': 'release-a', 'period': {'year': 2026, 'month': 9},
'comparisonPeriod': {'year': 2026, 'month': 8}, 'stage': 'black',
'filters': {'product': 'SYNTH-OLD', 'group': '合成分组', 'exclusions': {'product': ['排除品']}}}
bound, first = prepare_metric_context(QUESTION, base)
self.assertNotIn('stage', bound)
self.assertEqual(bound['filters']['product'], 'SYNTH-16X15+T')
record = {'baseMetricFrame': frozen_metric_frame(base), 'metricRequest': first}
context, changed = prepare_metric_context('换成SYNTH-NEW,2026年10月和9月,亮片投入量是多少?', base, record)
self.assertEqual(changed['product'], 'SYNTH-NEW')
self.assertEqual(context['period'], {'year': 2026, 'month': 10})
self.assertEqual(context['comparisonPeriod'], {'year': 2026, 'month': 9})
self.assertEqual(context['stage'], 'bright')
self.assertEqual(context['filters']['exclusions'], {'product': ['排除品']})
def test_unknown_language_and_ambiguous_dimensions_remain_outside_the_contract(self):
for question in [QUESTION + '仅统计合成分组。', QUESTION + '并提供全部原始明细。',
QUESTION.replace('2026年', ''), QUESTION.replace('8月', '8月和7月'),
QUESTION.replace('8月', '9月'), QUESTION + '并提供生产原因。',
QUESTION.replace('SYNTH-16X15+T', 'SYNTH-16X15+T和SYNTH-OTHER'),
QUESTION + '去年相比如何?']:
with self.subTest(question=question): self.assertIsNone(resolve_metric_request(question))
def test_only_requested_metrics_gate_coverage_and_appear_in_the_answer(self):
question = '请对比规格SYNTH-16X15+T在2026年9月和8月的亮片投入量。'
docs = metric_evidence()
for side in ('current', 'comparison'):
docs[0]['result'][side]['black'] = {}
docs[0]['result'][side]['coverage']['sourceMonths'][0]['blackRows'] = 0
docs[0]['result'][side]['compositeRate'] = None
docs[0]['result'][side]['bright']['rate'] = None
self.assertTrue(finalizer.coverage_sufficient(docs, question, {'releaseId': 'release-a'}))
reply = {'evidenceSufficient': True, 'missingEvidence': []}
result = finalizer.synthesize(base_url='https://synthetic.invalid/v1', api_key='synthetic',
model='mimo-v2.6-flash', question=question, evidence=docs, context={'releaseId': 'release-a'},
cancel_event=threading.Event(), opener=lambda *_a, **_kw: io.BytesIO(json.dumps({
'choices': [{'finish_reason': 'stop', 'message': {'content': json.dumps(reply)}}]}).encode()))
self.assertTrue(result['evidenceSufficient'])
self.assertIn('亮片投入重量', result['answer'])
self.assertNotIn('综合合格率', result['answer'])
self.assertNotIn('黑片', result['answer'])
self.assertEqual(result['modelMissingEvidence'], [])
docs[0]['result']['comparison']['bright']['totalWeight'] = None
self.assertFalse(finalizer.coverage_sufficient(docs, question, {'releaseId': 'release-a'}))
def test_fresh_query_detection_and_restatement_are_distinct(self):
for question in ['重新查询', '请重新分析刚才的数据', '再查一次', '再次查询亮片投入', '刷新数据']:
with self.subTest(question=question):
self.assertTrue(requires_fresh_query(question))
self.assertFalse(is_history_restatement(question))
for question in ['把刚才的结果再说一遍', '请复述上一轮结论', '总结刚才的结果']:
with self.subTest(question=question):
self.assertTrue(is_history_restatement(question))
self.assertFalse(requires_fresh_query(question))
if __name__ == '__main__': unittest.main()
+247
View File
@@ -0,0 +1,247 @@
"""Loopback fake-provider tests: no configured model, credential file or real network service."""
import json
import sys
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_model_gate import ModelGate
class HeldOpenProviderResponse:
"""A provider that sends a terminal SSE event but keeps its transport alive."""
status = 200
headers = {'Content-Type': 'text/event-stream'}
def __init__(self, chunks):
self.chunks = list(chunks)
self.read_count = 0
self.terminal_delivered = threading.Event()
self.read_after_terminal = threading.Event()
self.release = threading.Event()
self.closed = threading.Event()
def __enter__(self): return self
def __exit__(self, *_): self.closed.set()
def read1(self, _size):
self.read_count += 1
if self.chunks:
chunk = self.chunks.pop(0)
if not self.chunks:
self.terminal_delivered.set()
return chunk
self.read_after_terminal.set()
# Heartbeats keep the connection readable without adding completion data.
return b'' if self.release.wait(.02) else b': heartbeat\n\n'
class ModelGateTests(unittest.TestCase):
def setUp(self):
self.allowed = True
self.calls = []
self.provider_status = 200
owner = self
class Provider(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
owner.calls.append(json.loads(self.rfile.read(int(self.headers['Content-Length']))))
data = json.dumps({'choices': [{'message': {'content': 'fake model'}}]} if owner.provider_status == 200
else {'error': 'fake-sensitive-provider-details'}).encode()
self.send_response(owner.provider_status)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(data)))
self.end_headers(); self.wfile.write(data)
self.provider = ThreadingHTTPServer(('127.0.0.1', 0), Provider)
self.provider.daemon_threads = True
self.thread = threading.Thread(target=self.provider.serve_forever, daemon=True); self.thread.start()
self.gate = ModelGate(upstream_base=f'http://127.0.0.1:{self.provider.server_port}/v1', upstream_key='dummy-provider-key',
model='fake-model', port=0, client_key='dummy-native-key', authorize=self.authorize)
# Port zero is only used by this isolated test server.
self.base = f'http://127.0.0.1:{self.gate._server.server_port}/v1'
def tearDown(self):
self.gate.close(); self.provider.shutdown(); self.provider.server_close(); self.thread.join(timeout=1)
def authorize(self):
if not self.allowed: raise PermissionError('revoked or expired delegation')
def request(self, *, key='dummy-native-key', model='fake-model', extra=None, phase=None):
headers={'Content-Type':'application/json','Authorization':'Bearer '+key}
if phase:headers['X-Quality-Analysis-Phase']=phase
request = Request(self.base+'/chat/completions', data=json.dumps({'model': model, 'messages': [{'role': 'user', 'content': 'offline fixture'}],**(extra or {})}).encode(),
headers=headers)
try:
with urlopen(request, timeout=3) as response: return response.status, response.read().decode()
except HTTPError as error: return error.code, error.read().decode()
def test_revocation_blocks_native_and_final_synthesis_requests_before_fake_provider(self):
self.assertEqual(self.request()[0], 200)
self.assertEqual(len(self.calls), 1)
self.allowed = False
self.assertEqual(self.request()[0], 403) # next native provider request
self.assertEqual(self.request()[0], 403) # final synthesis uses this same connection
self.assertEqual(len(self.calls), 1)
def test_initial_denial_or_wrong_native_credential_never_reaches_provider(self):
self.allowed = False
self.assertEqual(self.request()[0], 403)
self.allowed = True
self.assertEqual(self.request(key='wrong-key')[0], 403)
self.assertEqual(self.request(model='different-model')[0], 400)
self.assertEqual(len(self.calls), 0)
def test_quality_api_charts_do_not_advertise_incompatible_sql_renderer(self):
data_tool={'type':'function','function':{'name':'mcp_call','parameters':{'type':'object'}}}
chart_tool={'type':'function','function':{'name':'display_chart','parameters':{'type':'object'}}}
self.assertEqual(self.request(extra={'tools':[data_tool,chart_tool],
'tool_choice':{'type':'function','function':{'name':'display_chart'}}})[0],200)
self.assertEqual(self.calls[-1]['tools'],[data_tool])
self.assertNotIn('tool_choice',self.calls[-1])
self.assertEqual(self.request(extra={'tools':[chart_tool],'tool_choice':'required'})[0],200)
self.assertNotIn('tools',self.calls[-1])
self.assertNotIn('tool_choice',self.calls[-1])
def test_provider_errors_are_bounded_public_failures(self):
self.provider_status = 503
code, body = self.request()
self.assertEqual(code, 503)
self.assertNotIn('fake-sensitive-provider-details', body)
self.assertEqual(len(self.calls), 1)
def test_previous_turn_background_call_cannot_enter_current_thinking_or_provider(self):
marker='quality-turn:current-request'
self.gate._native_turn_marker=marker
self.gate._thinking={'enabled':True,'effort':'medium','protocol':'thinking'}
self.assertEqual(self.request(extra={'messages':[{'role':'user','content':'quality-turn:old-request'}]})[0],403)
self.assertEqual(self.request(extra={'messages':[{'role':'system','content':marker},{'role':'user','content':'old request'}]})[0],403)
self.assertEqual(self.calls,[])
self.assertEqual(self.gate._request_number,0)
self.assertEqual(self.gate.request_audit,[])
self.assertEqual(self.gate.rejected_stale_requests,2)
self.assertEqual(self.request(extra={'messages':[{'role':'user','content':[{'type':'text','text':marker+' current task'}]}]})[0],200)
self.assertEqual(self.request(phase='planning')[0],200)
self.assertEqual(len(self.calls),2)
self.assertEqual(len(self.gate.request_audit),2)
def test_saved_enabled_effort_applies_to_every_request_despite_helper_defaults(self):
self.gate._thinking={'enabled':True,'effort':'high','protocol':'thinking'}
self.assertEqual(self.request(extra={'thinking':{'type':'disabled'},'reasoning_effort':'low','temperature':.2,'max_completion_tokens':4096})[0],200)
body=self.calls[-1]
self.assertEqual(body['thinking'],{'type':'enabled'})
self.assertEqual(body['reasoning_effort'],'high')
self.assertNotIn('temperature',body)
from nao_runtime_policy import MAX_OUTPUT_TOKENS
self.assertEqual(body['max_completion_tokens'],MAX_OUTPUT_TOKENS)
self.assertNotIn('offline fixture',json.dumps(self.gate.request_audit))
def test_qwen38_native_effort_uses_template_low_medium_and_xhigh(self):
self.gate._model='qwen3.8-27b-local'
for selected,native in [('low','low'),('medium','medium'),('high','xhigh')]:
self.gate._thinking={'enabled':True,'effort':selected,'protocol':'chat_template'}
self.assertEqual(self.request(model='qwen3.8-27b-local',extra={'reasoning_effort':'high','chat_template_kwargs':{'reasoning_effort':'low','preserve_thinking':True}})[0],200)
body=self.calls[-1]
self.assertEqual(body['chat_template_kwargs'],{'reasoning_effort':native,'enable_thinking':True,'preserve_thinking':True})
self.assertNotIn('reasoning_effort',body)
self.assertEqual(self.gate.request_audit[-1]['reasoningEffort'],native)
def test_qwen_switch_uses_chat_template_control_and_removes_conflicting_flags(self):
self.gate._thinking={'enabled':False,'effort':'medium','protocol':'chat_template'}
self.assertEqual(self.request(extra={'thinking':{'type':'enabled'},'reasoning_effort':'high','chat_template_kwargs':{'enable_thinking':True}})[0],200)
body=self.calls[-1]
self.assertEqual(body['chat_template_kwargs']['enable_thinking'],False)
self.assertNotIn('thinking',body)
self.assertNotIn('reasoning_effort',body)
def test_explicit_sse_done_finishes_without_eof_for_bridge_and_passthrough(self):
from unittest.mock import Mock
rows = [
{'id': 'fake-completion', 'model': 'fake-model', 'choices': [
{'index': 0, 'delta': {'reasoning_content': '先核对。', 'content': '{"ok":'}}]},
{'choices': [{'index': 0, 'delta': {'content': 'true}'}, 'finish_reason': 'stop'}]},
# The usage-only tail must survive even though the choice already finished.
{'choices': [], 'usage': {'prompt_tokens': 7, 'completion_tokens': 12}},
]
wire = [('data: ' + json.dumps(row, ensure_ascii=False) + '\n\n').encode() for row in rows]
chunks = [*wire, b'data: [DO', b'NE]\n\n']
for stream, observe in ((False, True), (True, True), (True, False)):
with self.subTest(stream=stream, observe=observe):
events = []
self.gate._on_thinking = events.append if observe else None
upstream = HeldOpenProviderResponse(chunks)
self.gate._opener = Mock()
self.gate._opener.open.return_value = upstream
response = []
returned = threading.Event()
def request():
try:
response.append(self.request(extra={'stream': stream}))
except Exception as error:
response.append(error)
finally:
returned.set()
worker = threading.Thread(target=request, daemon=True)
worker.start()
try:
self.assertTrue(upstream.terminal_delivered.wait(1), 'fake provider did not deliver DONE')
completed_before_eof = returned.wait(1)
finally:
upstream.release.set()
worker.join(timeout=2)
self.assertTrue(completed_before_eof, 'ModelGate still waits for provider EOF after explicit DONE')
self.assertFalse(upstream.read_after_terminal.is_set(), 'ModelGate read beyond explicit DONE')
self.assertTrue(upstream.closed.is_set())
self.assertEqual(response[0][0], 200)
if stream:
self.assertEqual(response[0][1].encode(), b''.join(chunks))
else:
completion = json.loads(response[0][1])
self.assertEqual(completion['choices'][0]['message']['content'], '{"ok":true}')
self.assertEqual(completion['choices'][0]['finish_reason'], 'stop')
self.assertEqual(completion['usage'], {'prompt_tokens': 7, 'completion_tokens': 12})
self.assertNotIn('reasoning_content', completion['choices'][0]['message'])
if observe:
self.assertEqual(events[-1]['event'], 'end')
self.assertEqual(events[-1]['finishReason'], 'stop')
self.assertFalse(events[-1]['interrupted'])
def test_explicit_done_without_finish_reason_is_not_a_successful_completion(self):
from unittest.mock import Mock
upstream = HeldOpenProviderResponse([
b'data: {"choices":[{"index":0,"delta":{"content":"incomplete"}}]}\n\n',
b'data: [DONE]\n\n',
])
self.gate._opener = Mock()
self.gate._opener.open.return_value = upstream
events = []
self.gate._on_thinking = events.append
response = []
returned = threading.Event()
def request():
try:
response.append(self.request())
finally:
returned.set()
worker = threading.Thread(target=request, daemon=True)
worker.start()
try:
self.assertTrue(upstream.terminal_delivered.wait(1))
completed_before_eof = returned.wait(1)
finally:
upstream.release.set()
worker.join(timeout=2)
self.assertTrue(completed_before_eof, 'invalid DONE must fail without waiting for EOF')
self.assertEqual(response[0][0], 502)
self.assertTrue(events[-1]['interrupted'])
if __name__ == '__main__': unittest.main()
+25
View File
@@ -0,0 +1,25 @@
import copy,json,sys,unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
from nao_quality_tools import QualityTools
class FineDefectProjectionTest(unittest.TestCase):
def test_repeated_field_coverage_is_compact_without_changing_evidence_or_top_weights(self):
fields=[{'key':f'镀废缺陷字段{i}重量','label':f'镀废缺陷字段{i}重量','category':'镀废','sourceField':f'镀废缺陷字段{i}重量'} for i in range(35)]
components=[{'key':f['key'],'label':f['label'],'weight':None if i==2 else i/10,'presentRows':0 if i==2 else 72,'missingRows':72 if i==2 else 0,'invalidRows':0,'negativeRows':0,'sourceRows':72} for i,f in enumerate(fields)]
category={'category':'镀废','sourceRows':72,'components':components,'subitemWeightSum':58.3,'diff':None,'reconciliationStatus':'INCOMPLETE','reconciliationReasons':['存在缺失字段'],'formalComponentCoverage':{'presentRows':72,'missingRows':0}}
original={'kind':'quality_fine_defects','releaseId':'r1','stage':'bright','filters':{'group':'消费电子-美洲','product':'D34.5X5.5+C+Z'},'period':{'year':2026,'month':9},'comparisonPeriod':{'year':2026,'month':8},'rows':[{'key':fields[1]['key'],'weight':.1,'missingRows':0}],'pairedRows':[{'key':fields[1]['key'],'current':{'weight':.1},'comparison':{'weight':None},'deltaWeight':None,'deltaAvailable':False}],'sourceFields':fields,'comparisonSourceFields':fields,'categoryReconciliation':[category],'comparisonCategoryReconciliation':[category],'coverage':{'available':True,'categories':[category]},'comparisonCoverage':{'available':True,'categories':[category]},'rankingComplete':True,'pairedRankingComplete':True,'evidenceRef':'Q12345678'}
untouched=copy.deepcopy(original);view=QualityTools._model_view(original)
self.assertLess(len(json.dumps(view,ensure_ascii=False)),len(json.dumps(original,ensure_ascii=False))*.6)
self.assertEqual(original,untouched)
self.assertEqual(view['coverage'],{'available':True,'categoriesRef':'categoryReconciliation'})
self.assertEqual(view['comparisonCoverage'],{'available':True,'categoriesRef':'comparisonCategoryReconciliation'})
for key in ('rows','pairedRows','filters','period','comparisonPeriod','rankingComplete','pairedRankingComplete','evidenceRef'):self.assertEqual(view[key],original[key])
for name in ('categoryReconciliation','comparisonCategoryReconciliation'):
row=view[name][0];columns=row['componentColumns'];restored=[dict(zip(columns,values)) for values in row['componentValues']]
self.assertEqual(restored,components)
for key in category:
if key!='components':self.assertEqual(row[key],category[key])
self.assertEqual([f['key'] for f in view['sourceFields']],[f['key'] for f in fields])
if __name__=='__main__':unittest.main()
+109
View File
@@ -0,0 +1,109 @@
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_model_settings as settings
class ModelSettingsTest(unittest.TestCase):
def test_mimo26_legacy_effort_preserves_switch_but_reports_automatic_strength(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'config.json'
with patch.object(settings,'SETTINGS_FILE',p):
for enabled in (True,False):
p.write_text(json.dumps(dict(baseUrl='http://localhost/v1',apiKey='synthetic',model='mimo-v2.6-flash',thinkingEnabled=enabled,thinkingEffort='high')))
self.assertEqual(settings.model_configuration()[3],{'enabled':enabled,'effort':'auto','protocol':'thinking'})
def test_thinking_controls_are_loaded_atomically_with_profile_credentials(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'config.json'
p.write_text(json.dumps(dict(baseUrl='http://localhost/v1',apiKey='synthetic',model='qwen-local',thinkingEnabled=False,thinkingEffort='high')))
with patch.object(settings,'SETTINGS_FILE',p):
result=settings.model_configuration()
self.assertEqual(result[2],'qwen-local')
self.assertEqual(result[3],{'enabled':False,'effort':'high','protocol':'chat_template'})
for value in ('extreme',False):
p.write_text(json.dumps(dict(baseUrl='http://localhost/v1',apiKey='synthetic',model='mimo-model',thinkingEnabled=True,thinkingEffort=value)))
with self.assertRaises(ValueError):settings.model_configuration()
def test_ui_settings_override_legacy_and_refresh_each_read(self):
with tempfile.TemporaryDirectory() as tmp:
p = Path(tmp) / 'config.json'
with patch.object(settings, 'SETTINGS_FILE', p), patch.object(settings, '_legacy_settings') as legacy:
p.write_text(json.dumps(dict(baseUrl='http://localhost:8000/v1', apiKey='private-one', model='local-model')))
self.assertEqual(settings.model_settings(), ('http://localhost:8000/v1', 'private-one', 'local-model'))
p.write_text(json.dumps(dict(baseUrl='http://localhost:8000/v1', apiKey='private-two', model='other-model')))
self.assertEqual(settings.model_settings()[1:], ('private-two', 'other-model'))
legacy.assert_not_called()
def test_missing_ui_config_bootstraps_selected_legacy(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / 'apikey.txt').write_text('https://first.example\nfirst-key\nfirst-model\nhttps://second.example/v1/chat/completions\nBearer selected-key\nsecond-model\n')
selection = root / '.runtime/dbgpt-trial/selected-profile.json'
selection.parent.mkdir(parents=True)
selection.write_text('{"profile":2}')
# A clean checkout has no dbgpt_trial module. Bootstrap must still work.
with patch.object(settings, 'ROOT', root), patch.object(settings, 'SETTINGS_FILE', root / 'absent.json'), patch.dict('os.environ', {}, clear=True), patch.dict(sys.modules, {'dbgpt_trial': None}):
self.assertEqual(settings.model_settings(), ('https://second.example/v1', 'selected-key', 'second-model'))
def test_invalid_saved_config_does_not_silently_use_legacy(self):
with tempfile.TemporaryDirectory() as tmp:
p = Path(tmp) / 'config.json'
p.write_text(json.dumps(dict(baseUrl='http://user:secret@localhost/v1', apiKey='key', model='model')))
with patch.object(settings, 'SETTINGS_FILE', p), patch.object(settings, '_legacy_settings') as legacy:
with self.assertRaises(ValueError): settings.model_settings()
legacy.assert_not_called()
def test_legacy_utf16_and_explicit_config_path(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
p = root / 'selected.txt'
p.write_text('https://model.example/v1\nBEARER selected-key\nprovider/model:latest\n', encoding='utf-16')
with patch.object(settings, 'ROOT', root), patch.object(settings, 'SETTINGS_FILE', root / 'absent.json'), patch.dict('os.environ', {'QUALITY_MODEL_CONFIG': str(p)}):
self.assertEqual(settings.model_settings(), ('https://model.example/v1', 'selected-key', 'provider/model:latest'))
def test_legacy_rejects_invalid_profile_without_exposing_content(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / 'apikey.txt').write_text('https://model.example\nprivate-secret\nmodel\n')
selection = root / '.runtime/dbgpt-trial/selected-profile.json'
selection.parent.mkdir(parents=True)
with patch.object(settings, 'ROOT', root), patch.object(settings, 'SETTINGS_FILE', root / 'absent.json'), patch.dict('os.environ', {}, clear=True):
for profile in (0, 2, True, 1.5, '1'):
with self.subTest(profile=profile):
selection.write_text(json.dumps({'profile': profile}))
with self.assertRaises(ValueError) as error:
settings.model_settings()
self.assertNotIn('private-secret', str(error.exception))
def test_legacy_validation_and_bounded_read(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
p = root / 'apikey.txt'
invalid = [
'https://user:password@model.example\nkey\nmodel',
'https://model.example#fragment\nkey\nmodel',
'https://model.example\nkey with spaces\nmodel',
'https://model.example\nkey\ninvalid model',
'x' * 16385,
]
with patch.object(settings, 'ROOT', root), patch.object(settings, 'SETTINGS_FILE', root / 'absent.json'), patch.dict('os.environ', {}, clear=True):
for content in invalid:
with self.subTest(content_type=invalid.index(content)):
p.write_text(content)
with self.assertRaises(ValueError): settings.model_settings()
def test_dangling_saved_config_symlink_does_not_fall_back(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
p = root / 'config.json'
p.symlink_to(root / 'missing.json')
with patch.object(settings, 'SETTINGS_FILE', p), patch.object(settings, '_legacy_settings') as legacy:
with self.assertRaises(ValueError): settings.model_settings()
legacy.assert_not_called()
if __name__ == '__main__': unittest.main()
+19
View File
@@ -0,0 +1,19 @@
import io,json,sys,threading,unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
from nao_agent_task import _model_json
class ThinkingTimeoutTest(unittest.TestCase):
def test_valid_slow_local_thinking_plan_is_not_abandoned_at_thirty_seconds(self):
calls=[]
def provider(request,timeout):
calls.append(timeout)
if timeout<50:raise TimeoutError('fixture: correct answer arrives at 49 seconds')
return io.BytesIO(json.dumps({'model':'qwen3.8-27b-local','choices':[{'finish_reason':'stop','message':{'content':'{"goal":"compare formal quality"}'}}]}).encode())
result=_model_json(base_url='http://127.0.0.1:1/v1',api_key='fixture',model='qwen3.8-27b-local',system='fixture',payload={},cancel_event=threading.Event(),opener=provider)
self.assertEqual(result,{'goal':'compare formal quality'})
self.assertEqual(len(calls),1)
from nao_runtime_policy import REQUEST_TIMEOUT_SECONDS
self.assertLessEqual(calls[0],REQUEST_TIMEOUT_SECONDS)
if __name__=='__main__':unittest.main()
@@ -0,0 +1,188 @@
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_product_identification import identify_products, resolve_clarification_choice, authorized_extra_fields
class ProductIdentificationTests(unittest.TestCase):
def test_explicit_negative_dimension_does_not_return_as_positive_clarification_option(self):
options={'products':['18X15X4+T','18X15X4+Z','18X15X4.1+Z']}
result=identify_products('18X15X4合格率,不要包含18X15X4.1',options,respect_exclusions=True)
self.assertEqual(result['productMatch']['status'],'ambiguous')
self.assertEqual([x['name'] for x in result['productMatch']['candidates']],['18X15X4+T','18X15X4+Z'])
self.assertEqual(result['explicitlyExcludedProducts'],['18X15X4.1+Z'])
cancelled=identify_products('18X15X4合格率,取消排除18X15X4.1',options,respect_exclusions=True)
self.assertEqual(cancelled['explicitlyExcludedProducts'],[])
negative_only=identify_products('看公司整体合格率,排除18X15X4.1+Z',options,respect_exclusions=True)
self.assertEqual(negative_only['productMatch']['status'],'none')
self.assertEqual(negative_only['productMatch']['candidates'],[])
self.assertEqual(negative_only['explicitlyExcludedProducts'],['18X15X4.1+Z'])
named=identify_products('查看CUSTOM_SPEC',{'products':['CUSTOM_SPEC']},respect_exclusions=True)
self.assertEqual(named['productMatch']['status'],'exact')
def test_round_two_axis_geometry_preserves_diameter_and_exact_suffix(self):
options={'products':['D37.2X6.5+C','D37.2X6.51+C','37.2X6.5X2+C','18X15X4+T']}
for text in ('D37.2x6.5','Φ37.20×6.50','Ø37.2*6.5'):
match=identify_products(text,options)['productMatch']
self.assertEqual(match['status'],'unique');self.assertEqual([item['name'] for item in match['candidates']],['D37.2X6.5+C'])
self.assertEqual(identify_products('D37.2x6.5+Q',options)['productMatch']['status'],'none')
self.assertEqual(identify_products('D18x15x4',options)['productMatch']['status'],'none')
def setUp(self):
self.options = {
'products': ['18X15X4+T', '18X15X4+Z', '18X15X4.1+Z', '28X20X3'],
'organizationPaths': [
{'product': '18X15X4+T', 'department': '汽车', 'group': '国内'},
{'product': '18X15X4+Z', 'department': '汽车', 'group': '海外'},
{'product': '18X15X4.1+Z', 'department': '工业', 'group': '美洲'},
],
}
def test_dimension_shorthand_produces_candidates_without_wrong_decimal(self):
result = identify_products('今年18x15x4合格率变化趋势', self.options)
self.assertEqual(result['mentionedProducts'], [])
match = result['productMatch']
self.assertEqual(match['status'], 'ambiguous')
self.assertEqual([item['name'] for item in match['candidates']], ['18X15X4+T', '18X15X4+Z'])
self.assertEqual(match['candidateCount'], 2)
self.assertNotIn('18X15X4.1+Z', [item['name'] for item in match['candidates']])
def test_full_name_wins_over_other_suffix_and_normalizes_multiplication(self):
result = identify_products('今年18×15×4+t的趋势', self.options)
self.assertEqual(result['mentionedProducts'], ['18X15X4+T'])
self.assertEqual(result['productMatch']['status'], 'exact')
self.assertEqual([item['name'] for item in result['productMatch']['candidates']], ['18X15X4+T'])
def test_unique_and_decimal_equivalent_dimensions(self):
match = identify_products('18.0*15*4.10的变化', self.options)['productMatch']
self.assertEqual(match['status'], 'unique')
self.assertEqual(match['candidates'][0]['name'], '18X15X4.1+Z')
self.assertEqual(match['candidates'][0]['departments'], ['工业'])
def test_unknown_dimension_does_not_guess(self):
match = identify_products('20x11x7的趋势', self.options)['productMatch']
self.assertEqual(match['status'], 'none')
self.assertEqual(match['candidates'], [])
self.assertEqual(match['mentions'][0]['query'], '20x11x7')
def test_only_authorized_options_can_be_candidates(self):
scoped = {'products': ['18X15X4+T'], 'organizationPaths': self.options['organizationPaths']}
match = identify_products('18x15x4', scoped)['productMatch']
self.assertEqual(match['status'], 'unique')
self.assertEqual([item['name'] for item in match['candidates']], ['18X15X4+T'])
self.assertEqual(match['candidates'][0]['groups'], ['国内'])
def test_multiple_specifications_stay_separate(self):
match = identify_products('比较18×15×4+T和28x20x3', self.options)['productMatch']
self.assertEqual(match['status'], 'multiple')
self.assertEqual([m['status'] for m in match['mentions']], ['exact', 'exact'])
self.assertEqual([[c['name'] for c in m['candidates']] for m in match['mentions']],
[['18X15X4+T'], ['28X20X3']])
def test_candidate_cap_does_not_misrepresent_ambiguity(self):
options = {'products': [f'18X15X4+S{i}' for i in range(30)]}
match = identify_products('18x15x4', options)['productMatch']
self.assertEqual(match['status'], 'ambiguous')
self.assertEqual(match['candidateCount'], 30)
self.assertEqual(len(match['candidates']), 20)
self.assertTrue(match['truncated'])
current=identify_products('18x15x4',options,respect_exclusions=True)['productMatch']
self.assertEqual(current['candidateCount'],30)
self.assertEqual(len(current['candidates']),20)
self.assertTrue(current['truncated'])
def test_non_product_question_and_empty_catalog_are_safe(self):
self.assertEqual(identify_products('你好', self.options)['productMatch']['status'], 'none')
self.assertEqual(identify_products('18x15x4', {'products': None})['productMatch']['status'], 'none')
def pending(self):
return {'needsClarification': True, 'productResolution': {
'originalQuestion': '今年18x15x4合格率变化趋势',
'productMatch': identify_products('今年18x15x4合格率变化趋势', self.options)['productMatch'],
}}
def test_followup_suffix_is_resolved_only_from_pending_candidates(self):
for reply, expected in (('选 +T', '18X15X4+T'), ('选择+z', '18X15X4+Z')):
with self.subTest(reply=reply):
result = resolve_clarification_choice(reply, self.options, self.pending())
self.assertEqual(result['mentionedProducts'], [expected])
self.assertEqual(result['productMatch']['status'], 'exact')
self.assertEqual(result['productMatch']['previousQuestion'], '今年18x15x4合格率变化趋势')
self.assertTrue(result['productMatch']['selectedFromClarification'])
def test_selection_never_searches_suffix_globally_or_uses_revoked_access(self):
options = {'products': ['18X15X4+Z', '28X20X3+T']}
self.assertIsNone(resolve_clarification_choice('选+T', options, self.pending()))
def test_failed_selected_question_retry_recovers_only_same_authorized_choice(self):
previous = self.pending()
previous['needsClarification'] = False
previous['productResolution'].update(selectedName='18X15X4+T', selectionQuestion='选 +T')
result = resolve_clarification_choice('选+T', self.options, previous)
self.assertEqual(result['mentionedProducts'], ['18X15X4+T'])
self.assertEqual(result['productMatch']['previousQuestion'], '今年18x15x4合格率变化趋势')
self.assertIsNone(resolve_clarification_choice('选+Z', self.options, previous))
self.assertIsNone(resolve_clarification_choice('选+T', {'products':['18X15X4+Z']}, previous))
previous['productResolution']['selectedName'] = '28X20X3'
self.assertIsNone(resolve_clarification_choice('选+T', self.options, previous))
def test_unrelated_or_multi_variant_followup_does_not_choose(self):
for reply in ('再看整体合格率', '我想比较+T和+Z', '只查18x15x4+T九月趋势'):
with self.subTest(reply=reply):
self.assertIsNone(resolve_clarification_choice(reply, self.options, self.pending()))
prior = self.pending(); prior['needsClarification'] = False
self.assertIsNone(resolve_clarification_choice('选+T', self.options, prior))
def test_complete_candidate_choice_from_ui_uses_the_same_pending_frame(self):
for reply in ('选择18x15x4+T','18X15X4+T','选择18X15X4+T。','继续上一轮分析,保留未修改的原目标和条件。\n我的选择:18X15X4+T'):
result=resolve_clarification_choice(reply,self.options,self.pending())
self.assertEqual(result['mentionedProducts'],['18X15X4+T'])
self.assertTrue(result['productMatch']['selectedFromClarification'])
self.assertEqual(result['productMatch']['previousQuestion'],'今年18x15x4合格率变化趋势')
def test_truncated_or_duplicate_suffix_candidates_cannot_select(self):
prior = self.pending(); prior['productResolution']['productMatch']['truncated'] = True
self.assertIsNone(resolve_clarification_choice('选+T', self.options, prior))
options = {'products': ['18X15X4+A+T', '18X15X4+B+T']}
prior = {'needsClarification': True, 'productResolution': {
'originalQuestion':'今年18x15x4合格率变化趋势',
'productMatch': identify_products('18x15x4', options)['productMatch']}}
self.assertIsNone(resolve_clarification_choice('选+T', options, prior))
def test_case_colliding_catalog_names_remain_ambiguous(self):
match = identify_products('18x15x4+t', {'products':['18X15X4+T', '18x15x4+t']})['productMatch']
self.assertEqual(match['status'], 'ambiguous')
self.assertEqual(match['candidateCount'], 2)
def test_customer_code_that_looks_like_specification_is_not_a_product(self):
for question in ('今年客户代码18x15x4的合格率变化趋势', '查客户编码为18X15X4+T的趋势',
'客户代码“18x15x4”今年合格率趋势', '查客户代码为「18X15X4+T」的趋势',
'客户代码 "18x15x4" 的合格率走势'):
with self.subTest(question=question):
result = identify_products(question, self.options)
self.assertEqual(result['mentionedProducts'], [])
self.assertEqual(result['productMatch']['status'], 'none')
def test_customer_code_and_explicit_product_are_kept_separate(self):
result = identify_products('客户代码18x15x4的18X15X4+T产品合格率趋势', self.options)
self.assertEqual(result['mentionedProducts'], ['18X15X4+T'])
self.assertEqual(len(result['productMatch']['mentions']), 1)
self.assertEqual(result['productMatch']['status'], 'exact')
quoted = identify_products('客户代码“18x15x4”的18X15X4+T产品合格率趋势', self.options)
self.assertEqual(quoted['mentionedProducts'], ['18X15X4+T'])
self.assertEqual(quoted['productMatch']['status'], 'exact')
def test_authorized_extra_fields_are_not_inferred_or_expanded(self):
catalog = {'kind':'analysis_catalog', 'extraFields':[
{'key':'customerCode', 'label':'客户代码', 'blackColumn':'客户代码', 'brightColumn':'客户代码', 'enabled':True},
{'key':'disabled', 'label':'禁用字段', 'enabled':False},
]}
self.assertEqual(authorized_extra_fields(catalog),
{'extraFieldsStatus':'ready', 'extraFields':[{'key':'customerCode','label':'客户代码'}]})
self.assertEqual(authorized_extra_fields({'kind':'analysis_catalog','extraFields':[]})['extraFields'], [])
self.assertEqual(authorized_extra_fields({'kind':'missing'})['extraFieldsStatus'], 'unavailable')
if __name__ == '__main__':
unittest.main()
+240
View File
@@ -0,0 +1,240 @@
import copy
import io
import json
import sys
import tempfile
import threading
import unittest
from urllib.parse import parse_qs, urlsplit
from contextlib import ExitStack
from pathlib import Path
from unittest.mock import MagicMock, patch
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
import nao_quality_runtime as runtime
from nao_protocol_execution import execute
from nao_agent_task import normalize_task, bind_task
from test_nao_agent_task import task_value
from test_nao_quality_tools import FakeApi, RELEASE
ACTOR='11111111-1111-4111-8111-111111111111'
SESSION='22222222-2222-4222-8222-222222222222'
TURN='55555555-5555-4555-8555-555555555555'
def task(filters=None):
value = normalize_task(task_value(kind='metrics',dimensions=[],stages=[],fields=['rate'],trendRequired=True,
period={'year':2026},filterUpdates=filters or {}),'今年合格率月度趋势',{'_protocol':{'version':1}})
if value['filters'].get('columnFilters') == []: value['filters'].pop('columnFilters')
return value
def ranking_task(**updates):
plan=task_value(kind='analysis',dimensions=['product','group'],
dimensionStages={'product':['bright','black'],'group':['bright']},stages=['black','bright'],
fields=['totalWeight','scrapWeight'],period={'year':2026},trendRequired=False,
visualPreference='auto',visualType='pareto')
plan.update(updates)
value=normalize_task(plan,
'按产品和业务分组查看各阶段重量排名',{'_protocol':{'version':1}})
if value['filters'].get('columnFilters') == []:value['filters'].pop('columnFilters')
return value
def identity():
return {'attemptId':SESSION,'sessionId':ACTOR,'turnId':TURN,'revision':3,'executionId':TURN}
class BoundApi(FakeApi):
def __init__(self,proof=True,cancel=None):
super().__init__();self.proof=proof;self.cancel=cancel
def call(self,path,payload=None,method=None):
result=super().call(path,payload,method)
if path.startswith('/api/metrics'):
parsed=parse_qs(urlsplit(path).query)
if 'columnFilters' in parsed:
result['context']['filters']={'columnFilters':json.loads(parsed['columnFilters'][0])}
if self.proof:result['protocolEvidence']={**identity(),'queryId':TURN,'responseHash':'a'*64}
if self.cancel:self.cancel.set()
return result
class BoundRankingApi(BoundApi):
def __init__(self,proof=True,cancel=None):
super().__init__(proof=proof,cancel=cancel);self.rank_query=0
def call(self,path,payload=None,method=None):
result=super().call(path,payload,method)
if path.startswith('/api/metrics'):
self.rank_query+=1
parsed=parse_qs(urlsplit(path).query)
view=parsed['view'][0]
rows=[]
for stage,total,scrap in (('black',100,10),('bright',150,12)):
result['summary'][stage]['totalWeight']=total
result['summary'][stage]['scrapWeight']=scrap
for month in result['coverage']['sourceMonths']:
month['blackRows']=10;month['brightRows']=12
for suffix,share in (('a',.6),('b',.4)):
row={'key':view+'-'+suffix,'label':view+' '+suffix,'coverage':{'blackRows':2,'brightRows':2}}
for stage in ('black','bright'):
summary=result['summary'][stage]
row[stage]={key:summary[key]*share for key in ('totalWeight','scrapWeight')}
rows.append(row)
result['rows']=rows;result['context']['rowCount']=len(rows)
if 'monthStart' in parsed:
result['context']['monthStart']=int(parsed['monthStart'][0])
result['context']['monthEnd']=int(parsed['monthEnd'][0])
if self.proof:
result['protocolEvidence']={**identity(),'queryId':f'{TURN}-{self.rank_query}',
'responseHash':f'{self.rank_query:064x}'}
return result
class ProtocolExecutionTests(unittest.TestCase):
def test_comparison_returns_both_proofs_and_the_confirmed_comparison_context(self):
value=task_value(kind='metrics',dimensions=[],stages=['black'],fields=['rate'],trendRequired=False,
comparisonRequired=True,period={'year':2026,'month':9},comparisonPeriod={'year':2026,'month':8},visualPreference='none')
t=normalize_task(value,'2026年9月对比8月黑片合格率',{'_protocol':{'version':1}})
t['filters'].pop('columnFilters',None)
class ComparisonApi(BoundApi):
def call(inner,path,payload=None,method=None):
response=super().call(path,payload,method)
if path.startswith('/api/metrics'):
month=parse_qs(urlsplit(path).query).get('month',['0'])[0]
response['protocolEvidence']['queryId']='query-'+month
return response
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',return_value={'answer':'已核对两个期间','evidenceSufficient':True,'missingEvidence':[]}):
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,authority=identity(),actor_root=root,
delegated=ComparisonApi(),connection=('offline','unused','model'),cancel_event=threading.Event(),emit=lambda _:None)
self.assertEqual(result['agentContext']['comparisonPeriod'],{'year':2026,'month':8})
self.assertEqual({p['queryId'] for p in result['protocolEvidence']},{'query-9','query-8'})
self.assertEqual(result['agentContext']['period'],{'year':2026,'month':9})
def exercise(self,filters=None,api=None):
t=task(filters);request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']}
event=threading.Event();api=api or BoundApi();emitted=[]
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
with patch('nao_evidence_finalizer.synthesize',return_value={'answer':'月度正式走势','evidenceSufficient':True,'missingEvidence':[],'usage':{}}) as synth:
result=execute(request=request,task=t,authority={**identity(),'task':t},actor_root=root,
delegated=api,connection=('offline','synthetic','model'),cancel_event=event,emit=emitted.append)
documents=[json.loads(p.read_text()) for p in (root/'evidence').glob('nao-quality-*.json')]
return result,api,documents,synth
def test_real_tools_and_visuals_preserve_bound_monthly_series_and_cleared_scope(self):
selected,api,docs,_=self.exercise({'product':'18X15X4+T'})
self.assertEqual(selected['status'],'COMPLETED')
self.assertEqual(selected['agentContext']['filters'],{'product':'18X15X4+T'})
self.assertTrue(any('product=18X15X4%2BT' in path for _,path,_ in api.calls))
self.assertEqual(len(docs),1)
self.assertEqual(len(docs[0]['result']['trend']),12)
self.assertEqual(docs[0]['result']['protocolEvidence']['executionId'],TURN)
self.assertTrue(selected['visuals'])
self.assertEqual(selected['protocolEvidence'][0]['queryId'],TURN)
cleared,api,docs,_=self.exercise()
self.assertEqual(cleared['agentContext']['filters'],{})
self.assertTrue(all('product=' not in path for _,path,_ in api.calls))
self.assertTrue(all('columnFilters=' not in path for _,path,_ in api.calls))
self.assertEqual(docs[0]['result']['filters'],{'includeUnassigned':False})
def test_query_without_matching_execution_proof_never_reaches_the_model(self):
with self.assertRaises(ValueError):self.exercise(api=BoundApi(proof=False))
def test_weight_ranking_reads_dimension_stage_field_cartesian_product_before_shared_finalizer(self):
t=ranking_task();api=BoundRankingApi();request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']}
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',
return_value={'answer':'已核验全量重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}) as synth:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
result=execute(request=request,task=t,authority={**identity(),'task':t},actor_root=root,
delegated=api,connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
docs=[json.loads(p.read_text()) for p in (root/'evidence').glob('nao-quality-*.json')]
docs.sort(key=lambda doc:int(doc['result']['protocolEvidence']['queryId'].rsplit('-',1)[1]))
reads=[parse_qs(urlsplit(path).query) for _,path,_ in api.calls if path.startswith('/api/metrics')]
self.assertEqual(len(reads),6)
self.assertEqual([doc['result']['dimension'] for doc in docs],['product','product','product','product','group','group'])
self.assertEqual([doc['result']['stage'] for doc in docs],['bright','bright','black','black','bright','bright'])
self.assertEqual([doc['result']['metric'] for doc in docs],['totalWeight','scrapWeight']*3)
self.assertEqual(len(result['protocolEvidence']),6)
self.assertEqual(len({proof['queryId'] for proof in result['protocolEvidence']}),6)
self.assertTrue(all(proof['executionId']==TURN for proof in result['protocolEvidence']))
from nao_weight_ranking import matching_rankings,ranking_gaps
bound_context=bind_task({'releaseId':RELEASE},t)
self.assertEqual(result['status'],'COMPLETED', (result.get('visualState'),
[(v.get('dimension'),v.get('stage'),v.get('metric')) for v in result.get('visuals',[])],
result.get('evidenceCompletion'),len(matching_rankings(docs,t,bound_context)),
ranking_gaps(docs,t,bound_context),[(d['result'].get('dimension'),d['result'].get('stage'),d['result'].get('metric'),d['result'].get('period'),d['result'].get('periodCoverage'),d['result'].get('rankingCoverage')) for d in docs]))
self.assertTrue(result['visuals'])
self.assertEqual(result['toolCount'],6)
synth.assert_called_once()
self.assertFalse(any('rowLimit' in params or 'rowOffset' in params for params in reads))
def test_ranking_and_trend_or_comparison_are_rejected_without_dropping_task_requirements(self):
for updates in ({'trendRequired':True},
{'comparisonRequired':True,'comparisonPeriod':{'year':2025}}):
with self.subTest(updates=updates):
# This test exercises execute's fail-closed guard directly.
# Planner normalization now correctly rejects the same legacy
# shape earlier, before it could be confirmed as a protocol task.
t=ranking_task();t.update(updates);api=BoundRankingApi()
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
with self.assertRaisesRegex(ValueError,'不能与趋势或期间比较合并'):
execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,
authority={**identity(),'task':t},actor_root=root,delegated=api,
connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
self.assertEqual(t['trendRequired'],updates.get('trendRequired',False))
self.assertEqual(t.get('comparisonPeriod'),updates.get('comparisonPeriod'))
self.assertFalse(any(path.startswith('/api/metrics') for _,path,_ in api.calls))
def test_ranking_with_ready_but_incomplete_visual_set_finishes_partial(self):
t=ranking_task();api=BoundRankingApi()
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',
return_value={'answer':'已核验重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}),\
patch('analysis_visual_evidence.visual_result',return_value={
'visualSchemaVersion':1,'visualState':'ready','visuals':[]}):
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
result=execute(request={'releaseId':RELEASE,'turnId':TURN,'question':t['question']},task=t,
authority={**identity(),'task':t},actor_root=root,delegated=api,
connection=('offline','synthetic','model'),cancel_event=threading.Event(),emit=lambda _:None)
self.assertEqual(result['visualState'],'ready')
self.assertEqual(result['status'],'PARTIAL')
self.assertTrue(any('重量排名图表未覆盖' in item for item in result['evidenceCompletion']['missingEvidence']))
def test_clarification_executes_no_tool_or_finalizer(self):
t=task();t.update(needsClarification=True,clarification='请选完整规格')
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize') as synth:
api=MagicMock()
result=execute(request={'releaseId':RELEASE,'question':t['question']},task=t,authority=identity(),actor_root=Path(folder),
delegated=api,connection=('offline','key','model'),cancel_event=threading.Event(),emit=lambda _:None)
self.assertEqual(result['status'],'NEEDS_CLARIFICATION');api.call.assert_not_called();synth.assert_not_called()
def test_backend_previous_task_wins_over_local_task_and_chat(self):
previous=task({'product':'18X15X4+T'})
planned=task();planned.update(needsClarification=True,clarification='请确认期间')
options={'products':['18X15X4+T','18X15X4+Z'],'organizationPaths':[]}
captured={}
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
local=root/'private'/'last-agent-task.json'
local.write_text(json.dumps({'sessionId':SESSION,'releaseId':RELEASE,'task':task({'product':'WRONG'})}));local.chmod(0o600)
manager=MagicMock();manager.acquire_actor.return_value=(root,None)
api=MagicMock();api.call.side_effect=lambda path,**kw: options if path.startswith('/api/options') else {'task':planned,**identity()} if path=='/api/analysis/protocol/task' else {}
delegated=MagicMock();delegated.login.return_value=api
gate=MagicMock();gate.base_url='http://127.0.0.1:1/v1'
def plan(**kw):captured.update(copy.deepcopy({k:v for k,v in kw.items() if k!='cancel_event'}));return planned
with ExitStack() as stack:
for name,value in {'RuntimeManager':MagicMock(return_value=manager),'write_access_file':MagicMock(return_value=root/'access.json'),
'SessionState':MagicMock(),'_input_cancel_listener':MagicMock(),'_model_settings':MagicMock(return_value=('offline','key','model',{})),
'_model_gate_settings':MagicMock(return_value=(1,'key')),'DelegatedApi':MagicMock(return_value=delegated),'plan_agent_task':plan}.items():
stack.enter_context(patch.object(runtime,name,value))
stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate))
result=runtime.run_turn({'turnId':TURN,'actorId':ACTOR,'sessionId':SESSION,'releaseId':'33333333-3333-4333-8333-333333333333',
'accessToken':'synthetic','question':'清除规格条件','context':{'_protocol':{'version':1,'attemptId':SESSION,'previousTask':previous},
'_serverHistory':[{'answer':'unverified'}],'lastVisibleAnswer':'unverified'}},stdout=io.StringIO(),stdin=io.StringIO())
self.assertEqual(result['status'],'NEEDS_CLARIFICATION')
self.assertEqual(captured['previous'],previous)
self.assertNotIn('_serverHistory',captured['context']);self.assertNotIn('lastVisibleAnswer',captured['context'])
manager.ensure_instance.assert_not_called()
self.assertEqual(json.loads(local.read_text())['task']['filters']['product'],'WRONG')
def test_new_task_does_not_inherit_old_filters_or_period(self):
context=bind_task({'_protocol':{'version':1}},task({'product':'OLD','department':'OLD'}))
t=normalize_task(task_value(newTask=True,kind='metrics',dimensions=[],period={'year':2025}),'另一个新任务',context)
self.assertEqual(t['filters'],{'columnFilters':[]});self.assertEqual(t['period'],{'year':2025})
if __name__=='__main__':unittest.main()
+433
View File
@@ -0,0 +1,433 @@
"""Security and stream-state tests for the delegated NAO runtime (no model calls)."""
from __future__ import annotations
import io
import base64
import json
import tempfile
import threading
import time
import unittest
import uuid
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from unittest.mock import patch
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
import nao_quality_runtime as runtime
class FakeResponse(io.BytesIO):
status = 200
def __enter__(self):
return self
def __exit__(self, *_):
self.close()
class FakeOpener:
def __init__(self, body=b"{}"):
self.body = body
self.calls = []
def open(self, request, timeout=None):
self.calls.append((request, timeout))
return FakeResponse(self.body)
class RuntimeAccessTests(unittest.TestCase):
def test_complete_batch_ranking_progress_does_not_claim_arbitrary_truncation(self):
text = runtime.public_tool_result({'kind':'quality_changes','completeRanking':True,
'rows':[{'key':'甲'}], 'rowsTruncated':True})
self.assertIn('完整授权范围排序',text)
self.assertNotIn('不代表全量排名',text)
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix="test-nao-users-")
self.users = Path(self.temp.name).resolve()
self.actor = str(uuid.uuid4())
self.release = str(uuid.uuid4())
self.actor_root = self.users / self.actor
self.actor_root.mkdir(mode=0o700)
self.access = self.actor_root / "access.json"
runtime._atomic_json(self.access, {
"token": "delegated-secret-for-test",
"releaseId": self.release,
"queryUrl": runtime.RUNTIME_QUERY_URL,
})
def tearDown(self):
self.temp.cleanup()
def test_delegated_login_only_validates_file_and_never_calls_admin_login(self):
class NoNetwork:
calls = 0
def open(self, *_args, **_kwargs):
self.calls += 1
raise AssertionError("login must not call NAO or admin auth")
opener = NoNetwork()
api = runtime.DelegatedApi(self.access, runtime_root=self.users,
expected_actor_id=self.actor, opener=opener)
self.assertIs(api.login(), api)
self.assertEqual(opener.calls, 0)
def test_every_call_uses_fixed_loopback_delegate_and_pinned_release(self):
opener = FakeOpener(b'{"items":[]}')
api = runtime.DelegatedApi(self.access, runtime_root=self.users,
expected_actor_id=self.actor, opener=opener).login()
self.assertEqual(api.call("/api/releases", method="GET"), {"items": []})
request, _ = opener.calls[0]
body = json.loads(request.data.decode())
self.assertEqual(request.full_url, runtime.RUNTIME_QUERY_URL)
self.assertEqual(set(body), {"token", "method", "path", "payload"})
self.assertEqual(body["token"], "delegated-secret-for-test")
with self.assertRaisesRegex(ValueError, "releaseId"):
api.call("/api/metrics?releaseId=" + str(uuid.uuid4()), method="GET")
with self.assertRaisesRegex(ValueError, "允许范围"):
api.call("/api/admin/config", method="GET")
self.assertEqual(len(opener.calls), 1)
echoed = runtime.DelegatedApi(
self.access, runtime_root=self.users, expected_actor_id=self.actor,
opener=FakeOpener(b'{"token":"delegated-secret-for-test","note":"echo delegated-secret-for-test"}'),
).call("/api/releases", method="GET")
self.assertEqual(echoed, {"note": "echo [REDACTED]"})
def test_scoped_query_envelope_binds_node_and_scope_without_changing_url(self):
opener = FakeOpener(b'{"ok":true}')
api = runtime.DelegatedApi(self.access, runtime_root=self.users,
expected_actor_id=self.actor, opener=opener).login()
self.assertEqual(api.call('/api/metrics?releaseId=' + self.release, method='GET',
node_id='trend_a', scope_id='execution:trend_a:1'), {'ok': True})
request, _ = opener.calls[0]
wrapper = json.loads(request.data.decode())
self.assertEqual(set(wrapper), {'token', 'method', 'path', 'payload', 'nodeId', 'scopeId'})
self.assertEqual(wrapper['nodeId'], 'trend_a')
self.assertEqual(wrapper['scopeId'], 'execution:trend_a:1')
self.assertEqual(wrapper['path'], '/api/metrics?releaseId=' + self.release)
with self.assertRaisesRegex(ValueError, '同时提供'):
api.call('/api/metrics?releaseId=' + self.release, method='GET', node_id='trend_a')
self.assertEqual(len(opener.calls), 1)
def test_batch_changes_reaches_delegate_with_exact_protocol_scope(self):
opener=FakeOpener(b'{"ok":true}')
api=runtime.DelegatedApi(self.access,runtime_root=self.users,
expected_actor_id=self.actor,opener=opener).login()
payload={'releaseId':self.release,'period':{'year':2026,'month':9},
'comparisonPeriod':{'year':2026,'month':8},'filters':{'group':'消费电子-3C'},
'dimension':'product','sort':'current_rate','stage':None,'limit':3,
'fields':['rate','totalWeight','scrapWeight']}
self.assertEqual(api.call('/api/analysis/changes',payload=payload,method='POST',
node_id='products',scope_id='scope:products'),{'ok':True})
request,_=opener.calls[0];wrapper=json.loads(request.data)
self.assertEqual(wrapper['payload'],payload)
self.assertEqual(wrapper['nodeId'],'products')
self.assertEqual(wrapper['scopeId'],'scope:products')
self.assertEqual(request.full_url,runtime.RUNTIME_QUERY_URL)
def test_malformed_access_and_non_loopback_endpoint_are_rejected(self):
runtime._atomic_json(self.access, {"token": "x", "releaseId": self.release,
"queryUrl": "http://example.com/api/analysis/runtime/query"})
with self.assertRaisesRegex(ValueError, "本机接口"):
runtime.load_access_file(self.access, runtime_root=self.users,
expected_actor_id=self.actor)
runtime._atomic_json(self.access, {"token": "", "releaseId": self.release,
"queryUrl": runtime.RUNTIME_QUERY_URL})
with self.assertRaisesRegex(ValueError, "不完整"):
runtime.load_access_file(self.access, runtime_root=self.users,
expected_actor_id=self.actor)
def test_access_file_cannot_escape_actor_directory(self):
outside = self.users.parent / self.actor / "access.json"
with self.assertRaisesRegex(ValueError, "越界"):
runtime.load_access_file(outside, runtime_root=self.users)
def test_access_symlinks_are_rejected(self):
link = self.actor_root / "linked-access.json"
link.symlink_to(self.access)
with self.assertRaisesRegex(ValueError, "路径无效"):
runtime.load_access_file(link, runtime_root=self.users)
root_link = self.users.parent / (self.users.name + "-link")
root_link.symlink_to(self.users, target_is_directory=True)
try:
with self.assertRaisesRegex(ValueError, "运行目录无效"):
runtime.load_access_file(self.access, runtime_root=root_link)
finally:
root_link.unlink()
def test_session_to_native_chat_mapping_is_actor_scoped(self):
session_id = str(uuid.uuid4())
first = runtime.SessionState(self.actor_root, self.actor)
other_actor = str(uuid.uuid4())
other_root = self.users / other_actor
other_root.mkdir(mode=0o700)
second = runtime.SessionState(other_root, other_actor)
first.set_chat_id(session_id, "native-chat-one")
second.set_chat_id(session_id, "native-chat-two")
self.assertEqual(first.get_chat_id(session_id), "native-chat-one")
self.assertEqual(second.get_chat_id(session_id), "native-chat-two")
self.assertNotEqual(first.path, second.path)
class RuntimeStreamTests(unittest.TestCase):
def test_accumulator_keeps_provider_reasoning_separate_from_answer(self):
updates = []
stream = runtime.ConversationAccumulator(updates.append)
stream.feed({"type": "reasoning-delta", "delta": "private thought"})
stream.feed({"type": "text-start", "id": "first"})
stream.feed({"type": "text-delta", "id": "first", "delta": "early block"})
stream.feed({"type": "text-start", "id": "final"})
stream.feed({"type": "text-delta", "id": "final", "delta": "final answer"})
self.assertEqual(stream.answer, "final answer")
self.assertNotIn("private thought", updates)
self.assertEqual(stream.model_reasoning, "private thought")
def test_provider_reasoning_is_bounded_and_never_inferred_from_answer(self):
stream = runtime.ConversationAccumulator()
stream.feed({"type": "text-delta", "delta": "Only a final answer"})
self.assertEqual(stream.model_reasoning, "")
stream.feed({"type": "reasoning-delta", "id": "r1", "delta": "x" * 199_999})
stream.feed({"type": "reasoning-delta", "id": "r1", "delta": "yz"})
stream.feed({"type": "reasoning-delta", "id": "r2", "delta": "more"})
self.assertEqual(len(stream.model_reasoning.strip()), 200_000)
self.assertTrue(stream.reasoning_truncated)
self.assertEqual(stream.answer, "Only a final answer")
def test_length_and_stringified_mcp_error_are_partial(self):
stream = runtime.ConversationAccumulator()
stream.feed({"type": "tool-input-available", "toolName": "quality_data", "toolCallId": "q1"})
stream.feed({"type": "tool-output-available", "toolCallId": "q1",
"output": '{"isError":true,"content":[{"text":"forbidden"}]}'})
stream.feed({"type": "text-delta", "id": "answer", "delta": "查询受限,说明原因"})
stream.feed({"type": "finish", "finishReason": "stop"})
self.assertEqual(stream.tool_calls[0]["status"], "failed")
self.assertEqual(stream.status()[0], "PARTIAL")
output_error = runtime.ConversationAccumulator()
output_error.feed({"type": "tool-input-available", "toolName": "details", "toolCallId": "q2"})
output_error.feed({"type": "tool-output-error", "toolCallId": "q2"})
self.assertEqual(output_error.tool_calls[0]["status"], "failed")
truncated = runtime.ConversationAccumulator()
truncated.feed({"type": "text-delta", "id": "a", "delta": "正文未完"})
truncated.feed({"type": "finish", "finishReason": "length"})
self.assertEqual(truncated.status(), ("PARTIAL", "length"))
def test_native_stop_controller_stops_before_closing_stream_on_cancel(self):
calls = []
response = FakeResponse()
controller = runtime.NativeStopController(lambda chat_id: calls.append(chat_id) or True, timeout=5)
controller.set_chat_id("native-id")
controller.set_response(response)
controller.start()
controller.request_cancel("user")
controller.close()
self.assertEqual(calls, ["native-id"])
self.assertTrue(response.closed)
stream = runtime.ConversationAccumulator()
self.assertEqual(stream.status(cancel_cause="user", stop_succeeded=True),
("CANCELLED", "user_cancel"))
def test_watchdog_timeout_invokes_native_stop(self):
calls = []
controller = runtime.NativeStopController(lambda chat_id: calls.append(chat_id) or True, timeout=0.03)
controller.set_chat_id("timed-native-id")
controller.start()
deadline = time.monotonic() + 1
while controller.thread.is_alive() and time.monotonic() < deadline:
time.sleep(0.01)
controller.close()
self.assertEqual(calls, ["timed-native-id"])
self.assertEqual(controller.cause, "timeout")
def test_complete_evidence_can_stop_native_before_turn_timeout(self):
calls = []
controller = runtime.NativeStopController(
lambda chat_id: calls.append(chat_id) or True, timeout=3,
completion_ready=lambda: True)
controller.set_chat_id("covered-native-id")
controller.start()
deadline = time.monotonic() + 2
while controller.thread.is_alive() and time.monotonic() < deadline:
time.sleep(0.02)
controller.close()
self.assertEqual(calls, ["covered-native-id"])
self.assertEqual(controller.cause, "coverage_ready")
stream = runtime.ConversationAccumulator()
self.assertEqual(stream.status(cancel_cause="coverage_ready", stop_succeeded=True),
("FAILED", "coverage_ready"))
def test_transport_failure_stops_native_and_falls_back_to_owned_instance(self):
class BrokenClient:
def open(self, *_args, **_kwargs):
raise OSError("private transport detail must not escape")
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
users = Path(temp).resolve()
actor_id = str(uuid.uuid4())
actor_root = users / actor_id
actor_root.mkdir(mode=0o700)
session_id = str(uuid.uuid4())
state = runtime.SessionState(actor_root, actor_id)
with patch.object(runtime, "_stop_native", return_value=False) as native_stop, \
patch.object(runtime, "_stop_instance", return_value=True) as instance_stop:
result = runtime._sse_request(
client=BrokenClient(), base_url="http://127.0.0.1:10000", model="test",
question="q", release_id=str(uuid.uuid4()), context=None, chat_id="existing-native-id",
session_state=state, session_id=session_id, actor_root=actor_root,
emit=lambda _event: None, timeout=5,
)
self.assertEqual(result[1], "FAILED")
native_stop.assert_called_once_with(unittest.mock.ANY, "http://127.0.0.1:10000", "existing-native-id")
instance_stop.assert_called_once_with(actor_root, grace_seconds=1.0)
def test_early_eof_without_finish_or_done_stops_native_task(self):
class EofClient:
def open(self, *_args, **_kwargs):
return FakeResponse(b'data: {"type":"text-delta","id":"a","delta":"partial"}\n')
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
users = Path(temp).resolve()
actor_id = str(uuid.uuid4())
actor_root = users / actor_id
actor_root.mkdir(mode=0o700)
session_id = str(uuid.uuid4())
with patch.object(runtime, "_stop_native", return_value=True) as native_stop:
result = runtime._sse_request(
client=EofClient(), base_url="http://127.0.0.1:10000", model="test",
question="q", release_id=str(uuid.uuid4()), context=None, chat_id="existing-native-id",
session_state=runtime.SessionState(actor_root, actor_id), session_id=session_id,
actor_root=actor_root, emit=lambda _event: None, timeout=5,
)
self.assertEqual(result[1], "PARTIAL")
self.assertEqual(result[2], "transport_or_stream_error")
native_stop.assert_called_once_with(unittest.mock.ANY, "http://127.0.0.1:10000", "existing-native-id")
def test_story_export_uses_last_successful_call_and_matching_markdown(self):
class StoryClient:
def __init__(self):
self.slugs = []
def open(self, request, timeout=None):
encoded_input = parse_qs(urlsplit(request.full_url).query)["input"][0]
query = json.loads(encoded_input)["json"]
slug = query["storySlug"]
self.slugs.append(slug)
html = f"<html>{slug}</html>".encode()
payload = {"result": {"data": {"json": {
"data": base64.b64encode(html).decode("ascii")
}}}}
return FakeResponse(json.dumps(payload).encode())
client = StoryClient()
calls = [
{"tool": "story", "toolCallId": "create"},
{"tool": "story", "toolCallId": "replace"},
{"tool": "story", "toolCallId": "failed-replacement"},
]
outputs = {
"create": {"success": True, "id": "old-report", "code": "# Initial revision"},
"replace": {"success": True, "id": "final-report", "code": "# Final revision"},
"failed-replacement": {"success": False, "id": "broken-report", "code": "# Failed"},
}
markdown, html = runtime._story_files(client, "http://127.0.0.1:10000", "chat", calls, outputs)
self.assertEqual(markdown, "# Final revision")
self.assertEqual(html, "<html>final-report</html>")
self.assertEqual(client.slugs, ["final-report"])
def test_sse_uses_idle_timeout_and_warm_session_configuration_is_reused(self):
class CaptureClient:
def __init__(self):
self.timeout = None
def open(self, request, timeout=None):
self.timeout = timeout
return FakeResponse(b'data: {"type":"reasoning-delta","id":"r","delta":"provider reasoning"}\n'
b'data: {"type":"reasoning-end","id":"r"}\n'
b'data: {"type":"text-delta","id":"a","delta":"answer"}\n'
b'data: {"type":"finish","finishReason":"stop"}\n'
b'data: [DONE]\n')
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
users = Path(temp).resolve()
actor_id = str(uuid.uuid4())
session_id = str(uuid.uuid4())
actor_root = users / actor_id
actor_root.mkdir(mode=0o700)
project = actor_root / "project"
project.mkdir()
current_session = actor_root / "private" / "current-session.json"
runtime._atomic_json(current_session, {"actorId": actor_id, "sessionId": session_id})
runtime._write_project(actor_root, session_id, "test-model", refresh_templates=True,
refresh_mcp=True, backup_schema=False)
mcp_path = project / "agent" / "mcps" / "mcp.json"
original_config = mcp_path.read_bytes()
runtime._write_project(actor_root, session_id, "test-model", refresh_templates=False,
refresh_mcp=False, backup_schema=False)
self.assertEqual(mcp_path.read_bytes(), original_config)
client = CaptureClient()
events = []
result = runtime._sse_request(
client=client, base_url="http://127.0.0.1:10000", model="test-model",
question="q", release_id=str(uuid.uuid4()), context=None, chat_id=None,
session_state=runtime.SessionState(actor_root, actor_id), session_id=session_id,
actor_root=actor_root, emit=events.append, timeout=5,
)
self.assertEqual(client.timeout, runtime.SSE_IDLE_TIMEOUT_SECONDS)
self.assertEqual(result[1], "COMPLETED")
self.assertEqual(result[0].answer, "answer")
self.assertFalse(any("modelReasoning" in event for event in events))
self.assertTrue(any(event.get("timeline") for event in events))
next_session = str(uuid.uuid4())
runtime._write_project(actor_root, next_session, "test-model", refresh_templates=False,
refresh_mcp=True, backup_schema=False)
updated = json.loads(mcp_path.read_text(encoding="utf-8"))
server_env = updated["mcpServers"]["quality"]["env"]
self.assertEqual(server_env["QUALITY_AGENT_SESSION_ID"], next_session)
self.assertEqual(json.loads(current_session.read_text())["sessionId"], next_session)
class RuntimeManagerTests(unittest.TestCase):
def test_ensure_instance_executes_project_setup_and_registry_path(self):
with tempfile.TemporaryDirectory(prefix="test-nao-users-") as temp:
users = Path(temp).resolve()
manager = runtime.RuntimeManager(users_root=users)
actor_id = str(uuid.uuid4())
session_id = str(uuid.uuid4())
actor_root, actor_lock = manager.acquire_actor(actor_id)
fake_port = 49123
def fake_start(root, model_base, model_key, model_name, fingerprint):
self.assertEqual(root, actor_root)
runtime._atomic_json(root / "instance.json", {
"actorId": actor_id, "serverPort": fake_port,
"fingerprint": fingerprint, "processes": [],
})
try:
with patch.object(runtime, "_model_settings",
return_value=("https://model.invalid/v1", "test-key", "test-model")), \
patch.object(runtime, "_config_fingerprint", return_value="test-fingerprint"), \
patch.object(manager, "_start", side_effect=fake_start) as start:
config = manager.ensure_instance(actor_root, session_id)
self.assertEqual(config, ("https://model.invalid/v1", "test-key", "test-model",
f"http://127.0.0.1:{fake_port}"))
start.assert_called_once()
project = actor_root / "project"
self.assertTrue((project / "RULES.md").is_file())
self.assertTrue((project / "nao_config.yaml").is_file())
mcp_config = json.loads((project / "agent" / "mcps" / "mcp.json").read_text())
self.assertEqual(mcp_config["mcpServers"]["quality"]["env"]["QUALITY_AGENT_SESSION_ID"],
session_id)
self.assertEqual(runtime.RuntimeManager._current_session(actor_root), session_id)
finally:
manager.release_actor(actor_lock)
if __name__ == "__main__":
unittest.main()
+731
View File
@@ -0,0 +1,731 @@
from __future__ import annotations
import io
import json
import logging
import sys
import tempfile
import unittest
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from dbgpt_trial_data import encoded_selection
from nao_quality_mcp import MCPServer, serve
from nao_quality_tools import (
QualityToolError,
QualityTools,
normalize_filters,
symmetric_stage_decomposition,
tool_definition,
)
RELEASE = "release-a"
def snapshot_period(year: int, month: int | None, view: str, params: dict, rows: list[dict] | None = None) -> dict:
months = [month] if month is not None else list(range(1, 13))
rate_by_month = {1: (0.94, 0.96), 2: (0.95, 0.97), 3: (0.96, 0.98)}
black_rate, bright_rate = rate_by_month.get(month or 2, (0.95, 0.97))
black_scrap = 11.0 + (month or 2)
bright_scrap = 7.0 + (month or 2)
summary = {
"key": "overall", "label": "总体", "compositeRate": black_rate * bright_rate,
"black": {"rate": black_rate, "qualifiedWeight": 1000.0, "scrapWeight": black_scrap,
"totalWeight": 1000.0 + black_scrap,
"defects": [{"name": "责废", "weight": black_scrap / 2, "rate": 0.5},
{"name": "料废", "weight": black_scrap / 3, "rate": 1 / 3},
{"name": "外废", "weight": black_scrap / 6, "rate": 1 / 6}]},
"bright": {"rate": bright_rate, "qualifiedWeight": 900.0, "scrapWeight": bright_scrap,
"totalWeight": 900.0 + bright_scrap,
"defects": [{"name": "镀废", "weight": bright_scrap / 4, "rate": 0.25},
{"name": "材料废", "weight": bright_scrap / 4, "rate": 0.25},
{"name": "加工废", "weight": bright_scrap / 4, "rate": 0.25},
{"name": "扩散废", "weight": bright_scrap / 4, "rate": 0.25}]},
"coverage": {"blackRows": 10, "brightRows": 12},
}
source_months = [{"year": year, "month": current, "blackRows": 10, "brightRows": 12} for current in months]
context = {"year": year, "month": month, "quarter": None, "view": view,
"department": params.get("department"), "group": params.get("group"),
"product": params.get("product"), "projectGroup": params.get("projectGroup"),
"includeUnassigned": params.get("includeUnassigned", False),
"exclusions": json.loads(params["exclusions"]) if "exclusions" in params else {}}
if rows is None:
rows = [summary] if view != "overall" else []
row_count = len(rows)
if "rowLimit" in params:
start = int(params.get("rowOffset", 0))
rows = rows[start:start + params["rowLimit"]]
context["rowCount"] = 1 if view != "overall" else len(rows)
context["rowCount"] = row_count
trend = [{"period": f"{year}-{current:02d}", "periodLabel": f"{year}-{current:02d}", **summary} for current in months]
return {
"release": {"id": params.get("releaseId", RELEASE), "name": "发布版本 A", "status": "PUBLISHED"},
"context": context, "summary": summary, "rows": rows, "trend": trend,
"defects": {"black": summary["black"]["defects"], "bright": summary["bright"]["defects"]},
"coverage": {"rawRows": len(months) * 22, "includedRows": len(months) * 22,
"excludedBlankRows": 0, "unmappedRows": 0, "projectGroupUnconfiguredRows": 0,
"missingStages": [], "sourceMonths": source_months},
}
class FakeApi:
def __init__(self, *, bad_coverage_month: int | None = None, zero_denominator_month: int | None = None,
products: dict[int, list[dict]] | None = None) -> None:
self.calls: list[tuple[str, str, dict | None]] = []
self.bad_coverage_month = bad_coverage_month
self.zero_denominator_month = zero_denominator_month
self.products = products or {}
self.current_release_id = RELEASE
def login(self):
return self
@staticmethod
def _product_row(key: str, product: str, black_scrap: float, bright_scrap: float,
black_rate: float = 0.95, bright_rate: float = 0.97) -> dict:
return {"key": key, "product": product, "label": product, "compositeRate": black_rate * bright_rate,
"black": {"rate": black_rate, "qualifiedWeight": 100.0, "scrapWeight": black_scrap,
"totalWeight": 100.0 + black_scrap, "defects": [{"name": "责废", "weight": black_scrap}]},
"bright": {"rate": bright_rate, "qualifiedWeight": 100.0, "scrapWeight": bright_scrap,
"totalWeight": 100.0 + bright_scrap, "defects": [{"name": "镀废", "weight": bright_scrap}]},
"coverage": {"blackRows": 4, "brightRows": 5}}
def call(self, path: str, payload=None, method=None):
self.calls.append((method or "GET", path, payload))
parsed = urlsplit(path)
params = {key: values[-1] for key, values in parse_qs(parsed.query).items()}
for key in ("year", "month", "quarter", "rowLimit", "rowOffset", "limit"):
if key in params:
params[key] = int(params[key])
if "includeUnassigned" in params:
params["includeUnassigned"] = params["includeUnassigned"].lower() == "true"
if parsed.path == "/api/releases":
return {"currentReleaseId": self.current_release_id, "items": [
{"id": RELEASE, "name": "发布版本 A", "status": "PUBLISHED"},
{"id": "candidate", "name": "候选", "status": "CANDIDATE"},
{"id": "release-b", "name": "发布版本 B", "status": "PUBLISHED"},
]}
if parsed.path == "/api/analysis/tools/catalog":
return {"kind": "analysis_catalog", "visibleWeightFields": ["black_scrap_weight"],
"visibleDefectFields": [{"code": "black_responsibility_scrap", "name": "责废"}],
"dimensions": [{"key": "product"}], "topDimensions": ["category", "product"],
"grouping": ["overall", "product"], "scope": "当前授权范围", "limitations": []}
if parsed.path == "/api/options":
return {"years": [2025, 2026], "departments": [], "groups": [], "products": [], "projectGroups": []}
if parsed.path == "/api/metrics":
year = int(params["year"])
month = int(params["month"]) if "month" in params else None
view = params.get("view", "overall")
rows = self.products.get(month or 0, []) if view == "product" else None
result = snapshot_period(year, month, view, params, rows)
if self.bad_coverage_month == month:
result["coverage"]["sourceMonths"][0]["brightRows"] = 0
result["coverage"]["missingStages"] = [{"year": year, "month": month, "stage": "bright"}]
if self.zero_denominator_month == month:
result["summary"]["bright"]["qualifiedWeight"] = 0
result["summary"]["bright"]["scrapWeight"] = 0
result["summary"]["bright"]["totalWeight"] = 0
result["summary"]["bright"]["rate"] = 0
result["summary"]["compositeRate"] = 0
return result
if parsed.path == "/api/analytics/top":
result_query = {key: value for key, value in params.items()
if key in {"releaseId", "stage", "dimension", "category", "year", "month", "quarter",
"department", "group", "product", "projectGroup", "includeUnassigned", "limit"}}
return {"releaseId": params.get("releaseId"), "stage": params.get("stage"),
"dimension": params.get("dimension"), "category": params.get("category", ""),
"denominator": 10, "shareBasis": "当前筛选下分组对象重量合计;不表示总体贡献",
"rankingCoverage": {"complete": False}, "coverage": {"rawRows": 22},
"rows": [{"rank": 1, "key": "规格 A", "label": "规格 A", "sourceCategory": "责废",
"weight": 10, "denominator": 10, "share": 1, "coverage": {"rows": 2}}],
"cumulativeThresholdApplied": False, "resultId": "top-id", "resultQuery": result_query}
if parsed.path == "/api/analysis/tools/details":
assert method == "POST"
return {"kind": "analysis_details", "rows": [{"product": "规格 A"}], "rowCount": 1,
"limit": payload["limit"], "sampled": True, "truncated": True,
"scopeExplanation": "有限样本不代表全部明细。",
"range": {**{key: payload.get(key) for key in ("releaseId", "stage", "year", "month", "quarter", "department", "group", "product", "projectGroup", "includeUnassigned")}, "exclusions": payload.get("exclusions", {})}}
if parsed.path == "/api/analysis/defects":
assert method == "POST"
if payload.get("mode") == "catalog":
return {"kind": "fine_defect_catalog", "releaseId": payload["releaseId"],
"release": {"id": payload["releaseId"], "status": "PUBLISHED"},
"context": {"stage": payload.get("stage"), "mode": "catalog"},
"grain": "source_defect_weight", "stages": [payload.get("stage")] if payload.get("stage") else ["black", "bright"],
"categories": [{"stage": "black", "category": "责废", "available": True,
"formalComponent": "black_responsibility_scrap"}],
"fields": [{"stage": "black", "key": "责废尺寸重量", "label": "责废尺寸重量",
"category": "责废", "sourceField": "责废尺寸重量", "available": True}],
"scanned": False, "limitations": ["固定源字段目录"]}
year, month = payload["year"], payload.get("month", 1)
if payload.get("dimension") == "product":
rows = [{"key": "product-a", "label": "规格 A", "category": payload.get("category") or "责废",
"defect": payload.get("defect"), "weight": 13 if month == 3 else 10, "sourceRows": 8, "presentRows": 8,
"missingRows": 0, "invalidRows": 0, "negativeRows": 0}]
else:
by_month = {
1: [
{"key": "责废尺寸重量", "label": "责废尺寸重量", "category": "责废", "weight": 10,
"sourceRows": 8, "presentRows": 8, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
{"key": "料废尺寸重量", "label": "料废尺寸重量", "category": "料废", "weight": 4,
"sourceRows": 8, "presentRows": 8, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
{"key": "责废缺角重量", "label": "责废缺角重量", "category": "责废", "weight": 2,
"sourceRows": 8, "presentRows": 8, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
],
3: [
{"key": "责废尺寸重量", "label": "责废尺寸重量", "category": "责废", "weight": 13,
"sourceRows": 9, "presentRows": 9, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
{"key": "料废尺寸重量", "label": "料废尺寸重量", "category": "料废", "weight": 5,
"sourceRows": 9, "presentRows": 9, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
{"key": "责废刀丝重量", "label": "责废刀丝重量", "category": "责废", "weight": 1,
"sourceRows": 9, "presentRows": 9, "missingRows": 0, "invalidRows": 0, "negativeRows": 0},
],
}
rows = by_month.get(month, by_month[1])
if payload.get("defect") and payload.get("dimension") != "product":
rows = [row for row in rows if row["key"] == payload["defect"]]
if payload.get("category"):
rows = [row for row in rows if row["category"] == payload["category"]]
context = {key: payload.get(key) for key in ("year", "month", "quarter", "monthStart", "monthEnd", "department", "group", "product",
"projectGroup", "includeUnassigned", "stage", "dimension", "defect", "category")}
context.update(extraFilters=payload.get("extraFilters", {}),
exclusions=payload.get("exclusions", {}),
columnFilters=payload.get("columnFilters", []),
filters={"columnFilters": payload.get("columnFilters", [])})
return {"kind": "fine_defect_top", "releaseId": payload["releaseId"],
"release": {"id": payload["releaseId"], "status": "PUBLISHED"},
"context": {**context, "mode": "top"}, "grain": "source_defect_weight",
"rows": rows[:payload["limit"]], "totalGroups": len(rows), "rowsTruncated": len(rows) > payload["limit"],
"sourceRows": 8,
"coverage": {"available": True, "sourceRows": 8, "hasValidWeights": True,
"categories": [{"category": "责废", "isReconciled": True}]},
"categoryReconciliation": [{"category": "责废", "isReconciled": True}],
"sourceFields": [{"key": row["key"], "label": row["label"], "category": row["category"],
"sourceField": row["key"]} for row in rows],
"notice": "来源字段聚合,不跨字段合并。"}
raise AssertionError(f"Unexpected API path: {path}")
class QualityToolsTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.api = FakeApi()
self.tools = QualityTools(api_factory=lambda: self.api, evidence_dir=self.temp.name)
def tearDown(self):
self.temp.cleanup()
def test_schema_has_one_tool_and_modes(self):
definition = tool_definition()
self.assertEqual(definition["name"], "quality_data")
self.assertEqual(definition["inputSchema"]["properties"]["mode"]["enum"],
["catalog", "query", "rank", "compare", "top", "changes", "defects", "details", "investigate"])
self.assertIn("defect", definition["inputSchema"]["properties"])
self.assertFalse(definition["inputSchema"]["additionalProperties"])
self.assertEqual(definition["inputSchema"]["allOf"][0]["then"]["required"], ["releaseId"])
def test_scope_wire_encoding_and_filter_conflict(self):
filters, scope = normalize_filters("auto_combined", {}, scope_was_explicit=True)
self.assertEqual(filters["department"], '新能源汽车磁钢')
self.assertNotIn('group', filters)
self.assertEqual(scope, "auto_combined")
with self.assertRaisesRegex(QualityToolError, "不能同时指定"):
normalize_filters("auto_domestic", {"group": "汽车交通-国内"}, scope_was_explicit=True)
explicit_group, _ = normalize_filters(None, {"group": "研发组"}, scope_was_explicit=False)
self.assertEqual(explicit_group["group"], "研发组")
def test_exclusions_reach_metrics_and_details_and_are_checked(self):
exclusions = {"product": ["规格+T", "__UNASSIGNED__"]}
filters = {"exclusions": exclusions}
result = self.tools.quality_data({"mode": "compare", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3}, "comparisonPeriod": {"year": 2026, "month": 1}, "filters": filters})
metric_calls = [parse_qs(urlsplit(path).query) for _, path, _ in self.api.calls if urlsplit(path).path == "/api/metrics"]
self.assertEqual(2, len(metric_calls))
self.assertTrue(all(json.loads(call["exclusions"][0]) == exclusions for call in metric_calls))
self.tools.quality_data({"mode": "details", "releaseId": RELEASE, "period": {"year": 2026, "month": 3}, "stage": "black", "filters": filters})
detail = next(payload for _, path, payload in self.api.calls if path == "/api/analysis/tools/details")
self.assertEqual(exclusions, detail["exclusions"])
snapshot = snapshot_period(2026, 3, "overall", {"releaseId": RELEASE})
with self.assertRaisesRegex(QualityToolError, "未确认排除"):
self.tools._validate_source_snapshot(snapshot, RELEASE, {"year": 2026, "month": 3}, {**filters, "includeUnassigned": False}, "overall")
def test_details_rejects_missing_exclusion_echo(self):
class MissingExclusionEchoApi(FakeApi):
def call(self, path, payload=None, method=None):
response = super().call(path, payload, method)
if path == "/api/analysis/tools/details":
response["range"].pop("exclusions", None)
return response
api = MissingExclusionEchoApi()
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
with self.assertRaisesRegex(QualityToolError, "未确认排除"):
tools.quality_data({"mode": "details", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
"stage": "black", "filters": {"exclusions": {"product": ["P"]}}})
def test_exclusions_are_validated_unsupported_modes_reject_and_defects_forwards_them(self):
for exclusions in ({"product": []}, {"unknown": ["x"]}, {"product": [None]}, None):
with self.subTest(exclusions=exclusions), self.assertRaises(QualityToolError):
normalize_filters(None, {"exclusions": exclusions}, scope_was_explicit=False)
with self.assertRaisesRegex(QualityToolError, "暂不支持排除条件"):
self.tools.quality_data({"mode": "top", "releaseId": RELEASE, "period": {"year": 2026},
"filters": {"exclusions": {"product": ["P"]}}, "stage": "black", "dimension": "product"})
result = self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3}, "filters": {"exclusions": {"product": ["P"]}},
"stage": "black", "dimension": "defect", "limit": 10})
request = next(payload for method, path, payload in reversed(self.api.calls)
if method == "POST" and urlsplit(path).path == "/api/analysis/defects")
self.assertEqual(request["exclusions"], {"product": ["P"]})
self.assertEqual(result["filters"]["exclusions"], {"product": ["P"]})
self.assertFalse(any(urlsplit(path).path == "/api/analytics/top" for _, path, _ in self.api.calls))
def test_symmetric_decomposition_is_exact_in_percentage_points(self):
result = symmetric_stage_decomposition(
0.96, 0.98, 0.94, 0.96, 0.96 * 0.98 - 0.94 * 0.96,
black_current_total_weight=100, bright_current_total_weight=90,
black_comparison_total_weight=80, bright_comparison_total_weight=70)
self.assertEqual(result["status"], "available")
self.assertAlmostEqual(float(result["blackPP"] + result["brightPP"]),
100 * (0.96 * 0.98 - 0.94 * 0.96), places=10)
def test_missing_or_invalid_stage_rate_blocks_decomposition(self):
for invalid in (None, "bad", -0.1, 1.01):
result = symmetric_stage_decomposition(invalid, 0.98, 0.94, 0.96, 0.02,
black_current_total_weight=100, bright_current_total_weight=100,
black_comparison_total_weight=100, bright_comparison_total_weight=100)
self.assertEqual(result["status"], "unavailable")
self.assertIsNone(result["blackPP"])
def test_symmetric_decomposition_refuses_zero_stage_denominator(self):
result = symmetric_stage_decomposition(
0.0, 0.98, 0.94, 0.96, 0.02,
black_current_total_weight=0, bright_current_total_weight=100,
black_comparison_total_weight=100, bright_comparison_total_weight=100)
self.assertEqual(result["status"], "unavailable")
self.assertIsNone(result["blackPP"])
def test_compare_pins_same_release_and_reuses_filters(self):
result = self.tools.quality_data({
"mode": "compare", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
"comparisonPeriod": {"year": 2026, "month": 1},
"filters": {"department": "汽车交通", "group": "汽车交通-国内", "product": "规格 A",
"projectGroup": "项目甲", "includeUnassigned": False},
})
self.assertEqual(result["releaseId"], RELEASE)
self.assertTrue(result["delta"]["available"])
self.assertEqual(result["stageDecomposition"]["status"], "available")
self.assertNotIn("blackPP", result["stageDecomposition"])
self.assertAlmostEqual(result["stageRateChangesPP"]["black"], 2.0)
self.assertAlmostEqual(result["stageRateChangesPP"]["bright"], 2.0)
self.assertNotAlmostEqual(result["stageDecomposition"]["blackContributionToCompositePP"], 2.0)
metric_calls = [call for call in self.api.calls if urlsplit(call[1]).path == "/api/metrics"]
self.assertEqual(len(metric_calls), 2)
for _, path, _ in metric_calls:
query = parse_qs(urlsplit(path).query)
self.assertEqual(query["releaseId"], [RELEASE])
self.assertEqual(query["department"], ["新能源汽车磁钢"])
self.assertEqual(query["group"], ["汽车交通-国内"])
self.assertEqual(query["product"], ["规格 A"])
self.assertEqual(query["projectGroup"], ["项目甲"])
self.assertTrue(result["evidencePath"].endswith(result["evidenceId"] + ".json"))
self.assertRegex(result["evidenceRef"], r"^Q[0-9A-F]{8}$")
evidence = json.loads(Path(result["evidencePath"]).read_text())
self.assertEqual(evidence["evidenceRef"], result["evidenceRef"])
def test_compare_single_stage_coverage_keeps_composite_but_not_missing_stage_delta(self):
self.api.bad_coverage_month = 3
result = self.tools.quality_data({
"mode": "compare", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
"comparisonPeriod": {"year": 2026, "month": 1},
})
self.assertTrue(result["delta"]["available"])
self.assertIsNotNone(result["delta"]["percentagePoints"])
self.assertIsNone(result["stageRateChangesPP"]["bright"])
self.assertEqual(result["stageDecomposition"]["status"], "unavailable")
def test_compare_zero_stage_denominator_uses_backend_composite_without_stage_fallback(self):
self.api.zero_denominator_month = 3
result = self.tools.quality_data({
"mode": "compare", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
"comparisonPeriod": {"year": 2026, "month": 1},
})
self.assertTrue(result["delta"]["available"])
self.assertIsNotNone(result["delta"]["percentagePoints"])
self.assertIsNone(result["stageRateChangesPP"]["bright"])
self.assertEqual(result["stageDecomposition"]["status"], "unavailable")
def test_data_modes_require_release_and_catalog_refreshes_current(self):
with self.assertRaisesRegex(QualityToolError, "显式传回"):
self.tools.quality_data({"mode": "query", "period": {"year": 2026, "month": 3}})
first = self.tools.quality_data({"mode": "catalog"})
self.api.current_release_id = "release-b"
second = self.tools.quality_data({"mode": "catalog"})
self.assertEqual(first["releaseId"], RELEASE)
self.assertEqual(second["releaseId"], "release-b")
explicit = self.tools.quality_data({"mode": "query", "releaseId": RELEASE,
"period": {"year": 2026, "month": 1}})
self.assertEqual(explicit["releaseId"], RELEASE)
metric_calls = [call for call in self.api.calls if urlsplit(call[1]).path == "/api/metrics"]
ids = [parse_qs(urlsplit(call[1]).query)["releaseId"][0] for call in metric_calls]
self.assertEqual(ids, [RELEASE, "release-b", RELEASE])
def test_scope_is_returned_and_metrics_cache_does_not_cross_calls(self):
args = {"mode": "query", "releaseId": RELEASE, "period": {"year": 2026, "month": 3},
"scope": "auto_combined"}
first = self.tools.quality_data(args)
second = self.tools.quality_data(args)
expected = encoded_selection(["汽车交通-国内", "汽车交通-海外"])
self.assertEqual(first["selectedScope"], "auto_combined")
self.assertEqual(first["filters"]["department"], '新能源汽车磁钢')
self.assertEqual(second["selectedScope"], "auto_combined")
metric_calls = [call for call in self.api.calls if urlsplit(call[1]).path == "/api/metrics"]
self.assertEqual(len(metric_calls), 2)
def test_candidate_release_is_rejected(self):
with self.assertRaisesRegex(QualityToolError, "不是已发布版本"):
self.tools.quality_data({"mode": "query", "releaseId": "candidate", "period": {"year": 2026, "month": 1}})
def test_top_product_category_preserves_basis_and_coverage(self):
result = self.tools.quality_data({"mode": "top", "period": {"year": 2026, "month": 3},
"releaseId": RELEASE, "stage": "black", "dimension": "product", "category": "黑片责废"})
self.assertEqual(result["category"], "责废")
self.assertIn("不表示总体贡献", result["shareBasis"])
self.assertIn("coverage", result)
self.assertIn("亮片类别份额不必等同阶段正式报废重量", result["meaning"])
call = next(item for item in self.api.calls if urlsplit(item[1]).path == "/api/analytics/top")
self.assertEqual(parse_qs(urlsplit(call[1]).query)["category"], ["责废"])
def test_catalog_describes_fine_defect_endpoint_without_calling_it(self):
result = self.tools.quality_data({"mode": "catalog"})
capability = result["fineDefects"]
self.assertEqual(capability["endpoint"], "POST /api/analysis/defects")
self.assertIn("defect", capability["dimensions"])
self.assertFalse(capability["scanned"])
self.assertEqual(capability["fields"][0]["key"], "责废尺寸重量")
catalog_call = next(call for call in self.api.calls if urlsplit(call[1]).path == "/api/analysis/defects")
self.assertEqual(catalog_call[2]["mode"], "catalog")
def test_defects_query_preserves_source_field_keys_and_filters(self):
result = self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3}, "scope": "auto_domestic",
"filters": {"department": "汽车交通", "product": "规格 A"},
"stage": "black", "dimension": "defect", "category": "黑片责废",
"limit": 2})
self.assertEqual(result["grain"], "source_defect_weight")
self.assertEqual(result["category"], "责废")
self.assertEqual(result["selectedScope"], "auto_domestic")
self.assertEqual(result["rows"][0]["key"], "责废尺寸重量")
self.assertEqual(result["rowsReturned"], 2)
self.assertIn("source_defect_weight", result["grain"])
call = next(item for item in self.api.calls if urlsplit(item[1]).path == "/api/analysis/defects")
self.assertEqual(call[0], "POST")
self.assertEqual(call[2]["releaseId"], RELEASE)
self.assertEqual(call[2]["group"], "汽车交通-国内")
self.assertEqual(call[2]["department"], "新能源汽车磁钢")
self.assertEqual(call[2]["category"], "责废")
def test_defects_accepts_java_normalized_scope_sets_but_rejects_added_values(self):
class NormalizedScopeApi(FakeApi):
def call(inner, path, payload=None, method=None):
response = super().call(path, payload, method)
if urlsplit(path).path == "/api/analysis/defects" and payload.get("mode") == "top":
for key in ("department", "group", "product", "projectGroup"):
value = response["context"].get(key)
response["context"][key] = [] if value is None else [value]
return response
api = NormalizedScopeApi()
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
# Java echoes the normalized empty sets for unset scope filters.
empty_result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3}, "stage": "black", "dimension": "defect"})
self.assertTrue(empty_result["coverage"]["available"])
# A selected scalar and its singleton collection have the same scope.
selected_result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3},
"filters": {"department": "部门A", "group": "组A", "product": "规格A",
"projectGroup": "项目A"},
"stage": "black", "dimension": "defect"})
self.assertTrue(selected_result["coverage"]["available"])
original_call = api.call
def added_scope_value(path, payload=None, method=None):
response = original_call(path, payload, method)
if urlsplit(path).path == "/api/analysis/defects" and payload.get("mode") == "top":
response["context"]["group"] = [payload["group"], "未请求组"]
return response
api.call = added_scope_value
with self.assertRaisesRegex(QualityToolError, "未确认 group"):
tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3}, "filters": {"group": "组A"},
"stage": "black", "dimension": "defect"})
def test_defects_preserves_month_range_exclusions_and_authorized_column_filters(self):
class AuthorizedColumnApi(FakeApi):
def call(inner, path, payload=None, method=None):
response = super().call(path, payload, method)
if path == "/api/analysis/tools/catalog":
response["sourceFields"] = [{"key": "customerCode", "label": "客户代码",
"type": "source_value", "stages": ["black"],
"columns": {"black": "customer_code"}, "operators": ["in"]}]
return response
api = AuthorizedColumnApi()
tools = QualityTools(api_factory=lambda: api, evidence_dir=self.temp.name)
rule = {"field": "customerCode", "stages": ["black"], "operator": "in", "values": ["C-17"]}
result = tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "monthStart": 3, "monthEnd": 9},
"filters": {"exclusions": {"product": ["规格甲"]}}, "columnFilters": [rule],
"stage": "black", "dimension": "defect", "limit": 100})
request = next(payload for method, path, payload in reversed(api.calls)
if method == "POST" and urlsplit(path).path == "/api/analysis/defects")
self.assertEqual((request["monthStart"], request["monthEnd"]), (3, 9))
self.assertEqual(request["exclusions"], {"product": ["规格甲"]})
self.assertEqual(request["columnFilters"], [rule])
self.assertEqual(result["period"], {"year": 2026, "monthStart": 3, "monthEnd": 9})
self.assertEqual(result["filters"]["columnFilters"], [rule])
def test_reported_stage_defect_calls_default_missing_dimension_without_changing_scope(self):
for stage in ("black", "bright"):
with self.subTest(stage=stage):
arguments = {"mode": "defects", "releaseId": RELEASE, "stage": stage,
"period": {"year": 2026, "month": 3},
"comparisonPeriod": {"year": 2026, "month": 1},
"filters": {"product": "规格 A"}, "limit": 20}
result = self.tools.quality_data(arguments)
self.assertEqual(result["dimension"], "defect")
self.assertEqual(result["stage"], stage)
self.assertEqual(result["filters"]["product"], "规格 A")
self.assertEqual(result["period"], arguments["period"])
self.assertEqual(result["comparisonPeriod"], arguments["comparisonPeriod"])
self.assertNotIn("dimension", arguments)
def test_missing_defect_dimension_is_not_guessed_for_specific_field_queries(self):
with self.assertRaisesRegex(QualityToolError, "dimension"):
self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3},
"stage": "black", "defect": "责废尺寸重量"})
def test_defects_compare_full_outer_pairs_without_zero_filling(self):
result = self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3},
"comparisonPeriod": {"year": 2026, "month": 1},
"stage": "black", "dimension": "defect", "limit": 10})
rows = {row["key"]: row for row in result["pairedRows"]}
self.assertEqual(result["pairedTotalRows"], 4)
self.assertAlmostEqual(rows["责废尺寸重量"]["deltaWeight"], 3)
self.assertEqual(rows["责废刀丝重量"]["changeType"], "new")
self.assertFalse(rows["责废刀丝重量"]["deltaAvailable"])
self.assertIsNone(rows["责废刀丝重量"]["deltaWeight"])
self.assertEqual(rows["责废缺角重量"]["changeType"], "exit")
self.assertIsNone(rows["责废缺角重量"]["deltaWeight"])
calls = [call for call in self.api.calls if urlsplit(call[1]).path == "/api/analysis/defects"]
self.assertEqual([call[2]["limit"] for call in calls], [100, 100])
def test_single_defect_top_fetches_bounded_candidates_before_presentation_limit(self):
result=self.tools.quality_data({'mode':'defects','releaseId':RELEASE,'period':{'year':2026,'month':3},
'stage':'black','dimension':'defect','limit':1})
calls=[call for call in self.api.calls if urlsplit(call[1]).path=='/api/analysis/defects']
self.assertEqual(calls[-1][2]['limit'],100)
self.assertEqual(len(result['rows']),1);self.assertEqual(result['totalRows'],3)
self.assertTrue(result['rankingComplete']);self.assertTrue(result['truncated'])
self.assertEqual(result['rows'][0]['weight'],13)
def test_defects_product_ranking_requires_concrete_defect(self):
with self.assertRaisesRegex(QualityToolError, "必须指定 defect"):
self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3},
"stage": "black", "dimension": "product"})
result = self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3},
"stage": "black", "dimension": "product",
"defect": "责废尺寸重量"})
self.assertEqual(result["rows"][0]["label"], "规格 A")
self.assertEqual(result["defect"], "责废尺寸重量")
def test_defect_delta_ranking_precedes_limit_and_keeps_negative_order(self):
def snapshot(values):
return {"available": True, "rowsTruncated": False,
"rows": [{"key": key, "weight": weight} for key, weight in values.items()]}
current = snapshot({"a": 8, "b": 4, "龘": 30, "missing": None})
previous = snapshot({"a": 10, "b": 5, "龘": 10, "missing": 2})
paired = QualityTools._pair_defect_snapshots(current, previous, "defect", "scrap_delta")
self.assertEqual([row["key"] for row in paired["rows"]], ["龘", "b", "a", "missing"])
self.assertEqual(paired["rows"][0]["deltaWeight"], 20)
def test_defects_rejects_duplicate_keys_or_unconfirmed_context(self):
original_call = self.api.call
def duplicate_call(path, payload=None, method=None):
result = original_call(path, payload, method)
if urlsplit(path).path == "/api/analysis/defects":
result["rows"].append(dict(result["rows"][0]))
return result
self.api.call = duplicate_call
with self.assertRaisesRegex(QualityToolError, "重复 key"):
self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3},
"stage": "black", "dimension": "defect"})
self.api.call = original_call
def wrong_context(path, payload=None, method=None):
result = original_call(path, payload, method)
if urlsplit(path).path == "/api/analysis/defects":
result["context"]["product"] = "另一个规格"
return result
self.api.call = wrong_context
with self.assertRaisesRegex(QualityToolError, "未确认 product"):
self.tools.quality_data({"mode": "defects", "releaseId": RELEASE,
"period": {"year": 2026, "month": 3}, "filters": {"product": "规格 A"},
"stage": "black", "dimension": "defect"})
def test_full_outer_change_marks_new_and_exit_and_does_not_fill_rates(self):
self.api.products = {
1: [FakeApi._product_row("a", "规格 A", 10, 5), FakeApi._product_row("b", "规格 B", 4, 2)],
3: [FakeApi._product_row("a", "规格 A", 13, 4), FakeApi._product_row("c", "规格 C", 2, 1)],
}
result = self.tools.quality_data({"mode": "changes", "period": {"year": 2026, "month": 3},
"releaseId": RELEASE,
"comparisonPeriod": {"year": 2026, "month": 1},
"dimension": "product", "limit": 10})
by_key = {row["key"]: row for row in result["rows"]}
self.assertEqual(result["totalRows"], 3)
self.assertEqual(by_key["c"]["changeType"], "new")
self.assertEqual(by_key["c"]["comparison"]["scrapWeight"], 0)
self.assertIsNone(by_key["c"]["comparison"]["rate"])
self.assertEqual(by_key["b"]["changeType"], "exit")
self.assertEqual(by_key["b"]["current"]["scrapWeight"], 0)
self.assertIsNone(by_key["b"]["current"]["rate"])
self.assertIn("black + bright 两阶段", result["weightBasis"])
self.assertNotIn("totalWeight", by_key["c"]["current"])
self.assertIn("blackInputWeight", by_key["c"]["current"])
self.assertIn("brightInputWeight", by_key["c"]["current"])
self.assertNotIn("crossStageWeightSum", by_key["c"]["current"])
self.assertNotIn("totalWeightDelta", by_key["c"])
self.assertEqual([row["rank"] for row in result["rows"]], [1, 2, 3])
self.assertIn("不等于独立投入", by_key["c"]["volumeNote"])
self.assertIn("正式 compositeRate", result["rateBasis"])
signals = result["prioritySignals"]
self.assertEqual(signals["scrapIncrease"][0]["label"], "规格 C")
self.assertIn("blackInputWeight", signals["scrapIncrease"][0]["current"])
self.assertIn("brightInputWeight", signals["scrapIncrease"][0]["comparison"])
self.assertIn("不是统一贡献分数", signals["basis"])
def test_duplicate_and_missing_row_keys_are_rejected(self):
with self.assertRaisesRegex(QualityToolError, "重复 key"):
QualityTools._index_rows({"rows": [{"key": "x"}, {"key": "x"}]}, "product", "current")
with self.assertRaisesRegex(QualityToolError, "空 key"):
QualityTools._index_rows({"rows": [{"key": None}]}, "product", "current")
def test_category_absence_is_null_and_present_row_missing_stage_is_null(self):
row = {"key": "x", "black": {"scrapWeight": None, "defects": []}, "bright": {"scrapWeight": 5}}
self.assertIsNone(QualityTools._row_scrap(row, "black", "责废"))
self.assertIsNone(QualityTools._row_scrap({"key": "x", "black": None, "bright": {"scrapWeight": 5}}, None, None))
self.assertEqual(QualityTools._row_scrap(None, None, None), 0)
def test_changes_rate_requires_row_level_stage_coverage(self):
row = {"compositeRate": 0.95,
"black": {"rate": 0.96, "totalWeight": 4, "qualifiedWeight": 3},
"bright": {"rate": 0.99, "totalWeight": 0, "qualifiedWeight": 0},
"coverage": {"blackRows": 3, "brightRows": 0}}
self.assertIsNone(QualityTools._row_rate(row, None))
self.assertAlmostEqual(float(QualityTools._row_rate(row, "black")), 0.96)
self.assertIsNone(QualityTools._row_rate(row, "bright"))
self.assertIsNone(QualityTools._row_rate({**row, "coverage": None}, "black"))
positive = {**row, "bright": {"rate": 0.99, "totalWeight": 5, "qualifiedWeight": 4},
"coverage": {"blackRows": 3, "brightRows": 2}}
self.assertAlmostEqual(float(QualityTools._row_rate(positive, None)), 0.95)
def test_details_always_labels_bounded_sample(self):
result = self.tools.quality_data({"mode": "details", "period": {"year": 2026, "month": 3},
"releaseId": RELEASE,
"stage": "black", "limit": 5})
self.assertTrue(result["sampled"])
self.assertTrue(result["truncated"])
self.assertIn("不能当总体排名", result["notice"])
def test_catalog_returns_concise_periods_and_release_binding(self):
result = self.tools.quality_data({"mode": "catalog"})
self.assertEqual(result["releaseId"], RELEASE)
self.assertEqual(result["periods"]["years"], [2025, 2026])
self.assertEqual(result["periods"]["availableMonthsInLatestYear"], list(range(1, 13)))
def test_empty_coverage_is_an_error_not_success(self):
original_call = self.api.call
def empty_call(path, payload=None, method=None):
parsed = urlsplit(path)
if parsed.path == "/api/metrics":
result = original_call(path, payload, method)
result["coverage"]["rawRows"] = 0
result["coverage"]["includedRows"] = 0
return result
return original_call(path, payload, method)
self.api.call = empty_call
with self.assertRaisesRegex(QualityToolError, "2026年3月.*没有可用的已发布来源数据"):
self.tools.quality_data({"mode": "query", "releaseId": RELEASE, "period": {"year": 2026, "month": 3}})
def test_custom_formula_metadata_survives_query_and_both_comparison_sides(self):
original=self.api.call
rules={'combined':'configured_custom_formula','supportsStageContribution':False}
descriptions={'rate':'正式自定义综合率'}
def with_rules(path,payload=None,method=None):
result=original(path,payload,method)
if urlsplit(path).path == '/api/metrics':
result['context'].update(calculationRules=rules,metricFormulaDescriptions=descriptions)
return result
self.api.call=with_rules
queried=self.tools.quality_data({'mode':'query','releaseId':RELEASE,'period':{'year':2026,'month':3}})
self.assertEqual(queried['calculationRules'],rules)
compared=self.tools.quality_data({'mode':'compare','releaseId':RELEASE,
'period':{'year':2026,'month':3},'comparisonPeriod':{'year':2026,'month':2}})
for side in ('current','comparison'):
self.assertEqual(compared[side]['calculationRules'],rules)
self.assertEqual(compared[side]['metricFormulaDescriptions'],descriptions)
def test_mcp_json_rpc_initialize_list_ping_and_notification(self):
server = MCPServer(tools=self.tools)
initialized = server.handle({"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {"protocolVersion": "2024-11-05"}})
self.assertEqual(initialized["result"]["serverInfo"]["name"], "quality")
listed = server.handle({"jsonrpc": "2.0", "id": 2, "method": "tools/list"})
self.assertEqual([tool["name"] for tool in listed["result"]["tools"]], ["quality_data"])
self.assertEqual(server.handle({"jsonrpc": "2.0", "method": "notifications/initialized"}), None)
self.assertEqual(server.handle({"jsonrpc": "2.0", "id": 3, "method": "ping"})["result"], {})
def test_mcp_initialize_negotiates_newer_client_protocol(self):
server = MCPServer(tools=self.tools)
response = server.handle({"jsonrpc": "2.0", "id": "new-client", "method": "initialize",
"params": {"protocolVersion": "2025-11-25"}})
self.assertEqual(response["result"]["protocolVersion"], "2025-03-26")
def test_mcp_tools_call_and_newline_stdio(self):
server = MCPServer(tools=self.tools, logger=logging.getLogger("test.nao_quality_mcp"))
call = server.handle({"jsonrpc": "2.0", "id": 8, "method": "tools/call",
"params": {"name": "quality_data", "arguments": {
"mode": "query", "releaseId": RELEASE, "period": {"year": 2026, "month": 3}}}})
self.assertEqual(call["result"]["structuredContent"]["releaseId"], RELEASE)
self.assertFalse(call["result"]["isError"])
request_lines = [
{"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2024-11-05"}},
{"jsonrpc": "2.0", "method": "notifications/initialized"},
{"jsonrpc": "2.0", "id": 2, "method": "tools/list"},
{"jsonrpc": "2.0", "id": 3, "method": "ping"},
]
source = io.StringIO("\n".join(json.dumps(item) for item in request_lines) + "\n")
output = io.StringIO()
from unittest.mock import patch
with patch("nao_quality_mcp._configure_logger", return_value=logging.getLogger("test.nao_quality_mcp")):
serve(source, output, tools=self.tools)
responses = [json.loads(line) for line in output.getvalue().splitlines()]
self.assertEqual([item["id"] for item in responses], [1, 2, 3])
self.assertEqual(responses[1]["result"]["tools"][0]["name"], "quality_data")
if __name__ == "__main__":
unittest.main()
+133
View File
@@ -0,0 +1,133 @@
import copy
import sys
import unittest
from pathlib import Path
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
str(Path(__file__).resolve().parent)]
from nao_agent_task import TaskUnavailable, normalize_task
from nao_query_plan import normalize_query_plan, scope_documents
from test_nao_agent_task import task_value
def ranking_to_trends_plan():
return [
{'nodeId': 'rank', 'purpose': 'ranking', 'dimension': 'product', 'limit': 5,
'fields': ['totalWeight'], 'stages': ['black'], 'selector': {'kind': 'base'},
'visualType': 'pareto', 'visualPreference': 'auto'},
{'nodeId': 'top_trend', 'purpose': 'trend', 'fields': ['scrapRate'],
'stages': ['black'], 'selector': {'kind': 'ranking_top', 'sourceNodeId': 'rank',
'stage': 'black', 'metric': 'totalWeight', 'count': 2},
'visualType': 'line', 'visualPreference': 'auto'},
]
class QueryPlanNormalizationTests(unittest.TestCase):
def normalize(self, value, context=None):
return normalize_task(value, '按黑片投入重量排名后查看前两项报废率趋势',
{'_protocol': {'version': 1}, **(context or {})})
def test_rank_then_top_trend_preserves_root_union_and_filters(self):
value = task_value(kind='analysis', queryPlan=ranking_to_trends_plan(),
dimensions=['product'], dimensionStages={'product': ['black']},
stages=['black'], fields=['scrapRate', 'totalWeight'],
trendRequired=True, comparisonRequired=False, period={'year': 2026},
filterUpdates={'department': '部门甲'})
root = self.normalize(value)
self.assertEqual(root['filters'], {'department': '部门甲', 'columnFilters': []})
self.assertEqual(root['dimensions'], ['product'])
self.assertEqual(root['fields'], ['scrapRate', 'totalWeight'])
self.assertEqual(root['stages'], ['black'])
self.assertTrue(root['trendRequired'])
self.assertEqual(root['periodCoverage'], 'available')
self.assertEqual(root['comparisonRequired'], False)
self.assertEqual(root['dimensionStages'], {'product': ['black']})
self.assertEqual(root['queryPlan'][1]['selector']['count'], 2)
self.assertNotIn('period', root['queryPlan'][1])
self.assertNotIn('filters', root['queryPlan'][1])
def test_true_root_trend_and_existing_comparison_cannot_be_silently_dropped(self):
value = task_value(kind='metrics', queryPlan=[
{'nodeId': 'one', 'purpose': 'metrics', 'fields': ['rate'], 'stages': ['black'],
'selector': {'kind': 'base'}, 'visualPreference': 'none'}],
dimensions=[], dimensionStages={}, stages=['black'], fields=['rate'],
trendRequired=True, period={'year': 2026, 'month': 9})
with self.assertRaisesRegex(TaskUnavailable, 'trendRequired'):
self.normalize(value)
value['trendRequired'] = False
value['comparisonPeriod'] = {'year': 2026, 'month': 8}
with self.assertRaisesRegex(TaskUnavailable, 'comparisonPeriod'):
self.normalize(value)
def test_two_month_root_period_is_valid_for_explicit_line_trend(self):
node = {'nodeId': 'short_trend', 'purpose': 'trend', 'fields': ['scrapRate'],
'stages': ['black'], 'selector': {'kind': 'base'},
'visualType': 'line', 'visualPreference': 'auto'}
value = task_value(kind='metrics', queryPlan=[node], dimensions=[],
dimensionStages={}, stages=['black'], fields=['scrapRate'],
trendRequired=True, comparisonRequired=False,
period={'year': 2026, 'monthStart': 7, 'monthEnd': 8})
root = self.normalize(value)
self.assertEqual(root['queryPlan'][0]['visualType'], 'line')
self.assertEqual(root['period']['monthStart'], 7)
self.assertEqual(root['period']['monthEnd'], 8)
def test_previous_ranking_requires_trusted_preview_and_does_not_restore_old_filters(self):
artifact = {'queryId': 'trusted-query', 'dimension': 'product', 'stage': 'black',
'metric': 'totalWeight', 'rows': [{'rank': 1, 'name': '旧规格A'},
{'rank': 2, 'name': '旧规格B'}],
'period': {'year': 2025}, 'filters': {'department': '旧部门'}}
context = {'_protocol': {'version': 1, 'availableRankings': [artifact]}}
node = {'nodeId': 'old_trend', 'purpose': 'trend', 'fields': ['scrapRate'],
'stages': ['black'], 'selector': {'kind': 'previous_ranking',
'queryId': 'trusted-query', 'stage': 'black', 'metric': 'totalWeight', 'ranks': [1, 2]},
'visualPreference': 'auto', 'visualType': 'line'}
value = task_value(kind='metrics', queryPlan=[node], dimensions=['product'],
dimensionStages={}, stages=['black'], fields=['scrapRate'], trendRequired=True,
comparisonRequired=False, period={'year': 2026}, filterUpdates={'department': '本轮部门'})
root = self.normalize(value, context)
self.assertEqual(root['filters'], {'department': '本轮部门', 'columnFilters': []})
self.assertEqual(root['queryPlan'][0]['selector']['queryId'], 'trusted-query')
value['queryPlan'][0]['selector']['queryId'] = 'answer-text-id'
with self.assertRaisesRegex(TaskUnavailable, 'previous_ranking'):
self.normalize(value, context)
def test_catalog_group_members_must_match_authorized_directory_and_root_department(self):
catalog = {'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'],
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}
node = {'nodeId': 'compare_groups', 'purpose': 'metrics', 'fields': ['rate'],
'stages': ['black'], 'selector': {'kind': 'catalog_members', 'dimension': 'group',
'names': ['部门甲-国内', '部门甲-海外']}, 'comparisonRequired': True,
'visualPreference': 'none'}
value = task_value(kind='metrics', queryPlan=[node], dimensions=['group'],
dimensionStages={}, stages=['black'], fields=['rate'], trendRequired=False,
comparisonRequired=True, period={'year': 2026, 'month': 9},
comparisonPeriod={'year': 2026, 'month': 8}, filterUpdates={'department': '部门甲'})
root = self.normalize(value, {'_businessCatalog': catalog})
self.assertEqual(root['queryPlan'][0]['selector']['names'], catalog['groups'])
self.assertEqual(root['filters']['department'], '部门甲')
value['queryPlan'][0]['selector']['names'] = ['其他部门-国内']
with self.assertRaisesRegex(TaskUnavailable, '目录中'):
self.normalize(value, {'_businessCatalog': catalog})
def test_exact_scope_document_selection_rejects_wrong_node_stage_and_period(self):
scope = {'nodeId': 'rank', 'scopeId': 'scope-rank', 'purpose': 'ranking',
'task': {'kind': 'analysis', 'period': {'year': 2026}, 'filters': {},
'dimensions': ['product'], 'fields': ['totalWeight'], 'stages': ['black'],
'dimensionStages': {'product': ['black']}}}
proof = {'nodeId': 'rank', 'scopeId': 'scope-rank', 'queryId': 'q1'}
good = {'releaseId': 'release', 'mode': 'rank', 'result': {
'releaseId': 'release', 'period': {'year': 2026}, 'filters': {},
'protocolEvidence': proof, 'kind': 'quality_weight_ranking', 'dimension': 'product',
'stage': 'black', 'metric': 'totalWeight'}}
self.assertEqual(scope_documents([good], scope, release_id='release'), [good])
forged = copy.deepcopy(good); forged['result']['protocolEvidence']['scopeId'] = 'other'
wrong_stage = copy.deepcopy(good); wrong_stage['result']['stage'] = 'bright'
wrong_period = copy.deepcopy(good); wrong_period['result']['period'] = {'year': 2025}
self.assertEqual(scope_documents([forged, wrong_stage, wrong_period], scope,
release_id='release'), [])
if __name__ == '__main__':
unittest.main()
+283
View File
@@ -0,0 +1,283 @@
import copy
import json
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
str(Path(__file__).resolve().parent)]
from nao_agent_task import normalize_task
from nao_protocol_execution import execute
from test_nao_agent_task import task_value
from test_nao_protocol_execution import BoundApi, RELEASE, TURN, identity
class ComparisonPlanApi(BoundApi):
def __init__(self, scope_tasks):
super().__init__()
self.scope_tasks = scope_tasks
self.envelopes = []
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
self.envelopes.append({'path': path, 'nodeId': node_id, 'scopeId': scope_id})
if path == '/api/analysis/protocol/queries':
return {'version': 1, 'scopes': [
{'nodeId': 'groups_compare', 'scopeId': 'scope-' + key,
'label': key, 'purpose': 'metrics', 'task': copy.deepcopy(task)}
for key, task in self.scope_tasks.items()], 'pending': [], 'gaps': []}
response = super().call(path, payload, method)
if path.startswith('/api/metrics'):
proof = response.get('protocolEvidence')
if isinstance(proof, dict):
proof['nodeId'] = node_id
proof['scopeId'] = scope_id
proof['queryId'] = f'{TURN}-{len(self.envelopes)}'
proof['responseHash'] = f'{len(self.envelopes):064x}'
return response
class QueryPlanComparisonExecutionTests(unittest.TestCase):
def test_missing_prior_year_composite_sources_finish_partial_with_root_frame_and_proofs(self):
rule = {'nodeId': 'groups_compare', 'purpose': 'metrics', 'fields': ['rate'], 'stages': [],
'selector': {'kind': 'catalog_members', 'dimension': 'group',
'names': ['部门甲-国内', '部门甲-海外']},
'visualPreference': 'none', 'visualType': None, 'comparisonRequired': True}
value = task_value(kind='metrics', queryPlan=[rule], dimensions=['group'],
dimensionStages={}, stages=[], fields=['rate'], trendRequired=False,
comparisonRequired=True, period={'year': 2026, 'month': 9},
comparisonPeriod={'year': 2025, 'month': 9}, visualPreference='none',
filterUpdates={'department': '部门甲'})
context = {'_protocol': {'version': 1}, '_businessCatalog': {
'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'],
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}}
task = normalize_task(value, '比较部门甲国内和海外2026年9月与2025年9月的综合合格率', context)
child_tasks = {}
for name in rule['selector']['names']:
child_tasks[name] = {'version': 1, 'kind': 'metrics', 'question': task['question'],
'goal': task['goal'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'],
'filters': {**task['filters'], 'group': name}, 'dimensions': [], 'dimensionStages': {},
'fields': ['rate'], 'stages': [], 'trendRequired': False, 'comparisonRequired': True,
'periodCoverage': task['periodCoverage'], 'visualPreference': 'none', 'visualType': None,
'freshQuery': True, 'historyMode': 'none'}
class MissingPriorYearApi(ComparisonPlanApi):
def __init__(self, scope_tasks):
super().__init__(scope_tasks)
self.actual_metric_responses = []
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
response = super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
if path.startswith('/api/metrics') and 'year=2025' in path:
response['coverage'].update(rawRows=0, includedRows=0,
sourceMonths=[], missingStages=['black', 'bright'])
response['summary']['compositeRate'] = None
if path.startswith('/api/metrics'):
self.actual_metric_responses.append(copy.deepcopy(response))
return response
api = MissingPriorYearApi(child_tasks)
api.task = task
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
events = []
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
with patch('nao_evidence_finalizer._open_model', side_effect=AssertionError(
'缺少正式来源时不应调用模型补造事实')) as open_model:
result = execute(request=request, task=task, authority={**identity(), 'task': task},
actor_root=root, delegated=api, connection=('offline', 'synthetic', 'model'),
cancel_event=threading.Event(), emit=events.append)
self.assertFalse(list((root / 'evidence').glob('nao-quality-*.json')))
private_completion = json.loads((root / 'private' / 'protocol-completion.json').read_text())
self.assertEqual(result['status'], 'PARTIAL')
self.assertEqual(result['agentContext'], {'releaseId': RELEASE, 'filters': task['filters'],
'period': task['period'], 'comparisonPeriod': task['comparisonPeriod']})
self.assertEqual(len(result['protocolEvidence']), 4, result.get('evidenceCompletion'))
self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'groups_compare'})
self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']},
{'scope-部门甲-国内', 'scope-部门甲-海外'})
self.assertEqual(len({proof['queryId'] for proof in result['protocolEvidence']}), 4)
self.assertTrue(result['evidenceCompletion']['missingEvidence'])
self.assertTrue(all('2025' in gap for gap in result['evidenceCompletion']['missingEvidence']),
result['evidenceCompletion']['missingEvidence'])
self.assertFalse(result['scopedReport']['facts'])
self.assertNotIn('100%', result['answer'])
self.assertNotIn('100.00%', result['answer'])
self.assertEqual(len([item for item in api.envelopes if item['path'].startswith('/api/metrics')
and 'year=2025' in item['path']]), 2)
self.assertEqual(len([item for item in api.envelopes if item['path'].startswith('/api/metrics')]), 4)
missing_responses = [response for response in api.actual_metric_responses
if response.get('context', {}).get('year') == 2025]
self.assertEqual(len(missing_responses), 2)
self.assertTrue(all(response['coverage'].get('rawRows') == 0
and response['coverage'].get('includedRows') == 0
and response['coverage'].get('sourceMonths') == []
for response in missing_responses))
from types import SimpleNamespace
from nao_protocol_execution import _verified_empty_comparison_receipts
domestic_responses = [response for response in api.actual_metric_responses
if response.get('context', {}).get('group') == '部门甲-国内']
def receipt_observations(responses):
return [{'release': copy.deepcopy(response.get('release')),
'context': copy.deepcopy(response.get('context')),
'filters': copy.deepcopy(response.get('filters')),
'coverage': copy.deepcopy(response.get('coverage')),
'protocolEvidence': copy.deepcopy(response.get('protocolEvidence')),
'hasSummary': isinstance(response.get('summary'), dict)}
for response in responses]
scope = {'nodeId': 'groups_compare', 'scopeId': 'scope-部门甲-国内',
'label': '部门甲-国内'}
receipts = _verified_empty_comparison_receipts(SimpleNamespace(
metric_responses=receipt_observations(domestic_responses)), scope=scope,
task=child_tasks['部门甲-国内'], identity=identity(), release_id=RELEASE)
self.assertIsNotNone(receipts)
for mutate in (
lambda response: response['context'].pop('year', None),
lambda response: response['coverage'].__setitem__('rawRows', -1),
lambda response: response['coverage'].__setitem__('includedRows', 2),
lambda response: response['coverage'].pop('includedRows', None),
lambda response: response['coverage'].pop('rawRows', None),
lambda response: response['protocolEvidence'].update(scopeId='forged-scope'),
lambda response: response['protocolEvidence'].update(executionId='forged-execution')):
invalid = copy.deepcopy(domestic_responses)
mutate(invalid[1])
self.assertIsNone(_verified_empty_comparison_receipts(SimpleNamespace(
metric_responses=receipt_observations(invalid)), scope=scope, task=child_tasks['部门甲-国内'],
identity=identity(), release_id=RELEASE))
self.assertTrue(any('正式查询 4/40' in event.get('message', '') for event in events))
diagnostics = private_completion['queryDiagnostics']
self.assertEqual(len(diagnostics), 4)
self.assertEqual(sum(row['rawRows'] == 0 for row in diagnostics), 2)
self.assertEqual(sum(row['includedRows'] == 0 for row in diagnostics), 2)
self.assertTrue(all(row['nodeId'] == 'groups_compare' and row['queryId'] for row in diagnostics))
open_model.assert_not_called()
def test_one_empty_group_keeps_other_group_charts_ready_while_turn_is_partial(self):
rule = {'nodeId': 'groups_compare', 'purpose': 'metrics',
'fields': ['rate', 'totalWeight', 'scrapWeight'], 'stages': ['black'],
'selector': {'kind': 'catalog_members', 'dimension': 'group',
'names': ['部门甲-国内', '部门甲-海外']},
'visualPreference': 'required', 'visualType': 'column', 'comparisonRequired': True}
value = task_value(kind='metrics', queryPlan=[rule], dimensions=['group'],
dimensionStages={}, stages=['black'], fields=rule['fields'], trendRequired=False,
comparisonRequired=True, period={'year': 2026, 'month': 9},
comparisonPeriod={'year': 2025, 'month': 9}, visualPreference='required',
visualType='column', filterUpdates={'department': '部门甲'})
context = {'_protocol': {'version': 1}, '_businessCatalog': {
'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'],
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}}
task = normalize_task(value, '比较部门甲国内和海外2026年9月与2025年9月黑片合格率和重量', context)
child_tasks = {name: {'version': 1, 'kind': 'metrics', 'question': task['question'],
'goal': task['goal'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'],
'filters': {**task['filters'], 'group': name}, 'dimensions': [], 'dimensionStages': {},
'fields': list(rule['fields']), 'stages': ['black'], 'trendRequired': False,
'comparisonRequired': True, 'periodCoverage': task['periodCoverage'],
'visualPreference': 'required', 'visualType': 'column', 'freshQuery': True, 'historyMode': 'none'}
for name in rule['selector']['names']}
class MissingOverseasApi(ComparisonPlanApi):
def __init__(self, scope_tasks):
super().__init__(scope_tasks)
self.actual_metric_responses = []
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
response = super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
if (path.startswith('/api/metrics')
and response.get('context', {}).get('group') == '部门甲-海外'
and response.get('context', {}).get('year') == 2025):
response['coverage'].update(rawRows=0, includedRows=0,
sourceMonths=[], missingStages=['black', 'bright'])
response['summary']['compositeRate'] = None
if path.startswith('/api/metrics'):
self.actual_metric_responses.append(copy.deepcopy(response))
return response
api = MissingOverseasApi(child_tasks)
api.task = task
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
with patch('nao_evidence_finalizer.synthesize', return_value={
'answer': '国内对照数据有效,海外缺少去年来源。',
'evidenceSufficient': True, 'missingEvidence': [], 'usage': {}}):
result = execute(request=request, task=task, authority={**identity(), 'task': task},
actor_root=root, delegated=api, connection=('offline', 'synthetic', 'model'),
cancel_event=threading.Event(), emit=lambda _: None)
self.assertEqual(result['status'], 'PARTIAL')
self.assertEqual(result['visualState'], 'ready')
self.assertTrue(result['visuals'])
self.assertEqual({chart.get('scope', {}).get('group') for chart in result['visuals']},
{'部门甲-国内'})
self.assertEqual({chart['metric'] for chart in result['visuals']},
{'black_rate', 'black_total_weight', 'black_scrap_weight'})
self.assertTrue(any('部门甲-海外' in gap and '2025' in gap
for gap in result['evidenceCompletion']['missingEvidence']))
self.assertEqual(len(result['protocolEvidence']), 4)
self.assertEqual(len(result['queryEvidenceRefs']), 1)
def test_catalog_group_compare_uses_current_periods_without_mutating_root_filters(self):
rule = {'nodeId': 'groups_compare', 'purpose': 'metrics', 'fields': ['rate', 'totalWeight', 'scrapWeight'],
'stages': ['black'], 'selector': {'kind': 'catalog_members',
'dimension': 'group', 'names': ['部门甲-国内', '部门甲-海外']},
'visualPreference': 'required', 'visualType': 'column', 'comparisonRequired': True}
value = task_value(kind='metrics', queryPlan=[rule], dimensions=['group'],
dimensionStages={}, stages=['black'], fields=['rate', 'totalWeight', 'scrapWeight'], trendRequired=False,
comparisonRequired=True, period={'year': 2026, 'month': 9},
comparisonPeriod={'year': 2026, 'month': 8}, filterUpdates={'department': '部门甲'})
context = {'_protocol': {'version': 1}, '_businessCatalog': {
'departments': ['部门甲'], 'groups': ['部门甲-国内', '部门甲-海外'],
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}}
task = normalize_task(value, '比较部门甲的国内和海外两期黑片合格率', context)
child_tasks = {}
for name in rule['selector']['names']:
child_tasks[name] = {'version': 1, 'kind': 'metrics', 'question': task['question'],
'goal': task['goal'], 'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'],
'filters': {**task['filters'], 'group': name}, 'dimensions': [], 'dimensionStages': {},
'fields': ['rate', 'totalWeight', 'scrapWeight'], 'stages': ['black'], 'trendRequired': False,
'comparisonRequired': True, 'periodCoverage': task['periodCoverage'],
'visualPreference': 'required', 'visualType': 'column', 'freshQuery': True, 'historyMode': 'none'}
api = ComparisonPlanApi(child_tasks)
api.task = task
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
with patch('nao_evidence_finalizer.synthesize', return_value={
'answer': '已核对国内和海外两期指标', 'evidenceSufficient': True,
'missingEvidence': [], 'usage': {}}) as synth:
result = execute(request=request, task=task, authority={**identity(), 'task': task},
actor_root=root, delegated=api, connection=('offline', 'synthetic', 'model'),
cancel_event=threading.Event(), emit=lambda _: None)
self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence'])
self.assertEqual(len(result['protocolEvidence']), 4)
self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'groups_compare'})
self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']},
{'scope-部门甲-国内', 'scope-部门甲-海外'})
metric_calls = [call for call in api.envelopes if call['path'].startswith('/api/metrics')]
self.assertTrue(metric_calls)
self.assertTrue(all(call['nodeId'] == 'groups_compare' and call['scopeId'] for call in metric_calls))
self.assertEqual(result['agentContext']['filters'], task['filters'])
self.assertNotIn('group', result['agentContext']['filters'])
self.assertEqual(result['agentContext']['comparisonPeriod'], task['comparisonPeriod'])
chart_keys = {'id', 'type', 'title', 'subtitle', 'unit', 'categories', 'series',
'evidenceRefs', 'requiredFields', 'scope', 'period', 'comparisonPeriod',
'stage', 'metric', 'coverage', 'note', 'originTurnId', 'sectionKey', 'dimension'}
self.assertTrue(result['visuals'])
self.assertTrue(all(set(chart) <= chart_keys for chart in result['visuals']))
self.assertTrue(all(not {'protocolNodeId', 'protocolScopeId', 'protocolScopeLabel'} & set(chart)
for chart in result['visuals']))
for group in ('部门甲-国内', '部门甲-海外'):
scoped_charts = [chart for chart in result['visuals']
if chart.get('scope', {}).get('group') == group]
self.assertEqual({chart['metric'] for chart in scoped_charts},
{'black_rate', 'black_total_weight', 'black_scrap_weight'})
self.assertTrue(all(chart['type'] == 'column' for chart in scoped_charts))
synth.assert_called_once()
if __name__ == '__main__':
unittest.main()
+194
View File
@@ -0,0 +1,194 @@
import copy
import sys
import threading
import unittest
from pathlib import Path
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
str(Path(__file__).resolve().parent)]
from nao_agent_task import TaskUnavailable, normalize_task, plan_task
from nao_query_contract import bind_query
from test_nao_agent_task import task_value
def defect_catalog():
fields = [
{'stage': 'black', 'key': '责废尺寸重量', 'label': '责废尺寸重量',
'sourceField': '责废尺寸重量', 'category': '责废',
'grain': 'source_defect_weight', 'available': True},
{'stage': 'black', 'key': '料废端面重量', 'label': '料废端面重量',
'sourceField': '料废端面重量', 'category': '料废',
'grain': 'source_defect_weight', 'available': True},
{'stage': 'bright', 'key': '加工缺口重量', 'label': '加工缺口重量',
'sourceField': '加工缺口重量', 'category': '加工废',
'grain': 'source_defect_weight', 'available': True},
{'stage': 'bright', 'key': '加工数量', 'label': '加工数量',
'sourceField': '加工数量', 'category': '加工废',
'grain': 'source_defect_count', 'available': True, 'countOnly': True},
]
categories = [
{'stage': 'black', 'category': '责废', 'available': True},
{'stage': 'black', 'category': '料废', 'available': True},
{'stage': 'bright', 'category': '加工废', 'available': True},
]
return {'fineDefects': {'fields': fields, 'categories': categories}}
def defect_node(*, dimension='defect', stages=None, selector=None, defect=None,
category=None, node_id='black_defects', comparison=False):
node = {'nodeId': node_id, 'purpose': 'defects', 'dimension': dimension,
'fields': ['scrapWeight'], 'stages': stages or ['black'],
'selector': selector or {'kind': 'base'}, 'visualType': 'bar',
'visualPreference': 'auto', 'comparisonRequired': comparison}
if defect is not None:
node['defect'] = defect
if category is not None:
node['category'] = category
return node
class QueryPlanDefectNormalizationTests(unittest.TestCase):
def normalize(self, value, question='查看来源具体缺陷重量', context=None):
supplied = context or {}
business = {**defect_catalog(), **supplied.get('_businessCatalog', {})}
return normalize_task(value, question, {'_protocol': {'version': 1},
**supplied, '_businessCatalog': business})
def test_defect_node_derives_root_output_dimensions_stages_and_fields(self):
value = task_value(kind='analysis', queryPlan=[defect_node()],
dimensions=['defect'], dimensionStages={'defect': ['black']},
stages=['black'], fields=['scrapWeight'], trendRequired=False,
comparisonRequired=False, period={'year': 2026, 'month': 9})
task = self.normalize(value)
self.assertEqual(task['dimensions'], ['defect'])
self.assertEqual(task['dimensionStages'], {'defect': ['black']})
self.assertEqual(task['stages'], ['black'])
self.assertEqual(task['fields'], ['scrapWeight'])
self.assertEqual(task['queryPlan'][0]['visualType'], 'bar')
self.assertEqual(task['periodCoverage'], 'available')
def test_product_output_adds_product_and_defect_but_selector_group_is_not_output_dimension(self):
selector = {'kind': 'catalog_members', 'dimension': 'group',
'names': ['部门甲-国内', '部门甲-海外']}
node = defect_node(dimension='product', defect='责废尺寸重量', category='责废', selector=selector)
value = task_value(kind='analysis', queryPlan=[node], dimensions=['product', 'group', 'defect'],
dimensionStages={'product': ['black'], 'group': [], 'defect': ['black']},
stages=['black'], fields=['scrapWeight'], trendRequired=False,
comparisonRequired=False, period={'year': 2026, 'month': 9})
context = {'_businessCatalog': {'groups': ['部门甲-国内', '部门甲-海外']}}
task = self.normalize(value, context=context)
self.assertEqual(task['dimensions'], ['product', 'group', 'defect'])
self.assertEqual(task['dimensionStages'], {'product': ['black'], 'group': [], 'defect': ['black']})
self.assertEqual(task['queryPlan'][0]['selector'], selector)
def test_defect_output_can_follow_a_trusted_product_ranking_selector(self):
nodes = [
{'nodeId': 'rank', 'purpose': 'ranking', 'dimension': 'product', 'limit': 10,
'fields': ['totalWeight'], 'stages': ['black'], 'selector': {'kind': 'base'},
'visualType': 'pareto', 'visualPreference': 'auto', 'comparisonRequired': False},
defect_node(dimension='product', defect='责废尺寸重量',
selector={'kind': 'ranking_top', 'sourceNodeId': 'rank',
'stage': 'black', 'metric': 'totalWeight', 'count': 2},
node_id='top_defects'),
]
value = task_value(kind='analysis', queryPlan=nodes, dimensions=['product', 'defect'],
dimensionStages={'product': ['black'], 'defect': ['black']}, stages=['black'],
fields=['totalWeight', 'scrapWeight'], trendRequired=False, comparisonRequired=False,
period={'year': 2026, 'month': 9})
task = self.normalize(value)
self.assertEqual(task['dimensions'], ['product', 'defect'])
self.assertEqual(task['queryPlan'][1]['selector']['kind'], 'ranking_top')
self.assertEqual(task['queryPlan'][1]['defect'], '责废尺寸重量')
def test_category_or_defect_must_be_authorized_for_every_stage_and_not_count_only(self):
cases = [
defect_node(stages=['black', 'bright'], category='责废'),
defect_node(stages=['bright'], defect='加工数量'),
defect_node(stages=['black'], defect='责废尺寸重量', category='料废'),
]
for node in cases:
with self.subTest(node=node):
dims = ['defect']
dimension_stages = {'defect': list(node['stages'])}
value = task_value(kind='analysis', queryPlan=[node], dimensions=dims,
dimensionStages=dimension_stages, stages=list(node['stages']),
fields=['scrapWeight'], trendRequired=False, comparisonRequired=False,
period={'year': 2026, 'month': 9})
with self.assertRaises(TaskUnavailable):
self.normalize(value)
def test_auto_plan_is_limited_to_unqualified_all_detail_defect_analysis(self):
value = task_value(kind='analysis', dimensions=['defect'], dimensionStages={'defect': ['black']},
stages=['black'], fields=['scrapWeight'], trendRequired=False,
comparisonRequired=False, period={'year': 2026, 'month': 9})
task = self.normalize(value, question='查看2026年9月黑片具体缺陷重量排名')
self.assertEqual(len(task['queryPlan']), 1)
self.assertEqual(task['queryPlan'][0]['purpose'], 'defects')
self.assertEqual(task['queryPlan'][0]['dimension'], 'defect')
self.assertEqual(task['queryPlan'][0]['limit'], 100)
self.assertEqual(task['queryPlan'][0]['visualType'], 'bar')
def test_named_category_or_mixed_goal_cannot_be_silently_expanded(self):
value = task_value(kind='analysis', dimensions=['defect'], dimensionStages={'defect': ['black']},
stages=['black'], fields=['scrapWeight'], trendRequired=False,
comparisonRequired=False, period={'year': 2026, 'month': 9})
with self.assertRaises(TaskUnavailable) as raised:
self.normalize(value, question='只查看黑片责废的具体缺陷重量')
self.assertEqual(raised.exception.field, 'queryPlan')
self.assertEqual(raised.exception.expected, 'bounded defects node plan')
def test_comparison_requires_root_period_and_true_node_union(self):
node = defect_node(category='责废', comparison=True)
value = task_value(kind='analysis', queryPlan=[node], dimensions=['defect'],
dimensionStages={'defect': ['black']}, stages=['black'], fields=['scrapWeight'],
trendRequired=False, comparisonRequired=True, period={'year': 2026, 'month': 9},
comparisonPeriod={'year': 2025, 'month': 9})
task = self.normalize(value)
self.assertTrue(task['queryPlan'][0]['comparisonRequired'])
self.assertEqual(task['comparisonPeriod'], {'year': 2025, 'month': 9})
value['comparisonRequired'] = False
with self.assertRaises(TaskUnavailable):
self.normalize(value)
def test_defect_query_contract_binds_comparison_frame_and_column_filters(self):
task = normalize_task(task_value(kind='analysis', queryPlan=[defect_node(comparison=True)],
dimensions=['defect'], dimensionStages={'defect': ['black']}, stages=['black'],
fields=['scrapWeight'], trendRequired=False, comparisonRequired=True,
period={'year': 2026, 'monthStart': 3, 'monthEnd': 9},
comparisonPeriod={'year': 2025, 'monthStart': 3, 'monthEnd': 9}),
'比较两期黑片来源具体缺陷重量', {'_protocol': {'version': 1},
'_businessCatalog': defect_catalog()})
rule = {'field': 'customerCode', 'stages': ['black'], 'operator': 'in', 'values': ['C-17']}
task['filters']['columnFilters'] = [rule]
bound = bind_query({'mode': 'defects', 'releaseId': 'release-defects',
'period': task['period'], 'comparisonPeriod': task['comparisonPeriod'],
'filters': {}, 'columnFilters': [rule], 'stage': 'black', 'dimension': 'defect',
'limit': 100}, {'task': task, 'releaseId': 'release-defects', 'exactScope': True,
'nodeId': 'black_defects', 'scopeId': 'server-issued-scope'})
self.assertEqual(bound['period'], task['period'])
self.assertEqual(bound['comparisonPeriod'], task['comparisonPeriod'])
self.assertEqual(bound['columnFilters'], [rule])
with self.assertRaisesRegex(ValueError, '期间偏离'):
bind_query({**bound, 'comparisonPeriod': {'year': 2024, 'month': 9}},
{'task': task, 'releaseId': 'release-defects', 'exactScope': True,
'nodeId': 'black_defects', 'scopeId': 'server-issued-scope'})
def test_planning_payload_keeps_authorized_fine_defect_catalog(self):
from unittest.mock import patch
plan = task_value(kind='analysis', queryPlan=[defect_node()], dimensions=['defect'],
dimensionStages={'defect': ['black']}, stages=['black'], fields=['scrapWeight'],
trendRequired=False, comparisonRequired=False, period={'year': 2026, 'month': 9})
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False}
context = {'_protocol': {'version': 1}, '_businessCatalog': defect_catalog()}
with patch('nao_agent_task._model_json', side_effect=[plan, review]) as model_call:
planned = plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question='比较黑片具体缺陷重量', context=context, cancel_event=threading.Event())
payload = model_call.call_args_list[0].kwargs['payload']
self.assertEqual(payload['businessCatalog']['fineDefects'], defect_catalog()['fineDefects'])
self.assertEqual(planned['queryPlan'][0]['purpose'], 'defects')
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,162 @@
import copy
import json
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
str(Path(__file__).resolve().parent)]
from nao_agent_task import normalize_task
from nao_protocol_execution import execute
from test_nao_agent_task import task_value
from test_nao_protocol_execution import RELEASE, TURN, identity
from test_nao_query_plan_defects import defect_catalog, defect_node
from test_nao_quality_tools import FakeApi
SOURCE_FIELD = '责废尺寸重量'
def java_category_reconciliation(fields, source_rows):
expected_counts = {'责废': 7, '料废': 5}
component_codes = {'责废': 'black_responsibility_scrap', '料废': 'black_material_scrap'}
by_category = {}
for field in fields:
by_category.setdefault(field['category'], []).append(field)
result = []
for category, selected in by_category.items():
components = [{'key': field['key'], 'label': field['label'], 'weight': 4.0,
'sourceRows': source_rows, 'presentRows': source_rows,
'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0}
for field in selected]
expected = expected_counts[category]
result.append({'category': category, 'sourceRows': source_rows, 'components': components,
'subitemWeightSum': 4.0 * len(selected), 'validSubitemCount': len(selected),
'subitemFieldCount': expected, 'coverage': {'valid': len(selected), 'total': expected},
'formalComponent': component_codes[category], 'formalComponentSum': 9.0,
'formalComponentCoverage': {'sourceRows': source_rows, 'presentRows': source_rows,
'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0},
'diff': 4.0 * len(selected) - 9.0, 'isReconciled': False,
'reconciliationStatus': 'INCOMPLETE',
'reconciliationReasons': ['部分原始分项字段受授权限制,不能完整回加']})
return result
class DefectProtocolApi(FakeApi):
def __init__(self, *, bad_proof=False, unavailable=False):
super().__init__()
self.root = None
self.node = None
self.bad_proof = bad_proof
self.unavailable = unavailable
self.envelopes = []
self.reads = []
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
self.envelopes.append({'path': path, 'nodeId': node_id, 'scopeId': scope_id})
if path == '/api/analysis/protocol/queries':
child = copy.deepcopy(self.root)
child.pop('queryPlan', None)
child['defectQuery'] = {'dimension': self.node['dimension'], 'limit': self.node['limit']}
scope = {'nodeId': self.node['nodeId'], 'scopeId': 'black-defects-scope',
'label': '黑片具体缺陷', 'purpose': 'defects', 'task': child}
return {'version': 1, 'scopes': [scope], 'pending': [], 'gaps': []}
if path == '/api/analysis/defects':
self.reads.append(copy.deepcopy(payload))
year = payload['year']
rows = [
{'key': SOURCE_FIELD, 'label': SOURCE_FIELD, 'category': '责废',
'weight': 8.0 if year == 2026 else 5.0, 'sourceRows': 3,
'presentRows': 3, 'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0},
{'key': '料废尺寸重量', 'label': '料废尺寸重量', 'category': '料废',
'weight': 4.0 if year == 2026 else 3.0, 'sourceRows': 3,
'presentRows': 3, 'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0},
]
fields = [{'key': row['key'], 'label': row['label'], 'sourceField': row['key'],
'category': row['category']} for row in rows]
context = {key: payload.get(key) for key in (
'year', 'month', 'quarter', 'monthStart', 'monthEnd', 'department', 'group',
'product', 'projectGroup', 'includeUnassigned', 'stage', 'dimension', 'defect', 'category')}
context['extraFilters'] = payload.get('extraFilters', {})
context['exclusions'] = payload.get('exclusions', {})
context['filters'] = {'columnFilters': payload.get('columnFilters', [])}
available = not self.unavailable
proof = {**identity(), 'queryId': f'defect-{year}', 'responseHash': f'{year:064x}',
'nodeId': 'forged' if self.bad_proof else node_id, 'scopeId': scope_id}
return {'kind': 'fine_defect_top', 'releaseId': payload['releaseId'],
'release': {'id': payload['releaseId'], 'status': 'PUBLISHED'},
'context': {**context, 'mode': 'top'}, 'grain': 'source_defect_weight',
'rows': rows if available else [], 'totalGroups': len(rows) if available else 0,
'rowsTruncated': False, 'sourceRows': 3,
'coverage': {'available': available, 'sourceRows': 3,
'hasValidWeights': available},
'categoryReconciliation': java_category_reconciliation(fields, 3) if available else [],
'sourceFields': fields if available else [], 'notice': '来源细项重量。',
'protocolEvidence': proof}
return super().call(path, payload, method)
class QueryPlanDefectExecutionTests(unittest.TestCase):
def task(self):
node = defect_node(comparison=True, node_id='faults')
value = task_value(kind='analysis', queryPlan=[node], dimensions=['defect'],
dimensionStages={'defect': ['black']}, stages=['black'], fields=['scrapWeight'],
trendRequired=False, comparisonRequired=True, period={'year': 2026, 'month': 9},
comparisonPeriod={'year': 2025, 'month': 9}, visualPreference='auto', visualType='bar')
task = normalize_task(value, '比较2026年9月和2025年9月黑片来源具体缺陷重量',
{'_protocol': {'version': 1}, '_businessCatalog': defect_catalog()})
if task['filters'].get('columnFilters') == []:
task['filters'].pop('columnFilters')
return task
def run_query(self, api):
task = self.task()
api.root, api.node = task, task['queryPlan'][0]
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
(root / 'evidence').mkdir()
(root / 'private').mkdir()
with patch('nao_evidence_finalizer.synthesize', return_value={
'answer': '已核对两个期间的来源缺陷重量。', 'evidenceSufficient': True,
'missingEvidence': [], 'usage': {}}):
result = execute(request=request, task=task,
authority={**identity(), 'task': task}, actor_root=root, delegated=api,
connection=('offline', 'synthetic', 'model'), cancel_event=threading.Event(),
emit=lambda _: None)
documents = [json.loads(path.read_text()) for path in (root / 'evidence').glob('nao-quality-*.json')]
return result, documents
def test_comparison_defect_node_uses_two_bound_requests_and_completes(self):
api = DefectProtocolApi()
result, documents = self.run_query(api)
self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence'])
self.assertEqual(result['stopReason'], 'evidence_finalized')
self.assertEqual(result['completionMethod'], 'bounded_scoped_query_plan')
self.assertEqual([item['year'] for item in api.reads], [2026, 2025])
self.assertTrue(all(item['limit'] == 100 for item in api.reads))
self.assertEqual(len(documents), 1)
self.assertEqual(len(documents[0]['result']['protocolEvidence']), 2)
self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'faults'})
self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']}, {'black-defects-scope'})
self.assertEqual([item['nodeId'] for item in api.envelopes
if item['path'] == '/api/analysis/defects'], ['faults', 'faults'])
self.assertEqual([item['scopeId'] for item in api.envelopes
if item['path'] == '/api/analysis/defects'],
['black-defects-scope', 'black-defects-scope'])
def test_unavailable_detail_or_forged_scope_receipt_stays_partial(self):
for api in (DefectProtocolApi(unavailable=True), DefectProtocolApi(bad_proof=True)):
with self.subTest(api=api):
result, _ = self.run_query(api)
self.assertEqual(result['status'], 'PARTIAL')
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertTrue(result['evidenceCompletion']['missingEvidence'])
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,360 @@
import copy
import sys
import unittest
from pathlib import Path
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
str(Path(__file__).resolve().parent)]
from nao_defect_report import protocol_defect_report
from nao_evidence_finalizer import coverage_sufficient_scoped
from nao_query_plan import scope_documents
from analysis_visual_evidence import protocol_visual_result
RELEASE = 'release-defects'
QUESTION = '查看黑片来源具体缺陷重量'
FIELD = '责废尺寸重量'
def source_field(key=FIELD, category='责废'):
return {'key': key, 'label': key, 'sourceField': key, 'category': category}
def java_category_reconciliation(fields, *, source_rows, expected_fields=None,
reconciled=True, diff=0.0, reasons=None,
formal_component='black_responsibility_scrap', field_weights=None,
status=None):
expected_fields = len(fields) if expected_fields is None else expected_fields
field_weights = field_weights or {field['key']: 4.5 for field in fields}
components = [{'key': field['key'], 'label': field['label'], 'weight': field_weights[field['key']],
'presentRows': source_rows, 'missingRows': 0, 'invalidRows': 0,
'negativeRows': 0, 'sourceRows': source_rows} for field in fields]
formal_source_rows = source_rows
return {'category': fields[0]['category'], 'sourceRows': source_rows,
'components': components, 'subitemWeightSum': sum(field_weights.values()),
'validSubitemCount': len(fields), 'subitemFieldCount': expected_fields,
'coverage': {'valid': len(fields), 'total': expected_fields},
'formalComponent': formal_component,
'formalComponentSum': sum(field_weights.values()) - diff,
'formalComponentCoverage': {'presentRows': formal_source_rows, 'missingRows': 0,
'invalidRows': 0, 'negativeRows': 0, 'sourceRows': formal_source_rows},
'diff': diff, 'isReconciled': reconciled,
'reconciliationStatus': status or ('RECONCILED' if reconciled else 'DIFF'),
'reconciliationReasons': list(reasons or [])}
def defect_result(*, dimension='defect', comparison=False, stage='black', filters=None,
rows=None, ranking_complete=True, fields=None):
fields = fields or [source_field()]
if rows is None:
if dimension == 'product':
rows = [{'key': '规格A', 'label': '规格A', 'category': fields[0]['category'],
'weight': 4.5, 'defect': fields[0]['key'], 'sourceRows': 4,
'presentRows': 4, 'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0}]
else:
rows = [{'key': field['key'], 'label': field['label'], 'category': field['category'],
'weight': 4.5, 'defect': field['key'], 'sourceRows': 4,
'presentRows': 4, 'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0}
for field in fields]
current_reconciliation = java_category_reconciliation(fields, source_rows=4)
result = {
'kind': 'quality_fine_defects', 'releaseId': RELEASE, 'dimension': dimension,
'defect': fields[0]['key'] if dimension == 'product' else None, 'category': None,
'grain': 'source_defect_weight', 'stage': stage,
'period': {'year': 2026, 'month': 9}, 'filters': filters or {},
'coverage': {'available': True, 'hasValidWeights': True, 'sourceRows': 4},
'sourceRows': 4, 'sourceFields': fields,
'categoryReconciliation': [current_reconciliation],
'rows': rows, 'totalRows': len(rows), 'rowsReturned': len(rows),
'truncated': False, 'rankingComplete': ranking_complete,
'protocolEvidence': [{'nodeId': 'faults', 'scopeId': 'fault-scope', 'queryId': 'q-black'}],
}
if comparison:
comparison_reconciliation = java_category_reconciliation(fields, source_rows=3)
result.update(comparisonPeriod={'year': 2025, 'month': 9},
comparisonCoverage={'available': True, 'hasValidWeights': True, 'sourceRows': 3},
comparisonSourceFields=fields,
comparisonCategoryReconciliation=[comparison_reconciliation],
pairedRows=[{'key': FIELD,
'current': {'key': FIELD, 'label': FIELD, 'category': '责废', 'weight': 4.5, 'defect': FIELD},
'comparison': {'key': FIELD, 'label': FIELD, 'category': '责废', 'weight': 3.0, 'defect': FIELD},
'present': {'current': True, 'comparison': True}, 'deltaWeight': 1.5,
'deltaAvailable': True}],
pairedTotalRows=1, pairedTruncated=False, pairingComplete=True,
pairedRankingComplete=True, sort='scrap_delta')
result['protocolEvidence'].append({'nodeId': 'faults', 'scopeId': 'fault-scope', 'queryId': 'q-old'})
return result
def document(result, *, mode='defects'):
return {'tool': 'quality_data', 'mode': mode, 'releaseId': RELEASE,
'evidenceRef': 'QABCDEF01', 'result': result}
def child_task(*, dimension='defect', comparison=False, stages=None, visual_type='bar'):
output_dimensions = ['defect'] if dimension == 'defect' else ['product', 'defect']
stages = stages or ['black']
task = {'version': 1, 'kind': 'analysis', 'question': QUESTION, 'goal': QUESTION,
'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2025, 'month': 9} if comparison else None,
'comparisonRequired': comparison, 'trendRequired': False,
'dimensions': output_dimensions, 'dimensionStages': {key: list(stages) for key in output_dimensions},
'fields': ['scrapWeight'], 'stages': list(stages), 'filters': {},
'defectQuery': {'dimension': dimension, 'limit': 100,
**({'defect': FIELD} if dimension == 'product' else {})},
'visualPreference': 'auto', 'visualType': visual_type,
'periodCoverage': 'available', 'freshQuery': True, 'historyMode': 'none'}
return task
def scope(task):
return {'nodeId': 'faults', 'scopeId': 'fault-scope', 'label': '黑片具体缺陷',
'purpose': 'defects', 'task': task}
def compact_comparison_result(result):
"""Mirror the lossless comparison shape used by evidence_summaries."""
compact = copy.deepcopy(result)
compact['sourceFields'] = copy.deepcopy(compact['sourceFields'])
compact['comparisonSourceFields'] = copy.deepcopy(compact['comparisonSourceFields'])
for fields in (compact['sourceFields'], compact['comparisonSourceFields']):
for field in fields:
field.pop('label', None)
field.pop('sourceField', None)
columns = ['field', 'category', 'currentWeight', 'comparisonWeight', 'deltaWeight',
'currentPresent', 'comparisonPresent', 'current_sourceRows', 'current_presentRows',
'current_missingRows', 'current_invalidRows', 'current_negativeRows',
'comparison_sourceRows', 'comparison_presentRows', 'comparison_missingRows',
'comparison_invalidRows', 'comparison_negativeRows', 'deltaAvailable',
'deltaUnavailableReason']
values = []
for pair in compact.pop('pairedRows'):
current, comparison = pair['current'], pair['comparison']
values.append([pair['key'], current['category'], current['weight'], comparison['weight'],
pair['deltaWeight'], pair['present']['current'], pair['present']['comparison'],
*[current.get(key) for key in ('sourceRows', 'presentRows', 'missingRows', 'invalidRows', 'negativeRows')],
*[comparison.get(key) for key in ('sourceRows', 'presentRows', 'missingRows', 'invalidRows', 'negativeRows')],
pair['deltaAvailable'], pair.get('deltaUnavailableReason')])
compact['defectColumns'] = columns
compact['defectValues'] = values
compact.pop('rows', None)
return compact
def compact_category_components(result):
compact = copy.deepcopy(result)
for name in ('categoryReconciliation', 'comparisonCategoryReconciliation'):
for category in compact.get(name, []):
components = category.pop('components', None)
if not components:
continue
columns = list(dict.fromkeys(key for row in components for key in row))
category['componentColumns'] = columns
category['componentValues'] = [[row.get(key) for key in columns] for row in components]
return compact
class QueryPlanDefectReportTests(unittest.TestCase):
def test_single_period_report_requires_authorized_rows_and_reconciles_top_denominator(self):
task = child_task()
doc = document(defect_result())
report = protocol_defect_report([doc], task, RELEASE)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertIn('Top 1 项 / 全范围 1 个分组', '\n'.join(report['sections']))
self.assertIn('不等同于正式报废重量或总体贡献', report['policy'])
self.assertEqual(len(report['requiredFactIds']), 2)
self.assertFalse(protocol_defect_report([document(defect_result(ranking_complete=False))],
task, RELEASE)['coverageSufficient'])
def test_comparison_requires_both_complete_periods_and_source_category_reconciliation(self):
task = child_task(comparison=True)
doc = document(defect_result(comparison=True))
report = protocol_defect_report([doc], task, RELEASE)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertIn('2025年9月', '\n'.join(report['sections']))
bad = copy.deepcopy(doc)
bad['result']['comparisonCategoryReconciliation'][0]['isReconciled'] = False
report = protocol_defect_report([bad], task, RELEASE)
self.assertFalse(report['coverageSufficient'])
self.assertTrue(report['missingEvidence'])
def test_product_single_field_accepts_java_partial_category_reconciliation(self):
field = source_field('镀废缺角重量', '镀废')
task = child_task(dimension='product', stages=['bright'])
task['defectQuery']['defect'] = field['key']
result = defect_result(dimension='product', stage='bright', fields=[field], rows=[
{'key': '规格A', 'label': '规格A', 'category': '镀废', 'weight': 35.055,
'defect': field['key'], 'sourceRows': 165, 'presentRows': 165,
'missingRows': 0, 'invalidRows': 0, 'negativeRows': 0}])
result['sourceRows'] = 165
result['coverage']['sourceRows'] = 165
partial = java_category_reconciliation([field], source_rows=165,
expected_fields=5, reconciled=False, diff=-127.553,
formal_component='bright_plating_scrap', field_weights={field['key']: 35.055},
status='INCOMPLETE',
reasons=['存在缺失、非法或负值,覆盖不完整',
'来源分项与正式分类数值不一致;需核对分类转换或单位口径',
'当前请求仅选了部分源缺陷字段,不能与大类完整回加'])
doc = document(result)
doc['result']['categoryReconciliation'] = [partial]
report = protocol_defect_report([doc], task, RELEASE)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertIn('不等同于正式报废重量或总体贡献', report['policy'])
self.assertTrue(any('不与正式大类整体回加' in fact['text'] for fact in report['facts']))
self.assertFalse(any('相差' in fact['text'] for fact in report['facts']))
compact = document(compact_category_components(doc['result']))
compact_report = protocol_defect_report([compact], task, RELEASE)
self.assertTrue(compact_report['coverageSufficient'], compact_report['missingEvidence'])
def test_comparison_single_field_java_partial_reconciliation_survives_columnar_summary(self):
task = child_task(dimension='product', comparison=True)
result = defect_result(dimension='product', comparison=True)
partial_current = java_category_reconciliation([source_field()], source_rows=4,
expected_fields=7, reconciled=False,
reasons=['当前请求仅选了部分源缺陷字段,不能与大类完整回加'])
partial_previous = java_category_reconciliation([source_field()], source_rows=3,
expected_fields=7, reconciled=False,
reasons=['当前请求仅选了部分源缺陷字段,不能与大类完整回加'])
result['categoryReconciliation'] = [partial_current]
result['comparisonCategoryReconciliation'] = [partial_previous]
compact = compact_comparison_result(result)
doc = document(compact_category_components(compact))
report = protocol_defect_report([doc], task, RELEASE)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertIn('2025年9月', '\n'.join(report['sections']))
def test_partial_category_still_requires_complete_valid_selected_field_rows(self):
task = child_task(dimension='product')
doc = document(defect_result(dimension='product'))
partial = java_category_reconciliation([source_field()], source_rows=4,
expected_fields=7, reconciled=False,
reasons=['当前请求仅选了部分源缺陷字段,不能与大类完整回加'])
partial['components'][0].update(presentRows=3, missingRows=1)
doc['result']['categoryReconciliation'] = [partial]
report = protocol_defect_report([doc], task, RELEASE)
self.assertFalse(report['coverageSufficient'])
self.assertTrue(report['missingEvidence'])
invalid_weight = document(defect_result(dimension='product'))
invalid_weight['result']['categoryReconciliation'] = [java_category_reconciliation(
[source_field()], source_rows=4, expected_fields=7, reconciled=False,
reasons=['当前请求仅选了部分源缺陷字段,不能与大类完整回加'])]
invalid_weight['result']['categoryReconciliation'][0]['components'][0]['weight'] = -0.01
invalid_report = protocol_defect_report([invalid_weight], task, RELEASE)
self.assertFalse(invalid_report['coverageSufficient'])
def test_full_category_diff_is_reported_without_claiming_kpi_reconciliation(self):
field_names = ['责废尺寸重量', '责废缺角重量', '责废垂直度重量', '责废刀丝重量',
'责废烂毛重量', '责废开裂重量', '责废其他重量']
fields = [source_field(key) for key in field_names]
task = child_task()
task['defectQuery'] = {'dimension': 'defect', 'limit': 100}
doc = document(defect_result(fields=fields))
diff = 1.25
reconciliation = java_category_reconciliation(fields, source_rows=4,
reconciled=False, diff=diff,
reasons=['来源分项与正式分类数值不一致;需核对分类转换或单位口径'])
reconciliation['reconciliationStatus'] = 'DIFF'
doc['result']['categoryReconciliation'] = [reconciliation]
report = protocol_defect_report([doc], task, RELEASE)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
combined = '\n'.join(report['sections'] + [fact['text'] for fact in report['facts']])
self.assertIn('相差 +1.25 kg', combined)
self.assertIn('不代表正式KPI贡献或已回加一致', combined)
def test_reconciled_flag_with_mismatched_full_category_diff_is_rejected(self):
field_names = ['责废尺寸重量', '责废缺角重量', '责废垂直度重量', '责废刀丝重量',
'责废烂毛重量', '责废开裂重量', '责废其他重量']
fields = [source_field(key) for key in field_names]
task = child_task()
task['defectQuery'] = {'dimension': 'defect', 'limit': 100}
doc = document(defect_result(fields=fields))
reconciliation = java_category_reconciliation(fields, source_rows=4,
reconciled=True, diff=1.25)
doc['result']['categoryReconciliation'] = [reconciliation]
report = protocol_defect_report([doc], task, RELEASE)
self.assertFalse(report['coverageSufficient'])
def test_columnar_comparison_summary_is_restored_for_report_scope_and_chart_gates(self):
task = child_task(comparison=True)
compact = compact_comparison_result(defect_result(comparison=True))
doc = document(compact)
report = protocol_defect_report([doc], task, RELEASE)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertIn('2025年9月', '\n'.join(report['sections']))
selected = scope_documents([doc], scope(task), release_id=RELEASE)
self.assertEqual(len(selected), 1)
visual = protocol_visual_result([doc], [scope(task)], QUESTION, 'turn-defect-compare',
release_id=RELEASE)
self.assertEqual(visual['visualState'], 'ready', visual['missingEvidence'])
self.assertEqual(visual['visuals'][0]['metric'], 'source_defect_delta_weight')
def test_malformed_columnar_comparison_is_rejected_by_report_and_scope(self):
task = child_task(comparison=True)
compact = compact_comparison_result(defect_result(comparison=True))
compact['defectValues'][0].pop()
doc = document(compact)
self.assertFalse(protocol_defect_report([doc], task, RELEASE)['coverageSufficient'])
self.assertEqual(scope_documents([doc], scope(task), release_id=RELEASE), [])
def test_missing_required_stage_is_partial_and_does_not_create_zero_rows(self):
task = child_task(stages=['black', 'bright'])
report = protocol_defect_report([document(defect_result())], task, RELEASE)
self.assertFalse(report['coverageSufficient'])
self.assertTrue(any('亮片' in gap for gap in report['missingEvidence']))
self.assertNotIn('0 kg', '\n'.join(report['sections']))
def test_scoped_coverage_reuses_defect_specific_gate_and_exact_proofs(self):
task = child_task()
doc = document(defect_result())
self.assertEqual(scope_documents([doc], scope(task), release_id=RELEASE), [doc])
self.assertTrue(coverage_sufficient_scoped([doc], [scope(task)], QUESTION, RELEASE))
bad = copy.deepcopy(doc)
bad['result']['protocolEvidence'][0]['scopeId'] = 'another-scope'
self.assertEqual(scope_documents([bad], scope(task), release_id=RELEASE), [])
self.assertFalse(coverage_sufficient_scoped([bad], [scope(task)], QUESTION, RELEASE))
class QueryPlanDefectVisualTests(unittest.TestCase):
def run_visual(self, visual_type):
task = child_task(dimension='product', visual_type=visual_type)
result = defect_result(dimension='product', rows=[
{'key': '规格A', 'label': '规格A', 'category': '责废', 'defect': FIELD,
'weight': 4.5, 'sourceRows': 4, 'presentRows': 4, 'missingRows': 0,
'invalidRows': 0, 'negativeRows': 0}
])
doc = document(result)
return protocol_visual_result([doc], [scope(task)], QUESTION, 'turn-defect',
release_id=RELEASE)
def test_product_defect_weight_chart_is_bound_to_exact_field_and_dimension(self):
result = self.run_visual('bar')
self.assertEqual(result['visualState'], 'ready')
self.assertEqual(result['missingEvidence'], [])
chart = result['visuals'][0]
self.assertEqual(chart['dimension'], 'product')
self.assertEqual(chart['metric'], 'source_defect_weight')
self.assertEqual(chart['requiredFields'], [FIELD])
self.assertEqual(chart['scope']['defect'], FIELD)
self.assertIn('不代表正式报废大类或总体贡献', chart['note'])
def test_unsupported_explicit_pareto_is_partial_not_silently_changed_to_bar(self):
result = self.run_visual('pareto')
self.assertEqual(result['visuals'], [])
self.assertEqual(result['visualState'], 'unavailable')
self.assertTrue(any('图' in gap for gap in result['missingEvidence']))
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,179 @@
import copy
import json
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from unittest.mock import patch
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'),
str(Path(__file__).resolve().parent)]
from nao_agent_task import normalize_task
from nao_protocol_execution import execute
from test_nao_agent_task import task_value
from test_nao_protocol_execution import BoundRankingApi, RELEASE, TURN, identity
def _scope_task(root, *, purpose, fields, stages, selector_kind, product=None,
dimension=None, visual_type=None, dimension_stages=None):
filters = copy.deepcopy(root['filters'])
if product is not None:
filters['product'] = product
task = {'version': 1, 'kind': 'analysis' if purpose == 'ranking' else 'metrics',
'question': root['question'], 'goal': root['goal'],
'period': copy.deepcopy(root['period']), 'comparisonPeriod': None,
'filters': filters, 'dimensions': [dimension] if purpose == 'ranking' else [],
'dimensionStages': ({dimension: list(stages)} if purpose == 'ranking' else {}),
'fields': list(fields), 'stages': list(stages),
'trendRequired': purpose == 'trend', 'comparisonRequired': False,
'visualPreference': 'auto', 'visualType': visual_type,
'periodCoverage': root.get('periodCoverage', 'available'),
'freshQuery': True, 'historyMode': 'none'}
return task
class QueryPlanApi(BoundRankingApi):
def __init__(self, *, bad_node_proof=False, missing_second_child=False):
super().__init__()
self.rank_done = False
self.envelopes = []
self.bad_node_proof = bad_node_proof
self.missing_second_child = missing_second_child
self.root = None
self.plan = None
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
self.envelopes.append({'path': path, 'nodeId': node_id, 'scopeId': scope_id})
if path == '/api/analysis/protocol/queries':
rank_node = next(node for node in self.plan if node['nodeId'] == 'rank')
trend_node = next(node for node in self.plan if node['nodeId'] == 'top_trend')
rank_scope = {'nodeId': 'rank', 'scopeId': 'rank-scope', 'label': '黑片投入重量排名',
'purpose': 'ranking', 'limit': 5,
'task': _scope_task(self.root, purpose='ranking', fields=rank_node['fields'],
stages=rank_node['stages'], selector_kind='base', dimension='product',
visual_type='pareto')}
scopes = [rank_scope]
pending = [{'nodeId': 'top_trend', 'reason': '等待正式排名结果'}]
if self.rank_done:
scopes = [rank_scope]
for suffix, product in [('a', 'product a'), ('b', 'product b')]:
scopes.append({'nodeId': 'top_trend', 'scopeId': 'trend-' + suffix,
'label': product, 'purpose': 'trend',
'task': _scope_task(self.root, purpose='trend', fields=trend_node['fields'],
stages=trend_node['stages'], selector_kind='ranking_top', product=product,
visual_type='line')})
if self.missing_second_child:
scopes.pop()
pending = []
else:
pending = []
return {'version': 1, 'scopes': scopes, 'pending': pending, 'gaps': []}
response = super().call(path, payload, method)
if path.startswith('/api/metrics'):
if isinstance(response.get('summary'), dict):
response['summary'].setdefault('black', {})['scrapRate'] = 0.05
for month in response.get('trend', []):
if isinstance(month, dict):
month.setdefault('black', {})['scrapRate'] = 0.05
parsed = parse_qs(urlsplit(path).query)
proof = response.get('protocolEvidence')
if isinstance(proof, dict):
proof['nodeId'] = 'untrusted' if self.bad_node_proof else node_id
proof['scopeId'] = scope_id
proof['queryId'] = proof['queryId'] + '-' + str(len(self.envelopes))
proof['responseHash'] = f'{len(self.envelopes):064x}'
if parsed.get('view') == ['product']:
self.rank_done = True
return response
class QueryPlanExecutionTests(unittest.TestCase):
def task(self):
plan = [
{'nodeId': 'rank', 'purpose': 'ranking', 'dimension': 'product', 'limit': 5,
'fields': ['totalWeight'], 'stages': ['black'], 'selector': {'kind': 'base'},
'visualType': 'pareto', 'visualPreference': 'auto'},
{'nodeId': 'top_trend', 'purpose': 'trend', 'fields': ['scrapRate'],
'stages': ['black'], 'selector': {'kind': 'ranking_top', 'sourceNodeId': 'rank',
'stage': 'black', 'metric': 'totalWeight', 'count': 2},
'visualType': 'line', 'visualPreference': 'auto'},
]
value = task_value(kind='analysis', queryPlan=plan, dimensions=['product'],
dimensionStages={'product': ['black']}, stages=['black'],
fields=['totalWeight', 'scrapRate'], trendRequired=True,
comparisonRequired=False, period={'year': 2026}, filterUpdates={'department': '部门甲'})
return normalize_task(value, '按黑片投入重量排名后查看前两项报废率趋势',
{'_protocol': {'version': 1}})
def run_query(self, api, *, final_answer='已核对排名和趋势'):
task = self.task()
api.root = task
api.plan = task['queryPlan']
request = {'releaseId': RELEASE, 'turnId': TURN, 'question': task['question']}
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
with patch('nao_evidence_finalizer.synthesize', return_value={
'answer': final_answer, 'evidenceSufficient': True,
'missingEvidence': [], 'usage': {}}) as synth:
result = execute(request=request, task=task,
authority={**identity(), 'task': task}, actor_root=root, delegated=api,
connection=('offline', 'synthetic', 'model'),
cancel_event=threading.Event(), emit=lambda _: None)
documents = [json.loads(path.read_text()) for path in (root / 'evidence').glob('nao-quality-*.json')]
return result, documents, synth
def test_rank_then_two_server_resolved_trends_use_scoped_envelopes_and_one_finalizer(self):
api = QueryPlanApi()
result, documents, synth = self.run_query(api)
self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence'])
self.assertEqual(result['stopReason'], 'evidence_finalized')
self.assertEqual(result['completionMethod'], 'bounded_scoped_query_plan')
self.assertEqual(result['finishReason'], 'stop')
self.assertEqual({doc['mode'] for doc in documents}, {'rank', 'query'})
self.assertEqual(len(documents), 3)
self.assertEqual(len(result['protocolEvidence']), 3)
self.assertEqual({proof['nodeId'] for proof in result['protocolEvidence']}, {'rank', 'top_trend'})
self.assertEqual({proof['scopeId'] for proof in result['protocolEvidence']},
{'rank-scope', 'trend-a', 'trend-b'})
expected_agent_context = {'releaseId': RELEASE,
'filters': api.root['filters'],
'period': api.root['period']}
if api.root.get('comparisonPeriod') is not None:
expected_agent_context['comparisonPeriod'] = api.root['comparisonPeriod']
self.assertEqual(result['agentContext'], expected_agent_context)
self.assertNotIn('_protocolScopes', result['agentContext'])
self.assertNotIn('product', result['agentContext']['filters'])
self.assertTrue(all(call['nodeId'] is not None and call['scopeId'] is not None
for call in api.envelopes if call['path'] != '/api/analysis/protocol/queries'))
self.assertEqual(len([call for call in api.envelopes
if call['path'] == '/api/analysis/protocol/queries']), 3)
synth.assert_called_once()
context = synth.call_args.kwargs['context']
self.assertEqual(context['filters'], api.root['filters'])
self.assertEqual(len(context['_protocolScopes']['scopes']), 3)
def test_forged_scope_proof_and_missing_resolved_child_finish_partial(self):
for api in (QueryPlanApi(bad_node_proof=True), QueryPlanApi(missing_second_child=True)):
with self.subTest(api=api):
result, documents, _ = self.run_query(api)
self.assertEqual(result['status'], 'PARTIAL')
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertTrue(result['evidenceCompletion']['missingEvidence'])
def test_requested_scope_visual_gap_finishes_partial_even_when_text_evidence_is_complete(self):
api = QueryPlanApi()
with patch('analysis_visual_evidence.protocol_visual_result', return_value={
'visualSchemaVersion': 1, 'visualState': 'ready', 'visuals': [],
'missingEvidence': ['节点趋势图未覆盖。']}):
result, _, _ = self.run_query(api)
self.assertEqual(result['evidenceCompletion']['serverCoverageSufficient'], True)
self.assertEqual(result['status'], 'PARTIAL')
self.assertIn('节点趋势图未覆盖。', result['evidenceCompletion']['missingEvidence'])
if __name__ == '__main__':
unittest.main()
+473
View File
@@ -0,0 +1,473 @@
import copy
import io
import json
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'scripts'))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from nao_agent_task import TaskUnavailable, normalize_task
from nao_protocol_execution import execute
from nao_query_contract import bind_query
from nao_query_plan import normalize_query_plan
from nao_quality_tools import QualityToolError, QualityTools
from nao_evidence_finalizer import _resolve_completed_protocol_raw_gaps
from test_nao_agent_task import task_value
from test_nao_protocol_execution import BoundApi, RELEASE, TURN, identity
class RawPlanTests(unittest.TestCase):
def root(self, **overrides):
value = {'period':{'year':2026,'monthStart':1,'monthEnd':2},
'comparisonPeriod':None,'filters':{'department':'部门甲'},
'fields':[],'dimensions':[],'stages':[],
'trendRequired':False,'comparisonRequired':False,
'verifySource':True,'fullDetails':True,'periodCoverage':'available'}
value.update(overrides)
return value
def plan(self):
return [
{'nodeId':'source','purpose':'source','fields':[],'stages':['black'],
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
'comparisonRequired':False},
{'nodeId':'records','purpose':'records','fields':[],'stages':['black'],
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
'comparisonRequired':False},
]
def test_raw_node_union_sets_root_stage_and_flags_without_kpi_fields(self):
root = self.root()
result = normalize_query_plan(self.plan(), root_task=root, root_value=self.root())
self.assertEqual([node['purpose'] for node in result], ['source','records'])
self.assertEqual(root['fields'], [])
self.assertEqual(root['stages'], ['black'])
self.assertTrue(root['verifySource'])
self.assertTrue(root['fullDetails'])
self.assertEqual(root['dimensions'], [])
def test_raw_flags_cannot_be_omitted_or_downgraded(self):
cases = [
(self.root(verifySource=False), self.plan()),
(self.root(fullDetails=False), self.plan()),
(self.root(fullDetails=True), self.plan()[:1]),
]
for root, plan in cases:
with self.subTest(root=root, plan=plan), self.assertRaises(ValueError):
normalize_query_plan(plan, root_task=copy.deepcopy(root), root_value=root)
def test_raw_nodes_require_empty_fields_and_no_visuals(self):
for update in ({'fields':['rate']}, {'visualType':'bar'}, {'visualPreference':'auto'},
{'stages':[]}):
raw = self.plan()[:1]
raw[0].update(update)
with self.subTest(update=update), self.assertRaises(ValueError):
normalize_query_plan(raw, root_task=self.root(fullDetails=False),
root_value=self.root(fullDetails=False))
def test_source_fields_are_stage_specific_authorized_stable_keys(self):
catalog = {'sourceFields':[
{'key':'order_no','type':'source_value','stages':['black'],
'columns':{'black':'委托单号'}},
{'key':'batch_code','type':'source_value','stages':['bright'],
'columns':{'bright':'批次号'}},
]}
plan = self.plan()[:1]
plan[0]['sourceFields'] = {'black':['order_no']}
root = self.root(fullDetails=False)
result = normalize_query_plan(plan, root_task=root, root_value=root,
context={'_businessCatalog':catalog})
self.assertEqual(result[0]['sourceFields'], {'black':['order_no']})
for selected in ({'black':['missing']}, {'bright':['batch_code']},
{'black':['order_no','order_no']}):
bad = copy.deepcopy(plan)
bad[0]['sourceFields'] = selected
with self.subTest(selected=selected), self.assertRaises(ValueError):
normalize_query_plan(bad, root_task=self.root(fullDetails=False),
root_value=self.root(fullDetails=False),
context={'_businessCatalog':catalog})
def test_legacy_pure_raw_kinds_synthesize_query_plan_and_keep_full_goal(self):
source = task_value(kind='source_verify',dimensions=[],fields=[],stages=['black'],
verifySource=True,fullDetails=False,period={'year':2026})
records = task_value(kind='details',dimensions=[],fields=[],stages=['bright'],
verifySource=False,fullDetails=True,period={'year':2026})
for value, purpose in ((source,'source'),(records,'records')):
task = normalize_task(value,'完整核验当前原始数据',
{'_protocol':{'version':1}})
self.assertEqual(len(task['queryPlan']),1)
self.assertEqual(task['queryPlan'][0]['purpose'],purpose)
self.assertEqual(task['queryPlan'][0]['fields'],[])
self.assertEqual(task['queryPlan'][0]['stages'],[value['stages'][0]])
self.assertEqual(task['verifySource'],purpose=='source')
self.assertEqual(task['fullDetails'],purpose=='records')
def test_legacy_metrics_with_explicit_raw_requirements_keeps_metric_and_scope(self):
value=task_value(kind='metrics',dimensions=[],stages=[],fields=['rate'],
verifySource=True,fullDetails=True,period={'year':2026},
filterUpdates={'department':'部门甲'},trendRequired=False,
comparisonRequired=False)
task=normalize_task(value,'在部门甲范围分析综合合格率并核对原表、读取所有记录',
{'_protocol':{'version':1}})
by_purpose={node['purpose']:node for node in task['queryPlan']}
self.assertEqual(by_purpose['metrics']['fields'],['rate'])
self.assertEqual(by_purpose['metrics']['stages'],[])
self.assertEqual(by_purpose['source']['fields'],[])
self.assertEqual(by_purpose['records']['fields'],[])
self.assertEqual(task['fields'],['rate'])
self.assertEqual(task['filters']['department'],'部门甲')
self.assertTrue(task['verifySource'])
self.assertTrue(task['fullDetails'])
def test_scoped_investigate_binds_month_range_columns_exclusions_and_stage(self):
columns = [{'field':'customer_code','stages':['black'],'operator':'in','values':['C01']}]
task = {'filters':{'department':'部门甲','exclusions':{'product':['P-2']},
'columnFilters':columns},
'dimensions':[],'stages':['black'],'dimensionStages':{},
'period':{'year':2026,'monthStart':1,'monthEnd':2},
'comparisonPeriod':None}
args = {'mode':'investigate','action':'rows','releaseId':RELEASE,
'period':copy.deepcopy(task['period']),
'filters':{'department':'部门甲','exclusions':{'product':['P-2']}},
'columnFilters':copy.deepcopy(columns),'stage':'black',
'limit':100,'offset':0,'recordFilter':'all'}
bound = bind_query(args, {'task':task,'releaseId':RELEASE,'exactScope':True})
self.assertEqual(bound['period'],task['period'])
self.assertEqual(bound['filters']['exclusions'],task['filters']['exclusions'])
self.assertEqual(bound['columnFilters'],columns)
self.assertEqual(bound['stage'],'black')
class InvestigationApi:
def __init__(self, *, total_rows=205, missing_proof=False, wrong_context_key=None):
self.total_rows = total_rows
self.missing_proof = missing_proof
self.wrong_context_key = wrong_context_key
self.calls = []
self.query_count = 0
self.scopes = []
@staticmethod
def source_fields():
return [{'key':'customer_code','type':'source_value','stages':['black'],
'columns':{'black':'客户代码'},'operators':['in']}]
def add_scope(self, node_id, scope_id, purpose, root):
child = copy.deepcopy(root)
child.pop('queryPlan',None)
child['kind'] = 'source_verify' if purpose == 'source' else 'details'
child['fields'] = []
child['dimensions'] = []
child['dimensionStages'] = {}
child['stages'] = ['black']
child['trendRequired'] = False
child['comparisonRequired'] = False
child['visualType'] = None
child['visualPreference'] = 'none'
child['verifySource'] = purpose == 'source'
child['fullDetails'] = purpose == 'records'
child['investigationFields'] = {'black':['委托单号','客户代码']}
self.scopes.append({'nodeId':node_id,'scopeId':scope_id,'label':scope_id,
'purpose':purpose,'task':child})
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
self.calls.append({'path':path,'payload':copy.deepcopy(payload),
'method':method,'nodeId':node_id,'scopeId':scope_id})
if path == '/api/analysis/protocol/queries':
return {'version':1,'scopes':copy.deepcopy(self.scopes),'pending':[],'gaps':[]}
if path == '/api/releases':
return {'items':[{'id':RELEASE,'name':'Release','status':'PUBLISHED'}]}
if path == '/api/analysis/tools/catalog':
return {'kind':'analysis_catalog','sourceFields':self.source_fields()}
if path != '/api/analysis/investigate':
raise AssertionError(path)
self.query_count += 1
expected_period = {key:payload[key] for key in
('year','month','quarter','monthStart','monthEnd') if key in payload}
context = {**expected_period,'releaseId':RELEASE,'stage':payload['stage'],
'category':None,'sourceRevisionId':None,'includeUnassigned':False,
'department':payload.get('department'),'group':payload.get('group'),
'product':payload.get('product'),'projectGroup':payload.get('projectGroup'),
'extraFilters':payload.get('extraFilters',{}),
'columnFilters':payload.get('columnFilters',[]),
'exclusions':payload.get('exclusions',{})}
if self.wrong_context_key:
context[self.wrong_context_key] = 'OUT_OF_SCOPE_SENTINEL'
proof = {'queryId':f'raw-{self.query_count}','attemptId':identity()['attemptId'],
'sessionId':identity()['sessionId'],'turnId':identity()['turnId'],
'revision':identity()['revision'],'executionId':identity()['executionId'],
'responseHash':f'{self.query_count:064x}','nodeId':node_id,'scopeId':scope_id}
action = payload['action']
response = {'kind':'quality_investigation','action':action,'releaseId':RELEASE,
'context':context,'totalRows':self.total_rows,'aggregateComplete':True,
'categories':[],'lineage':[]}
if action == 'source':
response['sourceVerification'] = {'expectedRows':self.total_rows,
'complete':self.total_rows > 0,'verifiedRows':self.total_rows,
'missingRows':0,'mismatchRows':0,'allSelectedValuesMatchStored':self.total_rows > 0,
'files':[]}
else:
offset = payload.get('offset',0)
limit = payload['limit']
count = min(limit,max(0,self.total_rows-offset))
rows = [{'rowNumber':offset+index+1,'product':'规格A',
'values':{'委托单号':f'ORDER-{offset+index+1}',
'客户代码':'PRIVATE_RAW_VALUE'}}
for index in range(count)]
response.update(rows=rows,offset=offset,returnedRows=count,
matchingRows=self.total_rows,hasMore=offset+count<self.total_rows,
nextOffset=offset+count if offset+count<self.total_rows else None)
if not self.missing_proof:
response['protocolEvidence'] = proof
return response
class RawPlanExecutionTests(unittest.TestCase):
def root_task(self, *, purposes=('source','records'), with_column_filters=True):
nodes = []
for purpose in purposes:
nodes.append({'nodeId':purpose,'purpose':purpose,'fields':[],'stages':['black'],
'selector':{'kind':'base'},'visualType':None,
'visualPreference':'none','comparisonRequired':False})
columns=[{'field':'customer_code','stages':['black'],'operator':'in','values':['C01']}]
value = task_value(kind='analysis',dimensions=[],stages=['black'],fields=[],
verifySource='source' in purposes,fullDetails='records' in purposes,
period={'year':2026,'monthStart':1,'monthEnd':2},queryPlan=nodes,
**({'columnFilters':columns} if with_column_filters else {}),
filterUpdates={'exclusions':{'product':['P-2']}},
comparisonRequired=False,trendRequired=False)
return normalize_task(value,'核验原表并读取完整记录',
{'_protocol':{'version':1},
'_businessCatalog':{'sourceFields':InvestigationApi.source_fields()}})
def run_query(self, api, purposes=('source','records'), *, with_column_filters=True,
return_completion=False):
task = self.root_task(purposes=purposes,with_column_filters=with_column_filters)
if not with_column_filters:
task['filters'].pop('columnFilters',None)
for purpose in purposes:
api.add_scope(purpose,purpose+'-scope',purpose,task)
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
with patch('nao_evidence_finalizer.synthesize') as synth:
result=execute(request={'releaseId':RELEASE,'turnId':TURN,
'question':task['question']},task=task,
authority={**identity(),'task':task},actor_root=root,delegated=api,
connection=('http://offline','synthetic','model'),
cancel_event=threading.Event(),emit=lambda _:None)
documents=[json.loads(path.read_text()) for path in (root/'evidence').glob('nao-quality-*.json')]
completion=json.loads((root/'private'/'protocol-completion.json').read_text())
return (result,documents,synth,completion) if return_completion else (result,documents,synth)
def test_raw_nodes_use_server_columns_envelopes_and_read_all_stable_pages(self):
api=InvestigationApi(total_rows=205)
result,documents,synth=self.run_query(api)
self.assertEqual(result['status'],'COMPLETED',result['evidenceCompletion']['missingEvidence'])
self.assertEqual(result['stopReason'],'evidence_finalized')
self.assertEqual(result['completionMethod'],'bounded_scoped_query_plan')
self.assertEqual(result['finishReason'],'stop')
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertEqual(len(result['protocolEvidence']),4)
self.assertFalse(synth.called)
reads=[call for call in api.calls if call['path']=='/api/analysis/investigate']
self.assertEqual(len(reads),4)
self.assertEqual({(call['nodeId'],call['scopeId']) for call in reads},
{('source','source-scope'),('records','records-scope')})
self.assertEqual([call['payload']['offset'] for call in reads if call['payload']['action']=='rows'],
[0,100,200])
self.assertTrue(all(call['payload']['includeUnassigned'] is False for call in reads))
self.assertTrue(all(call['payload']['limit']==100 for call in reads))
self.assertTrue(all(call['payload']['fields']==['委托单号','客户代码'] for call in reads))
self.assertTrue(all(call['payload']['columnFilters']==[
{'field':'customer_code','stages':['black'],'operator':'in','values':['C01']}
] for call in reads))
self.assertTrue(all(call['payload']['exclusions']=={'product':['P-2']} for call in reads))
self.assertFalse(any(call['payload'].get('recordFilter') for call in reads
if call['payload']['action']=='source'))
self.assertFalse(any('PRIVATE_RAW_VALUE' in json.dumps(value,ensure_ascii=False)
for value in (result['answer'],result['scopeDiagnostics'])))
self.assertNotIn('recordList',result)
self.assertEqual({doc['mode'] for doc in documents},{'investigate'})
def test_unselected_column_filters_accept_canonical_empty_context_echo(self):
api=InvestigationApi(total_rows=2)
result,_,_=self.run_query(api,with_column_filters=False)
self.assertEqual(result['status'],'COMPLETED',result['evidenceCompletion']['missingEvidence'])
reads=[call for call in api.calls if call['path']=='/api/analysis/investigate']
self.assertTrue(reads)
self.assertTrue(all('columnFilters' not in call['payload'] for call in reads))
def test_failed_raw_read_keeps_private_safe_error_diagnostic(self):
api=InvestigationApi(total_rows=2,wrong_context_key='product')
result,_,_,completion=self.run_query(api,return_completion=True)
self.assertEqual(result['status'],'PARTIAL')
self.assertEqual(result['stopReason'],'evidence_incomplete')
self.assertEqual(result['completionMethod'],'bounded_scoped_query_plan')
self.assertEqual(result['finishReason'],'stop')
diagnostic=next(item for item in completion['queryDiagnostics']
if item.get('nodeId')=='records')
self.assertEqual(diagnostic['errorType'],'QualityToolError')
self.assertIn('未确认product范围',diagnostic['errorSummary'])
self.assertNotIn('P-2',diagnostic['errorSummary'])
self.assertNotIn('C01',diagnostic['errorSummary'])
self.assertNotIn('OUT_OF_SCOPE_SENTINEL',diagnostic['errorSummary'])
self.assertNotIn('QualityToolError',result['answer'])
self.assertNotIn('OUT_OF_SCOPE_SENTINEL',result['answer'])
def test_records_stop_at_global_4000_row_bound_and_finish_partial(self):
api=InvestigationApi(total_rows=4500)
result,documents,_=self.run_query(api,purposes=('records',))
self.assertEqual(result['status'],'PARTIAL')
self.assertEqual(result['evidenceCompletion']['serverCoverageSufficient'],False)
reads=[call for call in api.calls if call['path']=='/api/analysis/investigate']
self.assertEqual(len(reads),40)
self.assertEqual(sum(call['payload'].get('limit',0) for call in reads
if call['payload']['action']=='rows'),4000)
self.assertTrue(any('4000条' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_missing_raw_protocol_proof_is_partial_and_never_promoted_to_complete(self):
api=InvestigationApi(total_rows=2,missing_proof=True)
result,documents,_=self.run_query(api)
self.assertEqual(result['status'],'PARTIAL')
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertFalse(result['protocolEvidence'])
self.assertTrue(result['evidenceCompletion']['missingEvidence'])
def test_only_completed_raw_requirement_gaps_are_resolved(self):
raw_gaps=[
'该范围内逐条原始记录(客户代码、委托订单号、加工单位,不去重)未返回可用事实',
'上述字段与归档原始Excel的一致性核验结果未返回可用事实',
'亮片报废率仍缺少正式来源月份',
]
complete={'records':{'requiredChecks':1,'verifiedChecks':1,'complete':True},
'source':{'requiredChecks':1,'verifiedChecks':1,'complete':True}}
retained,resolved=_resolve_completed_protocol_raw_gaps(raw_gaps,complete)
self.assertEqual(retained,[raw_gaps[2]])
self.assertEqual(resolved,raw_gaps[:2])
incomplete={'records':complete['records'],
'source':{'requiredChecks':1,'verifiedChecks':0,'complete':False}}
retained,resolved=_resolve_completed_protocol_raw_gaps(raw_gaps,incomplete)
self.assertEqual(retained,raw_gaps[1:])
self.assertEqual(resolved,[raw_gaps[0]])
class MixedRawMetricsApi(BoundApi):
def __init__(self, root, plan):
super().__init__()
self.plan=plan
self.raw_api=InvestigationApi(total_rows=165)
self.scopes=[]
for node in plan:
purpose=node['purpose']
child=copy.deepcopy(root)
child.pop('queryPlan',None)
child['kind']={'source':'source_verify','records':'details'}.get(purpose,'metrics')
child['fields']=copy.deepcopy(node['fields'])
child['dimensions']=[]
child['dimensionStages']={}
child['stages']=copy.deepcopy(node['stages'])
child['trendRequired']=purpose=='trend'
child['comparisonRequired']=node['comparisonRequired']
child['visualType']=None
child['visualPreference']='none'
child['verifySource']=purpose=='source'
child['fullDetails']=purpose=='records'
if purpose in ('source','records'):
child['investigationFields']={stage:['客户代码','委托订单号','加工单位']
for stage in node['stages']}
self.scopes.append({'nodeId':node['nodeId'],'scopeId':purpose+'-scope',
'label':purpose,'purpose':purpose,'task':child})
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
if path=='/api/analysis/protocol/queries':
return {'version':1,'scopes':copy.deepcopy(self.scopes),'pending':[],'gaps':[]}
if path=='/api/analysis/investigate':
return self.raw_api.call(path,payload,method,node_id=node_id,scope_id=scope_id)
result=super().call(path,payload,method)
if path.startswith('/api/metrics'):
result['summary']['bright']['scrapRate']=0.081
for month in result.get('coverage',{}).get('sourceMonths',[]):
month['blackRows']=10
month['brightRows']=12
proof=result.get('protocolEvidence')
if isinstance(proof,dict):
proof.update(nodeId=node_id,scopeId=scope_id,queryId='mixed-kpi',responseHash='b'*64)
return result
class MixedRawPlanExecutionTests(unittest.TestCase):
def test_mixed_plan_synthesizes_only_kpi_evidence_and_platform_reports_raw_completion(self):
plan=[
{'nodeId':'records','purpose':'records','fields':[],'stages':['bright'],
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
'comparisonRequired':False},
{'nodeId':'verify','purpose':'source','fields':[],'stages':['bright'],
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
'comparisonRequired':False},
{'nodeId':'metric','purpose':'metrics','fields':['scrapRate'],'stages':['bright'],
'selector':{'kind':'base'},'visualType':None,'visualPreference':'none',
'comparisonRequired':False},
]
value=task_value(kind='details',dimensions=[],stages=['bright'],fields=['scrapRate'],
verifySource=True,fullDetails=True,period={'year':2026,'month':9},queryPlan=plan,
visualPreference='none',visualType=None,comparisonRequired=False,trendRequired=False)
task=normalize_task(value,'查询2026年9月亮片报废率,并读取该范围逐条原始记录及核验归档原件',
{'_protocol':{'version':1}})
api=MixedRawMetricsApi(task,task['queryPlan'])
for node in task['queryPlan']:
self.assertIn(node['purpose'],{'records','source','metrics'})
raw_model_gaps=[
'该范围内逐条原始记录(客户代码、委托订单号、加工单位,不去重)未返回可用事实',
'上述字段与归档原始Excel的一致性核验结果未返回可用事实',
]
model_requests=[]
def model_opener(request,timeout):
body=json.loads(request.data.decode())
model_requests.append(body)
selector=json.loads(body['messages'][1]['content'])
fact_id=selector['facts'][0]['id']
completion={'model':'model','usage':{},'choices':[{'finish_reason':'stop','message':{
'content':json.dumps({'reportTitle':'2026年9月亮片报废率',
'answerBlocks':[{'type':'paragraph','factIds':[fact_id]}],
'evidenceSufficient':False,'missingEvidence':raw_model_gaps},ensure_ascii=False)}}]}
return io.BytesIO(json.dumps(completion,ensure_ascii=False).encode())
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
with patch('nao_evidence_finalizer._open_model',side_effect=model_opener):
result=execute(request={'releaseId':RELEASE,'turnId':TURN,
'question':task['question']},task=task,
authority={**identity(),'task':task},actor_root=root,delegated=api,
connection=('http://offline','synthetic','model'),
cancel_event=threading.Event(),emit=lambda _:None)
docs=[json.loads(path.read_text()) for path in (root/'evidence').glob('nao-quality-*.json')]
self.assertEqual(result['status'],'COMPLETED',result['evidenceCompletion']['missingEvidence'])
self.assertTrue(task['verifySource'])
self.assertTrue(task['fullDetails'])
self.assertEqual(len(result['protocolEvidence']),4)
self.assertEqual(result['evidenceCompletion']['missingEvidence'],[])
self.assertFalse(result['evidenceCompletion']['modelDeclaredEvidenceSufficient'])
self.assertEqual(len(model_requests),1)
selector=json.loads(model_requests[0]['messages'][1]['content'])
self.assertEqual([item['kind'] for item in selector['coverage']],['quality_metrics'])
raw_summary=selector['protocolRawCoverage']
self.assertTrue(selector['agentTask']['verifySource'])
self.assertTrue(selector['agentTask']['fullDetails'])
self.assertEqual(raw_summary,{
'source':{'requiredChecks':1,'verifiedChecks':1,'complete':True},
'records':{'requiredChecks':1,'verifiedChecks':1,'complete':True}})
self.assertIn('source/records节点由平台独立执行',model_requests[0]['messages'][0]['content'])
self.assertNotIn('PRIVATE_RAW_VALUE',model_requests[0]['messages'][1]['content'])
self.assertEqual({document['mode'] for document in docs},{'query','investigate'})
self.assertIn('已按稳定分页读全165条',result['answer'])
self.assertIn('原表值与存储记录一致',result['answer'])
self.assertNotIn('PRIVATE_RAW_VALUE',result['answer'])
if __name__ == '__main__':
unittest.main()
+192
View File
@@ -0,0 +1,192 @@
"""Read recovery retries only the failed request, within the original scope and budget."""
import copy
import sys
import threading
import unittest
from pathlib import Path
from urllib.error import HTTPError
from unittest.mock import patch
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).resolve().parent)]
import nao_protocol_execution as execution
import test_nao_query_plan_execution as fixtures
from nao_quality_runtime import RuntimeErrorSafe
class Delegate:
def __init__(self, outcomes):
self.outcomes = list(outcomes)
self.requests = []
def call(self, path, payload=None, method=None, **scope):
self.requests.append((path, copy.deepcopy(payload), method, scope))
value = self.outcomes.pop(0)
if isinstance(value, BaseException):
raise value
return value
def transient(status=503):
error = RuntimeErrorSafe('暂时无法读取')
error.retryable = True
error.http_status = status
return error
class QueryRecoveryTests(unittest.TestCase):
def scoped(self, delegate, **kwargs):
return execution.ScopedApi(delegate, 'node', 'scope', cancel_event=threading.Event(), **kwargs)
def test_transient_read_retries_identical_envelope_and_counts_each_attempt(self):
delegate = Delegate([transient(), {'protocolEvidence': {'queryId': 'success'}}])
events, diagnostics = [], []
api = self.scoped(delegate, emit=events.append, recovery_log=diagnostics, read_limit=2)
payload = {'releaseId': 'release', 'offset': 100, 'limit': 100}
response = api.call('/api/analysis/investigate', payload, 'POST')
self.assertEqual(response['protocolEvidence']['queryId'], 'success')
self.assertEqual(delegate.requests[0], delegate.requests[1])
self.assertEqual(api.read_count, 2)
self.assertEqual(len(api.investigation_responses), 1)
self.assertEqual(diagnostics[-1]['outcome'], 'recovered')
self.assertTrue(any('重试' in event['message'] for event in events))
def test_http_retry_statuses_are_bounded_to_three_actual_attempts(self):
for status in (429, 502, 503, 504):
with self.subTest(status=status):
delegate = Delegate([HTTPError('https://offline.invalid', status, '', {}, None) for _ in range(4)])
api = self.scoped(delegate)
with self.assertRaises(HTTPError), patch.object(execution, '_QUERY_RETRY_DELAYS', (0, 0)):
api.call('/api/metrics?year=2026', method='GET')
self.assertEqual(len(delegate.requests), 3)
self.assertEqual(api.read_count, 3)
def test_permission_parameter_binding_and_unknown_failures_are_not_retried(self):
failures = [HTTPError('https://offline.invalid', code, '', {}, None) for code in (400, 401, 403, 409, 422)]
failures += [ValueError('正式查询证据未绑定当前执行'), RuntimeErrorSafe('所选期间没有来源数据')]
explicit_permanent = transient()
explicit_permanent.retryable = False
failures.append(explicit_permanent)
for failure in failures:
with self.subTest(failure=str(failure)):
delegate = Delegate([failure, {}])
api = self.scoped(delegate)
with self.assertRaises(type(failure)):
api.call('/api/metrics?year=2026', method='GET')
self.assertEqual(api.read_count, 1)
self.assertEqual(len(delegate.requests), 1)
def test_mutating_endpoints_are_never_retried(self):
for path in ('/api/analysis/protocol/task', '/api/analysis/protocol/finish'):
delegate = Delegate([transient(), {}])
api = self.scoped(delegate)
with self.assertRaises(RuntimeErrorSafe):
api.call(path, {}, 'POST')
self.assertEqual(len(delegate.requests), 1)
self.assertEqual(api.read_count, 0)
def test_budget_exhaustion_does_not_issue_an_extra_request(self):
delegate = Delegate([transient(), {}])
api = self.scoped(delegate, read_limit=1)
with self.assertRaises(RuntimeErrorSafe):
api.call('/api/metrics?year=2026', method='GET')
self.assertEqual(api.read_count, 1)
self.assertEqual(len(delegate.requests), 1)
with self.assertRaisesRegex(ValueError, '查询上限'):
api.call('/api/metrics?year=2026', method='GET')
self.assertEqual(len(delegate.requests), 1)
def test_finalization_reserve_and_cancellation_stop_retries(self):
for remaining, cancelled in ((300, False), (3000, True)):
with self.subTest(remaining=remaining, cancelled=cancelled):
cancel = threading.Event()
class Failing(Delegate):
def call(self, *args, **kwargs):
if cancelled:
cancel.set()
return super().call(*args, **kwargs)
delegate = Failing([transient(), {}])
api = execution.ScopedApi(delegate, 'node', 'scope', cancel_event=cancel)
with patch.object(execution, 'remaining_seconds', side_effect=[3000, remaining]):
with self.assertRaises((RuntimeErrorSafe, RuntimeError)):
api.call('/api/metrics?year=2026', method='GET')
self.assertEqual(len(delegate.requests), 1)
def test_cancelled_before_query_does_not_call_delegate(self):
cancel = threading.Event(); cancel.set()
delegate = Delegate([{}])
api = execution.ScopedApi(delegate, 'node', 'scope', cancel_event=cancel)
with self.assertRaisesRegex(RuntimeError, 'cancelled'):
api.call('/api/metrics?year=2026', method='GET')
self.assertEqual(len(delegate.requests), 0)
def test_expired_query_time_preserves_finalization_without_starting_a_request(self):
delegate = Delegate([{}])
api = self.scoped(delegate)
with patch.object(execution, 'remaining_seconds', return_value=300):
with self.assertRaisesRegex(ValueError, '时间不足'):
api.call('/api/metrics?year=2026', method='GET')
self.assertEqual(api.read_count, 0)
self.assertEqual(len(delegate.requests), 0)
def test_server_receipt_with_missing_data_is_not_retried(self):
delegate = Delegate([{'coverage': {'rawRows': 0}}, {}])
api = self.scoped(delegate)
self.assertEqual(api.call('/api/metrics?year=2026', method='GET')['coverage']['rawRows'], 0)
self.assertEqual(len(delegate.requests), 1)
def test_query_socket_deadline_reserve_is_scoped_and_restored(self):
class Reserved(Delegate):
deadline_reserve_seconds = 60
def call(self, *args, **kwargs):
self.observed_reserve = self.deadline_reserve_seconds
return super().call(*args, **kwargs)
delegate = Reserved([{}])
api = self.scoped(delegate)
api.call('/api/metrics?year=2026', method='GET')
self.assertEqual(delegate.observed_reserve, execution.FINALIZATION_RESERVE_SECONDS + 5)
self.assertEqual(delegate.deadline_reserve_seconds, 60)
def test_only_failed_child_request_retries_and_completed_siblings_keep_their_evidence(self):
class Intermittent(fixtures.QueryPlanApi):
def __init__(self):
super().__init__(); self.failed = False; self.failed_requests = []
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
if path.startswith('/api/metrics') and scope_id == 'trend-a' and not self.failed:
self.failed = True
self.failed_requests.append((path, node_id, scope_id))
raise transient(502)
return super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
api = Intermittent()
with patch.object(execution, '_QUERY_RETRY_DELAYS', (0, 0)):
result, documents, _ = fixtures.QueryPlanExecutionTests().run_query(api)
self.assertEqual(result['status'], 'COMPLETED', result['evidenceCompletion']['missingEvidence'])
self.assertEqual(len(documents), 3)
self.assertEqual(len(result['protocolEvidence']), 3)
self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'rank-scope']), 1)
self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'trend-b']), 1)
self.assertTrue(result['queryRecovery'])
def test_exhausted_failed_child_stays_partial_and_other_children_are_not_repeated(self):
class Unavailable(fixtures.QueryPlanApi):
def __init__(self):
super().__init__(); self.failed_attempts = 0
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
if path.startswith('/api/metrics') and scope_id == 'trend-a':
self.failed_attempts += 1
raise transient(503)
return super().call(path, payload, method, node_id=node_id, scope_id=scope_id)
api = Unavailable()
with patch.object(execution, '_QUERY_RETRY_DELAYS', (0, 0)):
result, documents, _ = fixtures.QueryPlanExecutionTests().run_query(api)
self.assertEqual(result['status'], 'PARTIAL')
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertEqual(api.failed_attempts, 3)
self.assertEqual(len(documents), 2)
self.assertEqual(len(result['protocolEvidence']), 2)
self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'rank-scope']), 1)
self.assertEqual(len([r for r in api.envelopes if r['path'].startswith('/api/metrics') and r['scopeId'] == 'trend-b']), 1)
self.assertTrue(any(item['outcome'] == 'stopped' for item in result['queryRecovery']))
if __name__ == '__main__':
unittest.main()
+71
View File
@@ -0,0 +1,71 @@
"""One system policy must reach planning, native calls and finalization."""
import io
import ast
import json
import re
import sys
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
class RuntimePolicyTest(unittest.TestCase):
def test_production_model_callers_have_no_independent_literal_budget(self):
root = Path(__file__).resolve().parents[1]
for path in (root/'scripts').glob('nao*.py'):
for node in ast.walk(ast.parse(path.read_text())):
if isinstance(node,ast.Dict):
for key,value in zip(node.keys,node.values):
if isinstance(key,ast.Constant) and key.value in ('max_completion_tokens','max_tokens'):
self.assertIsInstance(value,ast.Name,str(path))
self.assertEqual(value.id,'MAX_OUTPUT_TOKENS',str(path))
for path in (root/'backend/src/main/java/cn/yunsheng/quality/analysis').glob('*.java'):
for value in re.findall(r'"max_(?:completion_)?tokens",\s*([^,\n]+)',path.read_text()):
self.assertEqual(value,'AnalysisRuntimePolicy.MAX_OUTPUT_TOKENS',str(path))
def test_planner_uses_global_budget(self):
from nao_agent_task import _model_json
from nao_quality_runtime import native_output_budget
bodies = []
def provider(request, timeout):
bodies.append(json.loads(request.data))
return io.BytesIO(b'{"choices":[{"finish_reason":"stop","message":{"content":"{}"}}]}')
_model_json(base_url='http://localhost/v1', api_key='fixture', model='fixture',
system='fixture', payload={}, cancel_event=threading.Event(), opener=provider)
self.assertEqual(bodies[0]['max_completion_tokens'], native_output_budget())
def test_stale_environment_cannot_create_another_budget(self):
from nao_quality_runtime import native_output_budget
baseline = native_output_budget()
with patch.dict('os.environ', {'QUALITY_ANALYSIS_MAX_OUTPUT_TOKENS': '2048'}):
self.assertEqual(native_output_budget(), baseline)
def test_gate_replaces_both_legacy_caps_even_when_thinking_off(self):
from nao_model_gate import ModelGate
from nao_quality_runtime import native_output_budget
gate = ModelGate(upstream_base='http://localhost/v1', upstream_key='fixture', model='fixture',
port=0, client_key='fixture', authorize=lambda: None,
thinking_options={'enabled':False, 'effort':'auto', 'protocol':'thinking'})
try:
body = gate.apply_thinking({'max_tokens':1024, 'max_completion_tokens':4096})
self.assertNotIn('max_tokens', body)
self.assertEqual(body['max_completion_tokens'], native_output_budget())
finally:
gate.close()
def test_finalization_time_is_reserved_and_deadline_is_shared(self):
from nao_runtime_policy import turn_scope, remaining_seconds, request_timeout, native_timeout
with patch('nao_runtime_policy.time.monotonic', return_value=100):
with turn_scope():
total = remaining_seconds()
self.assertLess(native_timeout(), total)
with patch('nao_runtime_policy.time.monotonic', return_value=100+total-40):
self.assertLessEqual(request_timeout('final'), 35)
with self.assertRaises(TimeoutError):
request_timeout('planning')
if __name__ == '__main__':
unittest.main()
+42
View File
@@ -0,0 +1,42 @@
import json,sys,threading,unittest
from pathlib import Path
from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
from urllib.request import Request,urlopen
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
from nao_model_gate import ModelGate
class ThinkingGateTest(unittest.TestCase):
def test_nonstream_planner_receives_json_while_thinking_is_delivered_before_answer(self):
callbacks=[];calls=[];thought_seen=threading.Event()
class Provider(BaseHTTPRequestHandler):
def log_message(self,*args):pass
def do_POST(self):
body=json.loads(self.rfile.read(int(self.headers['Content-Length'])));calls.append(body)
self.send_response(200);self.send_header('Content-Type','text/event-stream');self.end_headers()
for field,text in [('reasoning','先核对'),('reasoning','完整接口文本')]:
raw=('data: '+json.dumps({'model':'fixture','choices':[{'index':0,'delta':{field:text}}]},ensure_ascii=False)+'\n\n').encode()
self.wfile.write(raw);self.wfile.flush()
if not thought_seen.wait(2):return
self.wfile.write(b'data: {"choices":[{"index":0,"delta":{"content":"{\\"ok\\":true}"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n');self.wfile.flush()
server=ThreadingHTTPServer(('127.0.0.1',0),Provider);thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
def thinking(event):
callbacks.append(event)
if event['event']=='delta':thought_seen.set()
gate=None
try:
gate=ModelGate(upstream_base=f'http://127.0.0.1:{server.server_port}/v1',upstream_key='fixture',
model='fixture',port=0,client_key='fixture',authorize=lambda:None,on_thinking=thinking)
req=Request(f'http://127.0.0.1:{gate._server.server_port}/v1/chat/completions',
data=json.dumps({'model':'fixture','stream':False,'messages':[]}).encode(),
headers={'Authorization':'Bearer fixture','Content-Type':'application/json','X-Quality-Analysis-Phase':'planning'})
with urlopen(req,timeout=4) as response:result=json.loads(response.read())
self.assertTrue(calls[0]['stream'])
self.assertEqual(''.join(e.get('text','') for e in callbacks if e['event']=='delta'),'先核对完整接口文本')
self.assertEqual(result['choices'][0]['message']['content'],'{"ok":true}')
self.assertEqual(callbacks[-1]['event'],'end');self.assertTrue(all(e['phase']=='planning' for e in callbacks))
finally:
if gate:gate.close()
server.shutdown();server.server_close();thread.join(timeout=1)
if __name__=='__main__':unittest.main()
+90
View File
@@ -0,0 +1,90 @@
import json
import sys
import unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
from nao_thinking_stream import ProviderStream,ThinkingPublisher
class ThinkingStreamTest(unittest.TestCase):
def test_default_publisher_does_not_bundle_a_second_of_text_into_one_update(self):
from unittest.mock import patch
updates=[]
with patch('nao_thinking_stream.time.monotonic') as clock:
clock.return_value=10.0
p=ThinkingPublisher(updates.append)
base={'id':'stream-cadence','model':'fixture','phase':'analysis'}
p.receive({**base,'event':'start'})
for i in range(11):
clock.return_value=10.0+i/10
p.receive({**base,'event':'delta','text':'新字'})
self.assertGreaterEqual(len([r for r in updates if r['text']]),4)
def test_converting_to_stream_does_not_change_tool_names_or_arguments(self):
stream=ProviderStream(lambda text:None)
stream.document({'choices':[{'index':0,'delta':{'tool_calls':[{'index':0,'id':'call-1','function':{'name':'quality_','arguments':'{"mode":'}}]}}]})
stream.document({'choices':[{'index':0,'delta':{'tool_calls':[{'index':0,'function':{'name':'data','arguments':'"compare"}'}}]},'finish_reason':'tool_calls'}]})
message=stream.completion()['choices'][0]['message']
self.assertEqual(message['tool_calls'],[{'id':'call-1','type':'function','function':{'name':'quality_data','arguments':'{"mode":"compare"}'}}])
self.assertIsNone(message['content'])
def test_utf8_split_stream_preserves_full_thought_and_final_json(self):
thoughts=[];stream=ProviderStream(thoughts.append)
rows=[{'model':'fixture','choices':[{'index':0,'delta':{'reasoning':'先核对🙂。'}}]},
{'choices':[{'index':0,'delta':{'reasoning':'再比较。'}}]},
{'choices':[{'index':0,'delta':{'content':'{"ok":'}}]},
{'choices':[{'index':0,'delta':{'content':'true}'},'finish_reason':'stop'}]},
{'usage':{'completion_tokens':12},'choices':[]}]
wire=''.join('data: '+json.dumps(row,ensure_ascii=False)+'\n\n' for row in rows).encode()+b'data: [DONE]\n\n'
for byte in wire:stream.feed(bytes([byte]))
stream.finish();answer=stream.completion()
self.assertEqual(''.join(thoughts),'先核对🙂。再比较。')
self.assertEqual(json.loads(answer['choices'][0]['message']['content']),{'ok':True})
self.assertEqual(answer['usage']['completion_tokens'],12)
def test_answer_is_not_invented_as_thinking_and_aliases_are_not_duplicated(self):
thoughts=[];stream=ProviderStream(thoughts.append)
stream.document({'choices':[{'message':{'content':'正式答案','reasoning_content':'接口思考','reasoning':'接口思考'},'finish_reason':'stop'}]})
self.assertEqual(thoughts,['接口思考'])
self.assertEqual(stream.completion()['choices'][0]['message']['content'],'正式答案')
def test_publisher_preserves_more_than_eight_thousand_characters_and_redacts_credentials(self):
updates=[];p=ThinkingPublisher(updates.append,interval=0)
base={'id':'model-1','model':'fixture','phase':'analysis'}
p.receive({**base,'event':'start'});p.receive({**base,'event':'delta','text':'长文本'*5000+' token=do-not-display'})
p.receive({**base,'event':'end'})
self.assertGreater(len(updates[-1]['text']),8000)
self.assertNotIn('do-not-display',updates[-1]['text'])
self.assertEqual(updates[-1]['status'],'completed')
self.assertFalse(updates[-1]['truncated'])
def test_missing_and_interrupted_provider_outputs_are_explicit(self):
updates=[];p=ThinkingPublisher(updates.append,interval=999)
base={'id':'model-1','model':'fixture','phase':'planning'}
p.receive({**base,'event':'start'});p.receive({**base,'event':'end'})
self.assertEqual(updates[-1]['status'],'not_returned')
p.receive({**base,'id':'model-2','event':'start'});p.receive({**base,'id':'model-2','event':'delta','text':'未完成的接口文本'})
p.close();self.assertEqual(updates[-1]['text'],'未完成的接口文本')
self.assertEqual(updates[-1]['status'],'interrupted')
count=len(updates);p.receive({**base,'event':'delta','text':'late'});self.assertEqual(len(updates),count)
def test_thought_timestamps_keep_first_text_position_and_close_time(self):
updates=[];p=ThinkingPublisher(updates.append,interval=0)
base={'id':'model-1','model':'fixture','phase':'analysis'}
p.receive({**base,'event':'start'})
start=updates[-1]['startedAt'];self.assertNotIn('at',updates[-1])
p.receive({**base,'event':'delta','text':'第一段'})
first=updates[-1]['at']
p.receive({**base,'event':'delta','text':'第二段'})
self.assertEqual(updates[-1]['at'],first)
p.receive({**base,'event':'end'})
self.assertEqual(updates[-1]['startedAt'],start)
self.assertLessEqual(start,first);self.assertLessEqual(first,updates[-1]['finishedAt'])
def test_cutoff_is_never_claimed_to_be_full_text(self):
updates=[];p=ThinkingPublisher(updates.append,interval=0,max_chars=5)
base={'id':'model-1','model':'fixture','phase':'analysis'}
p.receive({**base,'event':'start'});p.receive({**base,'event':'delta','text':'123456'})
self.assertEqual(updates[-1]['text'],'12345');self.assertTrue(updates[-1]['truncated'])
if __name__=='__main__':unittest.main()
+85
View File
@@ -0,0 +1,85 @@
import copy
import sys
import unittest
from pathlib import Path
from urllib.parse import urlencode
from unittest.mock import patch
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).resolve().parent)]
from nao_visual_recovery import rebuild, SavedResponses
from test_nao_protocol_execution import task, ranking_task, BoundApi, BoundRankingApi, identity, RELEASE
def query(value,api=None,**period):
api=api or BoundApi()
selected_period=period or value['period']
params={'releaseId':RELEASE,**selected_period,'view':value.get('_view','overall'),'includeUnassigned':'false'}
params.update(value['filters'])
response=api.call('/api/metrics?'+urlencode(params))
key='query-'+str(selected_period.get('month','year'))+'-'+params['view']
proof={**identity(),'queryId':key,'responseHash':'a'*64}
return {'queryId':key,'target':'GET /api/metrics','parameters':{k:str(v) for k,v in params.items()},
'responseHash':'a'*64,'response':response,'proof':proof,'task':copy.deepcopy(value),'scope':{}}
def bundle(value,queries):
return {'version':1,**identity(),'releaseId':RELEASE,'question':value['question'],
'task':value,'queries':queries,'scopes':[],'missingEvidence':[]}
class VisualRecoveryTests(unittest.TestCase):
def test_trend_is_rebuilt_from_only_the_frozen_response_without_api_or_model(self):
value=task({'product':'18X15X4+T'});saved=query(value);original=copy.deepcopy(saved)
with patch.object(SavedResponses,'_client',side_effect=AssertionError('no network')):
result=rebuild(bundle(value,[saved]))
self.assertEqual(result['visualState'],'ready');self.assertEqual(len(result['visuals']),1)
visual=result['visuals'][0]
self.assertEqual(visual['series'][0]['values'],[float(row['compositeRate']) for row in saved['response']['trend']])
self.assertEqual(visual['scope']['product'],'18X15X4+T')
self.assertEqual(result['queryRefs'][0]['queryIds'],[saved['queryId']]);self.assertEqual(saved,original)
def test_comparison_uses_both_original_periods(self):
value=task();value.update(trendRequired=False,comparisonRequired=True,period={'year':2026,'month':9},comparisonPeriod={'year':2026,'month':8},visualPreference='required')
saved=[query(value,year=2026,month=month) for month in (9,8)]
result=rebuild(bundle(value,saved));self.assertEqual(result['visualState'],'ready')
self.assertEqual(set(result['queryRefs'][0]['queryIds']),{q['queryId'] for q in saved})
self.assertTrue(all(chart['comparisonPeriod'].get('year')==2026 and chart['comparisonPeriod'].get('month')==8 for chart in result['visuals']))
partial=rebuild(bundle(value,saved[:1]));self.assertNotEqual(partial['visualState'],'ready');self.assertTrue(partial['missingEvidence'])
def test_weight_ranking_retains_the_full_original_denominator(self):
value=ranking_task(dimensions=['product'],dimensionStages={'product':['black']},stages=['black'],fields=['totalWeight'])
raw=copy.deepcopy(value);raw['_view']='product';saved=query(raw,BoundRankingApi())
result=rebuild(bundle(value,[saved]));self.assertEqual(result['visualState'],'ready')
self.assertEqual(result['visuals'][0]['coverage']['denominator'],100)
self.assertEqual(result['visuals'][0]['series'][0]['values'],[60.0,40.0])
def test_each_scope_uses_only_its_original_product_and_node_proof(self):
root=task();root['queryPlan']=[{'nodeId':f'trend-{i}','purpose':'trend','comparisonRequired':False} for i in (1,2)]
queries=[];scopes=[]
for i,product in enumerate(('18X15X4+T','18X15X4+Z'),1):
child=task({'product':product});saved=query(child);saved['queryId']+=str(i);saved['proof']['queryId']=saved['queryId']
scope={'nodeId':f'trend-{i}','scopeId':f'scope-{i}','purpose':'trend','task':child}
for key in ('nodeId','scopeId'):saved[key]=saved['proof'][key]=scope[key]
saved['scope']=scope;queries.append(saved);scopes.append(scope)
request=bundle(root,queries);request['scopes']=scopes
result=rebuild(request);self.assertEqual(result['visualState'],'ready');self.assertEqual(len(result['visuals']),2)
self.assertEqual({chart['scope']['product'] for chart in result['visuals']},{'18X15X4+T','18X15X4+Z'})
request['queries']=queries[:1]
partial=rebuild(request);self.assertEqual(len(partial['visuals']),1);self.assertTrue(partial['missingEvidence'])
self.assertEqual(partial['visuals'][0]['scope']['product'],'18X15X4+T')
def test_execution_or_proof_changes_never_rebind_old_numbers(self):
value=task();base=bundle(value,[query(value)])
for key in ('executionId','attemptId','sessionId','turnId','revision','queryId','responseHash','nodeId'):
changed=copy.deepcopy(base);changed['queries'][0]['proof'][key]='different'
with self.subTest(key=key),self.assertRaises(ValueError):rebuild(changed)
changed=copy.deepcopy(base);changed['queries']*=2
with self.assertRaises(ValueError):rebuild(changed)
def test_an_unrequested_chart_or_unrecorded_endpoint_cannot_trigger_a_read(self):
value=task();value['visualPreference']='none'
with self.assertRaises(ValueError):rebuild(bundle(value,[query(value)]))
with self.assertRaises(Exception):SavedResponses([])._call('GET','/api/metrics',params={})
if __name__=='__main__':unittest.main()
File diff suppressed because it is too large Load Diff
+120
View File
@@ -0,0 +1,120 @@
"""Native synthesis through the real ModelGate and a loopback fake provider."""
import json
import sys
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
from nao_evidence_finalizer import synthesize_native, FinalizationUnavailable
from nao_model_gate import ModelGate
from test_agent_answer import evidence
class NativeAnswerGateTests(unittest.TestCase):
def setUp(self):
self.allowed = True
self.cancelled = threading.Event()
self.calls = []
self.events = []
self.reject_before_followup = False
self.malformed_synthesis = False
self.cancel_after_synthesis = False
owner = self
class Provider(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
owner.calls.append(body)
reviewing = body['messages'][0]['content'].startswith('逐句核验证据')
content = ({'reviews': [{'index': 0, 'supported': True}]} if reviewing else
{'blocks': [{'text': '本期正式率低于基期,需要继续核查阶段差异。',
'factIds': ['F1'], 'claimType': 'inference'}],
'summarySelection': {'leadIndex': 0, 'evidenceIndexes': []}})
if owner.malformed_synthesis and not reviewing:
content = 'invalid composition'
data = json.dumps({'model': 'fixture-model', 'choices': [{'finish_reason': 'stop',
'message': {'content': content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)}}]}).encode()
self.send_response(200)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(data)))
self.end_headers()
self.wfile.write(data)
self.provider = ThreadingHTTPServer(('127.0.0.1', 0), Provider)
self.provider.daemon_threads = True
self.thread = threading.Thread(target=self.provider.serve_forever, daemon=True)
self.thread.start()
def authorize():
if not self.allowed or self.cancelled.is_set():
raise PermissionError('fixture delegation revoked or cancelled')
def thinking(event):
self.events.append(event)
# Revoke after the gate has fully authorized/read the first
# completion, exactly before the finalizer's next model request.
if event.get('event') == 'end' and not event.get('interrupted'):
if self.reject_before_followup:
self.allowed = False
if self.cancel_after_synthesis:
self.cancelled.set()
self.gate = ModelGate(upstream_base=f'http://127.0.0.1:{self.provider.server_port}/v1',
upstream_key='fixture-provider-key', model='fixture-model', port=0,
client_key='fixture-gate-key', authorize=authorize,
native_turn_marker='quality-turn:fixture-current',
thinking_options={'enabled': True, 'effort': 'high', 'protocol': 'thinking'}, on_thinking=thinking)
self.base = f'http://127.0.0.1:{self.gate._server.server_port}/v1'
def tearDown(self):
self.gate.close()
self.provider.shutdown()
self.provider.server_close()
self.thread.join(timeout=1)
def synthesize(self):
# No mocked opener: the finalizer, HTTP gate and upstream all run.
return synthesize_native(base_url=self.base, api_key='fixture-gate-key', model='fixture-model',
question='解释质量变化', evidence=evidence(), cancel_event=self.cancelled)
def test_synthesis_and_review_use_authorized_finalizing_phase_without_native_marker(self):
result = self.synthesize()
self.assertTrue(result['evidenceSufficient'], result)
self.assertEqual(len(self.calls), 2)
self.assertEqual(self.gate.rejected_stale_requests, 0)
self.assertEqual(result['compositionDiagnostics']['modelPhases'], ['native_synthesis', 'evidence_review'])
self.assertTrue(all(call['thinking'] == {'type': 'enabled'} for call in self.calls))
self.assertTrue(all(call['reasoning_effort'] == 'high' for call in self.calls))
self.assertFalse(any('quality-turn:fixture-current' in json.dumps(call['messages']) for call in self.calls))
self.assertTrue(self.events)
self.assertEqual({event['phase'] for event in self.events}, {'finalizing'})
def test_revoked_delegation_blocks_initial_synthesis_before_upstream(self):
self.allowed = False
with self.assertRaisesRegex(FinalizationUnavailable, '授权已失效'):
self.synthesize()
self.assertEqual(self.calls, [])
def test_revocation_is_checked_again_before_review_and_format_repair(self):
for malformed in (False, True):
with self.subTest(malformed=malformed):
self.allowed = True
self.calls.clear()
self.reject_before_followup = True
self.malformed_synthesis = malformed
with self.assertRaisesRegex(FinalizationUnavailable, '授权已失效'):
self.synthesize()
self.assertEqual(len(self.calls), 1)
def test_cancellation_blocks_initial_and_subsequent_model_work(self):
self.cancelled.set()
with self.assertRaisesRegex(FinalizationUnavailable, '已取消'):
self.synthesize()
self.assertEqual(self.calls, [])
self.cancelled.clear()
self.cancel_after_synthesis = True
with self.assertRaises(FinalizationUnavailable):
self.synthesize()
self.assertEqual(len(self.calls), 1)
if __name__ == '__main__':
unittest.main()
+393
View File
@@ -0,0 +1,393 @@
"""Native tool loop boundaries, using actual tool adapters and captured-like APIs."""
import copy
import sys
import tempfile
import unittest
import threading
from unittest.mock import patch
from pathlib import Path
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).parent)]
from nao_native_exploration import prepare_step, NativeExplorationTools, _proofs, native_model_view
from nao_agent_task import normalize_task, plan_task
from nao_quality_tools import QualityToolError
from test_nao_protocol_execution import BoundApi, identity
from test_nao_quality_tools import RELEASE
from test_nao_agent_task import task_value
from test_nao_query_plan_raw import InvestigationApi
def root():
return {'version': 1, 'goal': '解释质量变化并根据结果调查', 'question': '9月份合格率下降原因',
'executionMode': 'native_agent', 'kind': 'analysis', 'period': {'year': 2026, 'month': 9},
'comparisonPeriod': {'year': 2026, 'month': 8}, 'filters': {}, 'fields': ['rate'],
'dimensions': [], 'stages': [], 'dimensionStages': {}, 'visualPreference': 'auto',
'verifySource': False, 'fullDetails': False, 'freshQuery': True,
'comparisonRequired': True, 'trendRequired': False}
class Delegated(BoundApi):
def __init__(self):
super().__init__()
self.steps, self.authorization_calls = [], 0
self.timeout, self.deadline_reserve_seconds = 30, None
def call(self, path, payload=None, method=None, *, node_id=None, scope_id=None):
if path == '/api/analysis/model-authorization':
self.authorization_calls += 1
return {}
if path == '/api/analysis/protocol/queries':
return {'scopeCoverage':[{'nodeId':step['node']['nodeId'], 'scopeId':'scope-'+step['node']['nodeId'],
'complete':True,'missingEvidence':[]} for step in self.steps], 'rootGoalGaps':[]}
if path == '/api/analysis/protocol/exploration/steps':
self.steps.append(copy.deepcopy(payload))
task = root()
task.update(comparisonRequired=payload['node']['comparisonRequired'])
task['filters'] = {}
node = payload['node']
selector = node['selector']
if selector['kind'] == 'catalog_members': task['filters'][selector['dimension']] = selector['names'][0]
return {'nodeId': node['nodeId'], 'scopeId': 'scope-' + node['nodeId'],
'task': task, 'scope': {'nodeId': node['nodeId'], 'task': task}}
response = super().call(path, payload=payload, method=method)
if path.startswith('/api/metrics'):
from urllib.parse import urlsplit, parse_qs
month = parse_qs(urlsplit(path).query).get('month', ['0'])[0]
response['protocolEvidence'] = {**identity(), 'nodeId': node_id, 'scopeId': scope_id,
'queryId': 'query-' + month, 'responseHash': 'a'*64}
return response
class NativeExplorationTests(unittest.TestCase):
def state(self): return {'releaseId': RELEASE, 'scopes': []}
def test_model_view_removes_only_transport_and_identical_signal_duplicates(self):
row={'key':'甲','label':'甲','rank':1,'current':{'rate':0,'scrapWeight':None},
'comparison':{'rate':None,'scrapWeight':0}}
source={'kind':'quality_changes','rows':[row],'totalRows':9,'sort':'current_rate',
'prioritySignals':{'scale':[{'key':'甲','label':'甲','current':copy.deepcopy(row['current']),
'comparison':copy.deepcopy(row['comparison'])},{'key':'乙','current':{'rate':0.9}}]},
'evidenceRef':'Q12345678','protocolEvidence':{'queryId':'private'},'evidencePath':'private/path'}
view=native_model_view(source)
self.assertEqual(view['rows'],[row]);self.assertEqual(view['totalRows'],9)
self.assertEqual(view['prioritySignals']['scale'][0]['sameAsReturnedRow'],'甲')
self.assertEqual(view['prioritySignals']['scale'][1],source['prioritySignals']['scale'][1])
self.assertNotIn('protocolEvidence',view);self.assertNotIn('evidencePath',view)
self.assertEqual(view['evidenceRef'],'Q12345678')
self.assertIn('protocolEvidence',source)
source['prioritySignals']['scale'][0]['rateDrop']=None
changed=native_model_view(source)
self.assertIn('rateDrop',changed['prioritySignals']['scale'][0])
self.assertIsNone(changed['prioritySignals']['scale'][0]['rateDrop'])
def test_explicit_cause_cannot_silently_become_a_static_recipe(self):
from nao_native_planning import explicit_investigation
self.assertTrue(explicit_investigation('为什么下降?请自行深挖'))
self.assertFalse(explicit_investigation('取消原因分析,只查合格率'))
native = task_value(executionMode='native_agent', dimensions=[], stages=[], fields=['rate'],
period={'year':2026,'month':9}, comparisonPeriod={'year':2026,'month':8},
comparisonRequired=True, trendRequired=False)
wrong = copy.deepcopy(native); wrong.pop('executionMode')
with patch('nao_agent_task._model_json', side_effect=[wrong,native]) as model:
task = plan_task(base_url='http://offline.invalid', api_key='fixture', model='fixture',
question='为什么下降?请深入分析', context={'_protocol':{'version':1,'autonomousEnabled':True}},
cancel_event=threading.Event())
self.assertEqual(task['executionMode'],'native_agent')
self.assertNotIn('queryPlan', task)
self.assertEqual(model.call_count,2)
self.assertNotIn('黑片投入重量排名后查看前两项',model.call_args.kwargs['system'])
def test_clear_ranking_rejects_native_route_before_confirmation(self):
from nao_native_planning import explicit_batch_ranking
question='2026年9月汽车交通哪些规格合格率最低?按本期综合合格率从低到高列出前5个,同时说明投入和报废规模。'
self.assertTrue(explicit_batch_ranking(question))
self.assertFalse(explicit_batch_ranking('产品合格率最低的原因是什么?请深入调查'))
self.assertFalse(explicit_batch_ranking('不要排名,只查产品合格率'))
native=task_value(kind='analysis',executionMode='native_agent',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9},comparisonRequired=True,
comparisonPeriod={'year':2026,'month':8})
batch={**native,'dimensions':['product'],'fields':['rate','totalWeight','scrapWeight'],
'dimensionStages':{'product':[]},'queryPlan':[{
'nodeId':'rank','purpose':'changes','dimension':'product','sort':'current_rate','limit':5,
'fields':['rate','totalWeight','scrapWeight'],'stages':[],
'selector':{'kind':'base'},'comparisonRequired':True,'visualType':'bar','visualPreference':'auto'}]}
batch.pop('executionMode')
with patch('nao_agent_task._model_json',side_effect=[native,batch]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
question=question,context={'_protocol':{'version':1,'autonomousEnabled':True}},
cancel_event=threading.Event())
self.assertNotEqual(task.get('executionMode'),'native_agent')
self.assertEqual(task['queryPlan'][0]['sort'],'current_rate')
self.assertEqual(task['queryPlan'][0]['limit'],5)
self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_root_cannot_invent_domestic_and_explicit_group_gets_its_catalog_parent(self):
from nao_native_planning import validate_root_scope
catalog = {'groups':['汽车交通-国内','汽车交通-海外'],
'groupDepartments':{'汽车交通-国内':'新能源汽车磁钢','汽车交通-海外':'新能源汽车磁钢'}}
narrowed = root(); narrowed['filters'] = {'group':'汽车交通-国内'}
with self.assertRaisesRegex(ValueError,'擅自缩'):
validate_root_scope(copy.deepcopy(narrowed), '汽车交通为什么下降', catalog)
valid = validate_root_scope(copy.deepcopy(narrowed), '那国内为什么下降', catalog)
self.assertEqual(valid['filters'],{'group':'汽车交通-国内','department':'新能源汽车磁钢'})
inherited = validate_root_scope(copy.deepcopy(narrowed),'继续分析原因',catalog,{'group':'汽车交通-国内'})
self.assertEqual(inherited['filters']['group'],'汽车交通-国内')
new = copy.deepcopy(narrowed); new['newTask'] = True
with self.assertRaises(ValueError):
validate_root_scope(new,'换个问题,公司为什么下降',catalog,{'group':'汽车交通-国内'})
from dbgpt_trial_data import encoded_selection
multi = root(); multi['filters'] = {'group':encoded_selection(['汽车交通-国内','汽车交通-海外'])}
with self.assertRaises(ValueError):
validate_root_scope(copy.deepcopy(multi),'公司为什么下降',catalog)
explicit = validate_root_scope(copy.deepcopy(multi),'汽车交通国内和海外为什么下降',catalog)
self.assertEqual(explicit['filters']['department'],'新能源汽车磁钢')
self.assertEqual(validate_root_scope(copy.deepcopy(multi),'继续分析',catalog,multi['filters'])['filters'],explicit['filters'])
def test_explicit_authorized_department_cannot_disappear_from_root(self):
from nao_native_planning import validate_root_scope
catalog={'departments':['新能源汽车磁钢','消费电子-3C'],'groups':[], 'groupDepartments':{}}
for question in ('汽车交通为什么下降','分析汽车交通不良原因','新能源汽车磁钢9月合格率', '3C合格率',
'只分析汽车交通,不需要全公司数据','仅看汽车交通,不和消费电子比较',
'汽车交通为什么下降,不看其他部门'):
with self.subTest(question=question), self.assertRaises(ValueError):
validate_root_scope({'filters':{}},question,catalog)
for question in ('汽车交通不要了,改看公司整体','排除汽车交通看其余',
'汽车交通和消费电子比较','比较3C与汽车交通',
'汽车交通相比其他产业部如何','不含汽车交通,查全公司'):
with self.subTest(question=question):
self.assertEqual(validate_root_scope({'filters':{}},question,catalog)['filters'],{})
self.assertEqual(validate_root_scope({'filters':{'department':'新能源汽车磁钢'}},
'分析汽车交通不良原因',catalog)['filters']['department'],'新能源汽车磁钢')
with self.assertRaises(ValueError):
validate_root_scope({'filters':{'department':'消费电子-3C'}},'汽车交通为什么下降',catalog)
# A product containing the suffix is not a department mention.
self.assertEqual(validate_root_scope({'filters':{}},'查规格A3CX1',catalog)['filters'],{})
def test_department_omission_is_repaired_before_confirmation(self):
value=task_value(kind='analysis',executionMode='native_agent',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9})
context={'_protocol':{'version':1,'autonomousEnabled':True},
'_businessCatalog':{'departments':['新能源汽车磁钢','消费电子-3C']}}
with patch('nao_agent_task._model_json',side_effect=[value,{**value,'filterUpdates':{'department':'新能源汽车磁钢'}}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
question='请深挖汽车交通为什么下降',context=context,cancel_event=threading.Event())
self.assertEqual(task['filters']['department'],'新能源汽车磁钢')
self.assertEqual(task['planRecovery']['modelAttempts'],2)
self.assertEqual(task['planRecovery']['errors'][0]['field'],'filterUpdates.department')
def test_negated_broad_scope_cannot_bypass_department_repair(self):
value=task_value(kind='analysis',executionMode='native_agent',dimensions=[],dimensionStages={},
fields=['rate'],stages=[],period={'year':2026,'month':9})
context={'_protocol':{'version':1,'autonomousEnabled':True},
'_businessCatalog':{'departments':['新能源汽车磁钢','消费电子-3C']}}
for question in ('只分析汽车交通,不需要全公司数据','仅看汽车交通,不和消费电子比较',
'汽车交通为什么下降,不看其他部门'):
with self.subTest(question=question), patch('nao_agent_task._model_json',side_effect=[
value,{**value,'filterUpdates':{'department':'新能源汽车磁钢'}}]):
task=plan_task(base_url='https://offline.invalid/v1',api_key='fixture',model='fixture',
question=question,context=context,cancel_event=threading.Event())
self.assertEqual(task['filters']['department'],'新能源汽车磁钢')
self.assertEqual(task['planRecovery']['modelAttempts'],2)
def test_business_cross_check_is_not_raw_file_permission(self):
value = task_value(kind='analysis', dimensions=[], stages=[], fields=['rate'],
period={'year': 2026, 'month': 9}, executionMode='native_agent',
verifySource=True, fullDetails=True, trendRequired=False)
context = {'_protocol': {'version': 1, 'autonomousEnabled': True}}
task = normalize_task(value, '核对主要线索,先给分析', context)
self.assertFalse(task['verifySource']); self.assertFalse(task['fullDetails'])
explicit = normalize_task(value, '请核验原表,并逐行读取全部原始记录', context)
self.assertTrue(explicit['verifySource']); self.assertTrue(explicit['fullDetails'])
context['_protocol']['previousTask'] = explicit
cancelled = normalize_task(value, '取消原表核验,不要逐行读取', context)
self.assertFalse(cancelled['verifySource']); self.assertFalse(cancelled['fullDetails'])
for wording in ('后面不要原始数据,只分析统计', '别再看原始行了', '原始数据不用了'):
cancelled = normalize_task(value, wording, context)
self.assertFalse(cancelled['verifySource'], wording)
self.assertFalse(cancelled['fullDetails'], wording)
for wording in ('不需要每行数据', '不要数据行', '不要每一条记录'):
cancelled = normalize_task(value, wording, context)
self.assertFalse(cancelled['fullDetails'], wording)
retained = normalize_task(value, '不要逐行记录,继续核验原表', context)
self.assertFalse(retained['fullDetails'])
self.assertTrue(retained['verifySource'])
for wording, flag in (('请核验不良品原表', 'verifySource'),
('分析不合格原因并核验原表', 'verifySource'),
('请逐行读取不合格的原始记录', 'fullDetails')):
allowed = normalize_task(value, wording, {'_protocol': {'version': 1, 'autonomousEnabled': True}})
self.assertTrue(allowed[flag], wording)
def test_disabled_comparison_does_not_revive_an_optional_old_comparison_date(self):
value = task_value(kind='metrics', dimensions=[], fields=['rate'], stages=[],
period={'year':2026,'month':10}, comparisonRequired=False,
comparisonPeriod={'year':2026,'month':9})
task = normalize_task(value,'只查十月,不比较',{'_protocol':{'version':1,'autonomousEnabled':True}})
self.assertIsNone(task['comparisonPeriod'])
def test_new_confirmed_goal_does_not_restore_cancelled_historical_ranking(self):
value=task_value(goal='只查询综合合格率,取消排名及规模要求',kind='metrics',
dimensions=[],stages=[],fields=['rate'],period={'year':2026,'month':9},comparisonRequired=False)
previous={'goal':'按低率排名五个规格及规模','rootGoal':'按低率排名五个规格及规模','kind':'analysis','filters':{}}
with patch('nao_agent_task._model_json',return_value=value):
planned=plan_task(base_url='http://offline.invalid',api_key='fixture',model='fixture',
question='取消排名和规模,只查合格率',previous=previous,
context={'_protocol':{'version':1,'autonomousEnabled':True},
'_serverHistory':[{'question':previous['goal'],'evidenceRefs':['Q12345678']}]},
cancel_event=threading.Event())
self.assertEqual(planned['rootGoal'],value['goal'])
def test_planner_failure_clarification_keeps_explicit_negative_product_constraint(self):
context={'year':2026,'month':9,'_protocol':{'version':1,'autonomousEnabled':True},
'_businessCatalog':{'explicitlyExcludedProducts':['18X15X4.1+Z'],
'productMatch':{'status':'ambiguous','query':'18x15x4','truncated':False,
'candidates':[{'name':'18X15X4+T'},{'name':'18X15X4+Z'}]}}}
with patch('nao_agent_task._model_json',return_value=None):
task=plan_task(base_url='http://offline.invalid',api_key='fixture',model='fixture',
question='18X15X4合格率,不要包含18X15X4.1',context=context,cancel_event=threading.Event())
self.assertTrue(task['needsClarification'])
self.assertEqual(task['period'],{'year':2026,'month':9})
self.assertEqual(task['filters']['exclusions']['product'],['18X15X4.1+Z'])
def test_root_scope_cannot_be_changed_or_old_filter_reintroduced(self):
task = root(); task['filters'] = {'department': '甲'}
with self.assertRaises(QualityToolError):
prepare_step({'mode': 'compare', 'filters': {'department': '乙'}}, task, self.state())
args, _ = prepare_step({'mode': 'compare'}, task, self.state())
self.assertEqual(args['filters'], {'department': '甲'})
self.assertEqual(task['filters'], {'department': '甲'})
def test_next_observation_can_choose_a_new_dimension_and_another_stage(self):
task = root()
first, a = prepare_step({'mode': 'changes', 'dimension': 'group', 'sort': 'rate_drop'}, task, self.state())
second, b = prepare_step({'mode': 'defects', 'stage': 'bright', 'filters': {'product': '规格乙'}}, task, self.state())
self.assertEqual(a['node']['dimension'], 'group')
self.assertEqual(b['node']['selector']['names'], ['规格乙'])
self.assertEqual(b['node']['stages'], ['bright'])
self.assertNotEqual(a['node']['nodeId'], b['node']['nodeId'])
self.assertEqual(task, root())
def test_raw_reads_require_explicit_root_request(self):
with self.assertRaises(QualityToolError):
prepare_step({'mode': 'investigate', 'stage': 'black', 'action': 'rows'}, root(), self.state())
def test_exact_product_without_defect_uses_its_defect_breakdown_not_product_ranking(self):
args, step=prepare_step({'mode':'defects','stage':'black','dimension':'product',
'filters':{'product':'规格甲'}},root(),self.state())
self.assertEqual(args['dimension'],'defect')
self.assertEqual(step['node']['dimension'],'defect')
self.assertEqual(args['filters'],{'product':'规格甲'})
unchanged, _=prepare_step({'mode':'defects','stage':'black','dimension':'product','defect':'责废缺角重量'},root(),self.state())
self.assertEqual(unchanged['dimension'],'product')
def test_raw_tools_use_protocol_endpoints_and_exact_defaults(self):
task = root(); task['fullDetails'] = True; task['verifySource'] = True
args, proposal = prepare_step({'mode': 'details', 'stage': 'black'}, task, self.state())
self.assertEqual(args['mode'], 'investigate')
self.assertEqual((args['action'], args['offset'], args['recordFilter']), ('rows', 0, 'all'))
self.assertEqual(proposal['node']['purpose'], 'records')
args, proposal = prepare_step({'mode': 'investigate', 'stage': 'bright', 'action': 'source'}, task, self.state())
self.assertEqual(args['limit'], 100)
self.assertEqual(proposal['node']['purpose'], 'source')
def test_multilevel_child_inherits_verified_parent(self):
state = self.state(); state['scopes'] = [{'filters': {'group': '甲'}, 'queryIds': ['q1']}]
_, step = prepare_step({'mode': 'defects', 'stage': 'black', 'filters': {'group': '甲', 'product': '乙'}}, root(), state)
self.assertEqual(step['parentQueryIds'], ['q1'])
self.assertEqual(step['node']['selector'], {'kind': 'catalog_members', 'dimension': 'product', 'names': ['乙']})
def test_equivalent_single_month_syntax_keeps_the_confirmed_frame(self):
task = root()
args, _ = prepare_step({'mode': 'compare', 'period': {'year': 2026, 'monthStart': 9, 'monthEnd': 9}}, task, self.state())
self.assertEqual(args['period'], {'year': 2026, 'month': 9})
with self.assertRaises(QualityToolError):
prepare_step({'mode': 'compare', 'period': {'year': 2026, 'monthStart': 8, 'monthEnd': 9}}, task, self.state())
def test_invalid_shape_is_rejected_before_registering_a_scope(self):
with tempfile.TemporaryDirectory() as folder:
api = Delegated(); p = Path(folder)
tools = NativeExplorationTools(delegated=api, evidence_dir=p/'evidence', task=root(),
identity={**identity(), 'releaseId': RELEASE}, state_path=p/'state.json')
with self.assertRaises(QualityToolError):
tools.quality_data({'mode': 'compare', 'dimension': 'product'})
self.assertEqual(api.steps, [])
def test_actual_compare_preserves_both_proofs_and_duplicate_reads_are_cached(self):
with tempfile.TemporaryDirectory() as folder:
api = Delegated(); p = Path(folder)
tools = NativeExplorationTools(delegated=api, evidence_dir=p/'evidence', task=root(),
identity={**identity(), 'releaseId': RELEASE}, state_path=p/'state.json')
result = tools.quality_data({'mode': 'compare'})
self.assertEqual(len(_proofs(result)), 2)
self.assertEqual(len(api.steps), 1)
for _ in range(3):
duplicate = tools.quality_data({'mode': 'compare'})
self.assertTrue(duplicate['reusedEvidence'])
self.assertEqual(duplicate['evidenceRef'], result['evidenceRef'])
self.assertEqual(len(api.steps), 1)
self.assertEqual(api.authorization_calls, 4)
with self.assertRaisesRegex(QualityToolError, '停止'):
tools.quality_data({'mode': 'compare'})
def test_new_execution_never_reuses_previous_attempt_cache(self):
with tempfile.TemporaryDirectory() as folder:
api = Delegated(); p = Path(folder)
kwargs = dict(delegated=api, evidence_dir=p/'evidence', task=root(), state_path=p/'state.json')
a = NativeExplorationTools(**kwargs, identity={**identity(), 'releaseId': RELEASE})
first = a.quality_data({'mode': 'compare'})
b = NativeExplorationTools(**kwargs, identity={**identity(), 'executionId': 'new-execution', 'releaseId': RELEASE})
second = b.quality_data({'mode': 'compare'})
self.assertNotEqual(first['evidenceRef'], second['evidenceRef'])
self.assertEqual(len(api.steps), 2)
def test_explicit_refresh_reexecutes_but_identical_responses_are_still_no_progress(self):
with tempfile.TemporaryDirectory() as folder:
p=Path(folder); api=Delegated()
tools=NativeExplorationTools(delegated=api,evidence_dir=p/'evidence',task=root(),
identity={**identity(),'releaseId':RELEASE},state_path=p/'state.json')
first=tools.quality_data({'mode':'compare'})
for _ in range(3):
checked=tools.quality_data({'mode':'compare','freshQuery':True})
self.assertNotEqual(first['evidenceRef'],checked['evidenceRef'])
self.assertIn('没有新增',checked['investigationNotice'])
self.assertEqual(len(api.steps),1)
with self.assertRaisesRegex(QualityToolError,'停止'):
tools.quality_data({'mode':'compare','freshQuery':True})
def test_record_pages_share_authorized_scope_but_have_distinct_cached_evidence(self):
task = root()
task.update(fullDetails=True, comparisonRequired=False, comparisonPeriod=None, stages=['black'])
class Pages(InvestigationApi):
timeout = 30
deadline_reserve_seconds = None
def __init__(self):
super().__init__(total_rows=165)
self.registered = []
def call(self, path, payload=None, method=None, **kwargs):
if path == '/api/analysis/model-authorization': return {}
if path == '/api/analysis/protocol/queries':
return {'scopeCoverage':[{'nodeId':s['nodeId'],'scopeId':s['scopeId'],
'complete':self.query_count >= 2,'missingEvidence':[] if self.query_count >= 2 else ['后续页尚未读取']}
for s in self.scopes], 'rootGoalGaps':[]}
if path == '/api/analysis/protocol/exploration/steps':
self.registered.append(copy.deepcopy(payload))
self.add_scope(payload['node']['nodeId'], 'record-scope', 'records', task)
scope = self.scopes[-1]
return {'nodeId':scope['nodeId'], 'scopeId':scope['scopeId'], 'task':scope['task'], 'scope':scope}
return super().call(path, payload, method, **kwargs)
with tempfile.TemporaryDirectory() as folder:
p = Path(folder); api = Pages()
tools = NativeExplorationTools(delegated=api, evidence_dir=p/'evidence', task=task,
identity={**identity(), 'releaseId':RELEASE}, state_path=p/'state.json')
first = tools.quality_data({'mode':'details', 'stage':'black', 'offset':0})
last = tools.quality_data({'mode':'details', 'stage':'black', 'offset':100})
self.assertEqual(len(api.registered), 1)
self.assertFalse(first['investigationCoverage']['scopeComplete'])
self.assertTrue(last['investigationCoverage']['scopeComplete'])
reads = [c for c in api.calls if c['path'] == '/api/analysis/investigate']
self.assertEqual([c['payload']['offset'] for c in reads], [0,100])
self.assertEqual(len({(c['nodeId'],c['scopeId']) for c in reads}), 1)
self.assertNotEqual(first['evidenceRef'], last['evidenceRef'])
cached = tools.quality_data({'mode':'details', 'stage':'black', 'offset':100})
self.assertTrue(cached['reusedEvidence'])
self.assertEqual(api.query_count, 2)
tools.quality_data({'mode':'details', 'stage':'black', 'offset':100, 'freshQuery':True})
self.assertEqual(api.query_count, 3)
self.assertEqual(len(api.registered), 1)
if __name__ == '__main__': unittest.main()
+276
View File
@@ -0,0 +1,276 @@
"""Exercise the native runner with isolated files and mocked upstream/bridge calls."""
import copy
import json
import sys
import tempfile
import threading
import unittest
from contextlib import ExitStack
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock, patch
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts')]
from nao_native_run import run_native_exploration
from nao_quality_runtime import _atomic_json
IDENTITY = {'attemptId': 'attempt-current', 'sessionId': 'session-current',
'turnId': 'turn-current', 'revision': 3, 'executionId': 'execution-current'}
RELEASE = 'release-current'
def root_task():
return {'version': 1, 'goal': '解释下降并根据实际结果调查', 'question': '本期下降原因',
'executionMode': 'native_agent', 'kind': 'analysis',
'period': {'year': 2026, 'month': 9}, 'comparisonPeriod': {'year': 2026, 'month': 8},
'filters': {'department': '当前产业部'}, 'fields': ['rate'], 'stages': [],
'dimensions': [], 'dimensionStages': {}, 'visualPreference': 'auto',
'comparisonRequired': True, 'trendRequired': False,
'verifySource': False, 'fullDetails': False}
def proof(query_id, *, node='root', scope='scope-root', execution='execution-current'):
return {**IDENTITY, 'executionId': execution, 'queryId': query_id,
'nodeId': node, 'scopeId': scope, 'responseHash': 'a' * 64}
def document(ref, *proofs):
result = ({'current': {'protocolEvidence': proofs[0]},
'comparison': {'protocolEvidence': proofs[1]}}
if len(proofs) == 2 else {'protocolEvidence': proofs[0]})
return {'evidenceRef': ref, 'result': result}
class NativeRunTests(unittest.TestCase):
def test_full_facts_remain_private_instead_of_being_appended_to_the_business_answer(self):
self.final['fullEvidenceFacts'] = [{'text': '后台保留的完整统计事实', 'evidenceRef': 'Q00000001'}]
result = self.invoke()
self.assertNotIn('完整统计依据', result['answer'])
self.assertNotIn('后台保留的完整统计事实', result['answer'])
stored = json.loads((self.actor_root / 'private/native-full-evidence-facts.json').read_text())
self.assertEqual(stored, self.final['fullEvidenceFacts'])
def setUp(self):
self.folder = tempfile.TemporaryDirectory()
self.addCleanup(self.folder.cleanup)
self.actor_root = Path(self.folder.name)
self.cancel_event = threading.Event()
self.task = root_task()
self.request = {'sessionId': IDENTITY['sessionId'], 'turnId': IDENTITY['turnId'],
'actorId': 'actor-current', 'releaseId': RELEASE, 'question': self.task['question'],
'context': {'product': '已取消的旧规格',
'_serverQuestions': [{'question': '上一轮问题'}],
'_protocol': {'autonomousEnabled': True}}}
self.p_current = proof('query-current')
self.p_compare = proof('query-compare')
self.p_group = proof('query-group', node='group', scope='scope-group')
self.p_failed = proof('query-failed', node='failed', scope='scope-failed')
self.p_old = proof('query-old', execution='execution-old')
self.good = document('Q00000001', self.p_current, self.p_compare)
self.unused = document('Q00000002', self.p_group)
self.documents = [self.good, self.unused]
self.snapshot = {'scopes': [{'nodeId': 'root', 'scopeId': 'scope-root', 'task': self.task},
{'nodeId': 'group', 'scopeId': 'scope-group', 'task': self.task}],
'queries': [{'proof': p} for p in (self.p_current, self.p_compare, self.p_group)],
'scopeCoverage': [{'nodeId': 'root', 'scopeId': 'scope-root', 'complete': True},
{'nodeId': 'group', 'scopeId': 'scope-group', 'complete': True}],
'rootGoalGaps': [], 'gaps': [], 'pending': [], 'explorationWarnings': []}
self.final = {'answer': '按已核实结果解释[Q00000001]', 'reportTitle': '质量数据分析',
'readingSummary': {'version': 1, 'lead': '已核实解释[Q00000001]',
'evidence': [], 'notices': [], 'evidenceRefs': ['Q00000001']},
'usedEvidenceRefs': ['Q00000001'], 'evidenceSufficient': True,
'missingEvidence': [], 'rawModelResponses': [], 'compositionDiagnostics': {}}
self.accumulator = SimpleNamespace(answer='原生草稿', finish_reasons=['stop'],
tool_calls=[{'tool': 'quality_data', 'status': 'failed', 'input': {'mode': 'compare'}},
{'tool': 'quality_data', 'status': 'completed', 'input': {'mode': 'compare'}}],
timeline=[{'kind': 'tool_error'}, {'kind': 'tool_completed'}, {'kind': 'stop'}])
self.native_status, self.native_reason, self.stop_succeeded = 'COMPLETED', 'stop', None
self.cancel_during_native = False
self.events = []
self.manager = Mock()
self.manager.ensure_instance.return_value = (None, None, 'fixture-model', 'http://native.invalid')
self.delegated = Mock()
self.delegated.call.side_effect = lambda *args, **kwargs: copy.deepcopy(self.snapshot)
stack = self.enterContext(ExitStack())
self.auth = stack.enter_context(patch('nao_quality_runtime._authenticate', return_value=object()))
self.sse = stack.enter_context(patch('nao_quality_runtime._sse_request', side_effect=self.native))
self.read_evidence = stack.enter_context(patch('nao_evidence_finalizer.evidence_summaries',
side_effect=lambda *args, **kwargs: copy.deepcopy(self.documents)))
self.synthesis = stack.enter_context(patch('nao_evidence_finalizer.synthesize_native',
side_effect=lambda **kwargs: copy.deepcopy(self.final)))
self.visuals = stack.enter_context(patch('analysis_visual_evidence.protocol_visual_result',
return_value={'visualState': 'not_requested', 'visuals': [], 'missingEvidence': []}))
stack.enter_context(patch('nao_runtime_policy.native_timeout', return_value=30))
def native(self, **kwargs):
# Let the real runner decide when a useful response first became available.
kwargs['emit']({'type': 'progress', 'toolCalls': [
{'status': 'completed', 'input': {'mode': 'catalog'}, 'outputSummary': '读取目录'}]})
kwargs['emit']({'type': 'progress', 'toolCalls': [
{'status': 'completed', 'input': {'mode': 'compare'}, 'outputSummary': '证据:Q00000001'}]})
if self.cancel_during_native:
self.cancel_event.set()
return self.accumulator, self.native_status, self.native_reason, self.stop_succeeded
def invoke(self):
return run_native_exploration(request=copy.deepcopy(self.request), task=copy.deepcopy(self.task),
authority=copy.deepcopy(IDENTITY), actor_root=self.actor_root, delegated=self.delegated,
connection=('https://model.invalid/v1', 'synthetic-fixture-key', 'fixture-model'),
manager=self.manager, state={}, cancel_event=self.cancel_event, emit=self.events.append,
prior_evidence_files={'prior-evidence.json'},
phase_timings={'planningAndReview': 10, 'taskConfirmation': 20})
def test_corrected_tool_error_and_natural_stop_can_complete_with_verified_root(self):
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
self.snapshot['explorationWarnings'] = ['先前一次探索失败,后续已用完整证据核实根目标。']
result = self.invoke()
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(result['nativeStopReason'], 'mcp_error')
self.assertEqual(result['nativeFinishReason'], 'stop')
self.assertIsNone(result['nativeStopSucceeded'])
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertEqual(result['timeline'], self.accumulator.timeline)
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current', 'query-compare'])
self.assertIn(self.snapshot['explorationWarnings'][0], result['readingSummary']['notices'])
self.assertIn('firstUsefulMs', result['phaseTimingsMs'])
self.assertGreaterEqual(result['phaseTimingsMs']['firstUsefulMs'], 30)
self.delegated.call.assert_called_once_with('/api/analysis/protocol/queries', payload={}, method='POST')
context = self.sse.call_args.kwargs['context']
self.assertEqual(context['filters'], self.task['filters'])
self.assertNotIn('product', context)
self.assertIsNone(self.sse.call_args.kwargs['chat_id'])
stored = json.loads((self.actor_root / 'private/native-completion.json').read_text())
self.assertEqual(stored['status'], 'COMPLETED')
self.assertEqual(self.events[-1], result)
def test_generated_chart_contract_is_available_before_the_single_synthesis(self):
self.task['visualPreference']='required'
self.visuals.return_value={'visualState':'ready','missingEvidence':[], 'visuals':[{
'id':'actual-chart','type':'line','metric':'composite_rate','title':'正式趋势',
'scope':{'department':'当前产业部'},'period':{'year':2026,'month':9},
'categories':[{'key':'2026-09'}],'series':[{'values':[.95]}],
'evidenceRefs':['Q00000001']}]}
def synthesize(**kwargs):
preview=kwargs['context']['_platformVisuals']
self.assertEqual(preview['visualState'],'ready')
self.assertEqual(preview['charts'][0]['id'],'actual-chart')
self.assertEqual(preview['charts'][0]['evidenceRefs'],['Q00000001'])
self.assertEqual(preview['charts'][0]['pointCount'],1)
self.assertNotIn('series',preview['charts'][0])
return copy.deepcopy(self.final)
self.synthesis.side_effect=synthesize
result=self.invoke()
self.assertEqual(result['status'],'COMPLETED')
self.visuals.assert_called_once();self.synthesis.assert_called_once()
def test_root_gap_remains_partial_even_with_stop_and_available_evidence(self):
self.snapshot['rootGoalGaps'] = ['亮片必答阶段尚无完整两期依据。']
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertIn(self.snapshot['rootGoalGaps'][0], result['answer'])
self.assertIn(self.snapshot['rootGoalGaps'][0], result['readingSummary']['notices'])
def test_three_no_progress_attempts_remain_partial_after_natural_stop(self):
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
{'executionId': IDENTITY['executionId'], 'halted': True,
'noProgress': 3, 'queryElapsedMs': 1234})
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
self.assertEqual(result['phaseTimingsMs']['formalQuery'], 1234)
self.assertTrue(any('连续三次' in notice for notice in result['readingSummary']['notices']))
def test_controlled_no_progress_stop_does_not_add_generic_tool_failure(self):
_atomic_json(self.actor_root / 'private/native-exploration-state.json',
{'executionId': IDENTITY['executionId'], 'halted': True,
'noProgress': 3, 'queryElapsedMs': 1234})
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
result = self.invoke()
self.assertEqual(result['status'],'PARTIAL')
self.assertEqual(result['nativeStopReason'],'mcp_error')
self.assertTrue(any('连续三次' in gap for gap in result['evidenceCompletion']['missingEvidence']))
self.assertFalse(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_corrected_error_still_requires_real_stop_and_successful_stop_control(self):
self.native_status, self.native_reason = 'PARTIAL', 'mcp_error'
for reasons, stopped in (([], None), (['stop'], False)):
with self.subTest(finish_reasons=reasons, stop_succeeded=stopped):
self.accumulator.finish_reasons = reasons
self.stop_succeeded = stopped
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertTrue(any('未正常结束' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_only_complete_current_scope_documents_reach_synthesis(self):
self.snapshot['queries'].extend([{'proof': self.p_failed}, {'proof': self.p_old}])
self.snapshot['scopeCoverage'].append(
{'nodeId': 'failed', 'scopeId': 'scope-failed', 'complete': False})
self.documents.extend([
document('Q00000003', self.p_failed),
document('Q00000004', self.p_old),
document('Q00000005', proof('query-not-in-current-receipts')),
document('Q00000006', self.p_current, self.p_failed),
{'evidenceRef': 'Q00000007', 'result': {'summary': {'unproved': True}}},
])
result = self.invoke()
admitted = self.synthesis.call_args.kwargs['evidence']
self.assertEqual([doc['evidenceRef'] for doc in admitted], ['Q00000001', 'Q00000002'])
self.assertEqual(result['queryEvidenceRefs'], ['Q00000001', 'Q00000002'])
self.read_evidence.assert_called_once_with(self.actor_root / 'evidence', RELEASE,
exclude_files={'prior-evidence.json'}, allow_empty=True)
def test_used_refs_map_only_to_actual_current_proofs_not_unused_or_foreign_documents(self):
self.snapshot['queries'].append({'proof': self.p_old})
self.documents.append(document('Q00000003', self.p_old))
self.final['usedEvidenceRefs'] = ['Q00000001', 'Q00000003', 'QDEADBEEF', 'Q00000001']
result = self.invoke()
self.assertEqual(result['conclusionProtocolQueryIds'], ['query-current', 'query-compare'])
self.assertNotIn('query-group', result['conclusionProtocolQueryIds'])
self.assertNotIn('query-old', result['conclusionProtocolQueryIds'])
def test_claimed_sufficient_answer_without_current_cited_proof_remains_partial(self):
self.final['usedEvidenceRefs'] = ['QDEADBEEF']
result = self.invoke()
self.assertEqual(result['status'], 'PARTIAL')
self.assertEqual(result['conclusionProtocolQueryIds'], [])
self.assertTrue(any('实际查询证明' in gap for gap in result['evidenceCompletion']['missingEvidence']))
def test_cancelled_native_turn_never_reads_query_snapshot_or_synthesizes(self):
for cancellation in ('event', 'native_status'):
with self.subTest(cancellation=cancellation):
self.cancel_event.clear()
self.cancel_during_native = cancellation == 'event'
self.native_status = 'CANCELLED' if cancellation == 'native_status' else 'COMPLETED'
result = self.invoke()
self.assertEqual(result['status'], 'CANCELLED')
self.assertEqual(result['protocolEvidence'], [])
self.assertEqual(result['answer'], '')
self.delegated.call.assert_not_called()
self.read_evidence.assert_not_called()
self.synthesis.assert_not_called()
self.visuals.assert_not_called()
def test_auto_visual_failure_stays_in_appendix_but_required_visual_failure_is_partial(self):
self.visuals.return_value = {'visualState': 'failed', 'visuals': [],
'missingEvidence': ['图表尚未完整生成,可独立恢复。',
'queryPlan共有15张图表,超过12张呈现上限,已截取前12张;结果为部分完成。']}
for preference, expected in (('auto', 'COMPLETED'), ('required', 'PARTIAL')):
with self.subTest(visualPreference=preference):
self.task['visualPreference'] = preference
result = self.invoke()
self.assertEqual(result['status'], expected)
self.assertIn('图表尚未完整生成,可独立恢复。', result['answer'])
if preference == 'auto':
self.assertNotIn('图表尚未完整生成,可独立恢复。', result['readingSummary']['notices'])
self.assertNotIn('结果为部分完成', result['answer'])
self.assertIn('共有15张图表', result['answer'])
else:
self.assertIn('图表尚未完整生成,可独立恢复。', result['readingSummary']['notices'])
self.assertIn('结果为部分完成', result['answer'])
if __name__ == '__main__':
unittest.main()
+125
View File
@@ -0,0 +1,125 @@
import copy
import io
import json
import sys
import threading
import unittest
from pathlib import Path
sys.path[:0] = [str(Path(__file__).resolve().parents[1] / 'scripts'), str(Path(__file__).resolve().parent)]
from nao_agent_task import plan_task, normalize_task, TaskUnavailable
from test_nao_agent_task import task_value, semantic_ok
def group_plan(**updates):
return task_value(kind='metrics', goal='比较9月与8月各业务分组综合合格率,找出下降最多的分组',
dimensions=['group'], dimensionStages={}, stages=[], fields=['rate'],
period={'year': 2026, 'month': 9}, comparisonPeriod={'year': 2026, 'month': 8},
comparisonRequired=True, trendRequired=False, queryPlan=[{
'nodeId': 'groups', 'purpose': 'metrics', 'fields': ['rate'], 'stages': [],
'comparisonRequired': True, 'selector': {'kind': 'catalog_members', 'dimension': 'group',
'names': ['部门甲-国内', '部门甲-海外']}, 'visualPreference': 'none'}], **updates)
CONTEXT = {'_protocol': {'version': 1}, '_businessCatalog': {
'groups': ['部门甲-国内', '部门甲-海外'], 'departments': ['部门甲'],
'groupDepartments': {'部门甲-国内': '部门甲', '部门甲-海外': '部门甲'}}}
class PlanRecoveryTests(unittest.TestCase):
def run_plan(self, replies, **kwargs):
self.payloads = []
def opener(request, timeout):
body = json.loads(request.data)
self.payloads.append(json.loads(body['messages'][1]['content']))
response = replies.pop(0)
return io.BytesIO(json.dumps({'choices': [{'finish_reason': 'stop', 'message': {
'content': json.dumps(response)}}]}).encode())
return plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question='9月份哪个业务分组下滑最多', context=copy.deepcopy(CONTEXT),
cancel_event=threading.Event(), opener=opener, **kwargs)
def test_derivable_dimension_stage_summary_is_repaired_without_another_generation(self):
broken = group_plan()
broken['dimensionStages'] = {'group': ['black']}
task = self.run_plan([broken, {**semantic_ok(), 'requiresComparison': True}])
self.assertEqual(task['dimensionStages'], {'group': []})
self.assertTrue(task['planRecovery']['repairs'])
self.assertEqual(task['queryPlan'][0]['selector']['names'], CONTEXT['_businessCatalog']['groups'])
def test_omitted_read_flags_are_safely_defaulted_then_independently_reviewed(self):
broken = group_plan(); del broken['verifySource']; broken['fullDetails'] = None
task = self.run_plan([broken, {**semantic_ok(), 'requiresComparison': True}])
self.assertFalse(task['verifySource']); self.assertFalse(task['fullDetails'])
self.assertEqual(task['planRecovery']['modelAttempts'], 1)
self.assertEqual(len(self.payloads), 2)
def test_root_group_cannot_be_silently_removed_when_nodes_only_query_overall(self):
broken = group_plan()
broken['queryPlan'][0]['selector'] = {'kind': 'base'}
with self.assertRaises(TaskUnavailable):
normalize_task(broken, '9月份哪个业务分组下滑最多', CONTEXT)
task = self.run_plan([broken, group_plan(), {**semantic_ok(), 'requiresComparison': True}])
self.assertEqual(task['dimensions'], ['group'])
self.assertEqual(self.payloads[1]['validationDetails']['expected'], [])
self.assertIn('authorizedCatalog', self.payloads[1]['repairContext'])
def test_three_distinct_protocol_errors_do_not_exhaust_a_progressing_repair(self):
a = group_plan(); del a['queryPlan']
b = group_plan(); b['queryPlan'][0]['nodeId'] = '业务分组'
c = group_plan(); c['queryPlan'][0]['selector']['names'] = ['未授权组']
task = self.run_plan([a, b, c, group_plan(), {**semantic_ok(), 'requiresComparison': True}])
self.assertEqual(task['dimensions'], ['group'])
self.assertEqual(task['planRecovery']['modelAttempts'], 4)
self.assertEqual(self.payloads[3]['repairMode'], 'rebuild_nodes')
def test_automatic_summary_repair_never_drops_requested_fields_or_scopes(self):
from nao_plan_recovery import repair_summary
broken = group_plan(); broken['fields'] = ['rate', 'scrapWeight']
self.assertIsNone(repair_summary(broken, CONTEXT))
broken = group_plan(); broken['dimensions'] = ['group', 'product']
self.assertIsNone(repair_summary(broken, CONTEXT))
broken = group_plan(); broken['comparisonRequired'] = True
broken['queryPlan'][0]['comparisonRequired'] = False
self.assertIsNone(repair_summary(broken, CONTEXT))
def test_malformed_model_shapes_get_another_attempt_instead_of_crashing_recovery(self):
for corrupt in ('dimensions', 'ranking_dimension', 'selector_dimension'):
broken = group_plan()
if corrupt == 'dimensions': broken['dimensions'] = 1
elif corrupt == 'ranking_dimension':
broken['queryPlan'][0].update(purpose='ranking', dimension={}, fields=['totalWeight'], stages=['black'], limit=3)
else: broken['queryPlan'][0]['selector']['dimension'] = []
with self.subTest(corrupt=corrupt):
planned = self.run_plan([broken, group_plan(), {**semantic_ok(), 'requiresComparison': True}])
self.assertEqual(planned['planRecovery']['modelAttempts'], 2)
def test_real_business_decision_returns_choices_without_executing_nodes(self):
pending = group_plan()
pending.pop('queryPlan'); pending.pop('comparisonPeriod')
pending.update(needsClarification=True, clarification='请确认要比较的年份',
clarificationRequest={'options': [{'label': '2026年', 'value': '2026年'}]})
task = self.run_plan([pending, {**semantic_ok(), 'requiresComparison': True}])
self.assertTrue(task['needsClarification'])
self.assertEqual(task['clarificationRequest']['options'], [{'label': '2026年', 'value': '2026年'}])
self.assertNotIn('queryPlan', task)
def test_all_group_query_accepts_only_its_catalog_bound_parent_department(self):
from nao_protocol_execution import _validate_protocol_scope
root = normalize_task(group_plan(), '9月份哪个业务分组下滑最多', CONTEXT)
node = root['queryPlan'][0]
child = {**root, 'kind': 'metrics', 'dimensions': [], 'dimensionStages': {},
'filters': {'group': '部门甲-国内', 'department': '部门甲'},
'visualPreference': node['visualPreference'], 'visualType': node['visualType']}
child.pop('queryPlan')
scope = {'nodeId': 'groups', 'scopeId': 'group-domestic', 'purpose': 'metrics', 'task': child}
catalog = {'groupDepartments': {'部门甲-国内': '部门甲'}}
accepted = _validate_protocol_scope(scope, node, root, 'release', catalog=catalog)
self.assertEqual(accepted['task']['filters']['department'], '部门甲')
child['filters']['department'] = '未授权部门'
with self.assertRaises(ValueError):
_validate_protocol_scope(scope, node, root, 'release', catalog=catalog)
if __name__ == '__main__': unittest.main()
@@ -0,0 +1,103 @@
"""A saved clarification selection survives only authorized fresh retries."""
import io
import json
import copy
import sys
import tempfile
import unittest
from contextlib import ExitStack
from pathlib import Path
from unittest.mock import MagicMock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_quality_runtime as runtime
from nao_product_identification import identify_products
ACTOR = '11111111-1111-4111-8111-111111111111'
SESSION = '22222222-2222-4222-8222-222222222222'
RELEASE = '33333333-3333-4333-8333-333333333333'
TURN = '55555555-5555-4555-8555-555555555555'
class ProductClarificationRuntimeTests(unittest.TestCase):
def exercise(self, *, restart=False, revoke=False, other_release=False):
opts = {'products': ['18X15X4+T', '18X15X4+Z'], 'organizationPaths': []}
pending_match = identify_products('今年18x15x4合格率变化趋势', opts)['productMatch']
previous = {'kind':'analysis', 'question':'选+T', 'goal':'今年18x15x4合格率变化趋势',
'rootGoal':'今年18x15x4合格率变化趋势', 'period':{'year':2026},
'filters':{'product':'18X15X4+T'}, 'stages':[], 'needsClarification':False,
'productResolution':{'originalQuestion':'今年18x15x4合格率变化趋势',
'productMatch':pending_match,'selectedName':'18X15X4+T',
'selectionQuestion':'选+T'}}
if revoke:
opts['products'] = ['18X15X4+Z', '28X20X3+T']
context = {'releaseId':RELEASE, '_serverQuestions':[
{'turnId':'old-hello', 'question':'你好', 'status':'COMPLETED'},
{'turnId':'failed-choice', 'question':'选+T', 'status':'FAILED'}],
'_serverHistory':[{'turnId':'failed-choice','question':'选+T','answer':'错误旧答复'}]}
if restart:
context['_restartConversation'] = True
payload = {'turnId':TURN,'sessionId':SESSION,'actorId':ACTOR,'releaseId':RELEASE,
'accessToken':'synthetic-delegation','question':'选+T','context':context}
planned = {'kind':'analysis','goal':'test','question':'选+T','filters':{},'stages':[],
'needsClarification':True,'clarification':'请选择规格','verifySource':False,'fullDetails':False}
captured = {}
with tempfile.TemporaryDirectory() as folder:
root=Path(folder);(root/'private').mkdir();(root/'evidence').mkdir()
saved=root/'private'/'last-agent-task.json'
saved.write_text(json.dumps({'releaseId':('44444444-4444-4444-8444-444444444444' if other_release else RELEASE),
'sessionId':SESSION,'baseMetricFrame':None,'task':previous}),encoding='utf8')
saved.chmod(0o600)
manager=MagicMock();manager.acquire_actor.return_value=(root,None)
state=MagicMock();state.get_chat_id.return_value=None
gate=MagicMock();gate.base_url='http://127.0.0.1:1/v1'
api=MagicMock()
api.call.side_effect=lambda path,**kwargs: (opts if path.startswith('/api/options') else
{'kind':'analysis_catalog','extraFields':[{'key':'customerCode','label':'客户代码'}]}
if path=='/api/analysis/tools/catalog' else {})
delegated=MagicMock();delegated.login.return_value=api
def plan(**kwargs):
captured.update({k:copy.deepcopy(v) for k,v in kwargs.items() if k != 'cancel_event'})
return dict(planned)
with ExitStack() as stack:
for name,value in {
'RuntimeManager':MagicMock(return_value=manager), 'SessionState':MagicMock(return_value=state),
'write_access_file':MagicMock(return_value=root/'access.json'), '_input_cancel_listener':MagicMock(),
'_model_settings':MagicMock(return_value=('https://offline.invalid/v1','key','model',
{'enabled':False,'effort':'auto','protocol':'thinking'})),
'_model_gate_settings':MagicMock(return_value=(1,'key')), 'DelegatedApi':MagicMock(return_value=delegated),
'plan_agent_task':plan,
}.items():
stack.enter_context(patch.object(runtime,name,value))
stack.enter_context(patch('nao_model_gate.ModelGate',return_value=gate))
result=runtime.run_turn(payload,stdout=io.StringIO(),stdin=io.StringIO())
self.assertEqual(result['status'],'NEEDS_CLARIFICATION')
return captured, previous
def test_duplicate_failed_selection_keeps_goal_but_discards_failed_answer(self):
captured, previous = self.exercise()
self.assertEqual(captured['previous'], previous)
self.assertTrue(captured['fresh_retry'])
self.assertEqual(captured['context']['_serverHistory'], [])
match = captured['context']['_businessCatalog']['productMatch']
self.assertTrue(match['selectedFromClarification'])
self.assertEqual(match['candidates'][0]['name'], '18X15X4+T')
self.assertEqual(match['previousQuestion'], '今年18x15x4合格率变化趋势')
self.assertEqual(captured['context']['_businessCatalog']['extraFields'], [{'key':'customerCode','label':'客户代码'}])
def test_explicit_retry_also_restores_only_authorized_selection(self):
captured, previous = self.exercise(restart=True)
self.assertEqual(captured['previous'], previous)
self.assertTrue(captured['fresh_retry'])
self.assertEqual(captured['context']['_businessCatalog']['productMatch']['candidates'][0]['name'], '18X15X4+T')
def test_revoked_or_different_release_never_restores_selection(self):
for kwargs in ({'restart':True,'revoke':True},{'restart':True,'other_release':True}):
with self.subTest(kwargs=kwargs):
captured, _ = self.exercise(**kwargs)
self.assertIsNone(captured['previous'])
self.assertEqual(captured['context']['_businessCatalog']['productMatch']['status'], 'none')
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,13 @@
"""An explicit local deployment setting survives the normal project launcher."""
import importlib.util
from pathlib import Path
import unittest
from unittest.mock import patch
spec=importlib.util.spec_from_file_location('project_launch',Path(__file__).resolve().parents[1]/'scripts/project.py')
project=importlib.util.module_from_spec(spec);spec.loader.exec_module(project)
class ProjectProtocolSettingTest(unittest.TestCase):
def test_setting_is_explicit_and_defaults_to_disabled(self):
base=dict(mysqlPort=53306,mysqlDatabase='synthetic',mysqlUser='u',mysqlPassword='unused',adminUsername='u',adminPassword='unused')
for setting,expected in [(None,'false'),(False,'false'),(True,'true')]:
value={**base,**({} if setting is None else {'analysisProtocolV1Enabled':setting})}
with self.subTest(setting=setting),patch.object(project,'config',return_value=value):self.assertEqual(project.environment()['QUALITY_ANALYSIS_PROTOCOL_V1_ENABLED'],expected)
@@ -0,0 +1,48 @@
"""An explicitly versioned request must never downgrade into actor-local state."""
import io
import sys
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
import nao_quality_runtime as runtime
class ProtocolRuntimeBoundaryTest(unittest.TestCase):
def payload(self, context):
return dict(turnId='55555555-5555-4555-8555-555555555555',
sessionId='22222222-2222-4222-8222-222222222222',
actorId='11111111-1111-4111-8111-111111111111',
releaseId='33333333-3333-4333-8333-333333333333',
accessToken='synthetic', question='取消客户筛选', context=context)
def test_invalid_protocol_marker_stops_before_any_actor_or_model_state(self):
invalid = [None, [], '1', {}, {'version': 2}, {'version': True},
{'version': 1.0}, {'version': '1'}]
for marker in invalid:
with self.subTest(marker=marker):
manager = MagicMock()
manager.acquire_actor.side_effect = AssertionError('legacy actor state reached')
with patch.object(runtime, 'RuntimeManager', return_value=manager), \
patch.object(runtime, 'plan_agent_task') as planner:
result = runtime.run_turn(self.payload({'_protocol': marker,
'_serverHistory': [{'answer': 'OLD'}], '_task': {'filters': {'product': 'OLD'}}}),
stdout=io.StringIO(), stdin=io.StringIO())
self.assertEqual(result['status'], 'FAILED')
self.assertIn('分析请求', result['message'])
self.assertNotIn('本机', result['message'])
manager.acquire_actor.assert_not_called()
planner.assert_not_called()
self.assertEqual(result['toolCalls'], [])
def test_valid_version_and_absent_legacy_marker_remain_distinct(self):
legacy = runtime._normalize_request(self.payload({'year': 2026}))
self.assertNotIn('_protocol', legacy['context'])
valid = runtime._normalize_request(self.payload({'_protocol': {'version': 1,
'attemptId': '22222222-2222-4222-8222-222222222222'}}))
self.assertEqual(valid['context']['_protocol']['version'], 1)
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,82 @@
import copy
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from analysis_visuals import build_visuals
from nao_agent_report import _build_trend_report
RELEASE = 'signed-release'
FILTERS = {'department': '部门甲'}
def trend_document(black_total, bright_total=100.0):
rows = []
for month, black_weight in ((1, 120.0), (2, black_total)):
rows.append({'period': f'2026-{month:02d}', 'periodLabel': f'2026-{month:02d}',
'black': {'totalWeight': black_weight, 'scrapWeight': 10.0,
'scrapRate': 0.08 if month == 1 else 0.09,
'rate': 0.95, 'qualifiedWeight': 110.0},
'bright': {'totalWeight': bright_total, 'scrapWeight': 8.0,
'scrapRate': 0.08, 'rate': 0.96, 'qualifiedWeight': 92.0},
'coverage': {'blackRows': 1, 'brightRows': 1}})
result = {'kind': 'quality_metrics', 'releaseId': RELEASE,
'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 2},
'filters': copy.deepcopy(FILTERS), 'view': 'overall',
'summary': {'black': {'totalWeight': black_total, 'scrapWeight': 20.0},
'bright': {'totalWeight': bright_total, 'scrapWeight': 16.0}},
'trend': rows, 'trendCount': 2, 'trendTruncated': False,
'coverage': {'sourceMonths': [
{'year': 2026, 'month': 1, 'blackRows': 1, 'brightRows': 1},
{'year': 2026, 'month': 2, 'blackRows': 1, 'brightRows': 1}]}}
return {'tool': 'quality_data', 'mode': 'query', 'releaseId': RELEASE,
'evidenceRef': 'Q1234ABCD', 'result': result}
def child_task(field, *, visual_type='line', period_coverage='available'):
return {'version': 1, 'kind': 'metrics', 'question': '黑片投入重量趋势',
'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 2},
'filters': copy.deepcopy(FILTERS), 'dimensions': [], 'dimensionStages': {},
'stages': ['black'], 'fields': [field], 'trendRequired': True,
'comparisonRequired': False, 'comparisonPeriod': None,
'periodCoverage': period_coverage, 'visualPreference': 'auto',
'visualType': visual_type}
class QueryPlanSignedWeightTrendTests(unittest.TestCase):
def test_black_total_weight_keeps_signed_correction_in_two_point_line_and_report(self):
document = trend_document(-10.05)
task = child_task('totalWeight', visual_type='line')
context = {'releaseId': RELEASE, 'period': task['period'], 'filters': FILTERS,
'_agentTask': task}
visuals = build_visuals([document], task['question'], 'turn-signed', context=context)
chart = next(item for item in visuals['visuals'] if item['metric'] == 'black_total_weight')
self.assertEqual(chart['type'], 'line')
self.assertEqual(chart['series'][0]['values'], [120.0, -10.05])
self.assertIn('负向修正原值保留', chart['subtitle'])
report = _build_trend_report([document], context, task['question'])
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertIn('2月-10.05 kg', report['facts'][0]['text'])
def test_negative_bright_total_is_rejected_and_area_still_requires_three_points(self):
document = trend_document(100.0, bright_total=-1.0)
bright_task = {**child_task('totalWeight', visual_type='line'), 'stages': ['bright'],
'question': '亮片投入重量趋势'}
bright_context = {'releaseId': RELEASE, 'period': bright_task['period'],
'filters': FILTERS, '_agentTask': bright_task}
bright_charts = build_visuals([document], bright_task['question'], 'turn-bright',
context=bright_context)['visuals']
self.assertFalse(any(item['metric'] == 'bright_total_weight' for item in bright_charts))
area_task = child_task('totalWeight', visual_type='area')
area_context = {'releaseId': RELEASE, 'period': area_task['period'],
'filters': FILTERS, '_agentTask': area_task}
area_charts = build_visuals([trend_document(-10.05)], area_task['question'], 'turn-area',
context=area_context)['visuals']
self.assertFalse(any(item['metric'] == 'black_total_weight' for item in area_charts))
if __name__ == '__main__':
unittest.main()
+101
View File
@@ -0,0 +1,101 @@
"""Regressions for source/row requirements that cannot be satisfied by a draft."""
import copy
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).resolve().parent)]
import nao_quality_runtime as runtime
from nao_agent_task import TaskUnavailable, bind_task, normalize_task, plan_task
from nao_evidence_finalizer import coverage_sufficient
import test_agent_runtime as agent_runtime_fixture
import test_weight_ranking as weight_fixture
from test_nao_agent_task import task_value
def rows_plan():
return task_value(kind='explain',goal='逐条提供全部原始记录',freshQuery=False,historyMode='none',
dimensions=[],dimensionStages={},stages=['black'],fields=[],fullDetails=True,
period={'year':2026,'month':9})
class RequiredReadCompletionTests(unittest.TestCase):
def test_full_details_always_requires_a_fresh_read(self):
planned=normalize_task(rows_plan(),'请逐条列出全部原始记录')
self.assertTrue(planned['fullDetails'])
self.assertTrue(planned['freshQuery'])
def test_reviewer_cannot_claim_all_rows_are_required_without_evidence(self):
inconsistent={'aligned':True,'requiresEvidence':False,
'requiresSourceVerification':False,'requiresAllSourceRows':True}
with patch('nao_agent_task._model_json',side_effect=[rows_plan(),inconsistent]*3):
with self.assertRaises(TaskUnavailable):
plan_task(base_url='https://offline.invalid',api_key='synthetic',model='m',
question='请逐条列出全部原始记录',context={},cancel_event=threading.Event())
def test_unverified_native_draft_cannot_finish_when_rows_required(self):
harness=agent_runtime_fixture.AgentRuntimeIntegrationTest()
with tempfile.TemporaryDirectory() as folder:
# Simulate a legacy or malformed plan even after planner safeguards.
planned=agent_runtime_fixture.task('请逐条列出全部原始记录',kind='explain',freshQuery=False,
historyMode='none',dimensions=[],dimensionStages={},fields=[],stages=['black'],fullDetails=True)
planned['freshQuery']=False;planned['requiresVerifiedEvidence']=False
def native(**kwargs):
stream=runtime.ConversationAccumulator()
stream.blocks['native']='已经列出全部原始记录。';stream.block_order.append('native')
stream.record_event('assistant',text='已经列出全部原始记录。')
return stream,'COMPLETED','stop',None
with patch('nao_agent_task.plan_repairs',return_value=[]):
result=harness.run_mocked(Path(folder),planned,native)
self.assertEqual(result['status'],'PARTIAL')
self.assertNotIn('已经列出全部原始记录',result['answer'])
self.assertFalse(any('已经列出全部原始记录' in entry.get('text','') for entry in result['timeline']))
def test_complete_rankings_do_not_satisfy_source_or_row_requirements(self):
doc,_=weight_fixture.WeightRankingTests().run_rank()
department=copy.deepcopy(doc);department['result']['dimension']='department';department['evidenceRef']='Q00000002'
for flags in ({'verifySource':True},{'fullDetails':True}):
with self.subTest(flags=flags):
planned=weight_fixture.planned(**flags);context=bind_task({'releaseId':'release-a'},planned)
self.assertFalse(coverage_sufficient([doc,department],weight_fixture.QUESTION,context))
def test_complete_rankings_with_required_source_or_all_rows_can_finish(self):
doc,_=weight_fixture.WeightRankingTests().run_rank()
department=copy.deepcopy(doc);department['result']['dimension']='department';department['evidenceRef']='Q00000002'
side_context={'releaseId':'release-a','year':2026,'stage':'bright','includeUnassigned':False}
def investigation(action,side):
return {'mode':'investigate','evidenceRef':'Q00000003','result':{'kind':'quality_investigation',
'releaseId':'release-a','period':{'year':2026},'filters':{},'stage':'bright','action':action,'current':side}}
source=investigation('source',{'context':side_context,'aggregateComplete':True,'totalRows':2,
'sourceVerification':{'complete':True,'allSelectedValuesMatchStored':True,'expectedRows':2,
'verifiedRows':2,'missingRows':0,'mismatchRows':0}})
def page(offset,more):
return investigation('rows',{'context':side_context,'recordFilter':'all','matchingRows':4,
'offset':offset,'returnedRows':2,'rows':[{'id':offset},{'id':offset+1}],'hasMore':more})
for flags,extra in (({'verifySource':True},[source]),({'fullDetails':True},[page(0,True),page(2,False)])):
with self.subTest(flags=flags):
planned=weight_fixture.planned(**flags);context=bind_task({'releaseId':'release-a'},planned)
self.assertTrue(coverage_sufficient([doc,department,*extra],weight_fixture.QUESTION,context))
def test_semantic_all_rows_accepts_complete_multiple_pages_but_not_gaps(self):
question='列出黑片全部原始记录'
planned=normalize_task(task_value(kind='details',fullDetails=True,dimensions=[],dimensionStages={},
stages=['black'],fields=[],period={'year':2026,'month':9},comparisonRequired=False),question)
context=bind_task({'releaseId':'release-a'},planned)
def page(offset, more, total=4):
side={'context':{'releaseId':'release-a','year':2026,'month':9,'stage':'black','includeUnassigned':False},
'recordFilter':'all','matchingRows':total,'offset':offset,'returnedRows':2,
'rows':[{'id':offset},{'id':offset+1}],'hasMore':more}
return {'mode':'investigate','evidenceRef':'QAAAAAAAA','result':{'kind':'quality_investigation',
'releaseId':'release-a','period':{'year':2026,'month':9},'filters':{},'stage':'black',
'action':'rows','current':side}}
pages=[page(0,True),page(2,False)]
self.assertTrue(coverage_sufficient(pages,question,context))
self.assertFalse(coverage_sufficient([pages[0],page(3,False)],question,context))
self.assertFalse(coverage_sufficient([pages[0],page(2,False,total=5)],question,context))
if __name__=='__main__':unittest.main()
+48
View File
@@ -0,0 +1,48 @@
import json,sys,unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
import nao_quality_runtime as runtime
class TimelineTests(unittest.TestCase):
def test_real_text_tools_results_are_interleaved_and_reasoning_is_private(self):
a=runtime.ConversationAccumulator()
a.feed({'type':'reasoning-delta','delta':'hidden chain'})
a.feed({'type':'text-delta','id':'intro','delta':'先核查期间与覆盖。'})
call={'type':'tool-input-available','toolName':'mcp_call','toolCallId':'q1','input':{'tool':'quality_data','arguments':{'mode':'defects','stage':'black','apiKey':'secret'}}}
a.feed(call); a.feed(call)
a.feed({'type':'tool-output-available','toolCallId':'q1','output':{'evidenceRef':'Q123','rows':[{'weight':12}], 'categoryReconciliation':[{'reconciliationStatus':'MISMATCH'}]}})
a.feed({'type':'text-delta','id':'final','delta':'来源缺陷只作为变化线索。'})
self.assertEqual([e['kind'] for e in a.timeline],['assistant','tool_start','tool_result','assistant'])
self.assertEqual([e['sequence'] for e in a.timeline],[1,2,3,4])
encoded=json.dumps(a.timeline,ensure_ascii=False)
self.assertNotIn('hidden chain',encoded); self.assertNotIn('secret',encoded)
self.assertIn('Q123',encoded); self.assertIn('未对账',encoded)
def test_unmatched_result_does_not_invent_a_tool_start(self):
a=runtime.ConversationAccumulator();a.feed({'type':'tool-output-available','toolCallId':'missing','output':{'rows':[]}})
self.assertEqual(a.timeline,[])
def test_resumed_provider_text_block_stays_after_the_tool(self):
a=runtime.ConversationAccumulator()
a.feed({'type':'text-delta','id':'same','delta':'先查询。'})
a.feed({'type':'tool-input-available','toolName':'read','toolCallId':'r1'})
a.feed({'type':'tool-output-available','toolCallId':'r1','output':{}})
a.feed({'type':'text-delta','id':'same','delta':'已查询。'})
self.assertEqual([e['kind'] for e in a.timeline],['assistant','tool_start','tool_result','assistant'])
self.assertEqual(a.timeline[0]['text'],'先查询。')
self.assertEqual(a.timeline[-1]['text'],'已查询。')
def test_comparison_side_reconciliation_is_also_disclosed_and_null_is_safe(self):
text=runtime.public_tool_result({'categoryReconciliation':None,'comparisonCategoryReconciliation':[{'reconciliationStatus':'DIFF'}],'truncated':True})
self.assertIn('未对账',text);self.assertIn('仅覆盖返回集合',text)
text=runtime.public_tool_result({'evidenceRef':'QABC12345','summary':{'compositeRate':0.95},'token':'private'})
self.assertIn('QABC12345',text);self.assertNotIn('private',text);self.assertIn('95.00%',text)
if __name__=='__main__': unittest.main()
class InvestigationEvidenceStatusTests(unittest.TestCase):
def test_expected_vcm_is_not_an_unresolved_warning_but_non_vcm_remains(self):
from nao_quality_runtime import public_evidence_checks
base={'kind':'quality_investigation','releaseId':'r','stage':'bright','period':{'year':2026,'month':9},'filters':{'group':'工业电机-VCM'},'current':{'categories':[{'category':'镀废','expectedTotalsOnly':{'rows':419},'unexpectedDifferenceRows':0,'requiresFurtherInvestigation':False}],'sourceVerification':{'complete':True,'allSelectedValuesMatchStored':True}}}
self.assertEqual(list(public_evidence_checks(base).values()),['expected'])
base['current']['categories'][0].update(unexpectedDifferenceRows=12,requiresFurtherInvestigation=True)
self.assertEqual(list(public_evidence_checks(base).values()),['unresolved'])
def test_source_verification_failure_cannot_be_hidden_by_vcm_policy(self):
from nao_quality_runtime import public_evidence_checks
r={'kind':'quality_investigation','current':{'categories':[{'category':'镀废','expectedTotalsOnly':{'rows':419},'unexpectedDifferenceRows':0,'requiresFurtherInvestigation':False}],'sourceVerification':{'complete':False}}}
self.assertEqual(list(public_evidence_checks(r).values()),['unresolved'])
+181
View File
@@ -0,0 +1,181 @@
import sys,unittest,json,io,threading
from pathlib import Path
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).resolve().parent)]
from nao_query_contract import bind_query,canonical_filters
from nao_answer_focus import answer_focus,focus_fallback,missing_focus,with_focus_conclusion
from nao_quality_tools import normalize_filters,QualityTools,QualityToolError
from nao_agent_task import normalize_task
from test_nao_agent_task import task_value
import nao_evidence_finalizer as finalizer
class ScopeContractTest(unittest.TestCase):
def task(self):
return normalize_task(task_value(dimensions=['group'],dimensionStages={'group':[]},
filterUpdates={'department':'新能源汽车磁钢'},period={'year':2026,'month':9},
comparisonPeriod={'year':2026,'month':8}), '主要是国内还是国外的')
def contract(self):return {'task':self.task(),'releaseId':'same-release'}
def test_business_alias_is_department_not_union_of_groups(self):
self.assertEqual(canonical_filters({'department':'汽车交通'}),{'department':'新能源汽车磁钢'})
self.assertEqual(canonical_filters({'group':'汽车交通'}),{'department':'新能源汽车磁钢'})
filters,_=normalize_filters('auto_combined',{},scope_was_explicit=True)
self.assertEqual(filters,{'department':'新能源汽车磁钢','includeUnassigned':False})
task=normalize_task(task_value(dimensions=['group'],filterUpdates={'department':'汽车交通'}),'汽车交通合格率')
self.assertEqual(task['filters'],{'department':'新能源汽车磁钢'})
def test_missing_query_scope_inherits_frozen_parent(self):
q=bind_query({'mode':'changes','dimension':'group'},self.contract())
self.assertEqual(q['filters']['department'],'新能源汽车磁钢')
self.assertEqual(q['period'],{'year':2026,'month':9})
self.assertEqual(q['comparisonPeriod'],{'year':2026,'month':8})
self.assertEqual(q['releaseId'],'same-release')
def test_native_cannot_replace_department_with_group_union(self):
with self.assertRaisesRegex(ValueError,'范围'):
bind_query({'mode':'changes','dimension':'group','filters':{'group':'union-domestic-overseas'}},self.contract())
def test_native_cannot_change_release_period_or_expand_scope(self):
for changed in ({'releaseId':'different'}, {'period':{'year':2026,'month':7}},
{'filters':{'department':'其他产业部'}},{'filters':{'includeUnassigned':True}}):
with self.subTest(changed=changed),self.assertRaises(ValueError):
bind_query({'mode':'changes','dimension':'group',**changed},self.contract())
def test_explicit_child_compare_retains_parent_and_catalog_remains_available(self):
q=bind_query({'mode':'compare','filters':{'group':'汽车交通-国内'}},self.contract())
self.assertEqual(q['filters']['department'],'新能源汽车磁钢')
self.assertEqual(q['filters']['group'],'汽车交通-国内')
self.assertEqual(bind_query({'mode':'catalog'},self.contract()),{'mode':'catalog'})
def test_query_guard_runs_before_any_http_call(self):
called=[]
tools=QualityTools(api_factory=lambda:called.append(True),query_contract=self.contract())
with self.assertRaises(QualityToolError):
tools.quality_data({'mode':'changes','dimension':'group','filters':{'group':'wrong-union'}})
self.assertEqual(called,[])
def test_rank_binds_authorized_column_filters_but_other_modes_still_reject_them(self):
contract=self.contract()
rule={'field':'customer_code','stages':['bright'],'operator':'in','values':['0012']}
contract['task']['filters']['columnFilters']=[rule]
bound=bind_query({'mode':'rank','dimension':'group','stage':'bright','metric':'totalWeight'},contract)
self.assertEqual(bound['columnFilters'],[rule])
self.assertEqual(bound['period'],contract['task']['period'])
self.assertEqual(bound['filters']['department'],'新能源汽车磁钢')
with self.assertRaisesRegex(ValueError,'columnFilters'):
bind_query({'mode':'changes','dimension':'group','columnFilters':[rule]},contract)
def test_requested_child_drilldown_keeps_parent_without_filtering_its_own_rank(self):
contract=self.contract();contract['task']['dimensions']=['group','product']
contract['task']['dimensionStages']['product']=[]
q=bind_query({'mode':'changes','dimension':'product','filters':{'group':'汽车交通-国内'}},contract)
self.assertEqual(q['filters']['department'],'新能源汽车磁钢')
self.assertEqual(q['filters']['group'],'汽车交通-国内')
with self.assertRaises(ValueError):
bind_query({'mode':'changes','dimension':'group','filters':{'group':'汽车交通-国内'}},contract)
def test_combined_coverage_allows_supplemental_stage_query_without_changing_contract(self):
contract=self.contract()
q=bind_query({'mode':'changes','dimension':'group','stage':'black'},contract)
self.assertEqual(q['stage'],'black')
self.assertEqual(q['filters']['department'],'新能源汽车磁钢')
self.assertEqual(contract['task']['dimensionStages']['group'],[])
def test_understood_followup_executes_missing_group_read_even_when_native_made_no_calls(self):
from unittest.mock import patch,MagicMock
from test_agent_runtime import comparison
from nao_agent_task import bind_task
import nao_quality_runtime as runtime
task=self.task();primary=comparison();release=primary['releaseId'];primary['filterBinding']=task['filters']
docs=[{'evidenceRef':'QAAAAAAAA','mode':'compare','result':primary}];calls=[]
def read(arguments):
calls.append(arguments)
result={'kind':'quality_changes','releaseId':release,'dimension':'group','stage':None,
'filters':task['filters'],'currentPeriod':task['period'],'comparisonPeriod':task['comparisonPeriod'],
'totalRows':2,'returnedRows':2,'truncated':False,'sort':'scrap_delta',
'rows':[{'key':'国内','label':'国内','current':{'rate':.95},'comparison':{'rate':.96},'rateDrop':.01,'scrapDelta':5},
{'key':'海外','label':'海外','current':{'rate':.96},'comparison':{'rate':.96},'rateDrop':0,'scrapDelta':0}]}
doc={'evidenceRef':'QBBBBBBBB','mode':'changes','result':result};docs.append(doc);return doc
request={'actorId':'actor','sessionId':'session','releaseId':release,'question':task['question'],
'context':bind_task({'releaseId':release},task)}
with patch('nao_evidence_finalizer.evidence_summaries',side_effect=lambda *a,**k:list(docs)),\
patch('nao_agent_task.plan_repairs',side_effect=AssertionError('Known missing query must execute directly')),\
patch.object(runtime,'DelegatedApi'),patch('nao_quality_tools.QualityTools') as tools:
tools.return_value.quality_data.side_effect=read
runtime.complete_agent_evidence(task=task,request=request,actor_root=Path('/offline/session'),
access_path=Path('/offline/access'),manager=MagicMock(),connection=('offline','synthetic','m'),
accumulator=runtime.ConversationAccumulator(),prior_files=frozenset(),cancel_event=threading.Event(),emit=lambda e:None)
self.assertEqual(len(calls),1)
self.assertEqual(calls[0]['dimension'],'group')
self.assertEqual(calls[0]['filters']['department'],'新能源汽车磁钢')
self.assertNotIn('stage',calls[0])
self.assertTrue(finalizer.coverage_sufficient(docs,task['question'],request['context']))
class AnswerFocusTest(unittest.TestCase):
def facts(self):
return [{'id':'F1','kind':'overall','text':'总体合格率下降。','evidenceRef':'QAAAAAAAA'},
{'id':'F2','kind':'group','dimension':'group','stage':None,'label':'国内','text':'国内:96.39%→95.76%,下降0.63个百分点。','evidenceRef':'QBBBBBBBB'},
{'id':'F3','kind':'group','dimension':'group','stage':None,'label':'海外','text':'海外:93.70%→93.61%,下降0.09个百分点。','evidenceRef':'QBBBBBBBB'}]
def task(self):return {'kind':'analysis','dimensions':['group'],'dimensionStages':{'group':[]},'question':'主要是国内还是国外的'}
def test_group_followup_requires_both_objects_not_only_overall(self):
focus=answer_focus(self.facts(),self.task())
self.assertEqual(set(focus['requiredFactIds']),{'F2','F3'})
self.assertTrue(missing_focus(['F1'],focus))
self.assertFalse(missing_focus(['F2','F3'],focus))
def test_department_fact_cannot_satisfy_group_followup(self):
facts=self.facts();facts.insert(1,{'id':'D1','kind':'group','dimension':'department','stage':None,
'label':'父产业部','text':'父产业部变化。','evidenceRef':'QAAAAAAAA'})
self.assertEqual(set(answer_focus(facts,self.task())['requiredFactIds']),{'F2','F3'})
def test_explicit_group_comparison_cannot_be_replaced_by_other_top_groups(self):
facts=self.facts();facts[1]['label']='汽车交通-国内';facts[2]['label']='汽车交通-海外'
facts[1]['text']='汽车交通-国内:下降0.62个百分点。';facts[2]['text']='汽车交通-海外:下降0.09个百分点。'
facts[1:1]=[{'id':'X1','kind':'group','dimension':'group','stage':None,'label':'其他甲','text':'其他甲','evidenceRef':'QBBBBBBBB'},
{'id':'X2','kind':'group','dimension':'group','stage':None,'label':'其他乙','text':'其他乙','evidenceRef':'QBBBBBBBB'}]
task={**self.task(),'targetEntities':{'group':['汽车交通-国内','汽车交通-海外']}}
focus=answer_focus(facts,task)
self.assertEqual(set(focus['requiredFactIds']),{'F2','F3'})
self.assertTrue(missing_focus(['X1','X2'],focus))
self.assertFalse(answer_focus([f for f in facts if f['id']!='F3'],task)['available'])
def test_failed_composition_fallback_answers_current_focus_without_big_parent_tables(self):
report={'facts':self.facts(),'sections':['### 总体与阶段\n\n一张大表'],'policy':''}
answer,ids=focus_fallback(report,self.task(),[])
self.assertIn('国内',answer);self.assertIn('海外',answer)
self.assertNotIn('总体与阶段',answer);self.assertNotIn('一张大表',answer)
self.assertEqual(set(ids),{'F2','F3'})
def test_missing_group_facts_explain_gap_instead_of_dumping_parent_report(self):
report={'facts':self.facts()[:1],'sections':['一张大表'],'policy':''}
answer,ids=focus_fallback(report,self.task(),['范围尚未一致'])
self.assertIn('业务分组',answer);self.assertIn('范围尚未一致',answer)
self.assertNotIn('一张大表',answer);self.assertEqual(ids,[])
def test_direct_priority_requires_both_verified_change_signals_to_agree(self):
facts=self.facts();facts[1].update(rateDrop=.0062,scrapDelta=5572.52);facts[2].update(rateDrop=.0009,scrapDelta=738.91)
focus=answer_focus(facts,self.task());answer=with_focus_conclusion('正式事实',focus)
self.assertTrue(answer.startswith('按本轮可比分组的变化,优先关注国内'))
self.assertIn('不代表对总体下降的精确贡献',answer)
facts.append({'id':'F4','kind':'group','stage':None,'label':'新进分组','text':'本期新增,缺少比较期。','evidenceRef':'QBBBBBBBB','rateDrop':None,'scrapDelta':None})
self.assertEqual(answer_focus(facts,self.task())['priority']['label'],'国内')
facts[2]['scrapDelta']=6000
self.assertIsNone(answer_focus(facts,self.task())['priority'])
facts[2].pop('rateDrop')
self.assertIsNone(answer_focus(facts,self.task())['priority'])
def test_real_synthesis_rejects_off_target_model_answer_and_falls_back_to_group_pair(self):
from test_agent_runtime import comparison
from nao_agent_task import bind_task
release='33333333-3333-4333-8333-333333333333';period={'year':2026,'month':9};prior={'year':2026,'month':8}
filters={'department':'新能源汽车磁钢'};primary=comparison();primary['filterBinding']=filters
changes={'kind':'quality_changes','releaseId':release,'dimension':'group','stage':None,
'filters':filters,'currentPeriod':period,'comparisonPeriod':prior,'sort':'scrap_delta',
'totalRows':2,'returnedRows':2,'truncated':False,
'rows':[{'key':'国内','label':'汽车交通-国内','comparison':{'rate':.9639},'current':{'rate':.9576},'rateDrop':.0063,'scrapDelta':5},
{'key':'海外','label':'汽车交通-海外','comparison':{'rate':.9370},'current':{'rate':.9361},'rateDrop':.0009,'scrapDelta':1}]}
docs=[{'evidenceRef':'QAAAAAAAA','mode':'compare','result':primary},
{'evidenceRef':'QBBBBBBBB','mode':'changes','result':changes}]
q='主要是国内还是国外的';task=normalize_task(task_value(dimensions=['group'],dimensionStages={'group':[]},
filterUpdates=filters,period=period,comparisonPeriod=prior,comparisonRequired=True),q)
task['rootGoal']='之前全面分析产品和缺陷';context=bind_task({'releaseId':release},task);calls=[]
def opener(request,timeout):
body=json.loads(request.data);payload=json.loads(body['messages'][1]['content']);calls.append(payload)
overall=next(f['id'] for f in payload['facts'] if f['kind']=='overall')
value={'reportTitle':'质量变化分析','answerBlocks':[{'type':'paragraph','factIds':[overall]}],
'evidenceSufficient':True,'missingEvidence':[]}
return io.BytesIO(json.dumps({'choices':[{'finish_reason':'stop','message':{'content':json.dumps(value)}}]}).encode())
result=finalizer.synthesize(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=q,evidence=docs,context=context,cancel_event=threading.Event(),opener=opener)
self.assertEqual(len(calls),2)
self.assertTrue(result['compositionRepairAttempted'])
self.assertIn('汽车交通-国内',result['answer']);self.assertIn('汽车交通-海外',result['answer'])
self.assertNotIn('总体与阶段',result['answer'])
self.assertTrue(result['evidenceSufficient'])
if __name__=='__main__':unittest.main()
+286
View File
@@ -0,0 +1,286 @@
"""Formal stage scrap-rate evidence, coverage, report, and chart contracts."""
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from analysis_visuals import _comparison_charts, _trend_chart_from_points, build_visuals
from nao_agent_report import _build_trend_report, build_noncomparison_report
from nao_analysis_coverage import formal_metric_coverage, semantic_metric_gaps
from nao_evidence_finalizer import _formal_metric_report
from nao_evidence_report import build_report
RELEASE = 'release-scrap-rate'
QUESTION = '查询2026年9月黑片报废率'
PERIOD = {'year': 2026, 'month': 9}
COMPARISON_PERIOD = {'year': 2026, 'month': 8}
def task(question=QUESTION, *, period=PERIOD, comparison_period=None, stages=None,
fields=None, trend=False, visual='none'):
return {
'version': 1,
'question': question,
'kind': 'metrics',
'filters': {'product': '产品甲'},
'period': period,
'comparisonPeriod': comparison_period,
'stages': stages if stages is not None else ['black'],
'fields': fields if fields is not None else ['scrapRate'],
'dimensions': [],
'trendRequired': trend,
'comparisonRequired': comparison_period is not None,
'freshQuery': True,
'verifySource': False,
'fullDetails': False,
'needsClarification': False,
'visualPreference': visual,
'visualType': None,
'responseStyle': 'standard',
'periodCoverage': 'available',
'historyMode': 'none',
}
def metric_doc(*, value=0.125, month=9, filters=None, stage='black', rate=None,
evidence='A', rows=2):
period = {'year': 2026, 'month': month}
summary_stage = {'scrapRate': value, 'totalWeight': 100,
'scrapWeight': value * 100 if isinstance(value, (int, float)) and value >= 0 else 0}
if rate is not None:
summary_stage['rate'] = rate
return {
'tool': 'quality_data', 'mode': 'query', 'releaseId': RELEASE,
'evidenceRef': 'Q' + evidence * 8,
'result': {
'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': period,
'filters': filters or {'product': '产品甲'}, 'view': 'overall',
'summary': {stage: summary_stage},
'coverage': {'sourceMonths': [{'year': 2026, 'month': month,
'blackRows': rows if stage == 'black' else 0,
'brightRows': rows if stage == 'bright' else 0}]},
},
}
def comparison_doc(*, black_values=(0.05, 0.0), release=RELEASE, filters=None,
black_rows=(2, 2), evidence='B'):
def snapshot(month, value, rows):
return {
'period': {'year': 2026, 'month': month},
'compositeRate': 0.94,
'black': {'scrapRate': value, 'rate': 0.95, 'totalWeight': 100, 'scrapWeight': 5},
'bright': {'scrapRate': 0.06, 'rate': 0.96, 'totalWeight': 90, 'scrapWeight': 4},
'coverage': {'sourceMonths': [{'year': 2026, 'month': month,
'blackRows': rows, 'brightRows': 2}]},
}
result = {
'kind': 'quality_comparison', 'releaseId': release,
'filterBinding': filters or {'product': '产品甲'},
'comparison': snapshot(8, black_values[0], black_rows[0]),
'current': snapshot(9, black_values[1], black_rows[1]),
'delta': {'available': True, 'percentagePoints': 0},
'stageRateChangesPP': {'black': 0, 'bright': 0},
}
return {'tool': 'quality_data', 'mode': 'compare', 'releaseId': release,
'evidenceRef': 'Q' + evidence * 8, 'result': result}
class ScrapRateEvidenceTest(unittest.TestCase):
def context(self, selected_task):
return {'releaseId': RELEASE, 'period': selected_task['period'],
'comparisonPeriod': selected_task['comparisonPeriod'],
'filters': selected_task['filters'], '_agentTask': selected_task}
def test_single_period_uses_scrap_rate_directly_and_keeps_zero_and_over_one(self):
for raw, expected in ((0, '0.00%'), (1.25, '125.00%')):
selected = task()
report = build_noncomparison_report(
[metric_doc(value=raw, rate=None)], self.context(selected), QUESTION)
self.assertEqual([fact['label'] for fact in report['facts']], ['黑片报废率'])
self.assertIn(expected, report['facts'][0]['text'])
self.assertTrue(report['coverageSufficient'])
def test_requested_numerator_and_denominator_are_displayed_as_independent_formal_values(self):
question='黑片报废率是多少,同时列出分子分母'
selected=task(question,fields=['scrapRate','scrapWeight','totalWeight'])
doc=metric_doc(value=.175,rate=.99)
doc['result']['summary']['black'].update(totalWeight=200,scrapWeight=35)
report=build_noncomparison_report([doc],self.context(selected),question)
rendered='\n'.join(report['sections'])
for expected in ('17.50%','35.00','200.00'):
self.assertIn(expected,rendered)
self.assertEqual([f['label'] for f in report['facts']],['黑片报废率','黑片报废重量','黑片投入重量'])
self.assertTrue(report['coverageSufficient'])
self.assertNotIn('1.00%',rendered)
def test_missing_requested_denominator_is_incomplete_even_when_formal_rate_exists(self):
question='黑片报废率及报废总量和修正总重'
selected=task(question,fields=['scrapRate','scrapWeight','totalWeight'])
doc=metric_doc();doc['result']['summary']['black'].pop('totalWeight')
report=build_noncomparison_report([doc],self.context(selected),question)
self.assertFalse(report['coverageSufficient'])
self.assertTrue(any('重量' in gap for gap in report['missingEvidence']))
def test_missing_or_negative_scrap_rate_is_not_treated_as_zero_or_clamped(self):
for raw in (None, -0.01):
selected = task()
report = build_noncomparison_report(
[metric_doc(value=raw, rate=0.99)], self.context(selected), QUESTION)
self.assertIn('未提供', report['facts'][0]['text'])
self.assertFalse(report['coverageSufficient'])
self.assertEqual(semantic_metric_gaps(
[metric_doc(value=None)], QUESTION, self.context(task())),
['未取得指定期间、范围的黑片报废率正式指标或来源覆盖。'])
def test_scrap_rate_coverage_does_not_require_qualified_rate_or_other_stage(self):
selected = task()
doc = metric_doc(value=0.02, rate=None)
self.assertEqual(semantic_metric_gaps([doc], QUESTION, self.context(selected)), [])
report = build_noncomparison_report([doc], self.context(selected), QUESTION)
self.assertEqual([fact['label'] for fact in report['facts']], ['黑片报废率'])
self.assertNotIn('亮片', '\n'.join(fact['text'] for fact in report['facts']))
self.assertNotIn('合格率', '\n'.join(fact['text'] for fact in report['facts']))
def test_scope_and_period_mismatch_are_rejected(self):
selected = task()
wrong_scope = metric_doc(value=0.02, filters={'product': '产品乙'})
wrong_period = metric_doc(value=0.02, month=8)
self.assertTrue(semantic_metric_gaps([wrong_scope], QUESTION, self.context(selected)))
self.assertTrue(semantic_metric_gaps([wrong_period], QUESTION, self.context(selected)))
def test_two_period_formal_contract_and_body_use_only_scrap_rate(self):
question = '只看产品甲,2026年9月与8月黑片报废率'
selected = task(question, period=PERIOD, comparison_period=COMPARISON_PERIOD)
context = self.context(selected)
doc = comparison_doc(black_values=(1.25, 0.0))
contract = formal_metric_coverage([doc], question, context)
self.assertEqual(contract['fields'], ['scrapRate'])
self.assertEqual(contract['missingEvidence'], [])
self.assertEqual([item['value'] for item in contract['metrics']], [0.0, 1.25])
report = _formal_metric_report(contract)
text = '\n'.join(report['sections']) + '\n' + '\n'.join(item['text'] for item in report['facts'])
self.assertIn('125.00%', text)
self.assertIn('0.00%', text)
self.assertNotIn('合格率', text)
wrong_scope = comparison_doc(filters={'product': '产品乙'})
rejected = formal_metric_coverage([wrong_scope], question, context)
self.assertTrue(rejected['missingEvidence'])
comparison_question = '对比2026年9月与8月黑片报废率'
selected = task(comparison_question, period=PERIOD,
comparison_period=COMPARISON_PERIOD, stages=['black'])
body = build_report([doc], self.context(selected), comparison_question)
body_text = '\n'.join(body['sections']) + '\n' + '\n'.join(fact['text'] for fact in body['facts'])
self.assertTrue(body['coverageSufficient'], body['missingEvidence'])
self.assertIn('125.00%', body_text)
self.assertIn('0.00%', body_text)
self.assertIn('黑片报废率', body_text)
self.assertNotIn('亮片报废率', body_text)
def test_formal_scrap_weight_contract_rejects_negative_backend_weight(self):
question = '只看产品甲,2026年9月与8月黑片报废重量'
selected = task(question, period=PERIOD, comparison_period=COMPARISON_PERIOD,
fields=['scrapWeight'])
document = comparison_doc()
document['result']['current']['black']['scrapWeight'] = -0.01
contract = formal_metric_coverage([document], question, self.context(selected))
self.assertTrue(contract['missingEvidence'])
invalid = [metric for metric in contract['metrics']
if metric['field'] == 'scrapWeight' and metric['value'] < 0]
self.assertEqual(len(invalid), 1)
self.assertFalse(invalid[0]['covered'])
def test_omitted_stage_scrap_rate_stays_two_stage_not_a_composite(self):
question = '查询2026年9月产品甲报废率'
selected = task(question, stages=[])
context = self.context(selected)
doc = metric_doc(value=0.04)
doc['result']['summary']['bright'] = {'scrapRate': 0.03, 'totalWeight': 90, 'scrapWeight': 2.7}
doc['result']['coverage']['sourceMonths'][0]['brightRows'] = 2
self.assertEqual(semantic_metric_gaps([doc], question, context), [])
report = build_noncomparison_report([doc], context, question)
self.assertEqual({fact['label'] for fact in report['facts']}, {'黑片报废率', '亮片报废率'})
self.assertFalse(any('综合报废率' in fact['text'] for fact in report['facts']))
def test_monthly_trend_uses_server_values_and_preserves_same_value(self):
question = '查询2026年黑片报废率趋势'
selected = task(question, period={'year': 2026}, trend=True, visual='required')
trend = []
source_months = []
for month in (1, 2, 3):
trend.append({'period': f'2026-{month:02d}',
'black': {'scrapRate': 0.08, 'totalWeight': 100, 'scrapWeight': 8},
'coverage': {'blackRows': 2, 'brightRows': 0}})
source_months.append({'year': 2026, 'month': month, 'blackRows': 2, 'brightRows': 0})
doc = metric_doc(value=0.08)
doc['result'].update(period={'year': 2026}, trend=trend, trendCount=3,
trendTruncated=False,
coverage={'sourceMonths': source_months})
report = _build_trend_report([doc], self.context(selected), question)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertEqual([fact['label'] for fact in report['facts']], ['黑片报废率月度趋势'])
self.assertIn('1月8.00%', report['facts'][0]['text'])
self.assertIn('3月8.00%', report['facts'][0]['text'])
self.assertIn('持平', report['facts'][0]['text'])
self.assertNotIn('合格率', report['facts'][0]['text'])
def test_trend_visual_uses_scrap_rate_contract_and_has_no_good_rate_series(self):
points = [
{'period': f'2026-0{month}', 'snapshot': {
'black': {'scrapRate': value, 'rate': 0.95,
'totalWeight': 100, 'scrapWeight': value * 100},
'coverage': {'blackRows': 2, 'brightRows': 0}}}
for month, value in ((1, 0.0), (2, 1.25), (3, 0.0))
]
charts = _trend_chart_from_points(
points, release_id=RELEASE, scope={'product': '产品甲'}, turn_id='turn-scrap',
evidence_refs=['QAAAAAAAA'], question='黑片报废率趋势', supplied_trend=True,
coverage_truncated=False, semantic_fields={'scrapRate'}, semantic_stages=['black'])
self.assertEqual([chart['metric'] for chart in charts], ['black_scrap_rate'])
self.assertEqual(charts[0]['unit'], 'ratio')
self.assertEqual(charts[0]['requiredFields'], ['black_scrap_weight', 'black_total_weight'])
self.assertEqual(charts[0]['series'][0]['values'], [0.0, 1.25, 0.0])
def test_comparison_chart_is_stage_specific_and_uses_scrap_authorization(self):
document = comparison_doc(black_values=(1.25, 0.0))
charts = _comparison_charts(
document, document['result'], 'turn-compare', '', requested_stage='black',
semantic_fields={'scrapRate'})
self.assertEqual([chart['metric'] for chart in charts], ['black_scrap_rate'])
self.assertEqual(charts[0]['unit'], 'ratio')
self.assertEqual(charts[0]['requiredFields'], ['black_scrap_weight', 'black_total_weight'])
self.assertEqual(charts[0]['series'][0]['values'], [1.25, 0.0])
self.assertEqual(charts[0]['type'], 'bar')
def test_comparison_chart_rejects_missing_coverage_and_semantic_output_has_no_extra_rates(self):
document = comparison_doc(black_rows=(0, 2))
charts = _comparison_charts(document, document['result'], 'turn-compare', '',
requested_stage='black', semantic_fields={'scrapRate'})
self.assertEqual(charts, [])
question = '对比2026年9月与8月黑片报废率'
selected = task(question, period=PERIOD, comparison_period=COMPARISON_PERIOD,
visual='required')
context = self.context(selected)
visuals = build_visuals([comparison_doc()], question, 'turn-semantic', context=context)['visuals']
self.assertEqual([chart['metric'] for chart in visuals], ['black_scrap_rate'])
both_question = '对比2026年9月与8月黑片和亮片报废率'
both_task = task(both_question, period=PERIOD, comparison_period=COMPARISON_PERIOD,
stages=['black', 'bright'], visual='required')
both_visuals = build_visuals([comparison_doc()], both_question, 'turn-both',
context=self.context(both_task))['visuals']
self.assertEqual({chart['metric'] for chart in both_visuals},
{'black_scrap_rate', 'bright_scrap_rate'})
plain_question = '比较黑片报废率'
plain_visuals = build_visuals([comparison_doc()], plain_question, 'turn-plain')['visuals']
self.assertEqual([chart['metric'] for chart in plain_visuals], ['black_scrap_rate'])
if __name__ == '__main__':
unittest.main()
+278
View File
@@ -0,0 +1,278 @@
"""Single-stage source references for official composite rates."""
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_agent_report import build_noncomparison_report, _build_trend_report
from nao_analysis_coverage import semantic_metric_gaps
from nao_evidence_report import _display_rate, build_report
from nao_quality_tools import QualityTools, _valid_composite_rate_reference
RELEASE = 'release-stage-reference'
def side(month, valid_stage='black', composite=.91, *, unknown=False,
black_total=100, bright_total=100, source_rows=True):
"""A backend snapshot where the missing stage may carry the default rate 1."""
black_valid = valid_stage == 'black'
bright_valid = valid_stage == 'bright'
if valid_stage == 'none':
black_valid = bright_valid = False
black_rows = (2 if black_valid else 0) if source_rows else 0
bright_rows = (2 if bright_valid else 0) if source_rows else 0
coverage = {} if unknown else {'sourceMonths': [
{'year': 2026, 'month': month, 'blackRows': black_rows, 'brightRows': bright_rows}]}
return {
'period': {'year': 2026, 'month': month},
'compositeRate': composite,
'black': {'rate': .97 if black_valid else 1, 'qualifiedWeight': 97 if black_valid else 0,
'totalWeight': black_total if black_valid else 0, 'scrapWeight': 3 if black_valid else 0},
'bright': {'rate': .98 if bright_valid else 1, 'qualifiedWeight': 98 if bright_valid else 0,
'totalWeight': bright_total if bright_valid else 0, 'scrapWeight': 2 if bright_valid else 0},
'coverage': coverage,
}
def metric_doc(month=9, valid_stage='black', composite=.91, **kwargs):
snapshot = side(month, valid_stage, composite, **kwargs)
return {'evidenceRef': 'QSTAGE001', 'mode': 'query', 'result': {
'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': snapshot['period'],
'filters': {}, 'view': 'overall', 'summary': {k: v for k, v in snapshot.items()
if k not in ('period', 'coverage')},
'coverage': snapshot['coverage'],
}}
def comparison_doc(current_stage='black', previous_stage='black', *, current_composite=.91,
previous_composite=.90, current_kwargs=None, previous_kwargs=None):
current = side(9, current_stage, current_composite, **(current_kwargs or {}))
previous = side(8, previous_stage, previous_composite, **(previous_kwargs or {}))
return {'evidenceRef': 'QSTAGE002', 'mode': 'compare', 'result': {
'kind': 'quality_comparison', 'releaseId': RELEASE, 'filterBinding': {},
'current': current, 'comparison': previous,
'delta': {'available': True, 'percentagePoints': (current_composite - previous_composite) * 100},
'stageRateChangesPP': {'black': None, 'bright': None},
}}
def metrics_context(question='综合合格率是多少', *, period=None, stages=None, fields=None):
period = period or {'year': 2026, 'month': 9}
task = {'version': 1, 'question': question, 'kind': 'metrics', 'historyMode': 'none',
'filters': {}, 'period': period, 'comparisonPeriod': None,
'dimensions': [], 'stages': stages or [], 'fields': fields or ['rate'],
'verifySource': False, 'fullDetails': False, 'freshQuery': False,
'needsClarification': False, 'comparisonRequired': False, 'trendRequired': False,
'visualPreference': 'none', 'periodCoverage': 'complete'}
return {'releaseId': RELEASE, 'period': period, 'filters': {}, '_agentTask': task}
def range_snapshot(source_by_month):
month_rows = []
for month, stages in source_by_month.items():
month_rows.append({'year': 2026, 'month': month,
'blackRows': stages.get('black', 0),
'brightRows': stages.get('bright', 0)})
black_present = any(row['blackRows'] > 0 for row in month_rows)
bright_present = any(row['brightRows'] > 0 for row in month_rows)
return {'period': {'year': 2026, 'monthStart': 1, 'monthEnd': 3},
'compositeRate': .89,
'black': {'rate': .97 if black_present else 1,
'qualifiedWeight': 97 if black_present else 0,
'totalWeight': 100 if black_present else 0},
'bright': {'rate': .98 if bright_present else 1,
'qualifiedWeight': 98 if bright_present else 0,
'totalWeight': 100 if bright_present else 0},
'coverage': {'sourceMonths': month_rows}}
class SingleStageReferenceTest(unittest.TestCase):
def test_composite_accepts_either_stage_and_preserves_server_zero(self):
for stage in ('black', 'bright'):
snapshot = side(9, stage, composite=0)
self.assertEqual(_valid_composite_rate_reference(snapshot, snapshot['period']), (True, None))
self.assertEqual(_display_rate(snapshot), 0)
def test_comparison_delta_accepts_both_single_stage_directions(self):
for current_stage, previous_stage in (('black', 'bright'), ('bright', 'black')):
current_side = side(9, current_stage, .93)
previous_side = side(8, previous_stage, .90)
current = {'summary': {key: value for key, value in current_side.items()
if key not in ('period', 'coverage')},
'coverage': current_side['coverage']}
previous = {'summary': {key: value for key, value in previous_side.items()
if key not in ('period', 'coverage')},
'coverage': previous_side['coverage']}
current['_period'] = {'year': 2026, 'month': 9}
previous['_period'] = {'year': 2026, 'month': 8}
delta = QualityTools._metric_difference(current, previous)
self.assertTrue(delta['available'], delta)
self.assertAlmostEqual(float(delta['percentagePoints']), 3.0)
def test_two_missing_unknown_and_zero_denominator_placeholder_are_rejected(self):
both_missing = side(9, 'none', composite=1)
self.assertFalse(_valid_composite_rate_reference(both_missing, both_missing['period'])[0])
unknown = side(9, 'black', unknown=True)
self.assertFalse(_valid_composite_rate_reference(unknown, unknown['period'])[0])
zero_denominators = side(9, 'none', composite=1)
zero_denominators['coverage']['sourceMonths'] = [
{'year': 2026, 'month': 9, 'blackRows': 2, 'brightRows': 2}]
self.assertFalse(_valid_composite_rate_reference(zero_denominators,
zero_denominators['period'])[0])
no_official_value = side(9, 'black')
no_official_value['compositeRate'] = None
self.assertFalse(_valid_composite_rate_reference(no_official_value,
no_official_value['period'])[0])
def test_mixed_stage_month_range_requires_known_source_each_month(self):
mixed = range_snapshot({1: {'black': 2}, 2: {'bright': 3}, 3: {'black': 1, 'bright': 4}})
self.assertTrue(_valid_composite_rate_reference(mixed, mixed['period'])[0])
missing_month = range_snapshot({1: {'black': 2}, 3: {'bright': 4}})
self.assertFalse(_valid_composite_rate_reference(missing_month, missing_month['period'])[0])
unknown_count = range_snapshot({1: {'black': 2}, 2: {'bright': 3}, 3: {'black': 1}})
unknown_count['coverage']['sourceMonths'][1]['blackRows'] = None
self.assertFalse(_valid_composite_rate_reference(unknown_count, unknown_count['period'])[0])
empty_month = range_snapshot({1: {'black': 2}, 2: {'bright': 3}, 3: {}})
self.assertFalse(_valid_composite_rate_reference(empty_month, empty_month['period'])[0])
question = '综合合格率是多少'
doc = {'evidenceRef': 'QSTAGE004', 'mode': 'query', 'result': {
'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': mixed['period'],
'filters': {}, 'view': 'overall',
'summary': {key: value for key, value in mixed.items() if key not in ('period', 'coverage')},
'coverage': mixed['coverage']}}
self.assertEqual(semantic_metric_gaps([doc], question,
metrics_context(question, period=mixed['period'])), [])
def test_legacy_positive_source_without_denominator_fields_remains_supported(self):
snapshot = side(9, 'black')
for stage in ('black', 'bright'):
snapshot[stage].pop('totalWeight')
self.assertTrue(_valid_composite_rate_reference(snapshot, snapshot['period'])[0])
def test_single_period_report_explains_missing_stage_without_claiming_100_percent(self):
question = '综合合格率是多少'
report = build_noncomparison_report([metric_doc(9, 'black')],
metrics_context(question), question)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
text = report['facts'][0]['text']
self.assertIn('综合合格率:91.00%', text)
self.assertIn('缺少亮片可用独立阶段指标', text)
self.assertIn('缺失阶段不作为实测100%', text)
self.assertNotIn('亮片合格率:100.00%', text)
self.assertEqual(semantic_metric_gaps([metric_doc(9, 'black')], question,
metrics_context(question)), [])
def test_explicit_stage_request_still_rejects_missing_stage_in_both_directions(self):
for selected, available in (('black', 'bright'), ('bright', 'black')):
label = '黑片' if selected == 'black' else '亮片'
question = f'只看{label}合格率'
context = metrics_context(question, stages=[selected])
report = build_noncomparison_report([metric_doc(9, available)], context, question)
self.assertFalse(report['coverageSufficient'])
self.assertIn(f'{label}合格率:未提供', report['facts'][0]['text'])
self.assertNotIn('100.00%', report['facts'][0]['text'])
def test_missing_stage_weight_and_scrap_rate_remain_uncovered(self):
for selected, label, field, question in (
('black', '黑片', 'totalWeight', '只看黑片投入重量'),
('bright', '亮片', 'scrapRate', '只看亮片报废率')):
context = metrics_context(question, stages=[selected], fields=[field])
report = build_noncomparison_report([metric_doc(9, 'bright' if selected == 'black' else 'black')],
context, question)
self.assertFalse(report['coverageSufficient'])
self.assertIn(f'{label}', report['facts'][0]['text'])
self.assertIn('未提供', report['facts'][0]['text'])
def test_scrap_rate_requires_positive_stage_total_and_keeps_true_zero(self):
question = '只看黑片报废率'
context = metrics_context(question, stages=['black'], fields=['scrapRate'])
valid = metric_doc(9, 'black')
valid['result']['summary']['black'].update(scrapRate=0, scrapWeight=0)
report = build_noncomparison_report([valid], context, question)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertIn('黑片报废率:0.00%', report['facts'][0]['text'])
self.assertEqual(semantic_metric_gaps([valid], question, context), [])
zero_denominator = metric_doc(9, 'black')
zero_denominator['result']['summary']['black'].update(
totalWeight=0, scrapRate=0, scrapWeight=0)
report = build_noncomparison_report([zero_denominator], context, question)
self.assertFalse(report['coverageSufficient'])
self.assertTrue(semantic_metric_gaps([zero_denominator], question, context))
negative_weight = metric_doc(9, 'black')
negative_weight['result']['summary']['black'].update(
scrapRate=0, scrapWeight=-0.1)
report = build_noncomparison_report([negative_weight], context, question)
self.assertFalse(report['coverageSufficient'])
self.assertTrue(semantic_metric_gaps([negative_weight], question, context))
def test_comparison_report_handles_both_directions_and_discloses_missing_stage(self):
for current_stage, previous_stage, missing_label in (
('black', 'bright', '亮片'), ('bright', 'black', '黑片')):
doc = comparison_doc(current_stage, previous_stage)
report = build_report([doc])
text = report['headline'] + '\n'.join(report['sections']) + report['policy']
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertIn('综合合格率:2026年8月90.00%→2026年9月91.00%', text)
self.assertIn(missing_label + '来源月份', text)
self.assertIn('解释为实测100%', text)
self.assertNotIn('| 黑片合格率 | 100.00%', text)
self.assertNotIn('| 亮片合格率 | 100.00%', text)
self.assertIn(f'| {missing_label}投入重量 | 100.00 | — |', text)
def test_stage_missing_composite_does_not_block_a_valid_explicit_stage(self):
question = '只看黑片合格率'
context = metrics_context(question, stages=['black'])
doc = metric_doc(9, 'black')
doc['result']['summary']['compositeRate'] = None
report = build_noncomparison_report([doc], context, question)
self.assertTrue(report['coverageSufficient'], report['missingEvidence'])
self.assertIn('黑片合格率:97.00%', report['facts'][0]['text'])
def test_trend_excludes_default_one_when_both_explicit_denominators_are_zero(self):
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 3}
question = '综合合格率月度趋势'
task = {'version': 1, 'question': question, 'kind': 'metrics', 'historyMode': 'none',
'filters': {}, 'period': period, 'comparisonPeriod': None, 'dimensions': [],
'stages': [], 'fields': ['rate'], 'verifySource': False, 'fullDetails': False,
'freshQuery': False, 'needsClarification': False, 'comparisonRequired': False,
'trendRequired': True, 'visualPreference': 'none', 'periodCoverage': 'complete'}
trend = []
source_months = []
for month in (1, 2, 3):
missing = month == 3
row = side(month, 'none' if missing else 'black', 1 if missing else .91)
row['period'] = f'2026-{month:02d}'
row['monthStatus'] = 'published'
row['noData'] = False
row['coverage'] = {'blackRows': 2, 'brightRows': 0}
row['black'] = {**row['black'], 'coverage': {'blackRows': 0 if missing else 2}}
row['bright'] = {**row['bright'], 'coverage': {'brightRows': 0}}
trend.append(row)
source_months.append({'year': 2026, 'month': month,
'blackRows': 2 if not missing else 2,
'brightRows': 2 if missing else 0})
document = {'evidenceRef': 'QSTAGE003', 'mode': 'query', 'result': {
'kind': 'quality_metrics', 'releaseId': RELEASE, 'period': period,
'filters': {}, 'view': 'overall', 'trend': trend, 'trendCount': 3,
'trendTruncated': False, 'coverage': {'sourceMonths': source_months}}}
report = _build_trend_report([document], {'releaseId': RELEASE, 'period': period,
'filters': {}, '_agentTask': task}, question)
self.assertFalse(report['coverageSufficient'])
self.assertTrue(any('全部月份' in item for item in report['missingEvidence']))
self.assertIn('3月未提供', report['facts'][0]['text'])
self.assertNotIn('3月100.00%', report['facts'][0]['text'])
if __name__ == '__main__':
unittest.main()
+501
View File
@@ -0,0 +1,501 @@
from __future__ import annotations
import json
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'scripts'))
sys.path.insert(0, str(ROOT / 'verification'))
from nao_agent_task import TaskUnavailable, bind_task, normalize_task
from nao_agent_task import plan_task
from nao_protocol_execution import execute
from nao_quality_tools import QualityToolError, QualityTools
from nao_query_contract import bind_query
from test_nao_agent_task import task_value
from test_nao_quality_tools import FakeApi, RELEASE
def source_field(key, label, black, bright, stages=('black', 'bright')):
return {'key': key, 'label': label, 'columns': {'black': black, 'bright': bright},
'stages': list(stages), 'type': 'source_value',
'operators': ['in', 'not_in', 'is_empty', 'not_empty']}
SOURCE_FIELDS = [
source_field('product', '产品', '黑片产品', '亮片产品'),
source_field('department', '产业部', '黑片产业部', '亮片产业部'),
source_field('group', '业务分组', '黑片分组', '亮片分组'),
source_field('projectGroup', '项目分组', '黑片项目分组', '亮片项目分组'),
source_field('src_hash', '来源哈希', '黑片行哈希', '亮片行哈希'),
source_field('batchNumber', '批号 | 代码', '黑片批次号', '亮片批号'),
source_field('customerSpec', '客户规格', '客户规格编码', '检包规格'),
source_field('dispatchOrder', '委托单', '黑片委托单号', '亮片工单委托'),
source_field('blackOnlyCode', '黑片专用代码', '黑片专用列', '', ('black',)),
]
CATALOG = {'sourceFields': SOURCE_FIELDS, 'departments': [], 'groups': []}
AUTHORITY = {'attemptId': 'attempt-1', 'sessionId': 'session-1', 'turnId': 'turn-1',
'revision': 4, 'executionId': 'execution-1'}
RULES = [
{'field': 'batchNumber', 'stages': ['black'], 'operator': 'in', 'values': ['0012', 'A|B']},
{'field': 'customerSpec', 'stages': ['bright'], 'operator': 'not_in', 'values': ['xY']},
{'field': 'dispatchOrder', 'stages': ['black', 'bright'], 'operator': 'not_empty'},
]
def protocol_task(**changes):
value = {**task_value(kind='metrics', dimensions=[], stages=['black', 'bright'], fields=['rate'],
period={'year': 2026, 'month': 3}, comparisonRequired=False,
trendRequired=False, columnFilters=RULES), **changes}
return normalize_task(value, '查询本期正式指标', {'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
class EchoColumnApi(FakeApi):
def __init__(self, *, echo='flat'):
super().__init__()
self.echo = echo
def call(self, path, payload=None, method=None):
result = super().call(path, payload, method)
parsed = urlsplit(path)
if parsed.path == '/api/analysis/tools/catalog':
result['sourceFields'] = SOURCE_FIELDS
if parsed.path == '/api/metrics' and 'columnFilters' in parse_qs(parsed.query):
requested = json.loads(parse_qs(parsed.query)['columnFilters'][0])
if self.echo == 'request':
result['context']['filters'] = {'columnFilters': requested}
elif self.echo == 'flat':
result['context']['columnFilters'] = requested
elif self.echo == 'mismatch':
result['context']['filters'] = {'columnFilters': []}
return result
class SourceColumnProtocolTests(unittest.TestCase):
def test_final_answer_selector_preserves_the_confirmed_column_scope(self):
from nao_evidence_finalizer import _agent_task_summary
task=protocol_task()
summary=_agent_task_summary(bind_task({'releaseId':RELEASE},task))
self.assertEqual(summary['filters']['columnFilters'],RULES)
self.assertEqual(summary['sourceFieldLabels'],task['sourceFieldLabels'])
def test_member_intrinsic_defaults_do_not_trigger_expensive_replanning(self):
value=task_value(kind='members',dimensions=[],fields=[],stages=['black'],memberFields=['dispatchOrder'],columnFilters=[],trendRequired=False)
for key in ('freshQuery','verifySource','fullDetails','responseStyle','historyMode','periodCoverage'):
value.pop(key,None)
task=normalize_task(value,'列出本期黑片委托单',{'_protocol':{'version':1},'_businessCatalog':CATALOG})
self.assertTrue(task['freshQuery']);self.assertFalse(task['verifySource']);self.assertFalse(task['fullDetails'])
self.assertEqual(task['responseStyle'],'standard')
def test_members_allow_authorized_canonical_dimension_projection_only(self):
canonical_product = {**SOURCE_FIELDS[0], 'type': 'dimension'}
catalog = {'sourceFields': [canonical_product, *SOURCE_FIELDS[1:]], 'departments': [], 'groups': []}
task = normalize_task(task_value(kind='members', dimensions=[], fields=[], stages=['black', 'bright'],
period={'year': 2026, 'month': 3}, comparisonRequired=False, trendRequired=False,
memberFields=['product'], columnFilters=[]), '列出客户规格清单',
{'_protocol': {'version': 1}, '_businessCatalog': catalog})
self.assertEqual(task['memberFields'], ['product'])
self.assertEqual(task['sourceFieldLabels'], {'product': '产品'})
# The same canonical directory entry is a member projection, not an
# authorized source_value filter rule.
with self.assertRaisesRegex(TaskUnavailable, '不是受支持的原表文本字段'):
normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3}, comparisonRequired=False, trendRequired=False,
columnFilters=[{'field': 'product', 'stages': ['black'], 'operator': 'in', 'values': ['18X15X4']}]),
'按产品筛选指标', {'_protocol': {'version': 1}, '_businessCatalog': catalog})
def test_column_filter_value_length_matches_backend_256_character_limit(self):
rule = {'field': 'batchNumber', 'stages': ['black'], 'operator': 'in', 'values': ['X' * 256]}
task = protocol_task(columnFilters=[rule])
self.assertEqual(task['filters']['columnFilters'][0]['values'], ['X' * 256])
with self.assertRaisesRegex(TaskUnavailable, '不超过256字符'):
protocol_task(columnFilters=[{**rule, 'values': ['X' * 257]}])
def test_protocol_presentation_enums_default_consistently_when_omitted_or_null(self):
base = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3}, comparisonRequired=False,
trendRequired=False, columnFilters=[])
base.pop('responseStyle')
base['visualPreference'] = None
base['historyMode'] = None
base['periodCoverage'] = None
task = normalize_task(base, '查询本期正式指标', {
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
self.assertEqual(task['responseStyle'], 'standard')
self.assertEqual(task['visualPreference'], 'auto')
self.assertEqual(task['historyMode'], 'none')
self.assertEqual(task['periodCoverage'], 'complete')
trend = normalize_task({**base, 'trendRequired': True}, '查询本期正式指标', {
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
self.assertEqual(trend['periodCoverage'], 'available')
for invalid in ('', 'verbose', False):
with self.subTest(invalid=invalid), self.assertRaises(TaskUnavailable):
normalize_task({**base, 'responseStyle': invalid}, '查询本期正式指标', {
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
with self.assertRaises(TaskUnavailable):
normalize_task({**base, 'historyMode': ''}, '查询本期正式指标', {
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
with self.assertRaises(TaskUnavailable):
normalize_task({**base, 'periodCoverage': ''}, '查询本期正式指标', {
'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
def test_protocol_short_prompt_lists_required_fields_and_enum_defaults(self):
# Capture the planner request without invoking a live model.
with patch('nao_agent_task._model_json', side_effect=[task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3}, comparisonRequired=False,
trendRequired=False, columnFilters=[]),
{'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
'requiresAllSourceRows': False, 'requiresTrend': False,
'requiresComparison': False, 'columnFiltersAligned': True}]) as model_call:
plan_task(base_url='https://offline.invalid/v1', api_key='synthetic', model='m',
question='查询本期正式指标',
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG},
cancel_event=threading.Event())
prompt = model_call.call_args_list[0].kwargs['system']
self.assertIn('responseStyle(brief/standard/detailed,可省略或null,默认standard)', prompt)
self.assertIn('historyMode(none/last/all,可省略或null,默认none)', prompt)
self.assertIn('periodCoverage(available/complete,可省略或null', prompt)
self.assertIn('freshQuery(bool)', prompt)
self.assertIn('needsClarification(bool)', prompt)
def test_three_arbitrary_source_keys_keep_literal_values_and_stage_scopes(self):
task = protocol_task()
self.assertNotIn('columnFilters', task)
self.assertEqual(task['filters']['columnFilters'], RULES)
self.assertEqual(task['sourceFieldLabels'], {
'batchNumber': '批号 | 代码', 'customerSpec': '客户规格', 'dispatchOrder': '委托单'})
self.assertEqual(task['filters']['columnFilters'][0]['values'], ['0012', 'A|B'])
self.assertEqual(task['filters']['columnFilters'][0]['stages'], ['black'])
self.assertEqual(task['filters']['columnFilters'][1]['stages'], ['bright'])
self.assertEqual(task['filters']['columnFilters'][2]['stages'], ['black', 'bright'])
def test_column_filter_rules_are_complete_replace_clear_and_inherit(self):
previous = protocol_task(columnFilters=[RULES[0]])
context = bind_task({'_protocol': {'version': 1}, '_businessCatalog': CATALOG}, previous)
inherited = normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3}, columnFilters=None), '继续查询', context)
self.assertEqual(inherited['filters']['columnFilters'], []) # explicit null clears
omitted = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3})
self.assertEqual(normalize_task(omitted, '继续查询', context)['filters']['columnFilters'], [RULES[0]])
omitted['columnFilters'] = [RULES[1]]
self.assertEqual(normalize_task(omitted, '改查亮片规格', context)['filters']['columnFilters'], [RULES[1]])
cleared = normalize_task({**omitted, 'columnFilters': []}, '清除原表列条件', context)
self.assertEqual(cleared['filters']['columnFilters'], [])
cleared_context = bind_task(context, cleared)
self.assertEqual(normalize_task(omitted | {'columnFilters': None}, '继续', cleared_context)['filters']['columnFilters'], [])
new_task = normalize_task({**omitted, 'newTask': True, 'columnFilters': None}, '另开任务', context)
self.assertEqual(new_task['filters']['columnFilters'], [])
def test_unauthorized_unknown_and_missing_stage_columns_fail_closed(self):
for bad in ([{**RULES[0], 'field': 'notInDirectory'}],
[{**RULES[0], 'field': 'blackOnlyCode', 'stages': ['bright']}],
[{**RULES[0], 'operator': 'contains'}]):
with self.subTest(bad=bad), self.assertRaises(TaskUnavailable):
normalize_task(task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026}, columnFilters=bad), '筛选原表字段',
{'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
with self.assertRaisesRegex(TaskUnavailable, '目录不可用'):
normalize_task(task_value(kind='metrics', dimensions=[], stages=[], fields=['rate'],
columnFilters=[RULES[0]]), '按原表列过滤', {'_protocol': {'version': 1}, '_businessCatalog': {}})
def test_semantic_review_must_confirm_source_filters_and_member_projection(self):
source_plan = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3}, columnFilters=[RULES[0]])
bad_review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False,
'columnFiltersAligned': False}
with patch('nao_agent_task._model_json', side_effect=[source_plan, bad_review] * 3) as model:
with self.assertRaisesRegex(TaskUnavailable, '字段、阶段或成员清单'):
plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
question='只看黑片批号 0012 的合格率',
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG},
cancel_event=threading.Event(), source_value_resolver=self._value_resolver)
self.assertEqual(model.call_count, 6)
self.assertTrue(model.call_args_list[1].kwargs['payload']['columnFilterReviewRequired'])
self.assertFalse(model.call_args_list[1].kwargs['payload']['memberListReviewRequired'])
member_plan = task_value(kind='members', dimensions=[], stages=['black'], fields=[],
period={'year': 2026, 'month': 3}, memberFields=['batchNumber'], columnFilters=[])
good_review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False,
'memberListAligned': True}
with patch('nao_agent_task._model_json', side_effect=[member_plan, good_review]) as model:
planned = plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
question='列出黑片批号清单',
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG},
cancel_event=threading.Event())
self.assertEqual(planned['kind'], 'members')
self.assertEqual(model.call_count, 2)
self.assertTrue(model.call_args_list[1].kwargs['payload']['memberListReviewRequired'])
@staticmethod
def _value_resolver(*, release_id, period, filters, field, stages, search, limit, offset):
return {'candidateOnly': True, 'releaseId': release_id, 'period': period, 'filters': filters,
'field': field, 'selectedStages': stages, 'offset': offset, 'limit': limit,
'totalCount': 1, 'hasMore': False, 'values': [search], 'rows': []}
def test_unconfirmed_source_value_is_clarified_without_partial_rule(self):
source_plan = task_value(kind='metrics', dimensions=[], stages=['black'], fields=['rate'],
period={'year': 2026, 'month': 3}, columnFilters=[
{'field': 'dispatchOrder', 'stages': ['black'], 'operator': 'in', 'values': ['宝鑫加工']}])
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False,
'columnFiltersAligned': True}
def resolver(**contract):
return {'candidateOnly': True, 'releaseId': contract['release_id'],
'period': contract['period'], 'filters': contract['filters'],
'field': contract['field'], 'selectedStages': contract['stages'],
'offset': contract['offset'], 'limit': contract['limit'],
'totalCount': 1, 'hasMore': False, 'values': ['宝鑫加工厂'], 'rows': []}
with patch('nao_agent_task._model_json', side_effect=[source_plan, review]) as model:
planned = plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
question='按委托单宝鑫加工查询黑片合格率',
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG,
'releaseId': RELEASE}, cancel_event=threading.Event(), source_value_resolver=resolver)
self.assertTrue(planned['needsClarification'])
self.assertEqual(planned['filters']['columnFilters'], [])
self.assertIn('不会自动替换', planned['clarification'])
resolution = planned['sourceValueResolution']
self.assertEqual(resolution['status'], 'unconfirmed')
self.assertEqual(resolution['candidates'], ['宝鑫加工厂'])
def test_candidate_lookup_scopes_other_rules_to_the_requested_stage_only(self):
source_plan = task_value(kind='metrics', dimensions=[], stages=['black', 'bright'], fields=['rate'],
period={'year': 2026, 'month': 3}, columnFilters=RULES)
review = {'aligned': True, 'requiresEvidence': True, 'requiresSourceVerification': False,
'requiresAllSourceRows': False, 'requiresTrend': False, 'requiresComparison': False,
'columnFiltersAligned': True}
seen = []
def resolver(**contract):
seen.append(contract)
return {'candidateOnly': True, 'releaseId': contract['release_id'],
'period': contract['period'], 'filters': contract['filters'],
'field': contract['field'], 'selectedStages': contract['stages'],
'offset': contract['offset'], 'limit': contract['limit'],
'totalCount': 1, 'hasMore': False, 'values': [contract['search']], 'rows': []}
with patch('nao_agent_task._model_json', side_effect=[source_plan, review]):
planned = plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
question='按黑片批号0012筛选,同时保留亮片客户规格条件',
context={'_protocol': {'version': 1}, '_businessCatalog': CATALOG,
'releaseId': RELEASE}, cancel_event=threading.Event(), source_value_resolver=resolver)
self.assertEqual(planned['filters']['columnFilters'], RULES)
self.assertEqual(len(seen), 3)
batch_lookup = next(call for call in seen if call['field'] == 'batchNumber')
self.assertEqual(batch_lookup['stages'], ['black'])
self.assertEqual(batch_lookup['filters']['columnFilters'], [
{'field': 'dispatchOrder', 'stages': ['black'], 'operator': 'not_empty'}])
customer_lookup = next(call for call in seen if call['field'] == 'customerSpec')
self.assertEqual(customer_lookup['stages'], ['bright'])
self.assertEqual(customer_lookup['filters']['columnFilters'], [
{'field': 'dispatchOrder', 'stages': ['bright'], 'operator': 'not_empty'}])
def test_missing_literal_offers_same_scope_alternatives_without_substitution(self):
from nao_agent_task import _resolve_source_column_values
task = protocol_task(columnFilters=[RULES[0]])
calls = []
def resolver(**contract):
calls.append(contract)
response = self._value_resolver(**contract)
response.update(values=[] if contract['search'] else ['亮片实际单位'],
totalCount=0 if contract['search'] else 1)
return response
result = _resolve_source_column_values(task, release_id=RELEASE, resolver=resolver)
self.assertEqual([c['search'] for c in calls], ['0012', ''])
self.assertEqual({k:v for k,v in calls[0].items() if k!='search'},
{k:v for k,v in calls[1].items() if k!='search'})
self.assertEqual(result['status'], 'unconfirmed')
self.assertEqual(result['candidates'], ['亮片实际单位'])
self.assertTrue(task['needsClarification'])
self.assertEqual(task['filters']['columnFilters'], [])
self.assertIn('亮片实际单位', task['clarification'])
def test_alternative_candidate_receipt_cannot_broaden_the_scope(self):
from nao_agent_task import _resolve_source_column_values
task = protocol_task(columnFilters=[RULES[0]])
def resolver(**contract):
response = self._value_resolver(**contract)
response.update(values=[] if contract['search'] else ['不能显示的其他阶段值'],
totalCount=0 if contract['search'] else 1)
if not contract['search']:response['selectedStages']=['bright']
return response
result = _resolve_source_column_values(task, release_id=RELEASE, resolver=resolver)
self.assertEqual(result['status'], 'unconfirmed')
self.assertEqual(result['candidates'], [])
self.assertNotIn('不能显示', task['clarification'])
self.assertEqual(task['filters']['columnFilters'], [])
def test_member_plan_repair_preserves_new_task_reset_and_customer_scope(self):
previous = protocol_task()
catalog = {**CATALOG, 'extraFields': [{'key': 'customerCode'}]}
context = bind_task({'_protocol': {'version': 1}, '_businessCatalog': catalog,
'releaseId': RELEASE}, previous)
good = task_value(kind='members', dimensions=[], stages=['black', 'bright'], fields=[],
memberFields=['product'], newTask=True, columnFilters=[],
period={'year':2026,'month':9}, filterUpdates={'extraFilters':{'customerCode':['800001']}})
bad = {**good, 'filterUpdates': {'customerCode':['800001']}}
review = {'aligned':True, 'requiresEvidence':True, 'requiresSourceVerification':False,
'requiresAllSourceRows':False, 'requiresTrend':False, 'requiresComparison':False,
'columnFiltersAligned':True, 'memberListAligned':True, 'customerCodes':['800001']}
with patch('nao_agent_task._model_json', side_effect=[bad,good,review]) as model:
planned = plan_task(base_url='https://offline.invalid/v1', api_key='never-used', model='m',
question='新任务:800001客户2026年9月有哪些规格,黑片亮片一起看',
context=context, previous=previous, cancel_event=threading.Event())
repair = model.call_args_list[1].kwargs['payload']
self.assertTrue(repair['invalidPlan']['newTask'])
self.assertEqual(planned['filters']['extraFilters'], {'customerCode':['800001']})
self.assertEqual(planned['filters']['columnFilters'], [])
self.assertEqual(planned['memberFields'], ['product'])
self.assertEqual(planned['stages'], ['black','bright'])
self.assertTrue(planned['newTask'])
def test_quality_tools_sends_column_filters_and_requires_exact_receipt(self):
with tempfile.TemporaryDirectory() as folder:
api = EchoColumnApi()
tools = QualityTools(api_factory=lambda: api, evidence_dir=folder)
result = tools.quality_data({'mode': 'query', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 3}, 'filters': {}, 'view': 'overall',
'columnFilters': RULES})
metrics_path = next(path for _, path, _ in api.calls if urlsplit(path).path == '/api/metrics')
sent = json.loads(parse_qs(urlsplit(metrics_path).query)['columnFilters'][0])
self.assertEqual(sent, RULES)
self.assertEqual(result['filters']['columnFilters'], RULES)
self.assertEqual(result['trend'][0]['period'], '2026-03')
with tempfile.TemporaryDirectory() as folder:
tools = QualityTools(api_factory=lambda: EchoColumnApi(echo='mismatch'), evidence_dir=folder)
with self.assertRaisesRegex(QualityToolError, '未原样确认 filters.columnFilters'):
tools.quality_data({'mode': 'query', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 3}, 'filters': {}, 'view': 'overall',
'columnFilters': RULES})
with tempfile.TemporaryDirectory() as folder:
tools = QualityTools(api_factory=lambda: EchoColumnApi(echo='missing'), evidence_dir=folder)
with self.assertRaisesRegex(QualityToolError, '未原样确认 filters.columnFilters'):
tools.quality_data({'mode': 'query', 'releaseId': RELEASE,
'period': {'year': 2026, 'month': 3}, 'filters': {}, 'view': 'overall',
'columnFilters': RULES})
def test_query_contract_extracts_authoritative_filters_and_rejects_changes(self):
task = protocol_task()
contract = {'task': task, 'releaseId': RELEASE}
args = {'mode': 'query', 'period': task['period'], 'filters': {}, 'view': 'overall'}
bound = bind_query(args, contract)
self.assertEqual(bound['columnFilters'], RULES)
self.assertNotIn('columnFilters', bound['filters'])
with self.assertRaisesRegex(ValueError, '偏离本轮任务约定'):
bind_query({**args, 'columnFilters': [RULES[0]]}, contract)
with self.assertRaisesRegex(ValueError, '只能用于已确认的正式query'):
bind_query({'mode': 'changes'}, contract)
def member_task(self, stages=('black', 'bright')):
value = task_value(kind='members', dimensions=[], fields=[], stages=list(stages),
period={'year': 2026, 'month': 3}, comparisonRequired=False, trendRequired=False,
memberFields=['batchNumber', 'dispatchOrder'], columnFilters=[])
task = normalize_task(value, '列出委托单和批号',
{'_protocol': {'version': 1}, '_businessCatalog': CATALOG})
task['filters'].pop('columnFilters', None) # confirmed backend state omits cleared filters
return task
def _member_page(self, payload, offset, total, rows, *, proof=True):
response = {'rows': rows, 'totalCount': total, 'offset': offset, 'limit': 200,
'hasMore': offset + len(rows) < total,
'complete': offset == 0 and offset + len(rows) >= total,
'pageComplete': True,
'releaseId': payload['releaseId'], 'filters': payload['filters'],
'period': payload['period'], 'memberFields': payload['memberFields'],
'selectedStages': payload['stages']}
if proof:
response['protocolEvidence'] = {'queryId': f'q-{offset}', **AUTHORITY, 'responseHash': 'a' * 64}
return response
def test_members_paginates_checks_each_proof_and_renders_without_model(self):
class MemberApi:
def __init__(self): self.calls = []
def call(self, path, payload=None, method=None):
self.calls.append((path, payload, method))
rows = ([{'values': {'batchNumber': '001|A' if i == 0 else f'B-{i}',
'dispatchOrder': 'https://bad.test/<img>' if i == 0 else f'委托-{i}'},
'stages': ['black', 'bright']} for i in range(200)]
if payload['offset'] == 0 else
[{'values': {'batchNumber': None, 'dispatchOrder': '委托-2'}, 'stages': ['bright']}])
return SourceColumnProtocolTests()._member_page(payload, payload['offset'], 201, rows)
api = MemberApi()
task = self.member_task()
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
with patch('nao_evidence_finalizer.synthesize') as synth:
result = execute(request={'releaseId': RELEASE, 'turnId': AUTHORITY['turnId'], 'question': task['question']},
task=task, authority=AUTHORITY, actor_root=root, delegated=api,
connection=('offline', 'secret-unused', 'model-id'), cancel_event=threading.Event(), emit=lambda _: None)
self.assertEqual(result['status'], 'COMPLETED')
self.assertEqual(len(result['protocolEvidence']), 2)
self.assertEqual(result['queryEvidenceRefs'], [])
self.assertEqual(result['visualState'], 'not_requested')
self.assertIn('共 201 个去重项,已查全', result['answer'])
self.assertNotIn('<img>', result['answer'])
self.assertIn('翻页查看全部', result['answer'])
self.assertTrue(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertEqual(len(api.calls), 2)
self.assertEqual(api.calls[0][1]['offset'], 0)
self.assertEqual(api.calls[1][1]['offset'], 200)
self.assertNotIn('columnFilters', api.calls[0][1]['filters'])
self.assertFalse(synth.called)
proof_store = json.loads((root / 'private' / 'protocol-members-evidence.json').read_text())
self.assertEqual(len(proof_store['pages']), 2)
self.assertEqual(proof_store['displayedCount'], 201)
self.assertTrue(proof_store['displayComplete'])
self.assertEqual(proof_store['pages'][0]['response']['rows'][0]['values']['dispatchOrder'], 'https://bad.test/<img>')
self.assertEqual(len(json.loads((root / 'private' / 'protocol-completion.json').read_text())['protocolEvidence']), 2)
def test_members_empty_result_and_proof_failure_are_distinct(self):
task = self.member_task(('black',))
class EmptyApi:
def call(inner, path, payload=None, method=None):
return SourceColumnProtocolTests()._member_page(payload, 0, 0, [])
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
empty = execute(request={'releaseId': RELEASE, 'question': task['question']}, task=task,
authority=AUTHORITY, actor_root=root, delegated=EmptyApi(),
connection=('offline', 'key', 'm'), cancel_event=threading.Event(), emit=lambda _: None)
self.assertEqual(empty['status'], 'COMPLETED')
self.assertIn('共 0 个去重项,已查全', empty['answer'])
class NoProof(EmptyApi):
def call(inner, path, payload=None, method=None):
return SourceColumnProtocolTests()._member_page(payload, 0, 0, [], proof=False)
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
with self.assertRaisesRegex(ValueError, '缺少完整协议证明'):
execute(request={'releaseId': RELEASE, 'question': task['question']}, task=task,
authority=AUTHORITY, actor_root=root, delegated=NoProof(),
connection=('offline', 'key', 'm'), cancel_event=threading.Event(), emit=lambda _: None)
def test_members_stops_after_twenty_pages_and_marks_truncation(self):
task = self.member_task(('black',))
class LargeApi:
def __init__(self): self.calls = 0
def call(inner, path, payload=None, method=None):
inner.calls += 1
rows = [{'values': {'batchNumber': str(payload['offset'] + i), 'dispatchOrder': 'D'},
'stages': ['black']} for i in range(200)]
return SourceColumnProtocolTests()._member_page(payload, payload['offset'], 5000, rows)
api = LargeApi()
with tempfile.TemporaryDirectory() as folder:
root = Path(folder); (root / 'evidence').mkdir(); (root / 'private').mkdir()
result = execute(request={'releaseId': RELEASE, 'question': task['question']}, task=task,
authority=AUTHORITY, actor_root=root, delegated=api,
connection=('offline', 'key', 'm'), cancel_event=threading.Event(), emit=lambda _: None)
self.assertEqual(api.calls, 20)
self.assertEqual(result['status'], 'PARTIAL')
self.assertEqual(len(result['protocolEvidence']), 20)
self.assertFalse(result['evidenceCompletion']['serverCoverageSufficient'])
self.assertIn('另有 1000 项未读取', result['answer'])
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,185 @@
import copy
import io
import sys
import threading
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
sys.path[:0]=[str(Path(__file__).resolve().parents[1]/'scripts'),str(Path(__file__).resolve().parent)]
import nao_quality_runtime as runtime
from nao_agent_task import TaskUnavailable, bind_task, normalize_task, plan_task
from nao_analysis_coverage import plan_missing_source_queries
from nao_evidence_finalizer import coverage_sufficient
from test_nao_agent_task import task_value
from test_agent_runtime import comparison
RELEASE='33333333-3333-4333-8333-333333333333'
CURRENT={'year':2026,'month':9}
PREVIOUS={'year':2026,'month':8}
FILTERS={'department':'新能源汽车磁钢'}
def review(source, rows, *, evidence=True):
return {'aligned':True,'requiresEvidence':evidence,
'requiresSourceVerification':source,'requiresAllSourceRows':rows,'requiresTrend':False}
def plan_value(**kwargs):
return task_value(dimensions=['group'],dimensionStages={'group':[]},stages=['black','bright'],
fields=['rate','scrapWeight'],period=CURRENT,comparisonPeriod=PREVIOUS,
comparisonRequired=True,filterUpdates=FILTERS,**kwargs)
def source_document(stage, group=None, *, bad=False, missing_comparison=False):
filters={**FILTERS,**({'group':group} if group else {})}
def side(period):
context={'releaseId':RELEASE,**period,**filters,'stage':stage,'includeUnassigned':False}
return {'context':context,'aggregateComplete':True,'totalRows':2,
'sourceVerification':{'complete':not bad,'allSelectedValuesMatchStored':not bad,
'expectedRows':2,'verifiedRows':1 if bad else 2,'missingRows':1 if bad else 0,'mismatchRows':0}}
result={'kind':'quality_investigation','action':'source','releaseId':RELEASE,'stage':stage,
'period':CURRENT,'comparisonPeriod':PREVIOUS,'filters':filters,'current':side(CURRENT)}
if not missing_comparison:result['comparison']=side(PREVIOUS)
return {'mode':'investigate','evidenceRef':'QAAAAAAAA','result':result}
class SemanticSourceIntentTests(unittest.TestCase):
def run_plan(self, question, replies, context=None):
with patch('nao_agent_task._model_json',side_effect=replies) as call:
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question=question,context=context or {},cancel_event=threading.Event())
return result,call
def test_unrequested_source_read_is_replanned_not_silently_rewritten(self):
initial=plan_value(verifySource=True)
corrected=plan_value()
context={'_serverQuestions':[{'question':'9月合格率为什么下降?','status':'PARTIAL'}]}
result,call=self.run_plan('主要是国内还是国外的',[initial,review(False,False),corrected,review(False,False)],context)
self.assertFalse(result['verifySource']);self.assertEqual(call.call_count,4)
self.assertTrue(call.call_args_list[2].kwargs['payload']['invalidPlan']['verifySource'])
self.assertIn('verifySource',call.call_args_list[2].kwargs['payload']['validationFeedback'])
def test_explicit_source_and_all_rows_requirements_are_preserved(self):
source=plan_value(verifySource=True)
result,_=self.run_plan('请对照原表核验两期数据',[source,review(True,False)])
self.assertTrue(result['verifySource'])
rows=plan_value(fullDetails=True)
result,_=self.run_plan('请逐条列出所有原始记录',[rows,review(False,True)])
self.assertTrue(result['fullDetails'])
def test_explicit_source_omitted_by_planner_is_replanned(self):
first=plan_value()
corrected=plan_value(verifySource=True)
result,call=self.run_plan('请对照原表核验',[first,review(True,False),corrected,review(True,False)])
self.assertTrue(result['verifySource']);self.assertEqual(call.call_count,4)
def test_missing_read_requirement_fields_and_explicit_conflict_fail_closed(self):
source=plan_value(verifySource=True)
with self.assertRaises(TaskUnavailable):
self.run_plan('请核验原表',[source,{'aligned':True,'requiresEvidence':True}]*3)
with self.assertRaises(TaskUnavailable):
self.run_plan('请核验原表',[plan_value(),review(False,False)]*3)
def test_explicit_cancellation_does_not_force_source_read(self):
result,_=self.run_plan('不要核对原表,只看合格率',[plan_value(),review(False,False)])
self.assertFalse(result['verifySource'])
def test_natural_first_turn_source_request_is_reviewed_even_when_plan_omits_flag(self):
question='把上传的xlsx中的数值与系统记录逐项比对'
result,call=self.run_plan(question,[plan_value(),review(True,False),plan_value(verifySource=True),review(True,False)])
self.assertTrue(result['verifySource']);self.assertEqual(call.call_count,4)
def test_previous_source_requirement_and_explicit_cancellation_follow_current_intent(self):
history={'_serverQuestions':[{'question':'把上传的xlsx中的数值与系统记录逐项比对','status':'PARTIAL'}]}
result,_=self.run_plan('按刚才要求继续,主要看国内还是海外',
[plan_value(),review(True,False),plan_value(verifySource=True),review(True,False)],history)
self.assertTrue(result['verifySource'])
result,_=self.run_plan('不用核对原表,只看合格率',[plan_value(),review(False,False)],history)
self.assertFalse(result['verifySource'])
def test_positive_optional_review_fields_are_never_ignored(self):
history={'_serverQuestions':[{'question':'将上传的资料和系统的值一项项核一遍','status':'PARTIAL'}]}
first=plan_value();corrected=plan_value(verifySource=True)
with patch('nao_agent_task._model_json',side_effect=[first,review(True,False),corrected,review(True,False)]) as call:
result=plan_task(base_url='https://offline.invalid/v1',api_key='synthetic',model='m',
question='按刚才要求继续,主要看国内还是海外',context=history,cancel_event=threading.Event(),semantic_review=True)
self.assertTrue(result['verifySource']);self.assertEqual(call.call_count,4)
class ParentSourceRepairTests(unittest.TestCase):
def setUp(self):
self.task=normalize_task(task_value(kind='source_verify',dimensions=[],dimensionStages={},
stages=['black','bright'],fields=[],period=CURRENT,comparisonPeriod=PREVIOUS,
comparisonRequired=True,filterUpdates=FILTERS),'对照原表核验')
self.context=bind_task({'releaseId':RELEASE},self.task)
root=comparison();root['releaseId']=RELEASE;root['filterBinding']=FILTERS
self.root={'mode':'compare','evidenceRef':'QBBBBBBBB','result':root}
self.children=[source_document('black','汽车交通-国内'),source_document('bright','汽车交通-国内'),
source_document('black','汽车交通-海外')]
def test_child_scopes_do_not_satisfy_parent_and_two_stages_cover_both_periods(self):
docs=[self.root,*self.children]
queries=plan_missing_source_queries(docs,self.task,self.context)
self.assertEqual([q['stage'] for q in queries],['black','bright'])
for q in queries:
self.assertEqual(q['filters'],FILTERS);self.assertEqual(q['period'],CURRENT)
self.assertEqual(q['comparisonPeriod'],PREVIOUS);self.assertEqual(q['action'],'source')
self.assertFalse(coverage_sufficient(docs,self.task['question'],self.context))
docs.append(source_document('black'));self.assertFalse(coverage_sufficient(docs,self.task['question'],self.context))
self.assertEqual([q['stage'] for q in plan_missing_source_queries(docs,self.task,self.context)],['bright'])
docs.append(source_document('bright'));self.assertTrue(coverage_sufficient(docs,self.task['question'],self.context))
def test_missing_period_or_mismatched_source_stays_incomplete(self):
for doc in (source_document('bright',bad=True),source_document('bright',missing_comparison=True)):
docs=[self.root,source_document('black'),doc]
self.assertFalse(coverage_sufficient(docs,self.task['question'],self.context))
self.assertEqual([q['stage'] for q in plan_missing_source_queries(docs,self.task,self.context)],['bright'])
def test_runtime_executes_both_parent_reads_with_longer_bounded_timeout(self):
docs=[self.root,*self.children];seen=[];api=MagicMock()
def read(arguments):
seen.append((copy.deepcopy(arguments),api.timeout));result=source_document(arguments['stage']);docs.append(result);return result
request={'actorId':'actor','sessionId':'session','releaseId':RELEASE,'question':self.task['question'],'context':self.context}
with patch('nao_evidence_finalizer.evidence_summaries',side_effect=lambda *a,**k:list(docs)),\
patch('nao_agent_task.plan_repairs',side_effect=AssertionError('parent checks should be planned directly')),\
patch.object(runtime,'DelegatedApi') as delegated,patch('nao_quality_tools.QualityTools') as tools,\
patch('nao_runtime_policy.remaining_seconds',return_value=1000):
delegated.return_value.login.return_value=api;tools.return_value.quality_data.side_effect=read
runtime.complete_agent_evidence(task=self.task,request=request,actor_root=Path('/offline/session'),
access_path=Path('/offline/access'),manager=MagicMock(),connection=('offline','synthetic','m'),
accumulator=runtime.ConversationAccumulator(),prior_files=frozenset(),cancel_event=threading.Event(),emit=lambda e:None)
self.assertEqual([q['stage'] for q,_ in seen],['black','bright'])
self.assertTrue(all(q['filters']==FILTERS and timeout==180 for q,timeout in seen))
self.assertTrue(coverage_sufficient(docs,self.task['question'],self.context))
def test_finalization_time_reserve_prevents_false_completion(self):
docs=[self.root,*self.children];seen=[];api=MagicMock()
request={'actorId':'actor','sessionId':'session','releaseId':RELEASE,'question':self.task['question'],'context':self.context}
with patch('nao_evidence_finalizer.evidence_summaries',side_effect=lambda *a,**k:list(docs)),\
patch.object(runtime,'DelegatedApi') as delegated,patch('nao_quality_tools.QualityTools') as tools,\
patch('nao_runtime_policy.remaining_seconds',return_value=90):
delegated.return_value.login.return_value=api;tools.return_value.quality_data.side_effect=lambda args:seen.append(args)
runtime.complete_agent_evidence(task=self.task,request=request,actor_root=Path('/offline/session'),
access_path=Path('/offline/access'),manager=MagicMock(),connection=('offline','synthetic','m'),
accumulator=runtime.ConversationAccumulator(),prior_files=frozenset(),cancel_event=threading.Event(),emit=lambda e:None)
self.assertEqual(seen,[]);self.assertFalse(coverage_sufficient(docs,self.task['question'],self.context))
def test_each_http_request_recalculates_source_budget(self):
timeouts=[]
class Opener:
def open(self, request, timeout):
timeouts.append(timeout);return io.BytesIO(b'{}')
api=runtime.DelegatedApi('/offline/access',opener=Opener(),timeout=180)
api.deadline_reserve_seconds=90
access=SimpleNamespace(release_id=RELEASE,token='synthetic',query_url='http://127.0.0.1/offline')
with patch.object(runtime,'load_access_file',return_value=access),\
patch('nao_runtime_policy.remaining_seconds',side_effect=[250,100,90]):
api.call('/api/options?releaseId='+RELEASE,method='GET')
api.call('/api/options?releaseId='+RELEASE,method='GET')
with self.assertRaises(runtime.RuntimeErrorSafe):api.call('/api/options?releaseId='+RELEASE,method='GET')
self.assertEqual(timeouts,[160,10])
if __name__=='__main__':unittest.main()
@@ -0,0 +1,101 @@
"""Per-turn diagnostic evidence must stay private and independently readable."""
import json
import os
import sys
import tempfile
import threading
import unittest
import uuid
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_agent_task import TaskUnavailable
from nao_quality_runtime import RuntimeErrorSafe, plan_agent_task
class TaskDiagnosticHistoryTests(unittest.TestCase):
def test_success_and_failure_are_distinct_private_redacted_records(self):
with tempfile.TemporaryDirectory() as tmp:
private = Path(tmp) / 'private'
private.mkdir()
legacy = private / 'task-plan-failure.json'
first, second = str(uuid.uuid4()), str(uuid.uuid4())
def success(**kwargs):
kwargs['on_validation_error']({'attempt': 1, 'field': 'filterUpdates.product',
'receivedType': 'dict', 'expected': 'password=private',
'message': 'Bearer private-token', 'rawValue': 'secret-value'})
return {'kind': 'metrics'}
def failure(**kwargs):
kwargs['on_validation_error']({'attempt': 2, 'field': 'goal', 'receivedType': 'str',
'message': 'api_key=private-secret'})
raise TaskUnavailable('password=private-secret', field='goal', received={'token': 'raw'})
with patch('nao_agent_task.plan_task', side_effect=success):
self.assertEqual(plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id=first)['kind'], 'metrics')
with patch('nao_agent_task.plan_task', side_effect=failure):
with self.assertRaisesRegex(RuntimeErrorSafe, '尚未查询数据'):
plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id=second)
first_path = private / 'task-plan-turns' / (first + '.json')
second_path = private / 'task-plan-turns' / (second + '.json')
first_data, second_data = json.loads(first_path.read_text()), json.loads(second_path.read_text())
self.assertEqual(first_data['status'], 'planned')
self.assertEqual(first_data['attempts'][0]['field'], 'filterUpdates.product')
self.assertEqual(second_data['status'], 'failed')
self.assertEqual(second_data['failure']['field'], 'goal')
for path in (first_path, second_path):
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
for sensitive in ('private-secret', 'private-token', 'secret-value', 'password='):
self.assertNotIn(sensitive, path.read_text())
self.assertTrue(legacy.exists())
self.assertTrue((private / 'task-plan-validation.json').exists())
def test_concurrent_turns_keep_independent_attempt_history(self):
with tempfile.TemporaryDirectory() as tmp:
private = Path(tmp) / 'private'
private.mkdir()
legacy = private / 'task-plan-failure.json'
barrier = threading.Barrier(2)
ids = [str(uuid.uuid4()), str(uuid.uuid4())]
errors = []
def planner(**kwargs):
barrier.wait(timeout=5)
kwargs['on_validation_error']({'attempt': kwargs['marker'], 'field': 'goal', 'receivedType': 'str'})
return {'kind': 'metrics'}
def run(index):
try:
plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id=ids[index], marker=index + 1)
except Exception as error:
errors.append(error)
with patch('nao_agent_task.plan_task', side_effect=planner):
threads = [threading.Thread(target=run, args=(i,)) for i in range(2)]
for thread in threads: thread.start()
for thread in threads: thread.join(timeout=10)
self.assertFalse(errors)
for index, turn_id in enumerate(ids):
data = json.loads((private / 'task-plan-turns' / (turn_id + '.json')).read_text())
self.assertEqual(data['status'], 'planned')
self.assertEqual(data['attempts'][0]['attempt'], index + 1)
def test_invalid_turn_and_symlink_path_never_write_outside_private_directory(self):
with tempfile.TemporaryDirectory() as tmp:
private = Path(tmp) / 'private'
outside = Path(tmp) / 'outside'
private.mkdir(); outside.mkdir()
legacy = private / 'task-plan-failure.json'
with patch('nao_agent_task.plan_task', return_value={'kind': 'metrics'}):
with self.assertRaises(ValueError):
plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id='../outside/pwn')
os.symlink(outside, private / 'task-plan-turns')
with self.assertRaises(ValueError):
plan_agent_task(diagnostic_path=legacy, diagnostic_turn_id=str(uuid.uuid4()))
self.assertEqual(list(outside.iterdir()), [])
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,80 @@
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from analysis_visuals import build_visuals
from test_month_range_evidence import QUESTION, context, metric_doc
class VisualStageReferenceTests(unittest.TestCase):
def test_explicit_empty_denominators_stay_a_gap_instead_of_fallback_100(self):
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
doc = metric_doc(period, range(1, 6))
for row in doc['result']['trend']:
row['black']['totalWeight'] = 100
row['bright']['totalWeight'] = 100
invalid = doc['result']['trend'][1]
invalid.update(compositeRate=1)
invalid['black']['totalWeight'] = 0
invalid['bright']['totalWeight'] = None
chart = build_visuals([doc], QUESTION, 'turn-reference', context=context(period))['visuals'][0]
self.assertEqual(chart['series'][0]['values'], [.71, None, .73, .74, .75])
def test_unknown_source_counts_cannot_turn_into_a_100_percent_point(self):
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
doc = metric_doc(period, range(1, 6))
doc['result']['trend'][1].update(compositeRate=1, coverage={'blackRows': None, 'brightRows': None})
chart = build_visuals([doc], QUESTION, 'turn-reference', context=context(period))['visuals'][0]
self.assertIsNone(chart['series'][0]['values'][1])
def test_one_unknown_side_or_parent_source_count_stays_a_gap(self):
for change_row in (True, False):
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
doc = metric_doc(period, range(1, 6))
doc['result']['coverage']['sourceMonths'][1]['brightRows'] = None
if change_row:
doc['result']['trend'][1]['coverage']['brightRows'] = None
chart = build_visuals([doc], QUESTION, 'turn-reference', context=context(period))['visuals'][0]
self.assertIsNone(chart['series'][0]['values'][1])
def test_stage_rate_zero_denominator_is_a_gap_even_with_positive_source_rows(self):
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
frame = context(period)
frame['_agentTask']['stages'] = ['black']
doc = metric_doc(period, range(1, 6))
for row in doc['result']['trend']:
row['black']['totalWeight'] = 100
doc['result']['trend'][1]['black'].update(rate=1, totalWeight=0)
chart = build_visuals([doc], QUESTION, 'turn-reference', context=frame)['visuals'][0]
self.assertEqual(chart['metric'], 'black_rate')
self.assertIsNone(chart['series'][0]['values'][1])
def test_scrap_rate_zero_denominator_is_a_gap_and_real_zero_is_kept(self):
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 5}
frame = context(period)
frame['_agentTask'].update(stages=['black'], fields=['scrapRate'])
doc = metric_doc(period, range(1, 6))
for row in doc['result']['trend']:
row['black'].update(totalWeight=100, scrapWeight=10, scrapRate=.1)
doc['result']['trend'][1]['black'].update(totalWeight=0, scrapWeight=0, scrapRate=0)
doc['result']['trend'][2]['black'].update(totalWeight=100, scrapWeight=0, scrapRate=0)
chart = build_visuals([doc], QUESTION, 'turn-reference', context=frame)['visuals'][0]
self.assertEqual(chart['metric'], 'black_scrap_rate')
self.assertEqual(chart['series'][0]['values'], [.1, None, 0, .1, .1])
def test_real_zero_with_one_positive_stage_is_preserved_in_both_directions(self):
for stage, other in [('black', 'bright'), ('bright', 'black')]:
period = {'year': 2026, 'monthStart': 1, 'monthEnd': 3}
doc = metric_doc(period, range(1, 4))
for row in doc['result']['trend']:
row['compositeRate'] = 0
row[stage]['totalWeight'] = 100
row[other]['totalWeight'] = None
row['coverage'][other + 'Rows'] = 0
chart = build_visuals([doc], QUESTION, 'turn-reference', context=context(period))['visuals'][0]
self.assertEqual(chart['series'][0]['values'], [0, 0, 0])
if __name__ == '__main__':
unittest.main()
+328
View File
@@ -0,0 +1,328 @@
import copy
import json
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
from nao_quality_tools import QualityTools, QualityToolError
from test_nao_quality_tools import FakeApi, snapshot_period
from nao_agent_task import normalize_task, bind_task
from test_nao_agent_task import task_value
from analysis_visuals import build_visuals
from nao_evidence_finalizer import coverage_sufficient
from nao_protocol_execution import execute
from nao_weight_ranking import rank_response, ranking_chart, ranking_report, missing_ranking_visuals
QUESTION = '做几张26年的亮片生产重量柏拉图,产品级、产业部级'
ACTOR='11111111-1111-4111-8111-111111111111'
SESSION='22222222-2222-4222-8222-222222222222'
TURN='55555555-5555-4555-8555-555555555555'
def signed_response(positive_rows=872, unavailable_rows=167, *, top_weight=None, metric='totalWeight', stage='black'):
period={'year':2026,'monthStart':1,'monthEnd':9}
total=sum([top_weight] if top_weight is not None else [])
rows=[]
for index in range(positive_rows):
weight=(top_weight if index==0 and top_weight is not None else 1)
total+=weight if not (index==0 and top_weight is not None) else 0
rows.append({'key':f'product-{index:04d}','label':f'规格{index:04d}',
stage:{metric:weight},'coverage':{stage+'Rows':1}})
rows.append({'key':'mimo-negative','label':'mimo-negative',stage:{metric:-10.05},'coverage':{stage+'Rows':1}})
for index in range(unavailable_rows):
rows.append({'key':f'no-stage-{index:04d}','label':f'未覆盖{index:04d}',stage:{metric:None},
'coverage':{stage+'Rows':0}})
total-=10.05
months=[{'year':2026,'month':month,'blackRows':1 if stage=='black' else 0,
'brightRows':1 if stage=='bright' else 0} for month in range(1,10)]
summary={stage:{metric:total}}
return {'rows':rows,'context':{'rowCount':len(rows)},'summary':summary,
'coverage':{'rawRows':len(rows),'includedRows':len(rows),'sourceMonths':months}},period
class WeightApi(FakeApi):
def __init__(self, *, partial_year=True, hole=False, mismatch=False):
super().__init__()
self.partial_year, self.hole, self.mismatch = partial_year, hole, mismatch
self.metric_query = 0
def call(self, path, payload=None, method=None):
result = super().call(path, payload, method)
if path == '/api/analysis/tools/catalog':
result['sourceFields'] = [{'key':'customer_code','label':'客户编码','type':'source_value',
'stages':['bright'],'columns':{'bright':'客户编码'},'operators':['in']}]
if path.startswith('/api/metrics?'):
self.metric_query += 1
params = parse_qs(urlsplit(path).query)
for month in result['coverage']['sourceMonths']:month['brightRows']=12
rows = []
for label, weight, scrap in [('scrap-leader', 10, 9), ('production-leader', 80, 1), ('third', 60, 2)]:
row = self._product_row(label, label, 1, scrap)
row['bright']['totalWeight'] = weight
rows.append(row)
result['rows'] = rows
result['context']['rowCount'] = 3
result['summary']['bright']['totalWeight'] = 151 if self.mismatch else 150
result['summary']['bright']['scrapWeight'] = 12
if self.partial_year:
result['coverage']['sourceMonths'] = result['coverage']['sourceMonths'][:9]
result['coverage']['missingStages'] = [{'year':2026,'month':m,'stage':'bright'} for m in (10,11,12)]
if self.hole:
result['coverage']['sourceMonths'] = [m for m in result['coverage']['sourceMonths'] if m['month'] != 5]
if 'monthStart' in params:
result['context']['monthStart'] = int(params['monthStart'][0])
result['context']['monthEnd'] = int(params['monthEnd'][0])
if 'columnFilters' in params:
result['filters'] = {'columnFilters':json.loads(params['columnFilters'][0])}
result['protocolEvidence'] = {'queryId':f'rank-query-{self.metric_query}',
'attemptId':'attempt','sessionId':'session','turnId':'turn','revision':1,
'executionId':'execution','responseHash':'a'*64}
return result
class NegativeWeightApi(WeightApi):
def call(self, path, payload=None, method=None):
result=super().call(path,payload,method)
if path.startswith('/api/metrics?'):
params=parse_qs(urlsplit(path).query)
if params.get('view',[''])[0]=='product':
rows=[]
for key,label,weight in (('large','大重量规格',100),('small','小重量规格',1),('mimo','mimo修正',-10.05)):
row=self._product_row(key,label,1,1)
row['black']['totalWeight']=weight
rows.append(row)
result['rows']=rows;result['context']['rowCount']=3
result['summary']['black']['totalWeight']=90.95
for month in result['coverage']['sourceMonths']:month['blackRows']=1
result['protocolEvidence']={'attemptId':SESSION,'sessionId':ACTOR,'turnId':TURN,
'revision':3,'executionId':TURN,'queryId':'signed-rank-query','responseHash':'b'*64}
return result
def planned(**updates):
value={'dimensions':['product','department'],'stages':['bright'],'fields':['totalWeight'],
'period':{'year':2026},'visualPreference':'required','visualType':'pareto','periodCoverage':'available'}
value.update(updates)
return normalize_task(task_value(**value), QUESTION)
class WeightRankingTests(unittest.TestCase):
def run_rank(self, metric='totalWeight', **options):
api = WeightApi(**options)
with tempfile.TemporaryDirectory() as folder:
QualityTools(api_factory=lambda:api, evidence_dir=Path(folder)).quality_data({
'mode':'rank','releaseId':'release-a','period':{'year':2026},
'dimension':'product','stage':'bright','metric':metric,'limit':2})
doc = json.loads(next(Path(folder).glob('nao-quality-*.json')).read_text())
return doc, api
def test_ranks_all_products_by_requested_stage_weight_before_top_cut(self):
doc, api = self.run_rank()
result = doc['result']
self.assertEqual([row['label'] for row in result['rows']], ['production-leader','third'])
self.assertEqual(result['totalWeight'], 150)
self.assertEqual(result['rows'][0]['share'], 80/150)
self.assertAlmostEqual(result['rows'][-1]['cumulativeShare'], 140/150)
self.assertLess(result['rows'][-1]['cumulativeShare'], 1)
self.assertEqual(result['rankingCoverage']['groups'], 3)
self.assertFalse(any('rowLimit=' in path for _,path,_ in api.calls if path.startswith('/api/metrics')))
def test_black_total_weight_signed_reconciliation_keeps_all_absent_groups_auditable(self):
response,period=signed_response()
result=rank_response(response,{'id':'release-a','name':'A','status':'PUBLISHED'},period,{},
'product','black','totalWeight',10)
audit=result['rankingCoverage']
self.assertEqual(response['context']['rowCount'],1040)
self.assertEqual(audit['sourceGroups'],1040)
self.assertEqual(audit['groups'],873)
self.assertEqual(audit['unavailableGroups'],167)
self.assertEqual(audit['positiveGroups'],872)
self.assertEqual(audit['negativeGroups'],1)
self.assertEqual(audit['negativeWeight'],-10.05)
self.assertEqual(audit['negativeWeightMagnitude'],10.05)
self.assertEqual(result['totalWeight'],861.95)
self.assertEqual(result['rows'][0]['weight'],1)
self.assertTrue(all(row['weight']>0 for row in result['rows']))
self.assertAlmostEqual(result['rows'][0]['share'],1/861.95)
doc={'evidenceRef':'QSIGNED10','result':result}
task=planned(dimensions=['product'],stages=['black'],fields=['totalWeight'],period=period)
chart=ranking_chart(doc,'signed-black-total',task)
self.assertIsNotNone(chart)
self.assertIn('1项负向投入重量修正',chart['note'])
bad=copy.deepcopy(response);bad['summary']['black']['totalWeight']+=.1
with self.assertRaisesRegex(QualityToolError,'合计与同范围正式总量不一致'):
rank_response(bad,{'id':'release-a'},period,{},'product','black','totalWeight',10)
def test_negative_weights_remain_rejected_outside_black_total_weight(self):
for stage,metric in (('black','scrapWeight'),('bright','totalWeight'),('bright','scrapWeight')):
with self.subTest(stage=stage,metric=metric):
response={'rows':[{'key':'negative','label':'负值',stage:{metric:-1},'coverage':{stage+'Rows':1}}],
'context':{'rowCount':1},'summary':{stage:{metric:10}},
'coverage':{'sourceMonths':[{'year':2026,'month':month,'blackRows':1,'brightRows':1}
for month in range(1,10)]}}
with self.assertRaisesRegex(QualityToolError,'重量不能为负'):
rank_response(response,{'id':'release-a'},{'year':2026,'monthStart':1,'monthEnd':9},
{},'product',stage,metric,10)
def test_signed_black_total_reports_correction_and_blocks_over_100_chart_as_partial(self):
response,period=signed_response(positive_rows=2,unavailable_rows=0,top_weight=100)
result=rank_response(response,{'id':'release-a','name':'A','status':'PUBLISHED'},period,{},
'product','black','totalWeight',10)
self.assertEqual(result['totalWeight'],90.95)
self.assertEqual([row['label'] for row in result['rows']],['规格0000','规格0001'])
self.assertGreater(result['rows'][-1]['cumulativeShare'],1)
document={'evidenceRef':'QNEGATIVE1','result':result}
task=planned(dimensions=['product'],stages=['black'],fields=['totalWeight'],period=period,
visualPreference='required')
context=bind_task({'releaseId':'release-a'},task)
self.assertIsNone(ranking_chart(document,'negative-correction',task))
gap=missing_ranking_visuals(task,[],[document])
self.assertEqual(len(gap),1)
self.assertIn('负向修正使正重量Top合计超过黑片净总重',gap[0])
report=ranking_report([document],task,context)
self.assertIn('1项负向投入重量修正,净额-10.05 kg仍计入正式净总重分母',report['facts'][0]['text'])
section=report['sections'][0]
self.assertGreater(section.index('排名表只展示正重量Top'),section.index('| 1 |'))
def test_protocol_execution_keeps_signed_denominator_but_returns_partial_without_over_100_chart(self):
t=normalize_task(task_value(kind='analysis',dimensions=['product'],stages=['black'],
dimensionStages={'product':['black']},fields=['totalWeight'],period={'year':2026},
trendRequired=False,visualPreference='auto',visualType='pareto'),
'按产品查看黑片投入重量排名',{'_protocol':{'version':1}})
if t['filters'].get('columnFilters')==[]:t['filters'].pop('columnFilters')
api=NegativeWeightApi()
with tempfile.TemporaryDirectory() as folder,patch('nao_evidence_finalizer.synthesize',return_value={
'answer':'已核验黑片修正重量排名','evidenceSufficient':True,'missingEvidence':[],'usage':{}}):
root=Path(folder);(root/'evidence').mkdir();(root/'private').mkdir()
result=execute(request={'releaseId':'release-a','turnId':TURN,'question':t['question']},task=t,
authority={'attemptId':SESSION,'sessionId':ACTOR,'turnId':TURN,'revision':3,'executionId':TURN},
actor_root=root,delegated=api,connection=('offline','synthetic','model'),
cancel_event=threading.Event(),emit=lambda _:None)
doc=json.loads(next((root/'evidence').glob('nao-quality-*.json')).read_text())
self.assertEqual(doc['result']['totalWeight'],90.95)
self.assertEqual(doc['result']['rankingCoverage']['negativeGroups'],1)
self.assertEqual(doc['result']['rankingCoverage']['negativeWeight'],-10.05)
self.assertEqual(result['status'],'PARTIAL')
self.assertTrue(any('负向修正使正重量Top合计超过黑片净总重' in item
for item in result['evidenceCompletion']['missingEvidence']))
self.assertFalse(result['visuals'])
def test_rank_retains_column_filter_month_range_and_execution_proof(self):
api=WeightApi()
rule={'field':'customer_code','stages':['bright'],'operator':'in','values':['0012']}
with tempfile.TemporaryDirectory() as folder:
QualityTools(api_factory=lambda:api,evidence_dir=Path(folder)).quality_data({
'mode':'rank','releaseId':'release-a','period':{'year':2026,'monthStart':1,'monthEnd':9},
'filters':{},'columnFilters':[rule],'dimension':'product','stage':'bright',
'metric':'totalWeight','limit':2})
doc=json.loads(next(Path(folder).glob('nao-quality-*.json')).read_text())
result=doc['result']
self.assertEqual(result['filters']['columnFilters'],[rule])
self.assertEqual(result['period'],{'year':2026,'monthStart':1,'monthEnd':9})
self.assertEqual(result['periodCoverage']['availableMonths'],list(range(1,10)))
self.assertTrue(result['periodCoverage']['complete'])
self.assertEqual(result['protocolEvidence']['queryId'],'rank-query-1')
metrics_paths=[path for _,path,_ in api.calls if path.startswith('/api/metrics')]
self.assertEqual(len(metrics_paths),1)
self.assertIn('columnFilters=',metrics_paths[0])
self.assertNotIn('rowLimit=',metrics_paths[0])
self.assertNotIn('rowOffset=',metrics_paths[0])
def test_rank_only_allows_column_filter_parameter_and_other_modes_reject_it(self):
api=WeightApi()
with tempfile.TemporaryDirectory() as folder:
tools=QualityTools(api_factory=lambda:api,evidence_dir=Path(folder))
with self.assertRaisesRegex(QualityToolError,'query/compare/rank'):
tools.quality_data({'mode':'changes','releaseId':'release-a','period':{'year':2026},
'comparisonPeriod':{'year':2025},'dimension':'group','columnFilters':[]})
self.assertEqual(api.calls,[])
def test_ranking_chart_scope_preserves_exact_continuous_month_range(self):
from nao_weight_ranking import ranking_chart
doc,_=self.run_rank()
doc['result']['period']={'year':2026,'monthStart':3,'monthEnd':5}
t=planned();t['period']=doc['result']['period']
chart=ranking_chart(doc,'range-ranking-turn',t)
self.assertIsNotNone(chart)
self.assertEqual({key:chart['scope'][key] for key in ('year','monthStart','monthEnd')},t['period'])
def test_pareto_keeps_top_rows_but_binds_ratio_to_full_population(self):
doc, _ = self.run_rank()
task = planned()
context = bind_task({'releaseId':'release-a'},task)
output = build_visuals([doc],QUESTION,'turn-ranking',context=context)
self.assertEqual(output['visualState'],'ready')
chart = output['visuals'][0]
self.assertEqual(chart['type'],'pareto')
self.assertEqual(chart['coverage']['denominator'],150)
self.assertEqual(chart['coverage']['shareBasis'],'all_scope_weight')
self.assertIn('1—9月',chart['subtitle'])
self.assertEqual(len(chart['categories']),2)
department=copy.deepcopy(doc);department['result']['dimension']='department';department['evidenceRef']='Q00000002'
output=build_visuals([doc,department],QUESTION,'turn-ranking',context=context)
self.assertEqual({v['dimension'] for v in output['visuals']},{'product','department'})
from nao_weight_ranking import missing_ranking_visuals
self.assertEqual(missing_ranking_visuals(task,output['visuals']),[])
self.assertTrue(missing_ranking_visuals(task,output['visuals'][:1]))
def test_pareto_emits_each_requested_weight_field_for_the_same_stage(self):
total_doc,_=self.run_rank()
scrap_doc,_=self.run_rank(metric='scrapWeight')
task=planned(fields=['totalWeight','scrapWeight'],dimensions=['product'],visualPreference='auto')
context=bind_task({'releaseId':'release-a'},task)
output=build_visuals([total_doc,scrap_doc],QUESTION,'two-field-ranking',context=context)
self.assertEqual(output['visualState'],'ready')
self.assertEqual({visual['metric'] for visual in output['visuals']},
{'bright_total_weight','bright_scrap_weight'})
from nao_weight_ranking import missing_ranking_visuals
self.assertEqual(missing_ranking_visuals(task,output['visuals']),[])
def test_missing_dimension_is_not_completion_and_year_data_is_not_called_absent(self):
doc, _ = self.run_rank()
task = planned()
context = bind_task({'releaseId':'release-a'},task)
self.assertFalse(coverage_sufficient([doc],QUESTION,context))
department = copy.deepcopy(doc)
department['result']['dimension'] = 'department'
department['evidenceRef'] = 'Q00000002'
self.assertTrue(coverage_sufficient([doc,department],QUESTION,context))
complete = planned(periodCoverage='complete')
self.assertFalse(coverage_sufficient([doc,department],QUESTION,bind_task({'releaseId':'release-a'},complete)))
def test_internal_month_holes_and_total_mismatch_do_not_pass_as_complete(self):
doc, _ = self.run_rank(hole=True)
context = bind_task({'releaseId':'release-a'},planned())
self.assertFalse(coverage_sufficient([doc],QUESTION,context))
with self.assertRaises(QualityToolError):self.run_rank(mismatch=True)
def test_enum_strings_are_normalized_without_guessing_business_intent(self):
task = normalize_task(task_value(dimensions='product,department',stages='bright',fields='totalWeight'),QUESTION)
self.assertEqual(task['dimensions'],['product','department'])
self.assertEqual(task['stages'],['bright'])
def test_each_ranking_dimension_uses_only_its_declared_stage(self):
doc, _ = self.run_rank()
department = copy.deepcopy(doc)
department['result'].update(dimension='department', stage='black')
department['evidenceRef'] = 'Q00000002'
task = planned()
task['stages'] = ['black', 'bright']
task['dimensionStages'] = {'product': ['bright'], 'department': ['black']}
context = bind_task({'releaseId': 'release-a'}, task)
self.assertTrue(coverage_sufficient([doc, department], QUESTION, context))
charts = build_visuals([doc, department], QUESTION, 'mixed-stage-ranking', context=context)['visuals']
self.assertEqual({(v['dimension'], v['stage']) for v in charts}, {('product', 'bright'), ('department', 'black')})
self.assertTrue(all(v['coverage']['denominator'] == 150 for v in charts))
from nao_weight_ranking import missing_ranking_visuals
self.assertEqual(missing_ranking_visuals(task, charts), [])
wrong = copy.deepcopy(department); wrong['result']['stage'] = 'bright'
self.assertFalse(coverage_sufficient([doc, wrong], QUESTION, context))
if __name__ == '__main__':unittest.main()
+13
View File
@@ -0,0 +1,13 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { defectWeightRate } from '../web/src/trendMetrics.ts';
test('defect rate uses total stage weight rather than the sum of defect categories', () => {
assert.equal(defectWeightRate(60, 3000), 0.02);
assert.notEqual(defectWeightRate(60, 3000), 0.6);
});
test('zero and missing denominators are unavailable, while zero defects is zero', () => {
assert.equal(defectWeightRate(0, 1000), 0);
for (const total of [0, null, undefined, NaN]) assert.equal(defectWeightRate(60, total), null);
assert.equal(defectWeightRate(null, 1000), null);
});